From ddc0487f74277eb81b0b4ab68178e296ace6abc4 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 11:46:20 -0400 Subject: [PATCH 01/22] feat(schema): can:// id + byte-offset span utils for v2 (#180) --- src/main/java/com/ibm/cldk/schema/CanId.java | 39 ++++++++++ src/main/java/com/ibm/cldk/schema/Spans.java | 75 +++++++++++++++++++ .../java/com/ibm/cldk/schema/CanIdTest.java | 45 +++++++++++ .../java/com/ibm/cldk/schema/SpansTest.java | 51 +++++++++++++ 4 files changed, 210 insertions(+) create mode 100644 src/main/java/com/ibm/cldk/schema/CanId.java create mode 100644 src/main/java/com/ibm/cldk/schema/Spans.java create mode 100644 src/test/java/com/ibm/cldk/schema/CanIdTest.java create mode 100644 src/test/java/com/ibm/cldk/schema/SpansTest.java diff --git a/src/main/java/com/ibm/cldk/schema/CanId.java b/src/main/java/com/ibm/cldk/schema/CanId.java new file mode 100644 index 0000000..f35f36f --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/CanId.java @@ -0,0 +1,39 @@ +package com.ibm.cldk.schema; + +/** + * Canonical {@code can://} id construction for schema v2. + * + *

Durable ids (≥ callable) are containment paths + * {@code can://java////}; ordinal ids (< callable) are + * {@code @} where {@code } is a source position {@code line:col} (real + * nodes) or a synthetic tag (e.g. {@code entry}). Pure functions; ids are opaque handles (the + * {@code } segment itself may contain {@code /}). + */ +public final class CanId { + + private CanId() {} + + /** The scheme + language segment for this analyzer's ids. */ + public static final String SCHEME = "can://java"; + + /** {@code can://java/}. */ + public static String applicationId(String appName) { + return SCHEME + "/" + appName; + } + + /** {@code /} (separators normalized to {@code /}). */ + public static String moduleId(String applicationId, String fileKey) { + String rel = fileKey.replace("\\", "/").replaceFirst("^[./]+", ""); + return applicationId + "/" + rel; + } + + /** {@code /} — one downward step in the containment path. */ + public static String childId(String parentId, String segment) { + return parentId + "/" + segment; + } + + /** {@code @} — an ordinal id for a body node within a callable. */ + public static String ordinalId(String callableId, String tag) { + return callableId + "@" + tag; + } +} diff --git a/src/main/java/com/ibm/cldk/schema/Spans.java b/src/main/java/com/ibm/cldk/schema/Spans.java new file mode 100644 index 0000000..15fb913 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/Spans.java @@ -0,0 +1,75 @@ +package com.ibm.cldk.schema; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * UTF-8 byte-offset computation for schema v2 {@code span.bytes}. + * + *

Converts source positions given as a 1-based line and a 0-based character column into byte + * offsets into the (UTF-8) module source. {@code span.bytes} carries these alongside + * {@code line:col} so the SDK can slice a node's text as {@code module.source[from:to]} in O(1). + * The column is a character offset within the line (multibyte characters count as one + * column but contribute their full UTF-8 width to the byte offset). + */ +public final class Spans { + + private Spans() {} + + /** Byte offset into {@code source} of the position at (1-based {@code line}, 0-based {@code col}). */ + public static int byteOffset(String source, int line, int col) { + List lines = splitKeepEnds(source); + int prefixBytes = 0; + for (int k = 0; k < line - 1 && k < lines.size(); k++) { + prefixBytes += utf8Length(lines.get(k)); + } + String current = (line - 1 >= 0 && line - 1 < lines.size()) ? lines.get(line - 1) : ""; + int c = Math.max(0, Math.min(col, current.length())); + return prefixBytes + utf8Length(current.substring(0, c)); + } + + /** {@code [from, to]} byte offsets for a span from (startLine,startCol) to (endLine,endCol). */ + public static int[] byteOffsets(String source, int startLine, int startCol, int endLine, int endCol) { + return new int[] {byteOffset(source, startLine, startCol), byteOffset(source, endLine, endCol)}; + } + + private static int utf8Length(String s) { + return s.getBytes(StandardCharsets.UTF_8).length; + } + + /** + * Split into lines keeping their terminators (universal newlines: {@code \n}, + * {@code \r\n}, {@code \r}), mirroring Python's {@code splitlines(keepends=True)}. A final line + * without a terminator is included. + */ + private static List splitKeepEnds(String s) { + List out = new ArrayList<>(); + int n = s.length(); + int start = 0; + int i = 0; + while (i < n) { + char ch = s.charAt(i); + if (ch == '\n') { + out.add(s.substring(start, i + 1)); + i++; + start = i; + } else if (ch == '\r') { + if (i + 1 < n && s.charAt(i + 1) == '\n') { + out.add(s.substring(start, i + 2)); + i += 2; + } else { + out.add(s.substring(start, i + 1)); + i++; + } + start = i; + } else { + i++; + } + } + if (start < n) { + out.add(s.substring(start)); + } + return out; + } +} diff --git a/src/test/java/com/ibm/cldk/schema/CanIdTest.java b/src/test/java/com/ibm/cldk/schema/CanIdTest.java new file mode 100644 index 0000000..0cbea45 --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/CanIdTest.java @@ -0,0 +1,45 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Tests for canonical schema v2 {@code can://} id construction (see the design spec, + * decision D8: {@code can://java////} with {@code @} ordinals). + */ +class CanIdTest { + + @Test + void applicationId_buildsCanJavaScheme() { + assertEquals("can://java/myapp", CanId.applicationId("myapp")); + } + + @Test + void moduleId_appendsRelativeFileKey() { + assertEquals( + "can://java/myapp/src/main/java/Foo.java", + CanId.moduleId("can://java/myapp", "src/main/java/Foo.java")); + } + + @Test + void moduleId_normalizesBackslashesAndLeadingDotSlash() { + assertEquals( + "can://java/myapp/a/b/C.java", + CanId.moduleId("can://java/myapp", "./a\\b\\C.java")); + } + + @Test + void childId_appendsSegmentWithSlash() { + assertEquals( + "can://java/myapp/src/Foo.java/com.example.Foo", + CanId.childId("can://java/myapp/src/Foo.java", "com.example.Foo")); + } + + @Test + void ordinalId_appendsTagAfterAt() { + String callableId = "can://java/myapp/src/Foo.java/com.example.Foo/bar(int)"; + assertEquals(callableId + "@15:2", CanId.ordinalId(callableId, "15:2")); + assertEquals(callableId + "@entry", CanId.ordinalId(callableId, "entry")); + } +} diff --git a/src/test/java/com/ibm/cldk/schema/SpansTest.java b/src/test/java/com/ibm/cldk/schema/SpansTest.java new file mode 100644 index 0000000..1b0653d --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/SpansTest.java @@ -0,0 +1,51 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Tests for UTF-8 byte-offset computation used by schema v2 {@code span.bytes}. Contract mirrors + * the Python pilot's {@code byte_offsets}: input is a 1-based line and a 0-based character column + * (the offset of the char before which the position sits); output is a UTF-8 byte offset + * into the module source, so {@code module.source[bytes]} slices the node's text. + */ +class SpansTest { + + @Test + void byteOffset_asciiStartOfFile() { + assertEquals(0, Spans.byteOffset("abc\ndef\n", 1, 0)); + } + + @Test + void byteOffset_asciiWithinFirstLine() { + assertEquals(3, Spans.byteOffset("abc\ndef\n", 1, 3)); + } + + @Test + void byteOffset_secondLineCountsPriorNewline() { + assertEquals(4, Spans.byteOffset("abc\ndef\n", 2, 0)); + assertEquals(7, Spans.byteOffset("abc\ndef\n", 2, 3)); + } + + @Test + void byteOffset_multibyteColumnIsCharsButResultIsBytes() { + // 'é' is one character but two bytes in UTF-8. + String src = "é = 1\n"; + assertEquals(2, Spans.byteOffset(src, 1, 1)); // after 'é' + assertEquals(4, Spans.byteOffset(src, 1, 3)); // after "é =" + } + + @Test + void byteOffset_priorMultibyteLineBytesCounted() { + String src = "é\nx\n"; + assertEquals(3, Spans.byteOffset(src, 2, 0)); // "é\n" = 2 + 1 bytes + assertEquals(4, Spans.byteOffset(src, 2, 1)); // + "x" + } + + @Test + void byteOffsets_returnsFromToPair() { + assertArrayEquals(new int[] {0, 3}, Spans.byteOffsets("abc\ndef\n", 1, 0, 1, 3)); + } +} From 84112b1f9d8469db7875f31a93157e3d2bffc62e Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 14:08:21 -0400 Subject: [PATCH 02/22] test: add real-world Java app fixtures as git submodules (#180) --- .gitmodules | 27 +++++++++++++++++++ .../resources/test-applications/cargotracker | 1 + .../resources/test-applications/commons-lang | 1 + .../test-applications/daytrader-microservices | 1 + .../quarkuscoffeeshop-barista | 1 + .../quarkuscoffeeshop-counter | 1 + .../quarkuscoffeeshop-domain | 1 + .../quarkuscoffeeshop-inventory | 1 + .../quarkuscoffeeshop-kitchen | 1 + .../test-applications/spring-petclinic | 1 + 10 files changed, 36 insertions(+) create mode 100644 .gitmodules create mode 160000 src/test/resources/test-applications/cargotracker create mode 160000 src/test/resources/test-applications/commons-lang create mode 160000 src/test/resources/test-applications/daytrader-microservices create mode 160000 src/test/resources/test-applications/quarkuscoffeeshop-barista create mode 160000 src/test/resources/test-applications/quarkuscoffeeshop-counter create mode 160000 src/test/resources/test-applications/quarkuscoffeeshop-domain create mode 160000 src/test/resources/test-applications/quarkuscoffeeshop-inventory create mode 160000 src/test/resources/test-applications/quarkuscoffeeshop-kitchen create mode 160000 src/test/resources/test-applications/spring-petclinic diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b78ff42 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,27 @@ +[submodule "src/test/resources/test-applications/spring-petclinic"] + path = src/test/resources/test-applications/spring-petclinic + url = https://github.com/spring-projects/spring-petclinic.git +[submodule "src/test/resources/test-applications/cargotracker"] + path = src/test/resources/test-applications/cargotracker + url = https://github.com/eclipse-ee4j/cargotracker.git +[submodule "src/test/resources/test-applications/commons-lang"] + path = src/test/resources/test-applications/commons-lang + url = https://github.com/apache/commons-lang.git +[submodule "src/test/resources/test-applications/daytrader-microservices"] + path = src/test/resources/test-applications/daytrader-microservices + url = https://github.com/sample-daytrader/sample.daytrader.microservices.git +[submodule "src/test/resources/test-applications/quarkuscoffeeshop-counter"] + path = src/test/resources/test-applications/quarkuscoffeeshop-counter + url = https://github.com/quarkuscoffeeshop/quarkuscoffeeshop-counter.git +[submodule "src/test/resources/test-applications/quarkuscoffeeshop-barista"] + path = src/test/resources/test-applications/quarkuscoffeeshop-barista + url = https://github.com/quarkuscoffeeshop/quarkuscoffeeshop-barista.git +[submodule "src/test/resources/test-applications/quarkuscoffeeshop-kitchen"] + path = src/test/resources/test-applications/quarkuscoffeeshop-kitchen + url = https://github.com/quarkuscoffeeshop/quarkuscoffeeshop-kitchen.git +[submodule "src/test/resources/test-applications/quarkuscoffeeshop-inventory"] + path = src/test/resources/test-applications/quarkuscoffeeshop-inventory + url = https://github.com/quarkuscoffeeshop/quarkuscoffeeshop-inventory.git +[submodule "src/test/resources/test-applications/quarkuscoffeeshop-domain"] + path = src/test/resources/test-applications/quarkuscoffeeshop-domain + url = https://github.com/quarkuscoffeeshop/quarkuscoffeeshop-domain.git diff --git a/src/test/resources/test-applications/cargotracker b/src/test/resources/test-applications/cargotracker new file mode 160000 index 0000000..4d26b8f --- /dev/null +++ b/src/test/resources/test-applications/cargotracker @@ -0,0 +1 @@ +Subproject commit 4d26b8fd59a7f0bac6e42d5d19a8cd1b379353b5 diff --git a/src/test/resources/test-applications/commons-lang b/src/test/resources/test-applications/commons-lang new file mode 160000 index 0000000..e66ad3d --- /dev/null +++ b/src/test/resources/test-applications/commons-lang @@ -0,0 +1 @@ +Subproject commit e66ad3dd2e8538e24940d97179c960a60dd25495 diff --git a/src/test/resources/test-applications/daytrader-microservices b/src/test/resources/test-applications/daytrader-microservices new file mode 160000 index 0000000..8a68b59 --- /dev/null +++ b/src/test/resources/test-applications/daytrader-microservices @@ -0,0 +1 @@ +Subproject commit 8a68b59430a94a242c54384763da9eb7682728b4 diff --git a/src/test/resources/test-applications/quarkuscoffeeshop-barista b/src/test/resources/test-applications/quarkuscoffeeshop-barista new file mode 160000 index 0000000..bb1f5af --- /dev/null +++ b/src/test/resources/test-applications/quarkuscoffeeshop-barista @@ -0,0 +1 @@ +Subproject commit bb1f5afc8b9c911ffe0634bfd7e6af42a0a738e2 diff --git a/src/test/resources/test-applications/quarkuscoffeeshop-counter b/src/test/resources/test-applications/quarkuscoffeeshop-counter new file mode 160000 index 0000000..a4b5d17 --- /dev/null +++ b/src/test/resources/test-applications/quarkuscoffeeshop-counter @@ -0,0 +1 @@ +Subproject commit a4b5d171c1dd72b09de71fa1def03ea53a8dcc89 diff --git a/src/test/resources/test-applications/quarkuscoffeeshop-domain b/src/test/resources/test-applications/quarkuscoffeeshop-domain new file mode 160000 index 0000000..e7e7cc7 --- /dev/null +++ b/src/test/resources/test-applications/quarkuscoffeeshop-domain @@ -0,0 +1 @@ +Subproject commit e7e7cc7b80f6b654557a97a297fadf5924c18238 diff --git a/src/test/resources/test-applications/quarkuscoffeeshop-inventory b/src/test/resources/test-applications/quarkuscoffeeshop-inventory new file mode 160000 index 0000000..1462a77 --- /dev/null +++ b/src/test/resources/test-applications/quarkuscoffeeshop-inventory @@ -0,0 +1 @@ +Subproject commit 1462a77780bcce1addba7290d51cf0a66188e4f9 diff --git a/src/test/resources/test-applications/quarkuscoffeeshop-kitchen b/src/test/resources/test-applications/quarkuscoffeeshop-kitchen new file mode 160000 index 0000000..19b6bfb --- /dev/null +++ b/src/test/resources/test-applications/quarkuscoffeeshop-kitchen @@ -0,0 +1 @@ +Subproject commit 19b6bfba229be3009df1b1b5e25f5a9b8df8caea diff --git a/src/test/resources/test-applications/spring-petclinic b/src/test/resources/test-applications/spring-petclinic new file mode 160000 index 0000000..88e37c1 --- /dev/null +++ b/src/test/resources/test-applications/spring-petclinic @@ -0,0 +1 @@ +Subproject commit 88e37c15cf6fc8490b01bc3e8e2c800cec1ac272 From fa147f1de3069f25056bfc64815095e861ce4848 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 14:08:21 -0400 Subject: [PATCH 03/22] =?UTF-8?q?feat(schema):=20v2=20L1=20module/type=20t?= =?UTF-8?q?ree=20=E2=80=94=20AST-driven=20modular=20builders=20(#180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/ibm/cldk/schema/Analysis.java | 16 ++++ .../com/ibm/cldk/schema/JApplication.java | 13 ++++ .../java/com/ibm/cldk/schema/JDecorator.java | 17 ++++ .../java/com/ibm/cldk/schema/JModule.java | 25 ++++++ src/main/java/com/ibm/cldk/schema/JType.java | 20 +++++ src/main/java/com/ibm/cldk/schema/Span.java | 15 ++++ .../java/com/ibm/cldk/schema/V2Emitter.java | 36 +++++++++ .../syntactic_analysis/DecoratorBuilder.java | 39 ++++++++++ .../syntactic_analysis/L1BuildContext.java | 51 ++++++++++++ .../syntactic_analysis/ModuleBuilder.java | 39 ++++++++++ .../cldk/syntactic_analysis/TypeBuilder.java | 73 +++++++++++++++++ .../com/ibm/cldk/schema/V2EmitterTest.java | 30 +++++++ .../syntactic_analysis/ModuleBuilderTest.java | 63 +++++++++++++++ .../syntactic_analysis/TypeBuilderTest.java | 78 +++++++++++++++++++ 14 files changed, 515 insertions(+) create mode 100644 src/main/java/com/ibm/cldk/schema/Analysis.java create mode 100644 src/main/java/com/ibm/cldk/schema/JApplication.java create mode 100644 src/main/java/com/ibm/cldk/schema/JDecorator.java create mode 100644 src/main/java/com/ibm/cldk/schema/JModule.java create mode 100644 src/main/java/com/ibm/cldk/schema/JType.java create mode 100644 src/main/java/com/ibm/cldk/schema/Span.java create mode 100644 src/main/java/com/ibm/cldk/schema/V2Emitter.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/DecoratorBuilder.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java create mode 100644 src/test/java/com/ibm/cldk/schema/V2EmitterTest.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java diff --git a/src/main/java/com/ibm/cldk/schema/Analysis.java b/src/main/java/com/ibm/cldk/schema/Analysis.java new file mode 100644 index 0000000..2b73d3c --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/Analysis.java @@ -0,0 +1,16 @@ +package com.ibm.cldk.schema; + +import lombok.Data; + +/** + * The canonical schema v2 payload root (the envelope): manifest fields plus the {@code application} + * tree node. Serialized with Gson's {@code LOWER_CASE_WITH_UNDERSCORES} policy, so + * {@code schemaVersion} → {@code schema_version}, {@code maxLevel} → {@code max_level}, etc. + */ +@Data +public class Analysis { + private String schemaVersion; + private String language; + private int maxLevel; + private JApplication application; +} diff --git a/src/main/java/com/ibm/cldk/schema/JApplication.java b/src/main/java/com/ibm/cldk/schema/JApplication.java new file mode 100644 index 0000000..df50541 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JApplication.java @@ -0,0 +1,13 @@ +package com.ibm.cldk.schema; + +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Data; + +/** The root {@code application} node of the v2 CPG. */ +@Data +public class JApplication { + private String id; + private String kind = "application"; + private Map symbolTable = new LinkedHashMap<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JDecorator.java b/src/main/java/com/ibm/cldk/schema/JDecorator.java new file mode 100644 index 0000000..26072de --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JDecorator.java @@ -0,0 +1,17 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A structured annotation/decorator: {@code name} + argument expressions + {@code span}. Java + * annotations carry meaningful arguments (e.g. {@code @RequestMapping("/x")}), so v2 keeps them + * structured rather than as flat strings (design decision D2). + */ +@Data +public class JDecorator { + private String name; + private List args = new ArrayList<>(); + private Span span; +} diff --git a/src/main/java/com/ibm/cldk/schema/JModule.java b/src/main/java/com/ibm/cldk/schema/JModule.java new file mode 100644 index 0000000..ad6a788 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JModule.java @@ -0,0 +1,25 @@ +package com.ibm.cldk.schema; + +import com.google.gson.annotations.SerializedName; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Data; + +/** + * A per-file {@code module} (compilation unit) node. Holds the whole file's text once as + * {@code source}; every descendant node's text is a byte-slice of it. + */ +@Data +public class JModule { + private String id; + private String kind = "module"; + + /** {@code package} is a Java keyword, so the field is {@code packageName} but serializes as {@code package}. */ + @SerializedName("package") + private String packageName; + + private String source; + + /** Top-level types declared in this file, keyed by simple name (nested types hang under them). */ + private Map types = new LinkedHashMap<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JType.java b/src/main/java/com/ibm/cldk/schema/JType.java new file mode 100644 index 0000000..7784a71 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JType.java @@ -0,0 +1,20 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A v2 {@code type} node. The specific flavor is the {@code kind} value + * ({@code class}|{@code interface}|{@code enum}|{@code record}|{@code annotation}) rather than a + * pile of {@code is_*} booleans (design decision D4). + */ +@Data +public class JType { + private String id; + private String kind; + private Span span; + private List baseTypes = new ArrayList<>(); + private List interfaces = new ArrayList<>(); + private List decorators = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/Span.java b/src/main/java/com/ibm/cldk/schema/Span.java new file mode 100644 index 0000000..115062a --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/Span.java @@ -0,0 +1,15 @@ +package com.ibm.cldk.schema; + +import lombok.Data; + +/** + * Where a node lives in source. {@code start}/{@code end} are {@code [line, column]} (JavaParser + * native: both 1-based) for addressing/display; {@code bytes} are {@code [from, to)} UTF-8 offsets + * into {@code module.source} for O(1) slicing. + */ +@Data +public class Span { + private int[] start; + private int[] end; + private int[] bytes; +} diff --git a/src/main/java/com/ibm/cldk/schema/V2Emitter.java b/src/main/java/com/ibm/cldk/schema/V2Emitter.java new file mode 100644 index 0000000..d5852f9 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/V2Emitter.java @@ -0,0 +1,36 @@ +package com.ibm.cldk.schema; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeSet; + +/** + * Assembles the canonical schema v2 envelope from per-file {@link JModule}s produced by + * {@link V2SymbolTableBuilder}. Pure wiring — the tree is built by the L1 builder straight from the + * JavaParser AST (so spans, structured decorators, and source come from where that data actually + * lives); this class only wraps the modules into the {@code application} + envelope. + */ +public final class V2Emitter { + + private V2Emitter() {} + + /** Wrap already-built modules (keyed by relative file key) into the v2 envelope. */ + public static Analysis emit(String appName, int maxLevel, Map modules) { + JApplication application = new JApplication(); + application.setId(CanId.applicationId(appName)); + + // Sort file keys so output is deterministic (the -j gate). + Map sorted = new LinkedHashMap<>(); + for (String fileKey : new TreeSet<>(modules.keySet())) { + sorted.put(fileKey, modules.get(fileKey)); + } + application.setSymbolTable(sorted); + + Analysis analysis = new Analysis(); + analysis.setSchemaVersion("2.0.0"); + analysis.setLanguage("java"); + analysis.setMaxLevel(maxLevel); + analysis.setApplication(application); + return analysis; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/DecoratorBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/DecoratorBuilder.java new file mode 100644 index 0000000..253daf8 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/DecoratorBuilder.java @@ -0,0 +1,39 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.expr.MemberValuePair; +import com.github.javaparser.ast.expr.NormalAnnotationExpr; +import com.github.javaparser.ast.expr.SingleMemberAnnotationExpr; +import com.ibm.cldk.schema.JDecorator; +import java.util.ArrayList; +import java.util.List; + +/** + * Builds a structured {@link JDecorator} ({@code name} + argument expressions + {@code span}) from a + * JavaParser {@link AnnotationExpr}. Handles marker, single-member, and normal annotations. + */ +public final class DecoratorBuilder { + + private final L1BuildContext ctx; + + public DecoratorBuilder(L1BuildContext ctx) { + this.ctx = ctx; + } + + public JDecorator build(AnnotationExpr annotation) { + JDecorator decorator = new JDecorator(); + decorator.setName(annotation.getNameAsString()); + decorator.setSpan(ctx.spanOf(annotation)); + + List args = new ArrayList<>(); + if (annotation instanceof SingleMemberAnnotationExpr) { + args.add(((SingleMemberAnnotationExpr) annotation).getMemberValue().toString()); + } else if (annotation instanceof NormalAnnotationExpr) { + for (MemberValuePair pair : ((NormalAnnotationExpr) annotation).getPairs()) { + args.add(pair.getNameAsString() + "=" + pair.getValue().toString()); + } + } // MarkerAnnotationExpr has no arguments + decorator.setArgs(args); + return decorator; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java new file mode 100644 index 0000000..81c88e6 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java @@ -0,0 +1,51 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.Range; +import com.github.javaparser.ast.Node; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.Span; +import com.ibm.cldk.schema.Spans; +import lombok.Getter; + +/** + * Shared, per-file context threaded through the L1 v2 builders (one cohesive builder per node kind). + * Holds the identity/source data every builder needs and offers the small helpers they share, so the + * builders stay focused on their node kind rather than re-deriving ids/spans. + */ +@Getter +public final class L1BuildContext { + + private final String applicationId; + private final String fileKey; + private final String source; + + public L1BuildContext(String applicationId, String fileKey, String source) { + this.applicationId = applicationId; + this.fileKey = fileKey; + this.source = source; + } + + /** The {@code can://java//} id for this module. */ + public String moduleId() { + return CanId.moduleId(applicationId, fileKey); + } + + /** + * Build the {@link Span} for an AST node from its source range: {@code start}/{@code end} as + * JavaParser {@code [line, column]} (1-based), {@code bytes} as {@code [from, to)} UTF-8 offsets + * into the module source. Returns {@code null} when the node has no range (absent = no fact). + */ + public Span spanOf(Node node) { + if (node.getRange().isEmpty()) { + return null; + } + Range r = node.getRange().get(); + Span span = new Span(); + span.setStart(new int[] {r.begin.line, r.begin.column}); + span.setEnd(new int[] {r.end.line, r.end.column}); + // JavaParser columns are 1-based and the end position is the last char (inclusive); convert + // to a [from, to) byte slice: begin col-1 (0-based start), end col (0-based char after last). + span.setBytes(Spans.byteOffsets(source, r.begin.line, r.begin.column - 1, r.end.line, r.end.column)); + return span; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java new file mode 100644 index 0000000..41d0ced --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java @@ -0,0 +1,39 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.TypeDeclaration; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JType; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * Builds a canonical schema v2 {@code module} node from a JavaParser {@link CompilationUnit}. Owns + * only module-level concerns (id, package, source, imports) and delegates each declared type to + * {@code TypeBuilder}; it does not inline type/callable walking. + */ +public final class ModuleBuilder { + + private final L1BuildContext ctx; + + public ModuleBuilder(L1BuildContext ctx) { + this.ctx = ctx; + } + + public JModule build(CompilationUnit cu) { + JModule module = new JModule(); + module.setId(ctx.moduleId()); + module.setPackageName(cu.getPackageDeclaration().map(pd -> pd.getNameAsString()).orElse("")); + module.setSource(ctx.getSource()); + + // Top-level types, keyed by simple name and sorted for deterministic output (the -j gate). + TypeBuilder typeBuilder = new TypeBuilder(ctx); + Map types = new TreeMap<>(); + for (TypeDeclaration td : cu.getTypes()) { + types.put(td.getNameAsString(), typeBuilder.build(td, module.getId())); + } + module.setTypes(new LinkedHashMap<>(types)); + return module; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java new file mode 100644 index 0000000..b7f7b8a --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -0,0 +1,73 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.body.AnnotationDeclaration; +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.EnumDeclaration; +import com.github.javaparser.ast.body.RecordDeclaration; +import com.github.javaparser.ast.body.TypeDeclaration; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JType; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Builds a v2 {@code type} node from a JavaParser {@link TypeDeclaration}: derives the {@code kind}, + * the byte-offset {@code span}, structured {@code decorators}, and {@code base_types}/ + * {@code interfaces}. Delegates annotation shaping to {@link DecoratorBuilder}. + */ +public final class TypeBuilder { + + private final L1BuildContext ctx; + private final DecoratorBuilder decoratorBuilder; + + public TypeBuilder(L1BuildContext ctx) { + this.ctx = ctx; + this.decoratorBuilder = new DecoratorBuilder(ctx); + } + + /** + * @param td the type declaration + * @param parentId the containing node's id (module id for top-level types) + */ + public JType build(TypeDeclaration td, String parentId) { + JType type = new JType(); + type.setId(CanId.childId(parentId, td.getNameAsString())); + type.setKind(kindOf(td)); + type.setSpan(ctx.spanOf(td)); + type.setDecorators( + td.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); + + List baseTypes = new ArrayList<>(); + List interfaces = new ArrayList<>(); + if (td instanceof ClassOrInterfaceDeclaration) { + ClassOrInterfaceDeclaration cls = (ClassOrInterfaceDeclaration) td; + cls.getExtendedTypes().forEach(t -> baseTypes.add(t.asString())); + cls.getImplementedTypes().forEach(t -> interfaces.add(t.asString())); + } else if (td instanceof EnumDeclaration) { + ((EnumDeclaration) td).getImplementedTypes().forEach(t -> interfaces.add(t.asString())); + } else if (td instanceof RecordDeclaration) { + ((RecordDeclaration) td).getImplementedTypes().forEach(t -> interfaces.add(t.asString())); + } + type.setBaseTypes(baseTypes); + type.setInterfaces(interfaces); + return type; + } + + /** Maps a JavaParser type declaration to its v2 {@code kind} (design decision D4). */ + private static String kindOf(TypeDeclaration td) { + if (td instanceof AnnotationDeclaration) { + return "annotation"; + } + if (td instanceof EnumDeclaration) { + return "enum"; + } + if (td instanceof RecordDeclaration) { + return "record"; + } + if (td instanceof ClassOrInterfaceDeclaration && ((ClassOrInterfaceDeclaration) td).isInterface()) { + return "interface"; + } + return "class"; + } +} diff --git a/src/test/java/com/ibm/cldk/schema/V2EmitterTest.java b/src/test/java/com/ibm/cldk/schema/V2EmitterTest.java new file mode 100644 index 0000000..c223e69 --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/V2EmitterTest.java @@ -0,0 +1,30 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** Tests the thin {@link V2Emitter} assembler wrapping pre-built modules into the v2 envelope. */ +class V2EmitterTest { + + @Test + void emit_wrapsModulesIntoEnvelopeAndApplication() { + JModule module = new JModule(); + module.setId("can://java/myapp/src/Foo.java"); + module.setPackageName("com.example"); + Map modules = new LinkedHashMap<>(); + modules.put("src/Foo.java", module); + + Analysis analysis = V2Emitter.emit("myapp", 1, modules); + + assertEquals("2.0.0", analysis.getSchemaVersion()); + assertEquals("java", analysis.getLanguage()); + assertEquals(1, analysis.getMaxLevel()); + assertEquals("can://java/myapp", analysis.getApplication().getId()); + assertEquals("application", analysis.getApplication().getKind()); + assertSame(module, analysis.getApplication().getSymbolTable().get("src/Foo.java")); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java new file mode 100644 index 0000000..833d1c7 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java @@ -0,0 +1,63 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JModule; +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** Tests the v2 {@link ModuleBuilder} building a {@code module} node directly from the AST. */ +class ModuleBuilderTest { + + private static CompilationUnit parse(String source) { + return new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(source) + .getResult() + .orElseThrow(); + } + + @Test + void build_setsModuleIdKindPackageAndSource() { + String source = "package com.example;\n\npublic class Foo {}\n"; + CompilationUnit cu = parse(source); + String fileKey = "src/main/java/com/example/Foo.java"; + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), fileKey, source); + + JModule module = new ModuleBuilder(ctx).build(cu); + + assertEquals("can://java/myapp/" + fileKey, module.getId()); + assertEquals("module", module.getKind()); + assertEquals("com.example", module.getPackageName()); + assertEquals(source, module.getSource()); + } + + @Test + void build_populatesTopLevelTypesKeyedBySimpleName() { + String source = "package com.example;\n\npublic class Foo {}\ninterface Bar {}\n"; + CompilationUnit cu = parse(source); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), "src/Foo.java", source); + + JModule module = new ModuleBuilder(ctx).build(cu); + + assertEquals(Set.of("Foo", "Bar"), module.getTypes().keySet()); + assertEquals("class", module.getTypes().get("Foo").getKind()); + assertEquals("interface", module.getTypes().get("Bar").getKind()); + assertEquals("can://java/myapp/src/Foo.java/Foo", module.getTypes().get("Foo").getId()); + } + + @Test + void build_defaultsPackageToEmptyWhenAbsent() { + String source = "public class Foo {}\n"; + CompilationUnit cu = parse(source); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), "Foo.java", source); + + JModule module = new ModuleBuilder(ctx).build(cu); + + assertEquals("", module.getPackageName()); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java new file mode 100644 index 0000000..a673e83 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -0,0 +1,78 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.TypeDeclaration; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JDecorator; +import com.ibm.cldk.schema.JType; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** Tests the v2 {@link TypeBuilder} — kind, byte-offset span, structured decorators, inheritance. */ +class TypeBuilderTest { + + private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; + + private static CompilationUnit parse(String source) { + return new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(source) + .getResult() + .orElseThrow(); + } + + private static JType buildFirstType(String source) { + CompilationUnit cu = parse(source); + TypeDeclaration td = cu.getType(0); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); + return new TypeBuilder(ctx).build(td, ctx.moduleId()); + } + + @Test + void build_setsIdAndClassKind() { + JType t = buildFirstType("package com.example;\n\npublic class Foo {}\n"); + assertEquals("can://java/myapp/" + FILE_KEY + "/Foo", t.getId()); + assertEquals("class", t.getKind()); + } + + @Test + void build_derivesKindForInterfaceEnumRecordAnnotation() { + assertEquals("interface", buildFirstType("package p;\npublic interface I {}\n").getKind()); + assertEquals("enum", buildFirstType("package p;\npublic enum E { A, B }\n").getKind()); + assertEquals("record", buildFirstType("package p;\npublic record R(int x) {}\n").getKind()); + assertEquals("annotation", buildFirstType("package p;\npublic @interface A {}\n").getKind()); + } + + @Test + void build_capturesInheritance() { + JType t = buildFirstType("package p;\nclass Foo extends Base implements Runnable {}\n"); + assertEquals(List.of("Base"), t.getBaseTypes()); + assertEquals(List.of("Runnable"), t.getInterfaces()); + } + + @Test + void build_spanBytesSliceToTheTypeText() { + String source = "package com.example;\n\npublic class Foo {}\n"; + JType t = buildFirstType(source); + assertNotNull(t.getSpan()); + int[] bytes = t.getSpan().getBytes(); + assertTrue(source.substring(bytes[0], bytes[1]).contains("class Foo"), + "span.bytes should slice module source to the type's declaration text"); + } + + @Test + void build_capturesStructuredDecoratorWithArgs() { + JType t = buildFirstType("package p;\n@SuppressWarnings(\"unchecked\")\nclass Foo {}\n"); + assertEquals(1, t.getDecorators().size()); + JDecorator d = t.getDecorators().get(0); + assertEquals("SuppressWarnings", d.getName()); + assertEquals(List.of("\"unchecked\""), d.getArgs()); + assertNotNull(d.getSpan()); + } +} From f4290eb1a75434bd9b9c38322a0bb3c75c1ad976 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 14:14:15 -0400 Subject: [PATCH 04/22] build: exclude test-application fixtures from spotless formatting (#180) --- build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2189bfe..a5b17b6 100644 --- a/build.gradle +++ b/build.gradle @@ -156,7 +156,10 @@ test { spotless { java { - target 'src/**/*.java' + // Format only the analyzer's own sources. Test-application fixtures under + // src/test/resources (vendored apps and git submodules) are third-party inputs and must + // not be reformatted — doing so mutates test inputs and dirties submodule working trees. + target 'src/main/java/**/*.java', 'src/test/java/**/*.java' trimTrailingWhitespace() endWithNewline() importOrder() From febe9d0465dfbddc8b0404ea1331f782a927bf8c Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 15:00:39 -0400 Subject: [PATCH 05/22] feat(schema): nested member types via containment; drop nesting field (D4) (#180) --- .claude/SCHEMA_DECISIONS.md | 16 ++++++++++++---- src/main/java/com/ibm/cldk/schema/JType.java | 9 +++++++++ .../ibm/cldk/syntactic_analysis/TypeBuilder.java | 13 +++++++++++++ .../cldk/syntactic_analysis/TypeBuilderTest.java | 16 ++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 6d32591..aa7757a 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -31,10 +31,18 @@ extensible. **Divergence:** the Python pilot and current Java keep these flat (`cyclomatic_complexity`, `referenced_types`, `accessed_fields`); SDK views expose the old flat names. -### D4 — Type kinds: single `kind` + `nesting` -`type.kind ∈ {class, interface, enum, record, annotation}` plus -`nesting:{parent?, is_local?}`, replacing the v1 `is_interface`/`is_enum`/ -`is_record`/`is_nested`/… boolean pile. +### D4 — Type kinds: single `kind`; nesting via containment +`type.kind ∈ {class, interface, enum, record, annotation}` replaces the v1 +`is_interface`/`is_enum`/`is_record`/`is_nested`/… boolean pile. + +**Nesting/locality is encoded by containment, not a `nesting` field** (refined +2026-08 after checking the Python pilot): member/inner types live under the +enclosing type's `types{}`; local classes under the enclosing callable's +`types{}`; and the `can://…/Outer/Inner` id path records the parent. Parent and +is-local are therefore derivable from tree position — no `nesting` object is +emitted. The keystone lists a `nesting:{parent?,is_local?}` field, but full +containment subsumes it, matching how `codeanalyzer-python` models it +(`PyClass.types` for inner classes, `PyCallable.types` for local classes). ### D5 — L3 CFG engine & granularity: WALA engine → source-statement nodes Use WALA as the analysis engine (`SSACFG` + dominance + SSA def-use — heap-ready for diff --git a/src/main/java/com/ibm/cldk/schema/JType.java b/src/main/java/com/ibm/cldk/schema/JType.java index 7784a71..f02324a 100644 --- a/src/main/java/com/ibm/cldk/schema/JType.java +++ b/src/main/java/com/ibm/cldk/schema/JType.java @@ -1,7 +1,9 @@ package com.ibm.cldk.schema; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import lombok.Data; /** @@ -17,4 +19,11 @@ public class JType { private List baseTypes = new ArrayList<>(); private List interfaces = new ArrayList<>(); private List decorators = new ArrayList<>(); + + /** + * Member/inner types declared directly inside this one, keyed by simple name. Nesting and + * parent are encoded by this containment position (and the {@code can://…/Outer/Inner} id path); + * local classes declared in method bodies live under the enclosing callable, not here. + */ + private Map types = new LinkedHashMap<>(); } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index b7f7b8a..2a27570 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -8,7 +8,10 @@ import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JType; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; /** @@ -51,6 +54,16 @@ public JType build(TypeDeclaration td, String parentId) { } type.setBaseTypes(baseTypes); type.setInterfaces(interfaces); + + // Recurse into member (inner) types; nesting/parent are encoded by this containment (and the + // id path). Local classes in method bodies are handled later by the callable builder. + Map nested = new TreeMap<>(); + td.getMembers().stream() + .filter(m -> m instanceof TypeDeclaration) + .map(m -> (TypeDeclaration) m) + .forEach(member -> nested.put(member.getNameAsString(), build(member, type.getId()))); + type.setTypes(new LinkedHashMap<>(nested)); + return type; } diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index a673e83..7a1730b 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -12,6 +12,7 @@ import com.ibm.cldk.schema.JDecorator; import com.ibm.cldk.schema.JType; import java.util.List; +import java.util.Set; import org.junit.jupiter.api.Test; /** Tests the v2 {@link TypeBuilder} — kind, byte-offset span, structured decorators, inheritance. */ @@ -66,6 +67,21 @@ void build_spanBytesSliceToTheTypeText() { "span.bytes should slice module source to the type's declaration text"); } + @Test + void build_recursesIntoMemberTypesViaContainment() { + // Nesting is encoded by containment (member types under the parent's `types`) and the id + // path — no separate nesting/is_local field. + String source = "package p;\nclass Outer {\n class Inner {}\n enum E { A }\n}\n"; + JType outer = buildFirstType(source); + + assertEquals(Set.of("Inner", "E"), outer.getTypes().keySet()); + assertEquals("enum", outer.getTypes().get("E").getKind()); + + JType inner = outer.getTypes().get("Inner"); + assertEquals(outer.getId() + "/Inner", inner.getId()); + assertEquals("class", inner.getKind()); + } + @Test void build_capturesStructuredDecoratorWithArgs() { JType t = buildFirstType("package p;\n@SuppressWarnings(\"unchecked\")\nclass Foo {}\n"); From 0dfed93c9647597520ea03aa937f59e099d004a8 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 18:30:21 -0400 Subject: [PATCH 06/22] feat(schema): callable/field/param builders; call sites keyed by local id (#180) Key body nodes by bare local id (line:col) per the keystone, and emit call nodes for constructor invocations and this()/super() chaining so L2 can resolve those edges. --- .claude/SCHEMA_DECISIONS.md | 18 +++ src/main/java/com/ibm/cldk/SymbolTable.java | 24 +-- .../java/com/ibm/cldk/schema/JBodyNode.java | 20 +++ .../java/com/ibm/cldk/schema/JCallable.java | 35 ++++ src/main/java/com/ibm/cldk/schema/JField.java | 20 +++ .../java/com/ibm/cldk/schema/JMetrics.java | 12 ++ .../java/com/ibm/cldk/schema/JParameter.java | 18 +++ src/main/java/com/ibm/cldk/schema/JRefs.java | 16 ++ src/main/java/com/ibm/cldk/schema/JType.java | 6 + .../syntactic_analysis/CallSiteBuilder.java | 124 ++++++++++++++ .../syntactic_analysis/CallableBuilder.java | 152 ++++++++++++++++++ .../cldk/syntactic_analysis/FieldBuilder.java | 54 +++++++ .../syntactic_analysis/ParameterBuilder.java | 32 ++++ .../cldk/syntactic_analysis/Signatures.java | 49 ++++++ .../cldk/syntactic_analysis/TypeBuilder.java | 30 ++++ .../CallSiteBuilderTest.java | 137 ++++++++++++++++ .../CallableBuilderTest.java | 108 +++++++++++++ .../syntactic_analysis/FieldBuilderTest.java | 75 +++++++++ .../ParameterBuilderTest.java | 71 ++++++++ .../syntactic_analysis/TypeBuilderTest.java | 25 +++ 20 files changed, 1003 insertions(+), 23 deletions(-) create mode 100644 src/main/java/com/ibm/cldk/schema/JBodyNode.java create mode 100644 src/main/java/com/ibm/cldk/schema/JCallable.java create mode 100644 src/main/java/com/ibm/cldk/schema/JField.java create mode 100644 src/main/java/com/ibm/cldk/schema/JMetrics.java create mode 100644 src/main/java/com/ibm/cldk/schema/JParameter.java create mode 100644 src/main/java/com/ibm/cldk/schema/JRefs.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index aa7757a..2d7fef3 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -68,6 +68,24 @@ L4 unit; lands last. Java analog of the pilot's `can://python/…`; built from the existing `signatureOf()`. Ordinal ids `…@:` (real) / `…@` (synthetic) within a callable. +**L1 refinements (2026-08, during CallableBuilder):** +- **Signature is shared, not duplicated.** The v1 type-erasure logic moved to + `syntactic_analysis.Signatures.typeErasure(CallableDeclaration)`; both the v1 + symbol table and the v2 `CallableBuilder` call it, so ids match. It falls back to + the plain AST signature when no symbol solver is configured (pure syntactic parse), + so it never throws. +- **Ordinal-id anchor = invoked-name position.** A `call` body node's tag (and the + local-ids in its `arguments`) use the *method-name* `line:col`, not the whole + expression's begin — so chained calls `a.b().c()` get distinct ids instead of + colliding on the shared expression start. +- **`refs` at L1 are syntactic names, not resolved ids.** Cross-module resolution is + L2+; at L1 `refs.types` are the AST spellings of referenced/instantiated types and + `refs.fields` are the simple names of enclosing-type fields accessed. Refined to + `can://` ids once resolution is available. Keystone shows `[id]`; L1 emits best-effort. +- **`callable.kind ∈ {method, constructor}`.** Direct members only (via + `getMethods()`/`getConstructors()`); nested-type methods hang under their own type, + local (method-body) classes under `callable.types` (D4 containment). + ### D9 — Neo4j namespace: keep the `J_` relationship prefix Existing convention (`J_CALLS`, …); dual-label `JSymbol` merge pattern retained. `SchemaCatalog` takes a major bump (families rename v1→v2). diff --git a/src/main/java/com/ibm/cldk/SymbolTable.java b/src/main/java/com/ibm/cldk/SymbolTable.java index 602160c..c6c3729 100644 --- a/src/main/java/com/ibm/cldk/SymbolTable.java +++ b/src/main/java/com/ibm/cldk/SymbolTable.java @@ -599,29 +599,7 @@ private static Pair processCallableDeclaration(CallableDeclara * @return String representing type erasure or regular signature */ private static String getTypeErasureSignature(CallableDeclaration callableDecl) { - try { - StringBuilder signature = new StringBuilder( - (callableDecl instanceof MethodDeclaration) ? callableDecl.getNameAsString() : "" - ); - List erasureParameterTypes = new ArrayList<>(); - for (Object param : callableDecl.getParameters()) { - Parameter parameter = (Parameter) param; - ResolvedType resolvedType = parameter.getType().resolve(); - if (parameter.isVarArgs()) { - erasureParameterTypes.add(resolvedType.erasure().describe() + "[]"); - } else { - erasureParameterTypes.add(resolvedType.erasure().describe()); - } - } - signature.append("("); - signature.append(String.join(", ", erasureParameterTypes)); - signature.append(")"); - return signature.toString(); - } catch (Throwable e) { - Log.warn("Could not compute type erasure signature for "+callableDecl.getSignature().asString()+ - "; computing regular signature"); - return callableDecl.getSignature().asString(); - } + return com.ibm.cldk.syntactic_analysis.Signatures.typeErasure(callableDecl); } /** diff --git a/src/main/java/com/ibm/cldk/schema/JBodyNode.java b/src/main/java/com/ibm/cldk/schema/JBodyNode.java new file mode 100644 index 0000000..ee584ea --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JBodyNode.java @@ -0,0 +1,20 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A node in a callable's {@code body}: at L1 only {@code call} nodes (an AST region for a method + * invocation). {@code callee} is the sanctioned {@code null}-then-id slot — left {@code null} at L1 + * and backfilled with the callee's {@code can://} id when the L2 call graph resolves the site. + * {@code arguments} are the local ids of the invocation's argument expressions. + */ +@Data +public class JBodyNode { + private String kind; + private Span span; + /** Only meaningful on {@code call} nodes; {@code null} at L1 (backfilled at L2). */ + private String callee; + private List arguments = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JCallable.java b/src/main/java/com/ibm/cldk/schema/JCallable.java new file mode 100644 index 0000000..0aee6ad --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JCallable.java @@ -0,0 +1,35 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.Data; + +/** + * A v2 {@code callable} node (method or constructor). Its {@code id} is the containment path + * {@code /} (design decision D8). Per D1 there is no per-callable {@code code}, + * no flat {@code start_line}/{@code end_line}, and no {@code call_sites[]} — the source is a slice of + * {@code module.source[span.bytes]} and call sites are {@code body} {@code call} nodes. Metrics and + * cross-refs are nested (D3). {@code thrown_exceptions} become {@code error_channel}. + */ +@Data +public class JCallable { + private String id; + private String kind; + private String signature; + private Span span; + private List parameters = new ArrayList<>(); + private String returnType; + private List errorChannel = new ArrayList<>(); + private List modifiers = new ArrayList<>(); + private List decorators = new ArrayList<>(); + private JMetrics metrics; + private JRefs refs; + + /** L1 emits only {@code call} nodes here, keyed by ordinal id; the rest of the body arrives at L3. */ + private Map body = new LinkedHashMap<>(); + + /** Local (method-body) classes, keyed by simple name — nesting encoded by containment (D4). */ + private Map types = new LinkedHashMap<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JField.java b/src/main/java/com/ibm/cldk/schema/JField.java new file mode 100644 index 0000000..c9cf360 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JField.java @@ -0,0 +1,20 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A v2 {@code field} node — one per declared variable, so {@code int a, b;} yields two fields. The + * {@code id} is the containment path {@code /}; {@code type} is the AST spelling + * (syntactic — no resolution at L1). {@code span} covers the whole field declaration text. + */ +@Data +public class JField { + private String id; + private String name; + private String type; + private Span span; + private List modifiers = new ArrayList<>(); + private List decorators = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JMetrics.java b/src/main/java/com/ibm/cldk/schema/JMetrics.java new file mode 100644 index 0000000..0eb1e7f --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JMetrics.java @@ -0,0 +1,12 @@ +package com.ibm.cldk.schema; + +import lombok.Data; + +/** + * Per-callable metrics, nested rather than flattened onto the callable (design decision D3) so the + * family can grow without churning the callable's top-level shape. At L1: {@code cyclomatic}. + */ +@Data +public class JMetrics { + private int cyclomatic; +} diff --git a/src/main/java/com/ibm/cldk/schema/JParameter.java b/src/main/java/com/ibm/cldk/schema/JParameter.java new file mode 100644 index 0000000..8d8d164 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JParameter.java @@ -0,0 +1,18 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A v2 {@code parameter} of a callable: {@code name}, syntactic declared {@code type}, byte-offset + * {@code span}, and structured {@code decorators} (e.g. {@code @RequestParam("q")}). At L1 the type + * is the AST spelling (no cross-module resolution); dataflow {@code formal_in} vertices arrive later. + */ +@Data +public class JParameter { + private String name; + private String type; + private Span span; + private List decorators = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JRefs.java b/src/main/java/com/ibm/cldk/schema/JRefs.java new file mode 100644 index 0000000..b114cb6 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JRefs.java @@ -0,0 +1,16 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * Cross-references out of a callable, nested per design decision D3: {@code types} referenced and + * {@code fields} accessed in the body. At L1 these are best-effort syntactic names (no cross-module + * resolution); they are refined to {@code can://} ids once resolution is available. + */ +@Data +public class JRefs { + private List types = new ArrayList<>(); + private List fields = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JType.java b/src/main/java/com/ibm/cldk/schema/JType.java index f02324a..99816b5 100644 --- a/src/main/java/com/ibm/cldk/schema/JType.java +++ b/src/main/java/com/ibm/cldk/schema/JType.java @@ -20,6 +20,12 @@ public class JType { private List interfaces = new ArrayList<>(); private List decorators = new ArrayList<>(); + /** Fields declared in this type, keyed by simple name (one entry per declared variable). */ + private Map fields = new LinkedHashMap<>(); + + /** Methods and constructors, keyed by type-erasure signature (keystone containment name). */ + private Map callables = new LinkedHashMap<>(); + /** * Member/inner types declared directly inside this one, keyed by simple name. Nesting and * parent are encoded by this containment position (and the {@code can://…/Outer/Inner} id path); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java new file mode 100644 index 0000000..f0164d2 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java @@ -0,0 +1,124 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.NodeList; +import com.github.javaparser.ast.body.BodyDeclaration; +import com.github.javaparser.ast.expr.Expression; +import com.github.javaparser.ast.expr.MethodCallExpr; +import com.github.javaparser.ast.expr.ObjectCreationExpr; +import com.github.javaparser.ast.stmt.BlockStmt; +import com.github.javaparser.ast.stmt.ExplicitConstructorInvocationStmt; +import com.ibm.cldk.schema.JBodyNode; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Builds the L1 slice of a callable's {@code body{}}: one {@code call} node per call site that + * belongs directly to this callable — method invocations, {@code new} constructor invocations, and + * explicit {@code this(...)}/{@code super(...)} chaining (all three are sites L2 resolves into + * {@code call_graph} edges, so omitting any of them would lose edges). + * + *

Nodes are keyed by their local id — a {@code line:col} source position, per the + * keystone ({@code body} is "keyed by the node's local id"). The full + * {@code @} form is derived only where cross-callable ids are needed (L4's + * application-scope {@code param_in}/{@code param_out}). + * + *

The addressing anchor is the invoked name (method name, or instantiated type name), + * not the enclosing expression's start, so chained calls {@code a.b().c()} get distinct ids instead + * of colliding. Invocations inside nested local/anonymous classes belong to their own callables and + * are excluded; lambda bodies (which have no separate callable) are kept. Nodes are ordered by + * source position so output is deterministic under parallel fan-out. + * + *

L3 completes {@code body} with the remaining statements. A bare call statement resolves to the + * same local id as the {@code call} node emitted here — L3 must therefore not overwrite an + * existing {@code call} node (the call node is that statement, as in the keystone's worked + * example); rewriting its kind would break the additive invariant. + */ +public final class CallSiteBuilder { + + private final L1BuildContext ctx; + + public CallSiteBuilder(L1BuildContext ctx) { + this.ctx = ctx; + } + + public Map build(BlockStmt body) { + List sites = new ArrayList<>(); + body.findAll(MethodCallExpr.class).stream().filter(n -> belongsDirectlyTo(n, body)).forEach(sites::add); + body.findAll(ObjectCreationExpr.class).stream().filter(n -> belongsDirectlyTo(n, body)).forEach(sites::add); + body.findAll(ExplicitConstructorInvocationStmt.class).stream() + .filter(n -> belongsDirectlyTo(n, body)) + .forEach(sites::add); + + sites.sort(Comparator.comparingInt(n -> anchorPosition(n)[0]) + .thenComparingInt(n -> anchorPosition(n)[1])); + + Map nodes = new LinkedHashMap<>(); + for (Node site : sites) { + JBodyNode node = new JBodyNode(); + node.setKind("call"); + node.setSpan(ctx.spanOf(site)); + // `callee` stays unset at L1 and is filled in when L2 resolves this site. + node.setArguments(argumentsOf(site).stream().map(CallSiteBuilder::localId).collect(Collectors.toList())); + nodes.put(localId(site), node); + } + return nodes; + } + + /** + * True when {@code node} is part of {@code body} itself and not of a nested type or anonymous + * class declared within it: no {@link BodyDeclaration} (which includes type declarations and + * member methods/initializers) lies between the node and the body block. + */ + static boolean belongsDirectlyTo(Node node, BlockStmt body) { + for (Node cur = node.getParentNode().orElse(null); + cur != null && cur != body; + cur = cur.getParentNode().orElse(null)) { + if (cur instanceof BodyDeclaration) { + return false; + } + } + return true; + } + + private static List argumentsOf(Node site) { + NodeList args; + if (site instanceof MethodCallExpr) { + args = ((MethodCallExpr) site).getArguments(); + } else if (site instanceof ObjectCreationExpr) { + args = ((ObjectCreationExpr) site).getArguments(); + } else { + args = ((ExplicitConstructorInvocationStmt) site).getArguments(); + } + return new ArrayList<>(args); + } + + /** The local id {@code line:col} of a node's addressing anchor. */ + private static String localId(Node node) { + int[] pos = anchorPosition(node); + return pos[0] + ":" + pos[1]; + } + + /** + * Addressing position: the invoked name for a method call, the instantiated type for a + * {@code new} expression, and the statement itself for {@code this(...)}/{@code super(...)} — + * so sites nested in one expression stay distinct. Falls back to the node's own begin. + */ + private static int[] anchorPosition(Node node) { + Node anchor = node; + if (node instanceof MethodCallExpr) { + anchor = ((MethodCallExpr) node).getName(); + } else if (node instanceof ObjectCreationExpr) { + anchor = ((ObjectCreationExpr) node).getType(); + } + return anchor.getRange() + .map(r -> new int[] {r.begin.line, r.begin.column}) + .orElseGet(() -> node.getRange() + .map(r -> new int[] {r.begin.line, r.begin.column}) + .orElse(new int[] {0, 0})); + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java new file mode 100644 index 0000000..784da1c --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -0,0 +1,152 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.body.CallableDeclaration; +import com.github.javaparser.ast.body.ConstructorDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.body.TypeDeclaration; +import com.github.javaparser.ast.body.VariableDeclarator; +import com.github.javaparser.ast.expr.ConditionalExpr; +import com.github.javaparser.ast.expr.FieldAccessExpr; +import com.github.javaparser.ast.expr.NameExpr; +import com.github.javaparser.ast.expr.ObjectCreationExpr; +import com.github.javaparser.ast.stmt.BlockStmt; +import com.github.javaparser.ast.stmt.CatchClause; +import com.github.javaparser.ast.stmt.DoStmt; +import com.github.javaparser.ast.stmt.ForEachStmt; +import com.github.javaparser.ast.stmt.ForStmt; +import com.github.javaparser.ast.stmt.IfStmt; +import com.github.javaparser.ast.stmt.SwitchStmt; +import com.github.javaparser.ast.stmt.WhileStmt; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JMetrics; +import com.ibm.cldk.schema.JRefs; +import com.ibm.cldk.schema.JType; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.Collectors; + +/** + * Builds a v2 {@code callable} node from a JavaParser {@link CallableDeclaration}: the type-erasure + * {@code signature} + containment {@code id}, {@code parameters}, {@code return_type}, the + * {@code error_channel} (declared {@code throws}), {@code modifiers}, structured {@code decorators}, + * nested {@code metrics}/{@code refs}, the L1 {@code body} {@code call} nodes, and local classes + * under {@code types} (containment, design decision D4). Delegates each concern to its focused + * builder ({@link ParameterBuilder}, {@link CallSiteBuilder}, {@link TypeBuilder}, {@link DecoratorBuilder}). + */ +public final class CallableBuilder { + + private final L1BuildContext ctx; + private final ParameterBuilder parameterBuilder; + private final DecoratorBuilder decoratorBuilder; + private final CallSiteBuilder callSiteBuilder; + + public CallableBuilder(L1BuildContext ctx) { + this.ctx = ctx; + this.parameterBuilder = new ParameterBuilder(ctx); + this.decoratorBuilder = new DecoratorBuilder(ctx); + this.callSiteBuilder = new CallSiteBuilder(ctx); + // TypeBuilder is constructed lazily in localClasses() to break the callable<->type + // construction cycle (a type builds callables; a callable builds its local-class types). + } + + /** + * @param cd the callable declaration + * @param parentTypeId the containing type's id + * @param classFieldNames simple names of the enclosing type's fields (for {@code refs.fields}) + */ + public JCallable build(CallableDeclaration cd, String parentTypeId, List classFieldNames) { + JCallable callable = new JCallable(); + String signature = Signatures.typeErasure(cd); + callable.setSignature(signature); + callable.setId(CanId.childId(parentTypeId, signature)); + callable.setKind(cd instanceof MethodDeclaration ? "method" : "constructor"); + callable.setSpan(ctx.spanOf(cd)); + callable.setParameters( + cd.getParameters().stream().map(parameterBuilder::build).collect(Collectors.toList())); + callable.setReturnType( + cd instanceof MethodDeclaration ? ((MethodDeclaration) cd).getType().asString() : null); + callable.setErrorChannel( + cd.getThrownExceptions().stream().map(t -> t.asString()).collect(Collectors.toList())); + callable.setModifiers( + cd.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); + callable.setDecorators( + cd.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); + + JMetrics metrics = new JMetrics(); + metrics.setCyclomatic(cyclomaticComplexity(cd)); + callable.setMetrics(metrics); + + Optional body = bodyOf(cd); + callable.setRefs(refs(body, classFieldNames)); + body.ifPresent(b -> callable.setBody(callSiteBuilder.build(b))); + body.ifPresent(b -> callable.setTypes(localClasses(b, callable.getId()))); + return callable; + } + + private static Optional bodyOf(CallableDeclaration cd) { + if (cd instanceof MethodDeclaration) { + return ((MethodDeclaration) cd).getBody(); + } + return Optional.of(((ConstructorDeclaration) cd).getBody()); + } + + /** Local (method-body) classes declared directly in the body, keyed by simple name (sorted). */ + private Map localClasses(BlockStmt body, String callableId) { + TypeBuilder typeBuilder = new TypeBuilder(ctx); + Map locals = new TreeMap<>(); + body.findAll(TypeDeclaration.class).stream() + .filter(td -> CallSiteBuilder.belongsDirectlyTo(td, body)) + .forEach(td -> locals.put(td.getNameAsString(), typeBuilder.build(td, callableId))); + return new LinkedHashMap<>(locals); + } + + /** Syntactic cross-refs: types referenced and enclosing-type fields accessed in the body. */ + private static JRefs refs(Optional body, List classFieldNames) { + JRefs refs = new JRefs(); + if (body.isEmpty()) { + return refs; + } + BlockStmt b = body.get(); + + TreeSet types = new TreeSet<>(); + b.findAll(VariableDeclarator.class).stream() + .filter(vd -> CallSiteBuilder.belongsDirectlyTo(vd, b) && vd.getType().isClassOrInterfaceType()) + .forEach(vd -> types.add(vd.getType().asString())); + b.findAll(ObjectCreationExpr.class).stream() + .filter(oce -> CallSiteBuilder.belongsDirectlyTo(oce, b)) + .forEach(oce -> types.add(oce.getType().asString())); + refs.setTypes(new ArrayList<>(types)); + + TreeSet fields = new TreeSet<>(); + b.findAll(FieldAccessExpr.class).stream() + .filter(fa -> CallSiteBuilder.belongsDirectlyTo(fa, b) + && !(fa.getParentNode().orElse(null) instanceof FieldAccessExpr)) + .forEach(fa -> fields.add(fa.getNameAsString())); + b.findAll(NameExpr.class).stream() + .filter(ne -> CallSiteBuilder.belongsDirectlyTo(ne, b) && classFieldNames.contains(ne.getNameAsString())) + .forEach(ne -> fields.add(ne.getNameAsString())); + refs.setFields(new ArrayList<>(fields)); + return refs; + } + + /** + * Cyclomatic complexity: one plus the number of branch points (if/loop/switch-case/ternary/catch) + * in the callable (mirrors the v1 symbol-table metric). + */ + private static int cyclomaticComplexity(CallableDeclaration cd) { + int ifCount = cd.findAll(IfStmt.class).size(); + int loopCount = cd.findAll(DoStmt.class).size() + cd.findAll(ForStmt.class).size() + + cd.findAll(ForEachStmt.class).size() + cd.findAll(WhileStmt.class).size(); + int switchCaseCount = + cd.findAll(SwitchStmt.class).stream().mapToInt(s -> s.getEntries().size()).sum(); + int ternaryCount = cd.findAll(ConditionalExpr.class).size(); + int catchCount = cd.findAll(CatchClause.class).size(); + return ifCount + loopCount + switchCaseCount + ternaryCount + catchCount + 1; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java new file mode 100644 index 0000000..2b0a0d9 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java @@ -0,0 +1,54 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.body.FieldDeclaration; +import com.github.javaparser.ast.body.VariableDeclarator; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JDecorator; +import com.ibm.cldk.schema.JField; +import com.ibm.cldk.schema.Span; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Builds v2 {@code field} nodes from a JavaParser {@link FieldDeclaration}. A single declaration may + * declare several variables ({@code int a, b;}), so this yields one {@link JField} per variable — + * each keyed/id'd by its own name but sharing the declaration's modifiers, decorators, and span. + * Delegates annotation shaping to {@link DecoratorBuilder}. + */ +public final class FieldBuilder { + + private final L1BuildContext ctx; + private final DecoratorBuilder decoratorBuilder; + + public FieldBuilder(L1BuildContext ctx) { + this.ctx = ctx; + this.decoratorBuilder = new DecoratorBuilder(ctx); + } + + /** + * @param fd the field declaration + * @param parentTypeId the containing type's id + */ + public List build(FieldDeclaration fd, String parentTypeId) { + String type = fd.getCommonType().asString(); + List modifiers = + fd.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList()); + List decorators = + fd.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList()); + Span span = ctx.spanOf(fd); + + List fields = new ArrayList<>(); + for (VariableDeclarator var : fd.getVariables()) { + JField field = new JField(); + field.setName(var.getNameAsString()); + field.setId(CanId.childId(parentTypeId, var.getNameAsString())); + field.setType(type); + field.setModifiers(modifiers); + field.setDecorators(decorators); + field.setSpan(span); + fields.add(field); + } + return fields; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java new file mode 100644 index 0000000..5fbb9d3 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java @@ -0,0 +1,32 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.body.Parameter; +import com.ibm.cldk.schema.JParameter; +import java.util.stream.Collectors; + +/** + * Builds a v2 {@code parameter} node from a JavaParser {@link Parameter}: {@code name}, the AST + * declared {@code type} (syntactic — no resolution at L1), byte-offset {@code span}, and structured + * {@code decorators}. Delegates annotation shaping to {@link DecoratorBuilder}. + */ +public final class ParameterBuilder { + + private final L1BuildContext ctx; + private final DecoratorBuilder decoratorBuilder; + + public ParameterBuilder(L1BuildContext ctx) { + this.ctx = ctx; + this.decoratorBuilder = new DecoratorBuilder(ctx); + } + + public JParameter build(Parameter param) { + JParameter parameter = new JParameter(); + parameter.setName(param.getNameAsString()); + // Varargs (`String...`) spell as the element type + "[]" so the type reads as a real Java type. + parameter.setType(param.isVarArgs() ? param.getType().asString() + "[]" : param.getType().asString()); + parameter.setSpan(ctx.spanOf(param)); + parameter.setDecorators( + param.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); + return parameter; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java b/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java new file mode 100644 index 0000000..5509aec --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java @@ -0,0 +1,49 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.body.CallableDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.body.Parameter; +import com.github.javaparser.resolution.types.ResolvedType; +import com.ibm.cldk.utils.Log; +import java.util.ArrayList; +import java.util.List; + +/** + * Type-erasure signature construction for callables — the durable {@code ()} + * segment of a callable's {@code can://} id (design decision D8). Shared by the v1 symbol table and + * the v2 builders so both mint identical signatures. When type resolution is unavailable (pure + * syntactic parse) it falls back to the AST signature, so it never throws. + */ +public final class Signatures { + + private Signatures() {} + + /** + * The type-erasure signature for {@code callableDecl}: the method name (or {@code } for a + * constructor) followed by erased parameter types. Falls back to the plain AST signature if the + * parameter types cannot be resolved (no symbol solver configured). + */ + public static String typeErasure(CallableDeclaration callableDecl) { + try { + StringBuilder signature = new StringBuilder( + (callableDecl instanceof MethodDeclaration) ? callableDecl.getNameAsString() : ""); + List erasureParameterTypes = new ArrayList<>(); + for (Parameter parameter : callableDecl.getParameters()) { + ResolvedType resolvedType = parameter.getType().resolve(); + if (parameter.isVarArgs()) { + erasureParameterTypes.add(resolvedType.erasure().describe() + "[]"); + } else { + erasureParameterTypes.add(resolvedType.erasure().describe()); + } + } + signature.append("("); + signature.append(String.join(", ", erasureParameterTypes)); + signature.append(")"); + return signature.toString(); + } catch (Throwable e) { + Log.warn("Could not compute type erasure signature for " + callableDecl.getSignature().asString() + + "; computing regular signature"); + return callableDecl.getSignature().asString(); + } + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index 2a27570..282f549 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -1,11 +1,15 @@ package com.ibm.cldk.syntactic_analysis; import com.github.javaparser.ast.body.AnnotationDeclaration; +import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.EnumDeclaration; +import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.RecordDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JField; import com.ibm.cldk.schema.JType; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -23,10 +27,14 @@ public final class TypeBuilder { private final L1BuildContext ctx; private final DecoratorBuilder decoratorBuilder; + private final FieldBuilder fieldBuilder; + private final CallableBuilder callableBuilder; public TypeBuilder(L1BuildContext ctx) { this.ctx = ctx; this.decoratorBuilder = new DecoratorBuilder(ctx); + this.fieldBuilder = new FieldBuilder(ctx); + this.callableBuilder = new CallableBuilder(ctx); } /** @@ -55,6 +63,28 @@ public JType build(TypeDeclaration td, String parentId) { type.setBaseTypes(baseTypes); type.setInterfaces(interfaces); + // Fields, keyed by simple name — one entry per declared variable (int a, b; -> a, b). + Map fields = new LinkedHashMap<>(); + for (FieldDeclaration fd : td.getFields()) { + fieldBuilder.build(fd, type.getId()).forEach(f -> fields.put(f.getName(), f)); + } + type.setFields(fields); + + // Callables (methods + constructors) declared directly in this type — getMethods()/ + // getConstructors() return only direct members, so nested-type methods are not swept in. + // Keyed by type-erasure signature. Field names are handed down so each callable's + // refs.fields can recognize accesses to this type's fields. + List fieldNames = new ArrayList<>(fields.keySet()); + List> declared = new ArrayList<>(); + declared.addAll(td.getConstructors()); + declared.addAll(td.getMethods()); + Map callables = new TreeMap<>(); + for (CallableDeclaration cd : declared) { + JCallable callable = callableBuilder.build(cd, type.getId(), fieldNames); + callables.put(callable.getSignature(), callable); + } + type.setCallables(new LinkedHashMap<>(callables)); + // Recurse into member (inner) types; nesting/parent are encoded by this containment (and the // id path). Local classes in method bodies are handled later by the callable builder. Map nested = new TreeMap<>(); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java new file mode 100644 index 0000000..503a019 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java @@ -0,0 +1,137 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.CallableDeclaration; +import com.github.javaparser.ast.body.ConstructorDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.stmt.BlockStmt; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JBodyNode; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Tests the v2 {@link CallSiteBuilder} — L1 emits only {@code call} nodes, keyed by the node's + * local id ({@code line:col}), covering method calls, constructor invocations, and explicit + * {@code this(...)}/{@code super(...)} chaining. + */ +class CallSiteBuilderTest { + + private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; + + private static Map build(String source) { + CompilationUnit cu = new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(source) + .getResult() + .orElseThrow(); + CallableDeclaration cd = cu.getType(0).findFirst(CallableDeclaration.class).orElseThrow(); + BlockStmt body = (cd instanceof MethodDeclaration) + ? ((MethodDeclaration) cd).getBody().orElseThrow() + : ((ConstructorDeclaration) cd).getBody(); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); + return new CallSiteBuilder(ctx).build(body); + } + + @Test + void build_keysAreBareLocalIdsNotFullIds() { + // Keystone: `body` is keyed by the node's LOCAL id (`line:col` / `@tag`); the full + // `@` form is only used at application scope (L4 param_in/param_out). + String source = "package p;\nclass Foo {\n void m() {\n bar(x);\n baz();\n }\n}\n"; + Map body = build(source); + assertEquals(List.of("4:5", "5:5"), new ArrayList<>(body.keySet())); + assertEquals("call", body.get("4:5").getKind()); + } + + @Test + void build_callNodeCarriesNoCalleeAtL1() { + JBodyNode bar = build("package p;\nclass Foo {\n void m() {\n bar(x);\n }\n}\n").get("4:5"); + assertNull(bar.getCallee(), "callee is absent at L1 and set when L2 resolves the site"); + } + + @Test + void build_callNodeArgumentsAreLocalIdsOfArgumentExpressions() { + JBodyNode bar = build("package p;\nclass Foo {\n void m() {\n bar(x, y);\n }\n}\n").get("4:5"); + // " bar(x, y);" -> x at col 9, y at col 12 + assertEquals(List.of("4:9", "4:12"), bar.getArguments()); + } + + @Test + void build_callNodeSpanSlicesToTheCallText() { + String source = "package p;\nclass Foo {\n void m() {\n bar(x);\n }\n}\n"; + JBodyNode bar = build(source).get("4:5"); + assertNotNull(bar.getSpan()); + int[] bytes = bar.getSpan().getBytes(); + assertEquals("bar(x)", source.substring(bytes[0], bytes[1])); + } + + @Test + void build_chainedCallsGetDistinctIdsFromTheInvokedNameAnchor() { + // a.b().c(): anchoring on the invoked name (not the expression start) keeps the two calls apart. + String source = "package p;\nclass Foo {\n void m() {\n a.b().c();\n }\n}\n"; + Map body = build(source); + assertEquals(2, body.size()); + assertEquals(List.of("4:7", "4:11"), new ArrayList<>(body.keySet())); + } + + @Test + void build_emitsCallNodeForConstructorInvocation() { + // `new Helper()` is a call site too — L2 resolves it to the constructor callable, so without + // it the v2 call graph would systematically miss constructor edges. + String source = "package p;\nclass Foo {\n void m() {\n Helper h = new Helper();\n }\n}\n"; + Map body = build(source); + // anchored at the instantiated type name, mirroring the invoked-name anchor for method calls + assertEquals(List.of("4:20"), new ArrayList<>(body.keySet())); + assertEquals("call", body.get("4:20").getKind()); + } + + @Test + void build_constructorCallCarriesArgumentLocalIds() { + String source = "package p;\nclass Foo {\n void m() {\n new Helper(a);\n }\n}\n"; + JBodyNode node = build(source).get("4:9"); + assertEquals(List.of("4:16"), node.getArguments()); + } + + @Test + void build_emitsCallNodeForExplicitConstructorChaining() { + // this(...) / super(...) are constructor calls that matter for call-graph completeness. + String source = "package p;\nclass Foo {\n Foo() {\n this(1);\n }\n}\n"; + Map body = build(source); + assertEquals(List.of("4:5"), new ArrayList<>(body.keySet())); + assertEquals("call", body.get("4:5").getKind()); + } + + @Test + void build_ordersMethodAndConstructorCallsBySourcePosition() { + String source = "package p;\nclass Foo {\n void m() {\n a();\n new B();\n c();\n }\n}\n"; + Map body = build(source); + assertEquals(List.of("4:5", "5:9", "6:5"), new ArrayList<>(body.keySet())); + } + + @Test + void build_excludesCallsInsideNestedLocalClasses() { + // hidden() belongs to Local.inner()'s own body (its own callable), not to m(). + String source = "package p;\nclass Foo {\n void m() {\n outer();\n class Local {\n" + + " void inner() { hidden(); }\n }\n }\n}\n"; + Map body = build(source); + assertEquals(List.of("4:5"), new ArrayList<>(body.keySet())); + } + + @Test + void build_includesCallsInsideLambdas() { + // A lambda has no separate callable; its calls are part of the enclosing method's body. + String source = "package p;\nclass Foo {\n void m() {\n run(() -> log());\n }\n}\n"; + Map body = build(source); + assertTrue(body.keySet().stream().anyMatch(k -> k.startsWith("4:"))); + assertEquals(2, body.size(), "both run(...) and log() are calls in m()'s body"); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java new file mode 100644 index 0000000..a55cf8c --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -0,0 +1,108 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.CallableDeclaration; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JCallable; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +/** Tests the v2 {@link CallableBuilder} — signature, params, return/error channel, metrics, refs, body. */ +class CallableBuilderTest { + + private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; + private static final String TYPE_ID = "can://java/myapp/" + FILE_KEY + "/Foo"; + + private static JCallable build(String memberSource, List classFieldNames) { + String source = "package com.example;\nimport java.io.IOException;\nimport java.util.*;\n" + + "class Foo {\n " + memberSource + "\n}\n"; + CompilationUnit cu = new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(source) + .getResult() + .orElseThrow(); + CallableDeclaration cd = cu.getType(0).findFirst(CallableDeclaration.class).orElseThrow(); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); + return new CallableBuilder(ctx).build(cd, TYPE_ID, classFieldNames); + } + + private static JCallable build(String memberSource) { + return build(memberSource, List.of()); + } + + @Test + void build_methodKindSignatureIdAndSpan() { + JCallable c = build("int add(int a, int b) { return a + b; }"); + assertEquals("method", c.getKind()); + assertEquals("add(int, int)", c.getSignature()); + assertEquals(TYPE_ID + "/add(int, int)", c.getId()); + assertNotNull(c.getSpan()); + } + + @Test + void build_constructorKindHasNullReturnType() { + JCallable c = build("Foo(int x) {}"); + assertEquals("constructor", c.getKind()); + assertNull(c.getReturnType()); + assertTrue(c.getId().startsWith(TYPE_ID + "/")); + } + + @Test + void build_capturesParametersReturnTypeModifiersAndDecorators() { + JCallable c = build("@Override public String greet(String name) { return \"hi\"; }"); + assertEquals(List.of("name"), c.getParameters().stream() + .map(p -> p.getName()).collect(Collectors.toList())); + assertEquals("String", c.getParameters().get(0).getType()); + assertEquals("String", c.getReturnType()); + assertEquals(List.of("public"), c.getModifiers()); + assertEquals("Override", c.getDecorators().get(0).getName()); + } + + @Test + void build_capturesErrorChannelFromThrows() { + JCallable c = build("void read() throws IOException, RuntimeException {}"); + assertEquals(List.of("IOException", "RuntimeException"), c.getErrorChannel()); + } + + @Test + void build_computesCyclomaticMetric() { + JCallable c = build("void m(int x) { if (x > 0) { } }"); + assertEquals(2, c.getMetrics().getCyclomatic()); + } + + @Test + void build_capturesBodyCallNodes() { + JCallable c = build("void m() { foo(); }"); + assertEquals(1, c.getBody().size()); + assertEquals("call", c.getBody().values().iterator().next().getKind()); + } + + @Test + void build_capturesRefsTypesAndAccessedFields() { + JCallable c = build("void m() { Helper h = new Helper(); this.count = h.value(); }", List.of("count")); + assertTrue(c.getRefs().getTypes().contains("Helper"), + "referenced types should include the syntactic type Helper"); + assertEquals(List.of("count"), c.getRefs().getFields()); + } + + @Test + void build_capturesLocalClassUnderCallableTypesViaContainment() { + JCallable c = build("void m() { class Local {} }"); + assertTrue(c.getTypes().containsKey("Local")); + assertEquals(c.getId() + "/Local", c.getTypes().get("Local").getId()); + } + + @Test + void build_abstractMethodHasEmptyBody() { + JCallable c = build("abstract void m();"); + assertTrue(c.getBody().isEmpty()); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java new file mode 100644 index 0000000..daaee8a --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java @@ -0,0 +1,75 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.FieldDeclaration; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JField; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +/** Tests the v2 {@link FieldBuilder} — one field node per declared variable, with id/type/span. */ +class FieldBuilderTest { + + private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; + private static final String TYPE_ID = "can://java/myapp/" + FILE_KEY + "/Foo"; + + private static List build(String memberSource) { + String source = "package com.example;\nclass Foo {\n " + memberSource + "\n}\n"; + CompilationUnit cu = new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(source) + .getResult() + .orElseThrow(); + FieldDeclaration fd = cu.getType(0).findFirst(FieldDeclaration.class).orElseThrow(); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); + return new FieldBuilder(ctx).build(fd, TYPE_ID); + } + + @Test + void build_capturesNameTypeAndContainmentId() { + List fields = build("private int count;"); + assertEquals(1, fields.size()); + JField f = fields.get(0); + assertEquals("count", f.getName()); + assertEquals("int", f.getType()); + assertEquals(TYPE_ID + "/count", f.getId()); + } + + @Test + void build_capturesModifiers() { + JField f = build("private static final String NAME = \"x\";").get(0); + assertEquals(List.of("private", "static", "final"), f.getModifiers()); + } + + @Test + void build_emitsOneFieldPerVariableInAMultiVariableDeclaration() { + List fields = build("int a, b;"); + assertEquals(List.of("a", "b"), fields.stream().map(JField::getName).collect(Collectors.toList())); + assertTrue(fields.stream().allMatch(f -> f.getType().equals("int"))); + assertEquals(TYPE_ID + "/a", fields.get(0).getId()); + assertEquals(TYPE_ID + "/b", fields.get(1).getId()); + } + + @Test + void build_spanBytesSliceToTheFieldDeclarationText() { + List fields = build("private int count;"); + int[] bytes = fields.get(0).getSpan().getBytes(); + String source = "package com.example;\nclass Foo {\n private int count;\n}\n"; + assertEquals("private int count;", source.substring(bytes[0], bytes[1])); + } + + @Test + void build_capturesStructuredDecorators() { + JField f = build("@Column(name = \"id\") private Long id;").get(0); + assertEquals(1, f.getDecorators().size()); + assertEquals("Column", f.getDecorators().get(0).getName()); + assertNotNull(f.getDecorators().get(0).getSpan()); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java new file mode 100644 index 0000000..4bcaed2 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java @@ -0,0 +1,71 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.body.Parameter; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JParameter; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** Tests the v2 {@link ParameterBuilder} — name, declared type, byte-offset span, decorators. */ +class ParameterBuilderTest { + + private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; + + private static Parameter firstParam(String source) { + CompilationUnit cu = new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(source) + .getResult() + .orElseThrow(); + return cu.getType(0).findFirst(MethodDeclaration.class).orElseThrow().getParameter(0); + } + + private static JParameter build(String source) { + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); + return new ParameterBuilder(ctx).build(firstParam(source)); + } + + @Test + void build_capturesNameAndDeclaredType() { + JParameter p = build("package p;\nclass Foo {\n void m(final String name) {}\n}\n"); + assertEquals("name", p.getName()); + assertEquals("String", p.getType()); + } + + @Test + void build_capturesGenericAndArrayTypesSyntactically() { + assertEquals("List", build("package p;\nclass Foo {\n void m(List xs) {}\n}\n").getType()); + assertEquals("int[]", build("package p;\nclass Foo {\n void m(int[] xs) {}\n}\n").getType()); + } + + @Test + void build_spanBytesSliceToTheParameterText() { + String source = "package p;\nclass Foo {\n void m(String name) {}\n}\n"; + JParameter p = build(source); + assertNotNull(p.getSpan()); + int[] bytes = p.getSpan().getBytes(); + assertEquals("String name", source.substring(bytes[0], bytes[1])); + } + + @Test + void build_capturesStructuredParameterDecorators() { + JParameter p = build("package p;\nclass Foo {\n void m(@RequestParam(\"q\") String query) {}\n}\n"); + assertEquals(1, p.getDecorators().size()); + assertEquals("RequestParam", p.getDecorators().get(0).getName()); + assertEquals(List.of("\"q\""), p.getDecorators().get(0).getArgs()); + } + + @Test + void build_hasNoDecoratorsForPlainParameter() { + JParameter p = build("package p;\nclass Foo {\n void m(String name) {}\n}\n"); + assertTrue(p.getDecorators().isEmpty()); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index 7a1730b..306ccb3 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -9,6 +9,7 @@ import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.body.TypeDeclaration; import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JCallable; import com.ibm.cldk.schema.JDecorator; import com.ibm.cldk.schema.JType; import java.util.List; @@ -82,6 +83,30 @@ void build_recursesIntoMemberTypesViaContainment() { assertEquals("class", inner.getKind()); } + @Test + void build_populatesFieldsKeyedBySimpleName() { + JType t = buildFirstType("package p;\nclass Foo {\n private int count;\n String name;\n}\n"); + assertEquals(Set.of("count", "name"), t.getFields().keySet()); + assertEquals("int", t.getFields().get("count").getType()); + assertEquals(t.getId() + "/count", t.getFields().get("count").getId()); + } + + @Test + void build_populatesCallablesKeyedBySignature() { + JType t = buildFirstType("package p;\nclass Foo {\n Foo() {}\n void inc() {}\n}\n"); + assertTrue(t.getCallables().containsKey("inc()")); + assertEquals(2, t.getCallables().size(), "constructor + method"); + assertEquals("method", t.getCallables().get("inc()").getKind()); + } + + @Test + void build_callableRefsSeeEnclosingTypeFields() { + // TypeBuilder must hand its field names to the callable builder so refs.fields resolves. + JType t = buildFirstType("package p;\nclass Foo {\n int count;\n void inc() { count = count + 1; }\n}\n"); + JCallable inc = t.getCallables().get("inc()"); + assertEquals(List.of("count"), inc.getRefs().getFields()); + } + @Test void build_capturesStructuredDecoratorWithArgs() { JType t = buildFirstType("package p;\n@SuppressWarnings(\"unchecked\")\nclass Foo {}\n"); From 7c94c9d19087fe44926bc7a373ea8d46c9e15d18 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 18:51:20 -0400 Subject: [PATCH 07/22] feat(schema): v2 JSON config, module imports/hash/span, field kind, is_variadic (#180) Add V2Json (snake_case keys, no nulls emitted), module span/imports/ content_hash, field kind discriminator and parameter is_variadic, with a serialization-contract test covering the emitted key names. --- .claude/SCHEMA_DECISIONS.md | 34 ++++ src/main/java/com/ibm/cldk/schema/JField.java | 1 + .../java/com/ibm/cldk/schema/JImport.java | 17 ++ .../java/com/ibm/cldk/schema/JModule.java | 11 ++ .../java/com/ibm/cldk/schema/JParameter.java | 3 + src/main/java/com/ibm/cldk/schema/V2Json.java | 46 +++++ .../syntactic_analysis/L1BuildContext.java | 38 +++++ .../syntactic_analysis/ModuleBuilder.java | 19 +++ .../syntactic_analysis/ParameterBuilder.java | 6 +- .../java/com/ibm/cldk/schema/V2JsonTest.java | 158 ++++++++++++++++++ .../syntactic_analysis/FieldBuilderTest.java | 6 + .../syntactic_analysis/ModuleBuilderTest.java | 53 ++++++ .../ParameterBuilderTest.java | 15 ++ 13 files changed, 405 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/ibm/cldk/schema/JImport.java create mode 100644 src/main/java/com/ibm/cldk/schema/V2Json.java create mode 100644 src/test/java/com/ibm/cldk/schema/V2JsonTest.java diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 2d7fef3..3778b96 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -86,6 +86,40 @@ Ordinal ids `…@:` (real) / `…@` (synthetic) within a callabl `getMethods()`/`getConstructors()`); nested-type methods hang under their own type, local (method-body) classes under `callable.types` (D4 containment). +### D10 — L1 emission: body keys, null policy, call sites, spans + +Refinements settled while building L1 (2026-08), each checked against the keystone **and** +`codeanalyzer-python`: + +- **`body` is keyed by the bare local id** (`line:col`), not the full `@line:col`. + The keystone keys `body` "by the node's local id" and its worked example shows `"15:2"` / + `"@entry"`; the pilot does `key = f"{cs.start_line}:{cs.start_column}"`. The full form is derived + only where cross-callable ids are needed (L4's application-scope `param_in`/`param_out`). +- **L3 must not overwrite an L1 `call` node.** A bare call statement resolves to the same local id + as its `call` node; per the keystone's example the call node *is* that statement, so L3 adds the + remaining statements around it and never rewrites its `kind` (rewriting would break the additive + invariant). A call nested in a larger statement (`int y = bar(x);`) yields two distinct nodes. +- **Call sites include constructors.** `new Foo()` and explicit `this(...)`/`super(...)` chaining are + emitted as `call` nodes alongside method invocations — L2 resolves all three into `call_graph` + edges, so omitting them would silently drop constructor edges. Anchor: the invoked name (method + name, or instantiated type name), which also keeps chained calls `a.b().c()` distinct. +- **`arguments` are positional addresses, not node references.** They carry argument `line:col` + local ids for tooling, but no `body` node need exist at those positions: expression nodes are + optional in the keystone (`--materialize-expressions`, **not implemented here**) and L4's + `actual_in{of:"argN", parent}` is the canonical way arguments become real nodes. The no-dangling + invariant governs *edges*, which these are not. +- **No nulls are emitted — absence encodes "no fact"** (`V2Json` deliberately omits + `serializeNulls()`). This includes the `callee` refinement slot: the key is absent at L1 and + appears once L2 resolves the site. The keystone's `callee: null` example is illustrative; the pilot + likewise drops it via `exclude_none`. +- **Varargs: `type` keeps the element type + `is_variadic` flag** (keystone's `param.is_variadic?`), + so `String...` stays distinguishable from a real `String[]` parameter. +- **`module.span` covers the whole file**, computed from the source rather than the compilation + unit's AST range (which ends inconsistently around trailing whitespace), so + `module.source[span.bytes] == module.source` always holds. +- **`module.content_hash` is SHA-256 hex of the UTF-8 source** — for incremental caching and the + Neo4j writer's per-module diffing; never identity (the `id` is). + ### D9 — Neo4j namespace: keep the `J_` relationship prefix Existing convention (`J_CALLS`, …); dual-label `JSymbol` merge pattern retained. `SchemaCatalog` takes a major bump (families rename v1→v2). diff --git a/src/main/java/com/ibm/cldk/schema/JField.java b/src/main/java/com/ibm/cldk/schema/JField.java index c9cf360..550f675 100644 --- a/src/main/java/com/ibm/cldk/schema/JField.java +++ b/src/main/java/com/ibm/cldk/schema/JField.java @@ -12,6 +12,7 @@ @Data public class JField { private String id; + private String kind = "field"; private String name; private String type; private Span span; diff --git a/src/main/java/com/ibm/cldk/schema/JImport.java b/src/main/java/com/ibm/cldk/schema/JImport.java new file mode 100644 index 0000000..f8bf05e --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JImport.java @@ -0,0 +1,17 @@ +package com.ibm.cldk.schema; + +import lombok.Data; + +/** + * An import declaration on a {@code module}. {@code path} is the imported name as written + * ({@code java.util.List}, or the package for a wildcard import); {@code name} is its last segment. + * {@code is_static} / {@code is_wildcard} are Java-specific additions to the keystone's import shape. + */ +@Data +public class JImport { + private String name; + private String path; + private Span span; + private boolean isStatic; + private boolean isWildcard; +} diff --git a/src/main/java/com/ibm/cldk/schema/JModule.java b/src/main/java/com/ibm/cldk/schema/JModule.java index ad6a788..1493d81 100644 --- a/src/main/java/com/ibm/cldk/schema/JModule.java +++ b/src/main/java/com/ibm/cldk/schema/JModule.java @@ -1,7 +1,9 @@ package com.ibm.cldk.schema; import com.google.gson.annotations.SerializedName; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import lombok.Data; @@ -13,6 +15,7 @@ public class JModule { private String id; private String kind = "module"; + private Span span; /** {@code package} is a Java keyword, so the field is {@code packageName} but serializes as {@code package}. */ @SerializedName("package") @@ -20,6 +23,14 @@ public class JModule { private String source; + private List imports = new ArrayList<>(); + /** Top-level types declared in this file, keyed by simple name (nested types hang under them). */ private Map types = new LinkedHashMap<>(); + + /** + * Content hash of {@code source} — used for incremental caching and the Neo4j writer's + * per-module diffing. Not identity (the {@code id} is). + */ + private String contentHash; } diff --git a/src/main/java/com/ibm/cldk/schema/JParameter.java b/src/main/java/com/ibm/cldk/schema/JParameter.java index 8d8d164..517955a 100644 --- a/src/main/java/com/ibm/cldk/schema/JParameter.java +++ b/src/main/java/com/ibm/cldk/schema/JParameter.java @@ -15,4 +15,7 @@ public class JParameter { private String type; private Span span; private List decorators = new ArrayList<>(); + + /** True for a varargs parameter ({@code String... names}); {@code type} stays the element type. */ + private boolean isVariadic; } diff --git a/src/main/java/com/ibm/cldk/schema/V2Json.java b/src/main/java/com/ibm/cldk/schema/V2Json.java new file mode 100644 index 0000000..7090be5 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/V2Json.java @@ -0,0 +1,46 @@ +package com.ibm.cldk.schema; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Gson configuration for canonical schema v2 output. + * + *

Two conventions from the keystone are encoded here: + * + *

+ */ +public final class V2Json { + + private V2Json() {} + + private static final Gson COMPACT = base().create(); + private static final Gson PRETTY = base().setPrettyPrinting().create(); + + private static GsonBuilder base() { + return new GsonBuilder() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .disableHtmlEscaping(); + // Deliberately NOT serializeNulls(): absent = no fact. + } + + /** Compact JSON — what goes to stdout when {@code -o} is omitted. */ + public static Gson compact() { + return COMPACT; + } + + /** Pretty-printed JSON — what is written to {@code analysis.json}. */ + public static Gson pretty() { + return PRETTY; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java index 81c88e6..d7e5822 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java @@ -5,6 +5,9 @@ import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.Span; import com.ibm.cldk.schema.Spans; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import lombok.Getter; /** @@ -30,6 +33,41 @@ public String moduleId() { return CanId.moduleId(applicationId, fileKey); } + /** + * The span covering the whole file — the module's own span. Computed from the source rather than + * the compilation unit's AST range (which ends inconsistently around trailing whitespace), so the + * invariant {@code module.source[span.bytes] == module.source} always holds. + */ + public Span wholeFileSpan() { + String[] lines = source.split("\n", -1); + int lastLine = Math.max(1, lines.length); + int lastCol = lines[lines.length - 1].length() + 1; + Span span = new Span(); + span.setStart(new int[] {1, 1}); + span.setEnd(new int[] {lastLine, lastCol}); + span.setBytes(new int[] {0, source.getBytes(StandardCharsets.UTF_8).length}); + return span; + } + + /** + * SHA-256 hex of the file's UTF-8 source — the module's {@code content_hash}, used for + * incremental caching and the Neo4j writer's per-module diffing (never for identity). + */ + public String contentHash() { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(source.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte b : digest) { + hex.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is required of every JVM; treat absence as unrecoverable rather than silently + // emitting a hash that would break cache/diff correctness. + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + /** * Build the {@link Span} for an AST node from its source range: {@code start}/{@code end} as * JavaParser {@code [line, column]} (1-based), {@code bytes} as {@code [from, to)} UTF-8 offsets diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java index 41d0ced..38d8554 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java @@ -1,10 +1,14 @@ package com.ibm.cldk.syntactic_analysis; import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.ImportDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; +import com.ibm.cldk.schema.JImport; import com.ibm.cldk.schema.JModule; import com.ibm.cldk.schema.JType; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -24,8 +28,23 @@ public ModuleBuilder(L1BuildContext ctx) { public JModule build(CompilationUnit cu) { JModule module = new JModule(); module.setId(ctx.moduleId()); + module.setSpan(ctx.wholeFileSpan()); module.setPackageName(cu.getPackageDeclaration().map(pd -> pd.getNameAsString()).orElse("")); module.setSource(ctx.getSource()); + module.setContentHash(ctx.contentHash()); + + List imports = new ArrayList<>(); + for (ImportDeclaration id : cu.getImports()) { + JImport imp = new JImport(); + String path = id.getNameAsString(); + imp.setPath(path); + imp.setName(path.contains(".") ? path.substring(path.lastIndexOf('.') + 1) : path); + imp.setStatic(id.isStatic()); + imp.setWildcard(id.isAsterisk()); + imp.setSpan(ctx.spanOf(id)); + imports.add(imp); + } + module.setImports(imports); // Top-level types, keyed by simple name and sorted for deterministic output (the -j gate). TypeBuilder typeBuilder = new TypeBuilder(ctx); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java index 5fbb9d3..4dcf992 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java @@ -22,8 +22,10 @@ public ParameterBuilder(L1BuildContext ctx) { public JParameter build(Parameter param) { JParameter parameter = new JParameter(); parameter.setName(param.getNameAsString()); - // Varargs (`String...`) spell as the element type + "[]" so the type reads as a real Java type. - parameter.setType(param.isVarArgs() ? param.getType().asString() + "[]" : param.getType().asString()); + // Varargs keep the declared ELEMENT type; the `is_variadic` flag carries the `...` instead, so + // `String...` stays distinguishable from a real `String[]` parameter. + parameter.setType(param.getType().asString()); + parameter.setVariadic(param.isVarArgs()); parameter.setSpan(ctx.spanOf(param)); parameter.setDecorators( param.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); diff --git a/src/test/java/com/ibm/cldk/schema/V2JsonTest.java b/src/test/java/com/ibm/cldk/schema/V2JsonTest.java new file mode 100644 index 0000000..7d6cb7b --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/V2JsonTest.java @@ -0,0 +1,158 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.google.gson.JsonObject; +import com.ibm.cldk.syntactic_analysis.L1BuildContext; +import com.ibm.cldk.syntactic_analysis.ModuleBuilder; +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Serialization contract for schema v2: the emitted JSON key names (snake_case, so one set of + * SDK models parses every analyzer) and the no-null convention (absence encodes "no fact"). These keys + * are the contract — a rename here breaks every consumer, so they are asserted explicitly. + */ +class V2JsonTest { + + private static final String FILE_KEY = "src/Foo.java"; + private static final String SOURCE = "package com.example;\n" + + "import java.util.List;\n" + + "class Foo {\n" + + " private int count;\n" + + " Foo() {}\n" + + " int add(int a, String... rest) throws IllegalStateException {\n" + + " helper(a);\n" + + " return count;\n" + + " }\n" + + "}\n"; + + private static JsonObject payload() { + CompilationUnit cu = new JavaParser( + new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) + .parse(SOURCE) + .getResult() + .orElseThrow(); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, SOURCE); + Map modules = new LinkedHashMap<>(); + modules.put(FILE_KEY, new ModuleBuilder(ctx).build(cu)); + return V2Json.compact().toJsonTree(V2Emitter.emit("myapp", 1, modules)).getAsJsonObject(); + } + + private static JsonObject module() { + return payload().getAsJsonObject("application").getAsJsonObject("symbol_table").getAsJsonObject(FILE_KEY); + } + + private static JsonObject fooType() { + return module().getAsJsonObject("types").getAsJsonObject("Foo"); + } + + /** The one callable whose kind is {@code method} (the constructor is the other entry). */ + private static JsonObject theMethod() { + JsonObject callables = fooType().getAsJsonObject("callables"); + return callables.keySet().stream() + .map(callables::getAsJsonObject) + .filter(c -> "method".equals(c.get("kind").getAsString())) + .findFirst() + .orElseThrow(); + } + + private static JsonObject theConstructor() { + JsonObject callables = fooType().getAsJsonObject("callables"); + return callables.keySet().stream() + .map(callables::getAsJsonObject) + .filter(c -> "constructor".equals(c.get("kind").getAsString())) + .findFirst() + .orElseThrow(); + } + + @Test + void envelopeUsesSnakeCaseManifestKeys() { + JsonObject root = payload(); + assertEquals("2.0.0", root.get("schema_version").getAsString()); + assertEquals("java", root.get("language").getAsString()); + assertEquals(1, root.get("max_level").getAsInt()); + assertTrue(root.has("application")); + } + + @Test + void applicationCarriesIdKindAndFileKeyedSymbolTable() { + JsonObject app = payload().getAsJsonObject("application"); + assertEquals("can://java/myapp", app.get("id").getAsString()); + assertEquals("application", app.get("kind").getAsString()); + assertTrue(app.getAsJsonObject("symbol_table").has(FILE_KEY)); + } + + @Test + void moduleUsesPackageSourceAndContentHashKeys() { + JsonObject module = module(); + assertEquals("module", module.get("kind").getAsString()); + assertEquals("com.example", module.get("package").getAsString(), "`package` is a Java keyword, aliased"); + assertEquals(SOURCE, module.get("source").getAsString()); + assertTrue(module.has("content_hash")); + assertTrue(module.has("imports")); + assertTrue(module.has("span")); + } + + @Test + void typeUsesBaseTypesAndInterfacesKeys() { + JsonObject type = fooType(); + assertEquals("class", type.get("kind").getAsString()); + assertTrue(type.has("base_types")); + assertTrue(type.has("interfaces")); + assertTrue(type.has("decorators")); + assertTrue(type.has("fields")); + assertTrue(type.has("callables")); + } + + @Test + void callableUsesErrorChannelAndNestedMetricsAndRefs() { + JsonObject method = theMethod(); + assertEquals("IllegalStateException", method.getAsJsonArray("error_channel").get(0).getAsString()); + assertTrue(method.has("return_type")); + assertTrue(method.getAsJsonObject("metrics").has("cyclomatic"), "metrics are nested, not flattened"); + assertTrue(method.getAsJsonObject("refs").has("types")); + assertTrue(method.getAsJsonObject("refs").has("fields")); + assertTrue(method.has("body")); + } + + @Test + void fieldCarriesKindAndParameterCarriesIsVariadic() { + assertEquals("field", fooType().getAsJsonObject("fields").getAsJsonObject("count").get("kind").getAsString()); + JsonObject variadic = theMethod().getAsJsonArray("parameters").get(1).getAsJsonObject(); + assertTrue(variadic.get("is_variadic").getAsBoolean()); + } + + @Test + void spanCarriesStartEndAndByteOffsets() { + JsonObject span = fooType().getAsJsonObject("span"); + assertEquals(2, span.getAsJsonArray("start").size()); + assertEquals(2, span.getAsJsonArray("end").size()); + assertEquals(2, span.getAsJsonArray("bytes").size(), "byte offsets make node text an O(1) slice"); + } + + @Test + void nullsAreOmittedRatherThanEmitted() { + // Absence encodes "no fact": a constructor has no return type, and at L1 no call site has a + // resolved callee (that key appears once L2 backfills it). + assertFalse(theConstructor().has("return_type"), "constructor must not carry a null return_type"); + + JsonObject body = theMethod().getAsJsonObject("body"); + JsonObject callNode = body.getAsJsonObject(body.keySet().iterator().next()); + assertEquals("call", callNode.get("kind").getAsString()); + assertFalse(callNode.has("callee"), "callee is absent at L1, not null"); + } + + @Test + void bodyIsKeyedByBareLocalId() { + // `line:col`, never the full `@line:col` form. + String key = theMethod().getAsJsonObject("body").keySet().iterator().next(); + assertTrue(key.matches("\\d+:\\d+"), "expected a bare local id, got: " + key); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java index daaee8a..15865b2 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java @@ -42,6 +42,12 @@ void build_capturesNameTypeAndContainmentId() { assertEquals(TYPE_ID + "/count", f.getId()); } + @Test + void build_carriesFieldKind() { + // Every v2 node carries a `kind` discriminator; the SDK models one Node keyed on it. + assertEquals("field", build("private int count;").get(0).getKind()); + } + @Test void build_capturesModifiers() { JField f = build("private static final String NAME = \"x\";").get(0); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java index 833d1c7..40d81af 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java @@ -1,13 +1,21 @@ package com.ibm.cldk.syntactic_analysis; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.github.javaparser.JavaParser; import com.github.javaparser.ParserConfiguration; import com.github.javaparser.ast.CompilationUnit; import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JImport; import com.ibm.cldk.schema.JModule; +import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; /** Tests the v2 {@link ModuleBuilder} building a {@code module} node directly from the AST. */ @@ -21,6 +29,12 @@ private static CompilationUnit parse(String source) { .orElseThrow(); } + /** Build a module from source using a fixed file key. */ + private static JModule build(String source) { + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), "src/Foo.java", source); + return new ModuleBuilder(ctx).build(parse(source)); + } + @Test void build_setsModuleIdKindPackageAndSource() { String source = "package com.example;\n\npublic class Foo {}\n"; @@ -50,6 +64,45 @@ void build_populatesTopLevelTypesKeyedBySimpleName() { assertEquals("can://java/myapp/src/Foo.java/Foo", module.getTypes().get("Foo").getId()); } + @Test + void build_setsContentHashThatIsStableAndSourceSensitive() { + String a = "package p;\nclass Foo {}\n"; + String b = "package p;\nclass Bar {}\n"; + JModule m1 = build(a); + JModule m2 = build(a); + JModule m3 = build(b); + + assertTrue(m1.getContentHash().matches("[0-9a-f]{64}"), "expected lowercase sha-256 hex"); + assertEquals(m1.getContentHash(), m2.getContentHash(), "same source -> same hash (caching + Neo4j diffing)"); + assertNotEquals(m1.getContentHash(), m3.getContentHash(), "different source -> different hash"); + } + + @Test + void build_capturesImports() { + String source = "package p;\nimport java.util.List;\nimport static java.util.Arrays.asList;\n" + + "import java.io.*;\nclass Foo {}\n"; + List imports = build(source).getImports(); + + assertEquals(List.of("java.util.List", "java.util.Arrays.asList", "java.io"), + imports.stream().map(JImport::getPath).collect(Collectors.toList())); + assertEquals("List", imports.get(0).getName()); + assertTrue(imports.get(1).isStatic()); + assertTrue(imports.get(2).isWildcard()); + assertNotNull(imports.get(0).getSpan()); + } + + @Test + void build_moduleSpanCoversTheWholeFile() { + // The invariant the SDK relies on: module.source[span.bytes] IS the whole file. + String source = "package com.example;\n\npublic class Foo {}\n"; + JModule module = build(source); + assertNotNull(module.getSpan()); + assertArrayEquals(new int[] {1, 1}, module.getSpan().getStart()); + int[] bytes = module.getSpan().getBytes(); + assertEquals(0, bytes[0]); + assertEquals(source.getBytes(StandardCharsets.UTF_8).length, bytes[1]); + } + @Test void build_defaultsPackageToEmptyWhenAbsent() { String source = "public class Foo {}\n"; diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java index 4bcaed2..bfa2623 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java @@ -1,6 +1,7 @@ package com.ibm.cldk.syntactic_analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -55,6 +56,20 @@ void build_spanBytesSliceToTheParameterText() { assertEquals("String name", source.substring(bytes[0], bytes[1])); } + @Test + void build_marksVariadicParameterAndKeepsElementType() { + JParameter p = build("package p;\nclass Foo {\n void m(String... names) {}\n}\n"); + assertTrue(p.isVariadic(), "String... must set is_variadic"); + assertEquals("String", p.getType(), "type stays the element type; the flag carries the ..."); + } + + @Test + void build_plainArrayParameterIsNotVariadic() { + JParameter p = build("package p;\nclass Foo {\n void m(String[] names) {}\n}\n"); + assertFalse(p.isVariadic(), "String[] is an array, not varargs"); + assertEquals("String[]", p.getType()); + } + @Test void build_capturesStructuredParameterDecorators() { JParameter p = build("package p;\nclass Foo {\n void m(@RequestParam(\"q\") String query) {}\n}\n"); From 7e7c723bebf75a3ae0cda82cc364acbe01a9ab38 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 19:30:24 -0400 Subject: [PATCH 08/22] feat(schema): type/param modifiers, field initializer, declaration, code_start_line, comments (#180) --- .../java/com/ibm/cldk/schema/JCallable.java | 10 +++++++ .../java/com/ibm/cldk/schema/JComment.java | 22 +++++++++++++++ src/main/java/com/ibm/cldk/schema/JField.java | 4 +++ .../java/com/ibm/cldk/schema/JModule.java | 1 + .../java/com/ibm/cldk/schema/JParameter.java | 1 + src/main/java/com/ibm/cldk/schema/JType.java | 2 ++ .../syntactic_analysis/CallableBuilder.java | 5 ++++ .../cldk/syntactic_analysis/FieldBuilder.java | 4 +++ .../syntactic_analysis/L1BuildContext.java | 24 ++++++++++++++++ .../syntactic_analysis/ModuleBuilder.java | 9 ++++++ .../syntactic_analysis/ParameterBuilder.java | 2 ++ .../cldk/syntactic_analysis/TypeBuilder.java | 3 ++ .../CallableBuilderTest.java | 28 +++++++++++++++++++ .../syntactic_analysis/FieldBuilderTest.java | 20 +++++++++++++ .../syntactic_analysis/ModuleBuilderTest.java | 7 +++++ .../ParameterBuilderTest.java | 6 ++++ .../syntactic_analysis/TypeBuilderTest.java | 25 +++++++++++++++++ 17 files changed, 173 insertions(+) create mode 100644 src/main/java/com/ibm/cldk/schema/JComment.java diff --git a/src/main/java/com/ibm/cldk/schema/JCallable.java b/src/main/java/com/ibm/cldk/schema/JCallable.java index 0aee6ad..61b6d3f 100644 --- a/src/main/java/com/ibm/cldk/schema/JCallable.java +++ b/src/main/java/com/ibm/cldk/schema/JCallable.java @@ -24,6 +24,16 @@ public class JCallable { private List errorChannel = new ArrayList<>(); private List modifiers = new ArrayList<>(); private List decorators = new ArrayList<>(); + /** Signature-with-parameter-names text (not recoverable from span.bytes, which covers the body). */ + private String declaration; + + /** First line of the body block, or -1 when there is no body (abstract/interface method). */ + private int codeStartLine = -1; + + /** True for compiler-generated members the source does not declare (e.g. a default constructor). */ + private boolean isImplicit; + + private List comments = new ArrayList<>(); private JMetrics metrics; private JRefs refs; diff --git a/src/main/java/com/ibm/cldk/schema/JComment.java b/src/main/java/com/ibm/cldk/schema/JComment.java new file mode 100644 index 0000000..093ddf3 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JComment.java @@ -0,0 +1,22 @@ +package com.ibm.cldk.schema; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; + +/** + * A comment attached to a node — the declaration's own javadoc or leading line/block comment. + * + *

Unlike the v1 model (which collected all contained comments, so a type repeated every + * comment inside every member), a node here carries only the comment attached to it; the module + * carries the file-level/orphan comments. Text is also recoverable from + * {@code module.source[span.bytes]}, but keeping comments addressable matters for doc-driven + * consumers. + */ +@Data +public class JComment { + private String content; + private Span span; + + @SerializedName("is_javadoc") + private boolean javadoc; +} diff --git a/src/main/java/com/ibm/cldk/schema/JField.java b/src/main/java/com/ibm/cldk/schema/JField.java index 550f675..21f7896 100644 --- a/src/main/java/com/ibm/cldk/schema/JField.java +++ b/src/main/java/com/ibm/cldk/schema/JField.java @@ -17,5 +17,9 @@ public class JField { private String type; private Span span; private List modifiers = new ArrayList<>(); + private List comments = new ArrayList<>(); private List decorators = new ArrayList<>(); + + /** The declarator's initializer expression text, if any (absent when uninitialized). */ + private String initializer; } diff --git a/src/main/java/com/ibm/cldk/schema/JModule.java b/src/main/java/com/ibm/cldk/schema/JModule.java index 1493d81..2c0c24e 100644 --- a/src/main/java/com/ibm/cldk/schema/JModule.java +++ b/src/main/java/com/ibm/cldk/schema/JModule.java @@ -23,6 +23,7 @@ public class JModule { private String source; + private List comments = new ArrayList<>(); private List imports = new ArrayList<>(); /** Top-level types declared in this file, keyed by simple name (nested types hang under them). */ diff --git a/src/main/java/com/ibm/cldk/schema/JParameter.java b/src/main/java/com/ibm/cldk/schema/JParameter.java index 517955a..8f53fa0 100644 --- a/src/main/java/com/ibm/cldk/schema/JParameter.java +++ b/src/main/java/com/ibm/cldk/schema/JParameter.java @@ -14,6 +14,7 @@ public class JParameter { private String name; private String type; private Span span; + private List modifiers = new ArrayList<>(); private List decorators = new ArrayList<>(); /** True for a varargs parameter ({@code String... names}); {@code type} stays the element type. */ diff --git a/src/main/java/com/ibm/cldk/schema/JType.java b/src/main/java/com/ibm/cldk/schema/JType.java index 99816b5..59314d3 100644 --- a/src/main/java/com/ibm/cldk/schema/JType.java +++ b/src/main/java/com/ibm/cldk/schema/JType.java @@ -16,6 +16,8 @@ public class JType { private String id; private String kind; private Span span; + private List comments = new ArrayList<>(); + private List modifiers = new ArrayList<>(); private List baseTypes = new ArrayList<>(); private List interfaces = new ArrayList<>(); private List decorators = new ArrayList<>(); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index 784da1c..31dfee4 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -75,14 +75,19 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List t.asString()).collect(Collectors.toList())); callable.setModifiers( cd.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); + callable.setComments(ctx.commentsOf(cd)); callable.setDecorators( cd.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); + // `declaration` mirrors v1: modifiers + return type + name + parameter names, no body. + callable.setDeclaration(cd.getDeclarationAsString(true, true, true).strip()); + JMetrics metrics = new JMetrics(); metrics.setCyclomatic(cyclomaticComplexity(cd)); callable.setMetrics(metrics); Optional body = bodyOf(cd); + body.flatMap(b -> b.getRange().map(r -> r.begin.line)).ifPresent(callable::setCodeStartLine); callable.setRefs(refs(body, classFieldNames)); body.ifPresent(b -> callable.setBody(callSiteBuilder.build(b))); body.ifPresent(b -> callable.setTypes(localClasses(b, callable.getId()))); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java index 2b0a0d9..0a7b399 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java @@ -3,6 +3,7 @@ import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.VariableDeclarator; import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JComment; import com.ibm.cldk.schema.JDecorator; import com.ibm.cldk.schema.JField; import com.ibm.cldk.schema.Span; @@ -37,6 +38,7 @@ public List build(FieldDeclaration fd, String parentTypeId) { List decorators = fd.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList()); Span span = ctx.spanOf(fd); + List comments = ctx.commentsOf(fd); List fields = new ArrayList<>(); for (VariableDeclarator var : fd.getVariables()) { @@ -47,6 +49,8 @@ public List build(FieldDeclaration fd, String parentTypeId) { field.setModifiers(modifiers); field.setDecorators(decorators); field.setSpan(span); + field.setComments(comments); + var.getInitializer().ifPresent(init -> field.setInitializer(init.toString())); fields.add(field); } return fields; diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java index d7e5822..f9a1a0b 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java @@ -2,12 +2,16 @@ import com.github.javaparser.Range; import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.comments.Comment; import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JComment; import com.ibm.cldk.schema.Span; import com.ibm.cldk.schema.Spans; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; import lombok.Getter; /** @@ -33,6 +37,26 @@ public String moduleId() { return CanId.moduleId(applicationId, fileKey); } + /** + * The comment attached to a declaration — its javadoc, or the leading line/block comment. Returns + * an empty list when the node has none. Deliberately the node's own comment rather than + * every comment contained within it (v1's behaviour, which duplicated member comments onto types). + */ + public List commentsOf(Node node) { + List comments = new ArrayList<>(); + node.getComment().ifPresent(c -> comments.add(comment(c))); + return comments; + } + + /** Convert a JavaParser comment into the v2 model. */ + public JComment comment(Comment c) { + JComment out = new JComment(); + out.setContent(c.getContent()); + out.setSpan(spanOf(c)); + out.setJavadoc(c.isJavadocComment()); + return out; + } + /** * The span covering the whole file — the module's own span. Computed from the source rather than * the compilation unit's AST range (which ends inconsistently around trailing whitespace), so the diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java index 38d8554..85a49c9 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ModuleBuilder.java @@ -3,6 +3,7 @@ import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.ImportDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; +import com.ibm.cldk.schema.JComment; import com.ibm.cldk.schema.JImport; import com.ibm.cldk.schema.JModule; import com.ibm.cldk.schema.JType; @@ -33,6 +34,14 @@ public JModule build(CompilationUnit cu) { module.setSource(ctx.getSource()); module.setContentHash(ctx.contentHash()); + // File-level comments: the unit's own comment plus orphans (e.g. a licence header that is not + // attached to any declaration). Declaration comments live on their own nodes. + List comments = new ArrayList<>(ctx.commentsOf(cu)); + cu.getAllComments().stream() + .filter(c -> c.getCommentedNode().isEmpty()) + .forEach(c -> comments.add(ctx.comment(c))); + module.setComments(comments); + List imports = new ArrayList<>(); for (ImportDeclaration id : cu.getImports()) { JImport imp = new JImport(); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java index 4dcf992..32dad01 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java @@ -27,6 +27,8 @@ public JParameter build(Parameter param) { parameter.setType(param.getType().asString()); parameter.setVariadic(param.isVarArgs()); parameter.setSpan(ctx.spanOf(param)); + parameter.setModifiers( + param.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); parameter.setDecorators( param.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); return parameter; diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index 282f549..4b41b40 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -46,6 +46,9 @@ public JType build(TypeDeclaration td, String parentId) { type.setId(CanId.childId(parentId, td.getNameAsString())); type.setKind(kindOf(td)); type.setSpan(ctx.spanOf(td)); + type.setComments(ctx.commentsOf(td)); + type.setModifiers( + td.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); type.setDecorators( td.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java index a55cf8c..ef1223b 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -66,6 +66,34 @@ void build_capturesParametersReturnTypeModifiersAndDecorators() { assertEquals("Override", c.getDecorators().get(0).getName()); } + @Test + void build_capturesJavadocComment() { + JCallable c = build("/** Adds two numbers. */\n int add(int a, int b) { return a + b; }"); + assertEquals(1, c.getComments().size()); + assertTrue(c.getComments().get(0).getContent().contains("Adds two numbers.")); + assertTrue(c.getComments().get(0).isJavadoc()); + } + + @Test + void build_capturesDeclarationString() { + // The signature-with-names text v1 exposed as `declaration` (useful verbatim in LLM prompts); + // it is not recoverable from span.bytes, which covers the body too. + JCallable c = build("public int add(int a, int b) { return a + b; }"); + assertEquals("public int add(int a, int b)", c.getDeclaration()); + } + + @Test + void build_capturesCodeStartLineOfTheBody() { + // "class Foo {" is line 4 of the wrapper, so the member starts on line 5. + JCallable c = build("void m() {\n x();\n }"); + assertEquals(5, c.getCodeStartLine()); + } + + @Test + void build_abstractMethodHasNoCodeStartLine() { + assertEquals(-1, build("abstract void m();").getCodeStartLine()); + } + @Test void build_capturesErrorChannelFromThrows() { JCallable c = build("void read() throws IOException, RuntimeException {}"); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java index 15865b2..a70baaf 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java @@ -1,7 +1,9 @@ package com.ibm.cldk.syntactic_analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.github.javaparser.JavaParser; @@ -63,6 +65,24 @@ void build_emitsOneFieldPerVariableInAMultiVariableDeclaration() { assertEquals(TYPE_ID + "/b", fields.get(1).getId()); } + @Test + void build_capturesFieldComment() { + JField f = build("// how many\n private int count;").get(0); + assertEquals(1, f.getComments().size()); + assertTrue(f.getComments().get(0).getContent().contains("how many")); + assertFalse(f.getComments().get(0).isJavadoc(), "a line comment is not javadoc"); + } + + @Test + void build_capturesPerVariableInitializer() { + // v1 kept variable_initializers keyed per declarator; v2 keeps one field per variable, each + // carrying its own initializer expression text. + List fields = build("int a = 1, b = 2;"); + assertEquals("1", fields.get(0).getInitializer()); + assertEquals("2", fields.get(1).getInitializer()); + assertNull(build("int c;").get(0).getInitializer(), "no initializer -> absent, not empty string"); + } + @Test void build_spanBytesSliceToTheFieldDeclarationText() { List fields = build("private int count;"); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java index 40d81af..32be755 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java @@ -77,6 +77,13 @@ void build_setsContentHashThatIsStableAndSourceSensitive() { assertNotEquals(m1.getContentHash(), m3.getContentHash(), "different source -> different hash"); } + @Test + void build_capturesFileLevelComments() { + JModule m = build("/*\n * Copyright ACME.\n */\npackage p;\nclass Foo {}\n"); + assertTrue(m.getComments().stream().anyMatch(c -> c.getContent().contains("Copyright ACME.")), + "the file header comment belongs to the module"); + } + @Test void build_capturesImports() { String source = "package p;\nimport java.util.List;\nimport static java.util.Arrays.asList;\n" diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java index bfa2623..b171309 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java @@ -41,6 +41,12 @@ void build_capturesNameAndDeclaredType() { assertEquals("String", p.getType()); } + @Test + void build_capturesParameterModifiers() { + JParameter p = build("package p;\nclass Foo {\n void m(final String name) {}\n}\n"); + assertEquals(List.of("final"), p.getModifiers()); + } + @Test void build_capturesGenericAndArrayTypesSyntactically() { assertEquals("List", build("package p;\nclass Foo {\n void m(List xs) {}\n}\n").getType()); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index 306ccb3..17c5805 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -51,6 +51,31 @@ void build_derivesKindForInterfaceEnumRecordAnnotation() { assertEquals("annotation", buildFirstType("package p;\npublic @interface A {}\n").getKind()); } + @Test + void build_capturesJavadocAsOwnComment() { + JType t = buildFirstType("package p;\n/** A widget. */\nclass Foo {}\n"); + assertEquals(1, t.getComments().size()); + assertTrue(t.getComments().get(0).getContent().contains("A widget.")); + assertTrue(t.getComments().get(0).isJavadoc()); + assertNotNull(t.getComments().get(0).getSpan()); + } + + @Test + void build_commentsAreOwnNotAllContained() { + // v1 used getAllContainedComments(), so a type listed every comment inside every member. + // v2 gives each node only its OWN attached comment. + JType t = buildFirstType("package p;\n/** Type doc. */\nclass Foo {\n /** Method doc. */\n void m() {}\n}\n"); + assertEquals(1, t.getComments().size(), "the method's javadoc belongs to the method, not the type"); + assertTrue(t.getComments().get(0).getContent().contains("Type doc.")); + } + + @Test + void build_capturesModifiers() { + // Keystone's type node lists modifiers[] — v1 had them and v2 must not drop them. + assertEquals(List.of("public", "abstract"), + buildFirstType("package p;\npublic abstract class Foo {}\n").getModifiers()); + } + @Test void build_capturesInheritance() { JType t = buildFirstType("package p;\nclass Foo extends Base implements Runnable {}\n"); From 005f2d6749296624aeff1b61bdd366a9ba29a16b Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 19:31:46 -0400 Subject: [PATCH 09/22] feat(schema): local variables on callable (#180) --- .../java/com/ibm/cldk/schema/JCallable.java | 2 ++ .../ibm/cldk/schema/JVariableDeclaration.java | 22 +++++++++++++++++++ .../syntactic_analysis/CallableBuilder.java | 20 +++++++++++++++++ .../CallableBuilderTest.java | 19 ++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 src/main/java/com/ibm/cldk/schema/JVariableDeclaration.java diff --git a/src/main/java/com/ibm/cldk/schema/JCallable.java b/src/main/java/com/ibm/cldk/schema/JCallable.java index 61b6d3f..579d2dc 100644 --- a/src/main/java/com/ibm/cldk/schema/JCallable.java +++ b/src/main/java/com/ibm/cldk/schema/JCallable.java @@ -37,6 +37,8 @@ public class JCallable { private JMetrics metrics; private JRefs refs; + private List localVariables = new ArrayList<>(); + /** L1 emits only {@code call} nodes here, keyed by ordinal id; the rest of the body arrives at L3. */ private Map body = new LinkedHashMap<>(); diff --git a/src/main/java/com/ibm/cldk/schema/JVariableDeclaration.java b/src/main/java/com/ibm/cldk/schema/JVariableDeclaration.java new file mode 100644 index 0000000..d0f66ec --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JVariableDeclaration.java @@ -0,0 +1,22 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A local variable declared in a callable's body: {@code name}, the AST-declared {@code type} + * (syntactic at L1), its {@code initializer} expression text if any, and {@code span}. + * + *

Kept as a named list on the callable (as the Python reference analyzer does) even though L3 will + * also emit the declaration statements into {@code body} — the two answer different + * questions ("what locals exist here" vs "what is the control flow"). + */ +@Data +public class JVariableDeclaration { + private String name; + private String type; + private String initializer; + private Span span; + private List comments = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index 31dfee4..bf871fb 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -22,6 +22,7 @@ import com.ibm.cldk.schema.JMetrics; import com.ibm.cldk.schema.JRefs; import com.ibm.cldk.schema.JType; +import com.ibm.cldk.schema.JVariableDeclaration; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -89,6 +90,7 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List body = bodyOf(cd); body.flatMap(b -> b.getRange().map(r -> r.begin.line)).ifPresent(callable::setCodeStartLine); callable.setRefs(refs(body, classFieldNames)); + body.ifPresent(b -> callable.setLocalVariables(localVariables(b))); body.ifPresent(b -> callable.setBody(callSiteBuilder.build(b))); body.ifPresent(b -> callable.setTypes(localClasses(b, callable.getId()))); return callable; @@ -101,6 +103,24 @@ private static Optional bodyOf(CallableDeclaration cd) { return Optional.of(((ConstructorDeclaration) cd).getBody()); } + /** Locals declared directly in this body, in source order (nested classes' locals are theirs). */ + private List localVariables(BlockStmt body) { + List locals = new ArrayList<>(); + for (VariableDeclarator vd : body.findAll(VariableDeclarator.class)) { + if (!CallSiteBuilder.belongsDirectlyTo(vd, body)) { + continue; + } + JVariableDeclaration local = new JVariableDeclaration(); + local.setName(vd.getNameAsString()); + local.setType(vd.getType().asString()); + vd.getInitializer().ifPresent(init -> local.setInitializer(init.toString())); + local.setSpan(ctx.spanOf(vd)); + local.setComments(ctx.commentsOf(vd)); + locals.add(local); + } + return locals; + } + /** Local (method-body) classes declared directly in the body, keyed by simple name (sorted). */ private Map localClasses(BlockStmt body, String callableId) { TypeBuilder typeBuilder = new TypeBuilder(ctx); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java index ef1223b..756e07a 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -11,6 +11,7 @@ import com.github.javaparser.ast.body.CallableDeclaration; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JVariableDeclaration; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; @@ -66,6 +67,24 @@ void build_capturesParametersReturnTypeModifiersAndDecorators() { assertEquals("Override", c.getDecorators().get(0).getName()); } + @Test + void build_capturesLocalVariables() { + JCallable c = build("void m() { int total = 0; String name; }"); + assertEquals(List.of("total", "name"), + c.getLocalVariables().stream().map(JVariableDeclaration::getName).collect(Collectors.toList())); + assertEquals("int", c.getLocalVariables().get(0).getType()); + assertEquals("0", c.getLocalVariables().get(0).getInitializer()); + assertNull(c.getLocalVariables().get(1).getInitializer(), "uninitialized -> absent"); + assertNotNull(c.getLocalVariables().get(0).getSpan()); + } + + @Test + void build_localVariablesExcludeThoseInNestedLocalClasses() { + JCallable c = build("void m() { int mine = 1; class Local { void inner() { int theirs = 2; } } }"); + assertEquals(List.of("mine"), + c.getLocalVariables().stream().map(JVariableDeclaration::getName).collect(Collectors.toList())); + } + @Test void build_capturesJavadocComment() { JCallable c = build("/** Adds two numbers. */\n int add(int a, int b) { return a + b; }"); From acd0fa841dcaaebb4a8aae4011825acb04084ff4 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 19:39:53 -0400 Subject: [PATCH 10/22] feat(schema): resolve types via the JavaParser symbol solver at L1 (#180) Types, supertypes, error_channel and refs.types are now resolved to qualified names, and callable signatures use erased resolved parameter types, matching the v1 symbol table. Tests parse with a symbol solver so the resolution path is actually exercised; unresolvable types degrade to their AST spelling. --- .claude/SCHEMA_DECISIONS.md | 13 +++-- .../syntactic_analysis/CallableBuilder.java | 12 ++--- .../cldk/syntactic_analysis/FieldBuilder.java | 2 +- .../syntactic_analysis/L1BuildContext.java | 51 +++++++++++++++++++ .../syntactic_analysis/ParameterBuilder.java | 2 +- .../cldk/syntactic_analysis/TypeBuilder.java | 8 +-- .../CallSiteBuilderTest.java | 6 +-- .../CallableBuilderTest.java | 26 +++++++--- .../syntactic_analysis/FieldBuilderTest.java | 6 +-- .../syntactic_analysis/ModuleBuilderTest.java | 6 +-- .../ParameterBuilderTest.java | 23 +++++---- .../cldk/syntactic_analysis/TestParsers.java | 29 +++++++++++ .../syntactic_analysis/TypeBuilderTest.java | 10 ++-- 13 files changed, 138 insertions(+), 56 deletions(-) create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/TestParsers.java diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 3778b96..34cb1d1 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -78,10 +78,15 @@ Ordinal ids `…@:` (real) / `…@` (synthetic) within a callabl local-ids in its `arguments`) use the *method-name* `line:col`, not the whole expression's begin — so chained calls `a.b().c()` get distinct ids instead of colliding on the shared expression start. -- **`refs` at L1 are syntactic names, not resolved ids.** Cross-module resolution is - L2+; at L1 `refs.types` are the AST spellings of referenced/instantiated types and - `refs.fields` are the simple names of enclosing-type fields accessed. Refined to - `can://` ids once resolution is available. Keystone shows `[id]`; L1 emits best-effort. +- **L1 resolves types with the JavaParser symbol solver** (corrected 2026-08 — an earlier note here + wrongly said L1 stayed syntactic). The keystone's L1 guide expects the resolver to populate type + fields when the structural tool resolves, and the v1 symbol table did exactly this, so v2 matches: + `base_types`/`interfaces`, field/parameter/return/local types, `error_channel`, and `refs.types` + are **resolved qualified names** (`java.lang.String`), and the callable `signature` uses + **erased** resolved parameter types (`m(java.util.List, java.lang.String)`) — which is why the + durable id depends on the solver being configured. Resolution failures degrade to the AST spelling + (never crash) and are memoized per spelling. `refs.fields` remain simple names for now; promoting + them to `can://` ids needs cross-module resolution (L2+). - **`callable.kind ∈ {method, constructor}`.** Direct members only (via `getMethods()`/`getConstructors()`); nested-type methods hang under their own type, local (method-body) classes under `callable.types` (D4 containment). diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index bf871fb..bb650d7 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -71,9 +71,9 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List t.asString()).collect(Collectors.toList())); + cd.getThrownExceptions().stream().map(ctx::resolveType).collect(Collectors.toList())); callable.setModifiers( cd.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); callable.setComments(ctx.commentsOf(cd)); @@ -112,7 +112,7 @@ private List localVariables(BlockStmt body) { } JVariableDeclaration local = new JVariableDeclaration(); local.setName(vd.getNameAsString()); - local.setType(vd.getType().asString()); + local.setType(ctx.resolveType(vd.getType())); vd.getInitializer().ifPresent(init -> local.setInitializer(init.toString())); local.setSpan(ctx.spanOf(vd)); local.setComments(ctx.commentsOf(vd)); @@ -132,7 +132,7 @@ private Map localClasses(BlockStmt body, String callableId) { } /** Syntactic cross-refs: types referenced and enclosing-type fields accessed in the body. */ - private static JRefs refs(Optional body, List classFieldNames) { + private JRefs refs(Optional body, List classFieldNames) { JRefs refs = new JRefs(); if (body.isEmpty()) { return refs; @@ -142,10 +142,10 @@ private static JRefs refs(Optional body, List classFieldNames TreeSet types = new TreeSet<>(); b.findAll(VariableDeclarator.class).stream() .filter(vd -> CallSiteBuilder.belongsDirectlyTo(vd, b) && vd.getType().isClassOrInterfaceType()) - .forEach(vd -> types.add(vd.getType().asString())); + .forEach(vd -> types.add(ctx.resolveType(vd.getType()))); b.findAll(ObjectCreationExpr.class).stream() .filter(oce -> CallSiteBuilder.belongsDirectlyTo(oce, b)) - .forEach(oce -> types.add(oce.getType().asString())); + .forEach(oce -> types.add(ctx.resolveType(oce.getType()))); refs.setTypes(new ArrayList<>(types)); TreeSet fields = new TreeSet<>(); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java index 0a7b399..a961021 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/FieldBuilder.java @@ -32,7 +32,7 @@ public FieldBuilder(L1BuildContext ctx) { * @param parentTypeId the containing type's id */ public List build(FieldDeclaration fd, String parentTypeId) { - String type = fd.getCommonType().asString(); + String type = ctx.resolveType(fd.getCommonType()); List modifiers = fd.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList()); List decorators = diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java index f9a1a0b..0314f65 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java @@ -3,15 +3,20 @@ import com.github.javaparser.Range; import com.github.javaparser.ast.Node; import com.github.javaparser.ast.comments.Comment; +import com.github.javaparser.ast.expr.Expression; +import com.github.javaparser.ast.type.Type; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JComment; import com.ibm.cldk.schema.Span; import com.ibm.cldk.schema.Spans; +import com.ibm.cldk.utils.Log; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import lombok.Getter; /** @@ -26,6 +31,10 @@ public final class L1BuildContext { private final String fileKey; private final String source; + /** Memoized resolution failures — retrying them is expensive and they recur across a project. */ + private final Set unresolvedTypes = new HashSet<>(); + private final Set unresolvedExpressions = new HashSet<>(); + public L1BuildContext(String applicationId, String fileKey, String source) { this.applicationId = applicationId; this.fileKey = fileKey; @@ -37,6 +46,48 @@ public String moduleId() { return CanId.moduleId(applicationId, fileKey); } + /** + * Resolve a declared type to its qualified name via the JavaParser symbol solver, falling back to + * the AST spelling when resolution fails (unresolvable dependency, missing classpath entry). This + * is what makes L1 type fields qualified — the keystone expects the resolver to populate them when + * the structural tool resolves, and the v1 symbol table did exactly this. + * + *

Resolution is attempted through the resolver attached to the parsed unit, so the caller must + * have parsed with a symbol-solver-configured {@code ParserConfiguration}. Failures are memoized: + * an unresolvable spelling is expensive to retry and appears repeatedly in real projects. + */ + public String resolveType(Type type) { + String spelling = type.asString(); + if (unresolvedTypes.contains(spelling)) { + return spelling; + } + try { + return type.resolve().describe(); + } catch (Throwable e) { + Log.debug("Could not resolve type: " + spelling + ": " + e.getMessage()); + unresolvedTypes.add(spelling); + return spelling; + } + } + + /** + * Resolve an expression's type to its qualified name, or {@code ""} when it cannot be resolved + * (mirrors the v1 behaviour, where an unresolved expression contributes no type fact). + */ + public String resolveExpressionType(Expression expression) { + String spelling = expression.toString(); + if (unresolvedExpressions.contains(spelling)) { + return ""; + } + try { + return expression.calculateResolvedType().describe(); + } catch (Throwable e) { + Log.debug("Could not resolve expression: " + spelling + ": " + e.getMessage()); + unresolvedExpressions.add(spelling); + return ""; + } + } + /** * The comment attached to a declaration — its javadoc, or the leading line/block comment. Returns * an empty list when the node has none. Deliberately the node's own comment rather than diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java index 32dad01..49b85b5 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/ParameterBuilder.java @@ -24,7 +24,7 @@ public JParameter build(Parameter param) { parameter.setName(param.getNameAsString()); // Varargs keep the declared ELEMENT type; the `is_variadic` flag carries the `...` instead, so // `String...` stays distinguishable from a real `String[]` parameter. - parameter.setType(param.getType().asString()); + parameter.setType(ctx.resolveType(param.getType())); parameter.setVariadic(param.isVarArgs()); parameter.setSpan(ctx.spanOf(param)); parameter.setModifiers( diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index 4b41b40..f32e374 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -56,12 +56,12 @@ public JType build(TypeDeclaration td, String parentId) { List interfaces = new ArrayList<>(); if (td instanceof ClassOrInterfaceDeclaration) { ClassOrInterfaceDeclaration cls = (ClassOrInterfaceDeclaration) td; - cls.getExtendedTypes().forEach(t -> baseTypes.add(t.asString())); - cls.getImplementedTypes().forEach(t -> interfaces.add(t.asString())); + cls.getExtendedTypes().forEach(t -> baseTypes.add(ctx.resolveType(t))); + cls.getImplementedTypes().forEach(t -> interfaces.add(ctx.resolveType(t))); } else if (td instanceof EnumDeclaration) { - ((EnumDeclaration) td).getImplementedTypes().forEach(t -> interfaces.add(t.asString())); + ((EnumDeclaration) td).getImplementedTypes().forEach(t -> interfaces.add(ctx.resolveType(t))); } else if (td instanceof RecordDeclaration) { - ((RecordDeclaration) td).getImplementedTypes().forEach(t -> interfaces.add(t.asString())); + ((RecordDeclaration) td).getImplementedTypes().forEach(t -> interfaces.add(ctx.resolveType(t))); } type.setBaseTypes(baseTypes); type.setInterfaces(interfaces); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java index 503a019..924163a 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java @@ -29,11 +29,7 @@ class CallSiteBuilderTest { private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; private static Map build(String source) { - CompilationUnit cu = new JavaParser( - new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) - .parse(source) - .getResult() - .orElseThrow(); + CompilationUnit cu = TestParsers.parseResolved(source); CallableDeclaration cd = cu.getType(0).findFirst(CallableDeclaration.class).orElseThrow(); BlockStmt body = (cd instanceof MethodDeclaration) ? ((MethodDeclaration) cd).getBody().orElseThrow() diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java index 756e07a..b51fb92 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -25,11 +25,7 @@ class CallableBuilderTest { private static JCallable build(String memberSource, List classFieldNames) { String source = "package com.example;\nimport java.io.IOException;\nimport java.util.*;\n" + "class Foo {\n " + memberSource + "\n}\n"; - CompilationUnit cu = new JavaParser( - new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) - .parse(source) - .getResult() - .orElseThrow(); + CompilationUnit cu = TestParsers.parseResolved(source); CallableDeclaration cd = cu.getType(0).findFirst(CallableDeclaration.class).orElseThrow(); L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); return new CallableBuilder(ctx).build(cd, TYPE_ID, classFieldNames); @@ -48,6 +44,20 @@ void build_methodKindSignatureIdAndSpan() { assertNotNull(c.getSpan()); } + @Test + void build_signatureUsesTypeErasure() { + // The durable id's last segment: parameter types are RESOLVED and ERASED (type arguments + // dropped), which is why a symbol solver is required — a syntactic signature would differ. + JCallable c = build("void m(List xs, String s) {}"); + assertEquals("m(java.util.List, java.lang.String)", c.getSignature()); + assertEquals(TYPE_ID + "/m(java.util.List, java.lang.String)", c.getId()); + } + + @Test + void build_signatureFallsBackToAstWhenParameterTypeUnresolvable() { + assertEquals("m(Mystery)", build("void m(Mystery x) {}").getSignature()); + } + @Test void build_constructorKindHasNullReturnType() { JCallable c = build("Foo(int x) {}"); @@ -61,8 +71,8 @@ void build_capturesParametersReturnTypeModifiersAndDecorators() { JCallable c = build("@Override public String greet(String name) { return \"hi\"; }"); assertEquals(List.of("name"), c.getParameters().stream() .map(p -> p.getName()).collect(Collectors.toList())); - assertEquals("String", c.getParameters().get(0).getType()); - assertEquals("String", c.getReturnType()); + assertEquals("java.lang.String", c.getParameters().get(0).getType()); + assertEquals("java.lang.String", c.getReturnType()); assertEquals(List.of("public"), c.getModifiers()); assertEquals("Override", c.getDecorators().get(0).getName()); } @@ -116,7 +126,7 @@ void build_abstractMethodHasNoCodeStartLine() { @Test void build_capturesErrorChannelFromThrows() { JCallable c = build("void read() throws IOException, RuntimeException {}"); - assertEquals(List.of("IOException", "RuntimeException"), c.getErrorChannel()); + assertEquals(List.of("java.io.IOException", "java.lang.RuntimeException"), c.getErrorChannel()); } @Test diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java index a70baaf..3c74b78 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/FieldBuilderTest.java @@ -24,11 +24,7 @@ class FieldBuilderTest { private static List build(String memberSource) { String source = "package com.example;\nclass Foo {\n " + memberSource + "\n}\n"; - CompilationUnit cu = new JavaParser( - new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) - .parse(source) - .getResult() - .orElseThrow(); + CompilationUnit cu = TestParsers.parseResolved(source); FieldDeclaration fd = cu.getType(0).findFirst(FieldDeclaration.class).orElseThrow(); L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); return new FieldBuilder(ctx).build(fd, TYPE_ID); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java index 32be755..24c5410 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java @@ -22,11 +22,7 @@ class ModuleBuilderTest { private static CompilationUnit parse(String source) { - return new JavaParser( - new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) - .parse(source) - .getResult() - .orElseThrow(); + return TestParsers.parseResolved(source); } /** Build a module from source using a fixed file key. */ diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java index b171309..8f47450 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ParameterBuilderTest.java @@ -21,11 +21,7 @@ class ParameterBuilderTest { private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; private static Parameter firstParam(String source) { - CompilationUnit cu = new JavaParser( - new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) - .parse(source) - .getResult() - .orElseThrow(); + CompilationUnit cu = TestParsers.parseResolved(source); return cu.getType(0).findFirst(MethodDeclaration.class).orElseThrow().getParameter(0); } @@ -38,7 +34,7 @@ private static JParameter build(String source) { void build_capturesNameAndDeclaredType() { JParameter p = build("package p;\nclass Foo {\n void m(final String name) {}\n}\n"); assertEquals("name", p.getName()); - assertEquals("String", p.getType()); + assertEquals("java.lang.String", p.getType(), "types are resolved to qualified names via the symbol solver"); } @Test @@ -48,11 +44,18 @@ void build_capturesParameterModifiers() { } @Test - void build_capturesGenericAndArrayTypesSyntactically() { - assertEquals("List", build("package p;\nclass Foo {\n void m(List xs) {}\n}\n").getType()); + void build_resolvesGenericTypeArguments() { + assertEquals("java.util.List", + build("package p;\nimport java.util.List;\nclass Foo {\n void m(List xs) {}\n}\n").getType()); assertEquals("int[]", build("package p;\nclass Foo {\n void m(int[] xs) {}\n}\n").getType()); } + @Test + void build_fallsBackToAstSpellingWhenTypeCannotBeResolved() { + // A missing dependency must degrade to the source spelling, never crash the build. + assertEquals("Mystery", build("package p;\nclass Foo {\n void m(Mystery x) {}\n}\n").getType()); + } + @Test void build_spanBytesSliceToTheParameterText() { String source = "package p;\nclass Foo {\n void m(String name) {}\n}\n"; @@ -66,14 +69,14 @@ void build_spanBytesSliceToTheParameterText() { void build_marksVariadicParameterAndKeepsElementType() { JParameter p = build("package p;\nclass Foo {\n void m(String... names) {}\n}\n"); assertTrue(p.isVariadic(), "String... must set is_variadic"); - assertEquals("String", p.getType(), "type stays the element type; the flag carries the ..."); + assertEquals("java.lang.String", p.getType(), "type stays the element type; the flag carries the ..."); } @Test void build_plainArrayParameterIsNotVariadic() { JParameter p = build("package p;\nclass Foo {\n void m(String[] names) {}\n}\n"); assertFalse(p.isVariadic(), "String[] is an array, not varargs"); - assertEquals("String[]", p.getType()); + assertEquals("java.lang.String[]", p.getType()); } @Test diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TestParsers.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TestParsers.java new file mode 100644 index 0000000..102af58 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TestParsers.java @@ -0,0 +1,29 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.symbolsolver.JavaSymbolSolver; +import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver; +import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver; + +/** + * Parses test sources with a symbol solver attached, so the builders exercise the same resolution path + * they use in production (qualified type names, erased signatures). Without this the builders silently + * fall back to AST spellings and the tests would not cover resolution at all. + */ +final class TestParsers { + + private TestParsers() {} + + static CompilationUnit parseResolved(String source) { + CombinedTypeSolver typeSolver = new CombinedTypeSolver(); + typeSolver.add(new ReflectionTypeSolver()); + ParserConfiguration config = new ParserConfiguration() + .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21) + .setStoreTokens(true) + .setAttributeComments(true) + .setSymbolResolver(new JavaSymbolSolver(typeSolver)); + return new JavaParser(config).parse(source).getResult().orElseThrow(); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index 17c5805..340bc32 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -22,11 +22,7 @@ class TypeBuilderTest { private static final String FILE_KEY = "src/main/java/com/example/Foo.java"; private static CompilationUnit parse(String source) { - return new JavaParser( - new ParserConfiguration().setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21)) - .parse(source) - .getResult() - .orElseThrow(); + return TestParsers.parseResolved(source); } private static JType buildFirstType(String source) { @@ -79,8 +75,8 @@ void build_capturesModifiers() { @Test void build_capturesInheritance() { JType t = buildFirstType("package p;\nclass Foo extends Base implements Runnable {}\n"); - assertEquals(List.of("Base"), t.getBaseTypes()); - assertEquals(List.of("Runnable"), t.getInterfaces()); + assertEquals(List.of("java.lang.Runnable"), t.getInterfaces(), "resolved to a qualified name"); + assertEquals(List.of("Base"), t.getBaseTypes(), "unresolvable supertype degrades to its spelling"); } @Test From df1f63f51e49908d66b9afdba483d58c38676977 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 20:08:06 -0400 Subject: [PATCH 11/22] feat(schema): enum constants, record components, initializer-block callables (#180) --- .../com/ibm/cldk/schema/JEnumConstant.java | 18 ++++++ .../com/ibm/cldk/schema/JRecordComponent.java | 23 ++++++++ src/main/java/com/ibm/cldk/schema/JType.java | 6 ++ .../syntactic_analysis/CallableBuilder.java | 55 ++++++++++++++++--- .../cldk/syntactic_analysis/TypeBuilder.java | 50 +++++++++++++++++ .../syntactic_analysis/TypeBuilderTest.java | 53 ++++++++++++++++++ 6 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/ibm/cldk/schema/JEnumConstant.java create mode 100644 src/main/java/com/ibm/cldk/schema/JRecordComponent.java diff --git a/src/main/java/com/ibm/cldk/schema/JEnumConstant.java b/src/main/java/com/ibm/cldk/schema/JEnumConstant.java new file mode 100644 index 0000000..2cb3705 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JEnumConstant.java @@ -0,0 +1,18 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * An enum constant declared on an {@code enum} type, with the argument expressions passed to the + * enum's constructor (empty for a plain constant). A Java-specific addition to the keystone's type + * node, which has no enum-member vocabulary (see cldk-devtools#40). + */ +@Data +public class JEnumConstant { + private String name; + private List arguments = new ArrayList<>(); + private Span span; + private List comments = new ArrayList<>(); +} diff --git a/src/main/java/com/ibm/cldk/schema/JRecordComponent.java b/src/main/java/com/ibm/cldk/schema/JRecordComponent.java new file mode 100644 index 0000000..1ad2345 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JRecordComponent.java @@ -0,0 +1,23 @@ +package com.ibm.cldk.schema; + +import java.util.ArrayList; +import java.util.List; +import lombok.Data; + +/** + * A component of a {@code record} type — its name, resolved {@code type}, modifiers and structured + * decorators. A Java-specific addition to the keystone's type node (see cldk-devtools#40). + * + *

v1 also carried a {@code defaultValue} derived from compact-constructor assignments; that is + * dropped deliberately — Java record components have no default values, so the field was misleading. + */ +@Data +public class JRecordComponent { + private String name; + private String type; + private Span span; + private List modifiers = new ArrayList<>(); + private List decorators = new ArrayList<>(); + private List comments = new ArrayList<>(); + private boolean isVariadic; +} diff --git a/src/main/java/com/ibm/cldk/schema/JType.java b/src/main/java/com/ibm/cldk/schema/JType.java index 59314d3..5708430 100644 --- a/src/main/java/com/ibm/cldk/schema/JType.java +++ b/src/main/java/com/ibm/cldk/schema/JType.java @@ -22,6 +22,12 @@ public class JType { private List interfaces = new ArrayList<>(); private List decorators = new ArrayList<>(); + /** Enum constants, in declaration order — present only on {@code enum} types. */ + private List enumConstants = new ArrayList<>(); + + /** Record components, in declaration order — present only on {@code record} types. */ + private List recordComponents = new ArrayList<>(); + /** Fields declared in this type, keyed by simple name (one entry per declared variable). */ private Map fields = new LinkedHashMap<>(); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index bb650d7..a91dcae 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -2,6 +2,7 @@ import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.ConstructorDeclaration; +import com.github.javaparser.ast.body.InitializerDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; import com.github.javaparser.ast.body.VariableDeclarator; @@ -96,6 +97,37 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List classFieldNames, String signature) { + JCallable callable = new JCallable(); + callable.setSignature(signature); + callable.setId(CanId.childId(parentTypeId, signature)); + callable.setKind("initializer"); + callable.setSpan(ctx.spanOf(id)); + callable.setComments(ctx.commentsOf(id)); + callable.setModifiers( + id.isStatic() ? List.of("static") : List.of()); + callable.setDecorators( + id.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); + + JMetrics metrics = new JMetrics(); + metrics.setCyclomatic(cyclomaticComplexity(id)); + callable.setMetrics(metrics); + + BlockStmt body = id.getBody(); + body.getRange().map(r -> r.begin.line).ifPresent(callable::setCodeStartLine); + callable.setRefs(refs(Optional.of(body), classFieldNames)); + callable.setLocalVariables(localVariables(body)); + callable.setBody(callSiteBuilder.build(body)); + callable.setTypes(localClasses(body, callable.getId())); + return callable; + } + private static Optional bodyOf(CallableDeclaration cd) { if (cd instanceof MethodDeclaration) { return ((MethodDeclaration) cd).getBody(); @@ -164,14 +196,23 @@ private JRefs refs(Optional body, List classFieldNames) { * Cyclomatic complexity: one plus the number of branch points (if/loop/switch-case/ternary/catch) * in the callable (mirrors the v1 symbol-table metric). */ + private static int cyclomaticComplexity(InitializerDeclaration id) { + return branchPoints(id) + 1; + } + private static int cyclomaticComplexity(CallableDeclaration cd) { - int ifCount = cd.findAll(IfStmt.class).size(); - int loopCount = cd.findAll(DoStmt.class).size() + cd.findAll(ForStmt.class).size() - + cd.findAll(ForEachStmt.class).size() + cd.findAll(WhileStmt.class).size(); + return branchPoints(cd) + 1; + } + + /** Branch points (if / loop / switch-case / ternary / catch) inside any node. */ + private static int branchPoints(com.github.javaparser.ast.Node node) { + int ifCount = node.findAll(IfStmt.class).size(); + int loopCount = node.findAll(DoStmt.class).size() + node.findAll(ForStmt.class).size() + + node.findAll(ForEachStmt.class).size() + node.findAll(WhileStmt.class).size(); int switchCaseCount = - cd.findAll(SwitchStmt.class).stream().mapToInt(s -> s.getEntries().size()).sum(); - int ternaryCount = cd.findAll(ConditionalExpr.class).size(); - int catchCount = cd.findAll(CatchClause.class).size(); - return ifCount + loopCount + switchCaseCount + ternaryCount + catchCount + 1; + node.findAll(SwitchStmt.class).stream().mapToInt(s -> s.getEntries().size()).sum(); + int ternaryCount = node.findAll(ConditionalExpr.class).size(); + int catchCount = node.findAll(CatchClause.class).size(); + return ifCount + loopCount + switchCaseCount + ternaryCount + catchCount; } } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index f32e374..ef7c10f 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -3,13 +3,18 @@ import com.github.javaparser.ast.body.AnnotationDeclaration; import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.EnumConstantDeclaration; import com.github.javaparser.ast.body.EnumDeclaration; import com.github.javaparser.ast.body.FieldDeclaration; +import com.github.javaparser.ast.body.InitializerDeclaration; +import com.github.javaparser.ast.body.Parameter; import com.github.javaparser.ast.body.RecordDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JEnumConstant; import com.ibm.cldk.schema.JField; +import com.ibm.cldk.schema.JRecordComponent; import com.ibm.cldk.schema.JType; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -66,6 +71,38 @@ public JType build(TypeDeclaration td, String parentId) { type.setBaseTypes(baseTypes); type.setInterfaces(interfaces); + if (td instanceof EnumDeclaration) { + List constants = new ArrayList<>(); + for (EnumConstantDeclaration ecd : ((EnumDeclaration) td).getEntries()) { + JEnumConstant constant = new JEnumConstant(); + constant.setName(ecd.getNameAsString()); + constant.setArguments( + ecd.getArguments().stream().map(Object::toString).collect(Collectors.toList())); + constant.setSpan(ctx.spanOf(ecd)); + constant.setComments(ctx.commentsOf(ecd)); + constants.add(constant); + } + type.setEnumConstants(constants); + } + + if (td instanceof RecordDeclaration) { + List components = new ArrayList<>(); + for (Parameter p : ((RecordDeclaration) td).getParameters()) { + JRecordComponent component = new JRecordComponent(); + component.setName(p.getNameAsString()); + component.setType(ctx.resolveType(p.getType())); + component.setSpan(ctx.spanOf(p)); + component.setModifiers( + p.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); + component.setDecorators( + p.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); + component.setComments(ctx.commentsOf(p)); + component.setVariadic(p.isVarArgs()); + components.add(component); + } + type.setRecordComponents(components); + } + // Fields, keyed by simple name — one entry per declared variable (int a, b; -> a, b). Map fields = new LinkedHashMap<>(); for (FieldDeclaration fd : td.getFields()) { @@ -86,6 +123,19 @@ public JType build(TypeDeclaration td, String parentId) { JCallable callable = callableBuilder.build(cd, type.getId(), fieldNames); callables.put(callable.getSignature(), callable); } + // Initializer blocks are callables too (keystone kind `initializer`) — L3 gives them their own + // CFGs. Numbered per kind so the id survives line edits; `$` marks the synthetic member. + int staticIndex = 0; + int instanceIndex = 0; + for (InitializerDeclaration id : td.getMembers().stream() + .filter(m -> m instanceof InitializerDeclaration) + .map(m -> (InitializerDeclaration) m) + .collect(Collectors.toList())) { + String signature = id.isStatic() + ? "$" + staticIndex++ + "()" + : "$" + instanceIndex++ + "()"; + callables.put(signature, callableBuilder.buildInitializer(id, type.getId(), fieldNames, signature)); + } type.setCallables(new LinkedHashMap<>(callables)); // Recurse into member (inner) types; nesting/parent are encoded by this containment (and the diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index 340bc32..c0f6849 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -11,9 +11,12 @@ import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; import com.ibm.cldk.schema.JDecorator; +import com.ibm.cldk.schema.JEnumConstant; +import com.ibm.cldk.schema.JRecordComponent; import com.ibm.cldk.schema.JType; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; /** Tests the v2 {@link TypeBuilder} — kind, byte-offset span, structured decorators, inheritance. */ @@ -128,6 +131,56 @@ void build_callableRefsSeeEnclosingTypeFields() { assertEquals(List.of("count"), inc.getRefs().getFields()); } + @Test + void build_capturesEnumConstants() { + JType t = buildFirstType("package p;\nenum Color { RED, GREEN(\"g\"); Color() {} Color(String s) {} }\n"); + assertEquals(List.of("RED", "GREEN"), + t.getEnumConstants().stream().map(JEnumConstant::getName).collect(Collectors.toList())); + assertEquals(List.of("\"g\""), t.getEnumConstants().get(1).getArguments()); + assertNotNull(t.getEnumConstants().get(0).getSpan()); + } + + @Test + void build_capturesRecordComponents() { + JType t = buildFirstType("package p;\nrecord Point(int x, String label) {}\n"); + assertEquals("record", t.getKind()); + assertEquals(List.of("x", "label"), + t.getRecordComponents().stream().map(JRecordComponent::getName).collect(Collectors.toList())); + assertEquals("int", t.getRecordComponents().get(0).getType()); + assertEquals("java.lang.String", t.getRecordComponents().get(1).getType(), "resolved like any other type"); + } + + @Test + void build_capturesVariadicRecordComponent() { + JType t = buildFirstType("package p;\nrecord Args(String... values) {}\n"); + assertTrue(t.getRecordComponents().get(0).isVariadic()); + } + + @Test + void build_emitsStaticInitializerAsCallable() { + // The keystone's callable kinds include `initializer`; L3 needs these to get their own CFGs. + JType t = buildFirstType("package p;\nclass Foo {\n static { setup(); }\n}\n"); + JCallable init = t.getCallables().get("$0()"); + assertNotNull(init, "static initializer must appear among the type's callables"); + assertEquals("initializer", init.getKind()); + assertEquals(1, init.getBody().size(), "its call sites belong to it, not to any constructor"); + } + + @Test + void build_emitsInstanceInitializerAsCallable() { + JType t = buildFirstType("package p;\nclass Foo {\n { prime(); }\n}\n"); + JCallable init = t.getCallables().get("$0()"); + assertNotNull(init); + assertEquals("initializer", init.getKind()); + } + + @Test + void build_numbersMultipleInitializersOfTheSameKind() { + JType t = buildFirstType("package p;\nclass Foo {\n static { a(); }\n static { b(); }\n}\n"); + assertTrue(t.getCallables().containsKey("$0()")); + assertTrue(t.getCallables().containsKey("$1()")); + } + @Test void build_capturesStructuredDecoratorWithArgs() { JType t = buildFirstType("package p;\n@SuppressWarnings(\"unchecked\")\nclass Foo {}\n"); From db3a030cb225c42937106be9ae8c24e96fe24142 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Tue, 18 Aug 2026 20:13:07 -0400 Subject: [PATCH 12/22] feat(schema): rich call-site facts on call nodes (receiver/arg types, callee signature, flags) (#180) --- src/main/java/com/ibm/cldk/SymbolTable.java | 10 +--- .../java/com/ibm/cldk/schema/JBodyNode.java | 24 +++++++++ .../com/ibm/cldk/schema/JEnumConstant.java | 6 ++- .../com/ibm/cldk/schema/JRecordComponent.java | 4 +- .../syntactic_analysis/CallSiteBuilder.java | 50 ++++++++++++++++++- .../cldk/syntactic_analysis/Signatures.java | 20 +++++++- .../CallSiteBuilderTest.java | 38 ++++++++++++++ 7 files changed, 138 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/ibm/cldk/SymbolTable.java b/src/main/java/com/ibm/cldk/SymbolTable.java index c6c3729..bac625c 100644 --- a/src/main/java/com/ibm/cldk/SymbolTable.java +++ b/src/main/java/com/ibm/cldk/SymbolTable.java @@ -610,15 +610,7 @@ private static String getTypeErasureSignature(CallableDeclaration callableDecl) * @return String representing type erasure signature */ private static String getTypeErasureSignature(ResolvedMethodLikeDeclaration methodDecl) { - StringBuilder signature = new StringBuilder(methodDecl.getName()); - List erasureParameterTypes = new ArrayList<>(); - for (int i = 0; i < methodDecl.getNumberOfParams(); i++) { - erasureParameterTypes.add(methodDecl.getParam(i).getType().erasure().describe()); - } - signature.append("("); - signature.append(String.join(", ", erasureParameterTypes)); - signature.append(")"); - return signature.toString(); + return com.ibm.cldk.syntactic_analysis.Signatures.typeErasure(methodDecl); } private static boolean isEntryPointMethod(CallableDeclaration callableDecl) { diff --git a/src/main/java/com/ibm/cldk/schema/JBodyNode.java b/src/main/java/com/ibm/cldk/schema/JBodyNode.java index ee584ea..05b87d3 100644 --- a/src/main/java/com/ibm/cldk/schema/JBodyNode.java +++ b/src/main/java/com/ibm/cldk/schema/JBodyNode.java @@ -17,4 +17,28 @@ public class JBodyNode { /** Only meaningful on {@code call} nodes; {@code null} at L1 (backfilled at L2). */ private String callee; private List arguments = new ArrayList<>(); + + // --- Rich call-site facts (only on `call` nodes) -------------------------------------------- + // The canonical `call` node carries just {callee, arguments}, which is thinner than every + // analyzer's real call-site data (the Python reference analyzer keeps a parallel rich + // `call_sites[]` for the same reason). These are therefore additive Java fields, retained because + // the framework/CRUD finders key on `receiver_type` and dropping them would regress against v1. + + /** The receiver expression as written ({@code "abc"}, {@code helper}, {@code this}). */ + private String receiverExpr; + + /** Resolved type of the receiver — or, for a {@code new} expression, the instantiated type. */ + private String receiverType; + + /** Resolved types of the argument expressions, positionally. */ + private List argumentTypes = new ArrayList<>(); + + /** The argument expressions as written, positionally. */ + private List argumentExpr = new ArrayList<>(); + + /** Erased signature of the resolved callee ({@code substring(int)}); absent when unresolvable. */ + private String calleeSignature; + + private boolean isStaticCall; + private boolean isConstructorCall; } diff --git a/src/main/java/com/ibm/cldk/schema/JEnumConstant.java b/src/main/java/com/ibm/cldk/schema/JEnumConstant.java index 2cb3705..bb61812 100644 --- a/src/main/java/com/ibm/cldk/schema/JEnumConstant.java +++ b/src/main/java/com/ibm/cldk/schema/JEnumConstant.java @@ -6,8 +6,10 @@ /** * An enum constant declared on an {@code enum} type, with the argument expressions passed to the - * enum's constructor (empty for a plain constant). A Java-specific addition to the keystone's type - * node, which has no enum-member vocabulary (see cldk-devtools#40). + * enum's constructor (empty for a plain constant). + * + *

The canonical schema has no enum-member vocabulary, so this is an additive Java field; it exists + * because dropping it would lose information the v1 symbol table carried. */ @Data public class JEnumConstant { diff --git a/src/main/java/com/ibm/cldk/schema/JRecordComponent.java b/src/main/java/com/ibm/cldk/schema/JRecordComponent.java index 1ad2345..d1d305a 100644 --- a/src/main/java/com/ibm/cldk/schema/JRecordComponent.java +++ b/src/main/java/com/ibm/cldk/schema/JRecordComponent.java @@ -6,7 +6,9 @@ /** * A component of a {@code record} type — its name, resolved {@code type}, modifiers and structured - * decorators. A Java-specific addition to the keystone's type node (see cldk-devtools#40). + * decorators. + * + *

The canonical schema has no record-member vocabulary, so this is an additive Java field. * *

v1 also carried a {@code defaultValue} derived from compact-constructor assignments; that is * dropped deliberately — Java record components have no default values, so the field was misleading. diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java index f0164d2..a6547aa 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java @@ -8,7 +8,9 @@ import com.github.javaparser.ast.expr.ObjectCreationExpr; import com.github.javaparser.ast.stmt.BlockStmt; import com.github.javaparser.ast.stmt.ExplicitConstructorInvocationStmt; +import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration; import com.ibm.cldk.schema.JBodyNode; +import com.ibm.cldk.utils.Log; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; @@ -63,12 +65,58 @@ public Map build(BlockStmt body) { node.setKind("call"); node.setSpan(ctx.spanOf(site)); // `callee` stays unset at L1 and is filled in when L2 resolves this site. - node.setArguments(argumentsOf(site).stream().map(CallSiteBuilder::localId).collect(Collectors.toList())); + List args = argumentsOf(site); + node.setArguments(args.stream().map(CallSiteBuilder::localId).collect(Collectors.toList())); + node.setArgumentExpr(args.stream().map(Object::toString).collect(Collectors.toList())); + node.setArgumentTypes( + args.stream().map(ctx::resolveExpressionType).collect(Collectors.toList())); + enrich(node, site); nodes.put(localId(site), node); } return nodes; } + /** + * Fill in the resolved call-site facts, degrading silently when resolution fails (a missing + * dependency must thin the node's data, never drop the node or fail the build). + */ + private void enrich(JBodyNode node, Node site) { + if (site instanceof MethodCallExpr) { + MethodCallExpr call = (MethodCallExpr) site; + call.getScope().ifPresent(scope -> { + node.setReceiverExpr(scope.toString()); + String type = ctx.resolveExpressionType(scope); + if (!type.isEmpty()) { + node.setReceiverType(type); + } + }); + try { + ResolvedMethodDeclaration resolved = call.resolve(); + node.setCalleeSignature(Signatures.typeErasure(resolved)); + node.setStaticCall(resolved.isStatic()); + } catch (Throwable e) { + Log.debug("Could not resolve call: " + call + ": " + e.getMessage()); + } + } else if (site instanceof ObjectCreationExpr) { + ObjectCreationExpr creation = (ObjectCreationExpr) site; + node.setConstructorCall(true); + node.setReceiverType(ctx.resolveType(creation.getType())); + try { + node.setCalleeSignature(Signatures.typeErasure(creation.resolve())); + } catch (Throwable e) { + Log.debug("Could not resolve constructor call: " + creation + ": " + e.getMessage()); + } + } else if (site instanceof ExplicitConstructorInvocationStmt) { + node.setConstructorCall(true); + try { + node.setCalleeSignature( + Signatures.typeErasure(((ExplicitConstructorInvocationStmt) site).resolve())); + } catch (Throwable e) { + Log.debug("Could not resolve constructor invocation: " + site + ": " + e.getMessage()); + } + } + } + /** * True when {@code node} is part of {@code body} itself and not of a nested type or anonymous * class declared within it: no {@link BodyDeclaration} (which includes type declarations and diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java b/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java index 5509aec..79b69d4 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java @@ -3,6 +3,7 @@ import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.body.Parameter; +import com.github.javaparser.resolution.declarations.ResolvedMethodLikeDeclaration; import com.github.javaparser.resolution.types.ResolvedType; import com.ibm.cldk.utils.Log; import java.util.ArrayList; @@ -18,6 +19,23 @@ public final class Signatures { private Signatures() {} + /** + * The type-erasure signature of an already-resolved method/constructor — used to name the + * callee of a call site. Mirrors the declaration-side format so a call site's + * {@code callee_signature} matches the target callable's {@code signature}. + */ + public static String typeErasure(ResolvedMethodLikeDeclaration methodDecl) { + StringBuilder signature = new StringBuilder(methodDecl.getName()); + List erasureParameterTypes = new ArrayList<>(); + for (int i = 0; i < methodDecl.getNumberOfParams(); i++) { + erasureParameterTypes.add(methodDecl.getParam(i).getType().erasure().describe()); + } + signature.append("("); + signature.append(String.join(", ", erasureParameterTypes)); + signature.append(")"); + return signature.toString(); + } + /** * The type-erasure signature for {@code callableDecl}: the method name (or {@code } for a * constructor) followed by erased parameter types. Falls back to the plain AST signature if the @@ -41,7 +59,7 @@ public static String typeErasure(CallableDeclaration callableDecl) { signature.append(")"); return signature.toString(); } catch (Throwable e) { - Log.warn("Could not compute type erasure signature for " + callableDecl.getSignature().asString() + Log.debug("Could not compute type erasure signature for " + callableDecl.getSignature().asString() + "; computing regular signature"); return callableDecl.getSignature().asString(); } diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java index 924163a..eab90e2 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java @@ -1,6 +1,7 @@ package com.ibm.cldk.syntactic_analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -113,6 +114,43 @@ void build_ordersMethodAndConstructorCallsBySourcePosition() { assertEquals(List.of("4:5", "5:9", "6:5"), new ArrayList<>(body.keySet())); } + @Test + void build_callNodeCapturesReceiverAndResolvedTypes() { + // The rich call-site facts v1 exposed on CallSite: framework/CRUD finders key on receiver_type, + // and LLM consumers want the receiver/argument expressions verbatim. + JBodyNode node = build("package p;\nclass Foo {\n void m() {\n \"abc\".substring(1);\n }\n}\n") + .get("4:11"); + assertEquals("\"abc\"", node.getReceiverExpr()); + assertEquals("java.lang.String", node.getReceiverType()); + assertEquals(List.of("int"), node.getArgumentTypes()); + assertEquals(List.of("1"), node.getArgumentExpr()); + assertEquals("substring(int)", node.getCalleeSignature()); + assertFalse(node.isStaticCall()); + assertFalse(node.isConstructorCall()); + } + + @Test + void build_flagsStaticCall() { + JBodyNode node = build("package p;\nclass Foo {\n void m() {\n Math.max(1, 2);\n }\n}\n").get("4:10"); + assertTrue(node.isStaticCall(), "Math.max is static"); + assertEquals("java.lang.Math", node.getReceiverType()); + } + + @Test + void build_flagsConstructorCall() { + JBodyNode node = build("package p;\nclass Foo {\n void m() {\n new String(\"x\");\n }\n}\n").get("4:9"); + assertTrue(node.isConstructorCall()); + assertEquals("java.lang.String", node.getReceiverType(), "the instantiated type"); + } + + @Test + void build_unresolvableCallStillEmitsNodeWithoutResolvedFacts() { + // Honest degradation: an unresolvable callee must not drop the call node or crash. + JBodyNode node = build("package p;\nclass Foo {\n void m() {\n mystery(x);\n }\n}\n").get("4:5"); + assertEquals("call", node.getKind()); + assertNull(node.getCalleeSignature()); + } + @Test void build_excludesCallsInsideNestedLocalClasses() { // hidden() belongs to Local.inner()'s own body (its own callable), not to m(). From 577faef4f3f0a9cd7021bf4bf5df0aa453689132 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 09:04:10 -0400 Subject: [PATCH 13/22] feat(schema): entrypoint flags, qualified field refs, broader type refs, AstScopes (#180) --- .../java/com/ibm/cldk/schema/JCallable.java | 3 ++ src/main/java/com/ibm/cldk/schema/JType.java | 3 ++ .../cldk/syntactic_analysis/AstScopes.java | 31 ++++++++++++ .../syntactic_analysis/CallSiteBuilder.java | 23 ++------- .../syntactic_analysis/CallableBuilder.java | 50 ++++++++++++++----- .../cldk/syntactic_analysis/TypeBuilder.java | 8 ++- .../CallableBuilderTest.java | 23 ++++++++- .../syntactic_analysis/TypeBuilderTest.java | 11 +++- 8 files changed, 114 insertions(+), 38 deletions(-) create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/AstScopes.java diff --git a/src/main/java/com/ibm/cldk/schema/JCallable.java b/src/main/java/com/ibm/cldk/schema/JCallable.java index 579d2dc..ca19992 100644 --- a/src/main/java/com/ibm/cldk/schema/JCallable.java +++ b/src/main/java/com/ibm/cldk/schema/JCallable.java @@ -34,6 +34,9 @@ public class JCallable { private boolean isImplicit; private List comments = new ArrayList<>(); + /** True when a framework finder recognises this callable as an entrypoint (e.g. a Spring route). */ + private boolean isEntrypoint; + private JMetrics metrics; private JRefs refs; diff --git a/src/main/java/com/ibm/cldk/schema/JType.java b/src/main/java/com/ibm/cldk/schema/JType.java index 5708430..451e5fd 100644 --- a/src/main/java/com/ibm/cldk/schema/JType.java +++ b/src/main/java/com/ibm/cldk/schema/JType.java @@ -22,6 +22,9 @@ public class JType { private List interfaces = new ArrayList<>(); private List decorators = new ArrayList<>(); + /** True when a framework finder recognises this type as an entrypoint (e.g. a Spring controller). */ + private boolean isEntrypointClass; + /** Enum constants, in declaration order — present only on {@code enum} types. */ private List enumConstants = new ArrayList<>(); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/AstScopes.java b/src/main/java/com/ibm/cldk/syntactic_analysis/AstScopes.java new file mode 100644 index 0000000..0858774 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/AstScopes.java @@ -0,0 +1,31 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ast.Node; +import com.github.javaparser.ast.body.BodyDeclaration; +import com.github.javaparser.ast.stmt.BlockStmt; + +/** + * Scope questions the L1 builders share: deciding which AST nodes belong to the callable being built + * rather than to a type or callable nested inside it. + */ +final class AstScopes { + + private AstScopes() {} + + /** + * True when {@code node} belongs to {@code body} itself and not to a nested type or anonymous + * class declared within it: no {@link BodyDeclaration} (which includes type declarations and + * member methods/initializers) lies between the node and the body block. Lambda bodies have no + * {@code BodyDeclaration} of their own, so their contents stay with the enclosing callable. + */ + static boolean belongsDirectlyTo(Node node, BlockStmt body) { + for (Node cur = node.getParentNode().orElse(null); + cur != null && cur != body; + cur = cur.getParentNode().orElse(null)) { + if (cur instanceof BodyDeclaration) { + return false; + } + } + return true; + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java index a6547aa..e0bd621 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java @@ -2,7 +2,6 @@ import com.github.javaparser.ast.Node; import com.github.javaparser.ast.NodeList; -import com.github.javaparser.ast.body.BodyDeclaration; import com.github.javaparser.ast.expr.Expression; import com.github.javaparser.ast.expr.MethodCallExpr; import com.github.javaparser.ast.expr.ObjectCreationExpr; @@ -50,10 +49,10 @@ public CallSiteBuilder(L1BuildContext ctx) { public Map build(BlockStmt body) { List sites = new ArrayList<>(); - body.findAll(MethodCallExpr.class).stream().filter(n -> belongsDirectlyTo(n, body)).forEach(sites::add); - body.findAll(ObjectCreationExpr.class).stream().filter(n -> belongsDirectlyTo(n, body)).forEach(sites::add); + body.findAll(MethodCallExpr.class).stream().filter(n -> AstScopes.belongsDirectlyTo(n, body)).forEach(sites::add); + body.findAll(ObjectCreationExpr.class).stream().filter(n -> AstScopes.belongsDirectlyTo(n, body)).forEach(sites::add); body.findAll(ExplicitConstructorInvocationStmt.class).stream() - .filter(n -> belongsDirectlyTo(n, body)) + .filter(n -> AstScopes.belongsDirectlyTo(n, body)) .forEach(sites::add); sites.sort(Comparator.comparingInt(n -> anchorPosition(n)[0]) @@ -117,22 +116,6 @@ private void enrich(JBodyNode node, Node site) { } } - /** - * True when {@code node} is part of {@code body} itself and not of a nested type or anonymous - * class declared within it: no {@link BodyDeclaration} (which includes type declarations and - * member methods/initializers) lies between the node and the body block. - */ - static boolean belongsDirectlyTo(Node node, BlockStmt body) { - for (Node cur = node.getParentNode().orElse(null); - cur != null && cur != body; - cur = cur.getParentNode().orElse(null)) { - if (cur instanceof BodyDeclaration) { - return false; - } - } - return true; - } - private static List argumentsOf(Node site) { NodeList args; if (site instanceof MethodCallExpr) { diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index a91dcae..163a5a0 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -6,8 +6,10 @@ import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; import com.github.javaparser.ast.body.VariableDeclarator; +import com.github.javaparser.ast.expr.CastExpr; import com.github.javaparser.ast.expr.ConditionalExpr; import com.github.javaparser.ast.expr.FieldAccessExpr; +import com.github.javaparser.ast.expr.InstanceOfExpr; import com.github.javaparser.ast.expr.NameExpr; import com.github.javaparser.ast.expr.ObjectCreationExpr; import com.github.javaparser.ast.stmt.BlockStmt; @@ -18,6 +20,7 @@ import com.github.javaparser.ast.stmt.IfStmt; import com.github.javaparser.ast.stmt.SwitchStmt; import com.github.javaparser.ast.stmt.WhileStmt; +import com.ibm.cldk.javaee.EntrypointsFinderFactory; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; import com.ibm.cldk.schema.JMetrics; @@ -62,7 +65,8 @@ public CallableBuilder(L1BuildContext ctx) { * @param parentTypeId the containing type's id * @param classFieldNames simple names of the enclosing type's fields (for {@code refs.fields}) */ - public JCallable build(CallableDeclaration cd, String parentTypeId, List classFieldNames) { + public JCallable build( + CallableDeclaration cd, String parentTypeId, String typeFqn, List classFieldNames) { JCallable callable = new JCallable(); String signature = Signatures.typeErasure(cd); callable.setSignature(signature); @@ -77,6 +81,8 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List m.getKeyword().asString()).collect(Collectors.toList())); + callable.setEntrypoint( + EntrypointsFinderFactory.getEntrypointFinders().anyMatch(f -> f.isEntrypointMethod(cd))); callable.setComments(ctx.commentsOf(cd)); callable.setDecorators( cd.getAnnotations().stream().map(decoratorBuilder::build).collect(Collectors.toList())); @@ -90,7 +96,7 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List body = bodyOf(cd); body.flatMap(b -> b.getRange().map(r -> r.begin.line)).ifPresent(callable::setCodeStartLine); - callable.setRefs(refs(body, classFieldNames)); + callable.setRefs(refs(body, typeFqn, classFieldNames)); body.ifPresent(b -> callable.setLocalVariables(localVariables(b))); body.ifPresent(b -> callable.setBody(callSiteBuilder.build(b))); body.ifPresent(b -> callable.setTypes(localClasses(b, callable.getId()))); @@ -103,7 +109,11 @@ public JCallable build(CallableDeclaration cd, String parentTypeId, List classFieldNames, String signature) { + InitializerDeclaration id, + String parentTypeId, + String typeFqn, + List classFieldNames, + String signature) { JCallable callable = new JCallable(); callable.setSignature(signature); callable.setId(CanId.childId(parentTypeId, signature)); @@ -121,7 +131,7 @@ public JCallable buildInitializer( BlockStmt body = id.getBody(); body.getRange().map(r -> r.begin.line).ifPresent(callable::setCodeStartLine); - callable.setRefs(refs(Optional.of(body), classFieldNames)); + callable.setRefs(refs(Optional.of(body), typeFqn, classFieldNames)); callable.setLocalVariables(localVariables(body)); callable.setBody(callSiteBuilder.build(body)); callable.setTypes(localClasses(body, callable.getId())); @@ -139,7 +149,7 @@ private static Optional bodyOf(CallableDeclaration cd) { private List localVariables(BlockStmt body) { List locals = new ArrayList<>(); for (VariableDeclarator vd : body.findAll(VariableDeclarator.class)) { - if (!CallSiteBuilder.belongsDirectlyTo(vd, body)) { + if (!AstScopes.belongsDirectlyTo(vd, body)) { continue; } JVariableDeclaration local = new JVariableDeclaration(); @@ -158,13 +168,13 @@ private Map localClasses(BlockStmt body, String callableId) { TypeBuilder typeBuilder = new TypeBuilder(ctx); Map locals = new TreeMap<>(); body.findAll(TypeDeclaration.class).stream() - .filter(td -> CallSiteBuilder.belongsDirectlyTo(td, body)) + .filter(td -> AstScopes.belongsDirectlyTo(td, body)) .forEach(td -> locals.put(td.getNameAsString(), typeBuilder.build(td, callableId))); return new LinkedHashMap<>(locals); } /** Syntactic cross-refs: types referenced and enclosing-type fields accessed in the body. */ - private JRefs refs(Optional body, List classFieldNames) { + private JRefs refs(Optional body, String typeFqn, List classFieldNames) { JRefs refs = new JRefs(); if (body.isEmpty()) { return refs; @@ -173,21 +183,35 @@ private JRefs refs(Optional body, List classFieldNames) { TreeSet types = new TreeSet<>(); b.findAll(VariableDeclarator.class).stream() - .filter(vd -> CallSiteBuilder.belongsDirectlyTo(vd, b) && vd.getType().isClassOrInterfaceType()) + .filter(vd -> AstScopes.belongsDirectlyTo(vd, b) && vd.getType().isClassOrInterfaceType()) .forEach(vd -> types.add(ctx.resolveType(vd.getType()))); b.findAll(ObjectCreationExpr.class).stream() - .filter(oce -> CallSiteBuilder.belongsDirectlyTo(oce, b)) + .filter(oce -> AstScopes.belongsDirectlyTo(oce, b)) .forEach(oce -> types.add(ctx.resolveType(oce.getType()))); + b.findAll(CastExpr.class).stream() + .filter(ce -> AstScopes.belongsDirectlyTo(ce, b)) + .forEach(ce -> types.add(ctx.resolveType(ce.getType()))); + b.findAll(InstanceOfExpr.class).stream() + .filter(ie -> AstScopes.belongsDirectlyTo(ie, b)) + .forEach(ie -> types.add(ctx.resolveType(ie.getType()))); + b.findAll(CatchClause.class).stream() + .filter(cc -> AstScopes.belongsDirectlyTo(cc, b)) + .forEach(cc -> types.add(ctx.resolveType(cc.getParameter().getType()))); refs.setTypes(new ArrayList<>(types)); + // Field refs are qualified by their declaring type (as v1 did), so `other.count` and + // `this.count` stay distinguishable; unresolvable scopes fall back to the bare name. TreeSet fields = new TreeSet<>(); b.findAll(FieldAccessExpr.class).stream() - .filter(fa -> CallSiteBuilder.belongsDirectlyTo(fa, b) + .filter(fa -> AstScopes.belongsDirectlyTo(fa, b) && !(fa.getParentNode().orElse(null) instanceof FieldAccessExpr)) - .forEach(fa -> fields.add(fa.getNameAsString())); + .forEach(fa -> { + String declaring = ctx.resolveExpressionType(fa.getScope()); + fields.add(declaring.isEmpty() ? fa.getNameAsString() : declaring + "." + fa.getNameAsString()); + }); b.findAll(NameExpr.class).stream() - .filter(ne -> CallSiteBuilder.belongsDirectlyTo(ne, b) && classFieldNames.contains(ne.getNameAsString())) - .forEach(ne -> fields.add(ne.getNameAsString())); + .filter(ne -> AstScopes.belongsDirectlyTo(ne, b) && classFieldNames.contains(ne.getNameAsString())) + .forEach(ne -> fields.add(typeFqn + "." + ne.getNameAsString())); refs.setFields(new ArrayList<>(fields)); return refs; } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index ef7c10f..0f37841 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -10,6 +10,7 @@ import com.github.javaparser.ast.body.Parameter; import com.github.javaparser.ast.body.RecordDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; +import com.ibm.cldk.javaee.EntrypointsFinderFactory; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; import com.ibm.cldk.schema.JEnumConstant; @@ -51,6 +52,8 @@ public JType build(TypeDeclaration td, String parentId) { type.setId(CanId.childId(parentId, td.getNameAsString())); type.setKind(kindOf(td)); type.setSpan(ctx.spanOf(td)); + type.setEntrypointClass( + EntrypointsFinderFactory.getEntrypointFinders().anyMatch(f -> f.isEntrypointClass(td))); type.setComments(ctx.commentsOf(td)); type.setModifiers( td.getModifiers().stream().map(m -> m.getKeyword().asString()).collect(Collectors.toList())); @@ -115,12 +118,13 @@ public JType build(TypeDeclaration td, String parentId) { // Keyed by type-erasure signature. Field names are handed down so each callable's // refs.fields can recognize accesses to this type's fields. List fieldNames = new ArrayList<>(fields.keySet()); + String typeFqn = td.getFullyQualifiedName().orElse(td.getNameAsString()); List> declared = new ArrayList<>(); declared.addAll(td.getConstructors()); declared.addAll(td.getMethods()); Map callables = new TreeMap<>(); for (CallableDeclaration cd : declared) { - JCallable callable = callableBuilder.build(cd, type.getId(), fieldNames); + JCallable callable = callableBuilder.build(cd, type.getId(), typeFqn, fieldNames); callables.put(callable.getSignature(), callable); } // Initializer blocks are callables too (keystone kind `initializer`) — L3 gives them their own @@ -134,7 +138,7 @@ public JType build(TypeDeclaration td, String parentId) { String signature = id.isStatic() ? "$" + staticIndex++ + "()" : "$" + instanceIndex++ + "()"; - callables.put(signature, callableBuilder.buildInitializer(id, type.getId(), fieldNames, signature)); + callables.put(signature, callableBuilder.buildInitializer(id, type.getId(), typeFqn, fieldNames, signature)); } type.setCallables(new LinkedHashMap<>(callables)); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java index b51fb92..e421b55 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -1,6 +1,7 @@ package com.ibm.cldk.syntactic_analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -28,7 +29,7 @@ private static JCallable build(String memberSource, List classFieldNames CompilationUnit cu = TestParsers.parseResolved(source); CallableDeclaration cd = cu.getType(0).findFirst(CallableDeclaration.class).orElseThrow(); L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); - return new CallableBuilder(ctx).build(cd, TYPE_ID, classFieldNames); + return new CallableBuilder(ctx).build(cd, TYPE_ID, "com.example.Foo", classFieldNames); } private static JCallable build(String memberSource) { @@ -77,6 +78,13 @@ void build_capturesParametersReturnTypeModifiersAndDecorators() { assertEquals("Override", c.getDecorators().get(0).getName()); } + @Test + void build_flagsEntrypointMethod() { + assertTrue(build("@GetMapping(\"/x\") String get() { return \"\"; }").isEntrypoint(), + "@GetMapping is a Spring entrypoint method"); + assertFalse(build("String plain() { return \"\"; }").isEntrypoint()); + } + @Test void build_capturesLocalVariables() { JCallable c = build("void m() { int total = 0; String name; }"); @@ -135,6 +143,17 @@ void build_computesCyclomaticMetric() { assertEquals(2, c.getMetrics().getCyclomatic()); } + @Test + void build_refsTypesIncludeCastsInstanceofAndCatchTypes() { + // v1 only scanned variable declarators and object creations; a cast/instanceof/catch type is + // just as much a referenced type. + JCallable c = build("void m(Object o) { try { String s = (String) o; if (o instanceof Integer) {} }" + + " catch (IllegalStateException e) {} }"); + assertTrue(c.getRefs().getTypes().contains("java.lang.String")); + assertTrue(c.getRefs().getTypes().contains("java.lang.Integer"), "instanceof type"); + assertTrue(c.getRefs().getTypes().contains("java.lang.IllegalStateException"), "catch type"); + } + @Test void build_capturesBodyCallNodes() { JCallable c = build("void m() { foo(); }"); @@ -147,7 +166,7 @@ void build_capturesRefsTypesAndAccessedFields() { JCallable c = build("void m() { Helper h = new Helper(); this.count = h.value(); }", List.of("count")); assertTrue(c.getRefs().getTypes().contains("Helper"), "referenced types should include the syntactic type Helper"); - assertEquals(List.of("count"), c.getRefs().getFields()); + assertEquals(List.of("com.example.Foo.count"), c.getRefs().getFields()); } @Test diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index c0f6849..923a7d6 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -1,6 +1,7 @@ package com.ibm.cldk.syntactic_analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -128,7 +129,15 @@ void build_callableRefsSeeEnclosingTypeFields() { // TypeBuilder must hand its field names to the callable builder so refs.fields resolves. JType t = buildFirstType("package p;\nclass Foo {\n int count;\n void inc() { count = count + 1; }\n}\n"); JCallable inc = t.getCallables().get("inc()"); - assertEquals(List.of("count"), inc.getRefs().getFields()); + assertEquals(List.of("p.Foo.count"), inc.getRefs().getFields(), + "field refs are qualified by their declaring type"); + } + + @Test + void build_flagsEntrypointClass() { + assertTrue(buildFirstType("package p;\n@RestController\nclass Api {}\n").isEntrypointClass(), + "@RestController is a Spring entrypoint class"); + assertFalse(buildFirstType("package p;\nclass Plain {}\n").isEntrypointClass()); } @Test From 9df81731723427a50efd8fa879d0bdaddbc14521 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 09:08:48 -0400 Subject: [PATCH 14/22] =?UTF-8?q?feat(schema):=20L1=20extractor=20?= =?UTF-8?q?=E2=80=94=20v2=20modules=20from=20the=20project=20parse=20loop?= =?UTF-8?q?=20(#180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cldk/syntactic_analysis/L1Extractor.java | 103 +++++++++++++++++ .../syntactic_analysis/L1ExtractorTest.java | 106 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java new file mode 100644 index 0000000..88748fb --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java @@ -0,0 +1,103 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.github.javaparser.ParseResult; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.symbolsolver.utils.SymbolSolverCollectionStrategy; +import com.github.javaparser.utils.ProjectRoot; +import com.github.javaparser.utils.SourceRoot; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.utils.Log; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Pattern; + +/** + * Orchestrates L1: discovers a project's source roots, parses each file with a symbol solver + * (so type resolution and erased signatures work), and builds one canonical schema v2 {@code module} + * per file via {@link ModuleBuilder}. + * + *

The symbol solver is configured from the project itself, which is what lets types declared in + * other files of the same project resolve to qualified names. Modules are keyed by the file's path + * relative to the project root, normalised to {@code /} — the key must be stable across runs + * and machines for caching and SDK lookups to work. + */ +public final class L1Extractor { + + private L1Extractor() {} + + /** Source roots that hold test data rather than analysable project code. */ + private static final String[] EXCLUDED_SOURCE_ROOTS = { + Paths.get("src", "test", "resources").toString(), + Paths.get("src", "it", "resources").toString(), + Paths.get("src", "xdocs-examples").toString() + }; + + /** + * Build the v2 symbol table for a project. + * + * @param projectRoot the project's root directory + * @param appName the application name — the {@code can://java/} segment of every id + * @return modules keyed by relative file path, iterated in sorted key order for determinism + */ + public static Map extractAll(Path projectRoot, String appName) throws IOException { + ParserConfiguration config = new ParserConfiguration() + .setStoreTokens(true) + .setAttributeComments(true) + .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21); + SymbolSolverCollectionStrategy strategy = new SymbolSolverCollectionStrategy(config); + ProjectRoot root = strategy.collect(projectRoot); + String applicationId = CanId.applicationId(appName); + + // Collect into a sorted map first: source roots and directory listings are not ordered, and + // `-j N` output must be byte-identical to `-j 1`. + Map modules = new TreeMap<>(); + for (SourceRoot sourceRoot : root.getSourceRoots()) { + if (isExcluded(sourceRoot.getRoot(), projectRoot)) { + continue; + } + sourceRoot.setParserConfiguration(config); + for (ParseResult parseResult : sourceRoot.tryToParse()) { + if (parseResult.getResult().isEmpty()) { + Log.debug("Skipping unparsable file: " + parseResult.getProblems()); + continue; + } + CompilationUnit cu = parseResult.getResult().get(); + if (cu.getStorage().isEmpty()) { + continue; + } + Path path = cu.getStorage().get().getPath(); + String fileKey = fileKey(projectRoot, path); + // Read the file's own text rather than printing the AST: `span.bytes` must index the + // real file, byte for byte. + String source = Files.readString(path, StandardCharsets.UTF_8); + L1BuildContext ctx = new L1BuildContext(applicationId, fileKey, source); + modules.put(fileKey, new ModuleBuilder(ctx).build(cu)); + } + } + return new LinkedHashMap<>(modules); + } + + /** The {@code symbol_table} key: path relative to the project root, always {@code /}-separated. */ + private static String fileKey(Path projectRoot, Path file) { + Path relative = projectRoot.toAbsolutePath().normalize().relativize(file.toAbsolutePath().normalize()); + return relative.toString().replace('\\', '/'); + } + + private static boolean isExcluded(Path sourceRoot, Path projectRoot) { + Path relative = projectRoot.toAbsolutePath().relativize(sourceRoot.toAbsolutePath()); + for (String excluded : EXCLUDED_SOURCE_ROOTS) { + if (Pattern.compile(Pattern.quote(excluded)).matcher(relative.toString()).find()) { + return true; + } + } + return false; + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java new file mode 100644 index 0000000..0e6134e --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java @@ -0,0 +1,106 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JType; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Orchestration-level tests for {@link L1Extractor}: walking a real project directory, keying the + * symbol table by stable relative paths, and retaining source so node text is a byte-slice. These are + * the L1 gate checks stated in the design spec, exercised over a real (if small) project on disk. + */ +class L1ExtractorTest { + + private static Path writeProject(Path root) throws IOException { + Path pkg = root.resolve("src/main/java/com/example"); + Files.createDirectories(pkg); + Files.writeString(pkg.resolve("Greeter.java"), + "package com.example;\n" + + "\n" + + "/** Greets. */\n" + + "public class Greeter {\n" + + " private String name;\n" + + " public String greet(String who) {\n" + + " return \"hi \" + who;\n" + + " }\n" + + "}\n", + StandardCharsets.UTF_8); + Files.writeString(pkg.resolve("Caller.java"), + "package com.example;\n" + + "\n" + + "public class Caller {\n" + + " void run() {\n" + + " new Greeter().greet(\"world\");\n" + + " }\n" + + "}\n", + StandardCharsets.UTF_8); + return root; + } + + @Test + void extractAll_keysModulesByStableRelativePaths(@TempDir Path tmp) throws IOException { + Map modules = L1Extractor.extractAll(writeProject(tmp), "myapp"); + + assertEquals(2, modules.size()); + for (String key : modules.keySet()) { + assertFalse(key.startsWith("/"), "symbol_table keys must not be absolute: " + key); + assertFalse(key.contains(".."), "symbol_table keys must not escape the root: " + key); + assertFalse(key.contains("\\"), "separators must be normalised: " + key); + } + assertTrue(modules.containsKey("src/main/java/com/example/Greeter.java")); + } + + @Test + void extractAll_retainsSourceSoNodeTextIsAByteSlice(@TempDir Path tmp) throws IOException { + Map modules = L1Extractor.extractAll(writeProject(tmp), "myapp"); + JModule module = modules.get("src/main/java/com/example/Greeter.java"); + + JType greeter = module.getTypes().get("Greeter"); + assertNotNull(greeter); + JCallable greet = greeter.getCallables().get("greet(java.lang.String)"); + assertNotNull(greet, "signature should use resolved, erased parameter types"); + + int[] bytes = greet.getSpan().getBytes(); + String sliced = new String( + module.getSource().getBytes(StandardCharsets.UTF_8), bytes[0], bytes[1] - bytes[0], + StandardCharsets.UTF_8); + assertTrue(sliced.startsWith("public String greet(String who)"), "got: " + sliced); + assertTrue(sliced.endsWith("}")); + } + + @Test + void extractAll_resolvesAcrossFilesInTheProject(@TempDir Path tmp) throws IOException { + // Caller references Greeter from another file: the project's own sources must be on the + // solver's path, otherwise cross-file types silently degrade to bare spellings. + Map modules = L1Extractor.extractAll(writeProject(tmp), "myapp"); + JCallable run = modules.get("src/main/java/com/example/Caller.java") + .getTypes().get("Caller").getCallables().get("run()"); + + assertTrue(run.getRefs().getTypes().contains("com.example.Greeter"), + "cross-file type should resolve to its qualified name, got: " + run.getRefs().getTypes()); + assertTrue(run.getBody().values().stream() + .anyMatch(n -> "com.example.Greeter".equals(n.getReceiverType())), + "the greet(...) call's receiver type should resolve across files"); + } + + @Test + void extractAll_producesStableIdsAndDeterministicOutputAcrossRuns(@TempDir Path tmp) throws IOException { + Path root = writeProject(tmp); + assertEquals( + com.ibm.cldk.schema.V2Json.compact().toJson(L1Extractor.extractAll(root, "myapp")), + com.ibm.cldk.schema.V2Json.compact().toJson(L1Extractor.extractAll(root, "myapp")), + "two runs over unchanged source must produce byte-identical output"); + } +} From b8dd5e99659c7584c35cfd89ef6a38367d5cf158 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 10:32:07 -0400 Subject: [PATCH 15/22] feat(cli): --schema v2 emits the canonical envelope, with flag validation (#180) v2 is opt-in (v1 stays the default until the rest of the migration lands). Unsupported combinations (-a > 1, --emit neo4j, --source-analysis, --target-files, unknown --schema) fail with a clear non-zero error rather than silently emitting a different shape. stdout carries compact JSON only. --- src/main/java/com/ibm/cldk/CodeAnalyzer.java | 90 +++++++++++++- .../java/com/ibm/cldk/schema/Analysis.java | 1 + .../com/ibm/cldk/schema/JAnalyzerInfo.java | 13 ++ .../java/com/ibm/cldk/schema/V2Emitter.java | 11 ++ .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 115 ++++++++++++++++++ 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/ibm/cldk/schema/JAnalyzerInfo.java create mode 100644 src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index e475003..7f0b7e5 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -24,6 +24,11 @@ import com.ibm.cldk.entities.JavaCompilationUnit; import com.ibm.cldk.neo4j.BoltConfig; import com.ibm.cldk.neo4j.Neo4jEmitter; +import com.ibm.cldk.schema.Analysis; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.V2Emitter; +import com.ibm.cldk.schema.V2Json; +import com.ibm.cldk.syntactic_analysis.L1Extractor; import com.ibm.cldk.utils.BuildProject; import com.ibm.cldk.utils.Log; import java.io.File; @@ -40,7 +45,10 @@ import org.apache.commons.lang3.tuple.Pair; import picocli.CommandLine; import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Option; +import picocli.CommandLine.ParameterException; +import picocli.CommandLine.Spec; class VersionProvider implements CommandLine.IVersionProvider { @@ -116,6 +124,17 @@ public class CodeAnalyzer implements Runnable { @Option(names = { "--neo4j-database" }, description = "Neo4j database name (env: NEO4J_DATABASE, default: server default).") private static String neo4jDatabase; + @Option(names = { + "--schema" }, description = "Output schema: v1 (legacy, default) | v2 (canonical CPG). " + + "v2 currently covers analysis level 1 only.") + // Deliberately an INSTANCE field: the pre-existing options on this class are static, which leaks + // values between CommandLine instances in the same JVM. New flags do not add to that. + private String schema = "v1"; + + /** Handle used to report flag-validation errors as clean, non-zero picocli failures. */ + @Spec + private CommandSpec spec; + private static final String outputFileName = "analysis.json"; public static Gson gson = new GsonBuilder() @@ -156,7 +175,7 @@ public void run() { } } - private static void analyze() throws Exception { + private void analyze() throws Exception { // The Neo4j schema contract is a static artifact — no project analysis required. if ("schema".equalsIgnoreCase(emit)) { @@ -164,6 +183,11 @@ private static void analyze() throws Exception { return; } + if (isV2Schema()) { + analyzeV2(); + return; + } + JsonObject combinedJsonObject = new JsonObject(); Map symbolTable; projectRootPom = projectRootPom == null ? input : projectRootPom; @@ -290,6 +314,70 @@ private static void analyze() throws Exception { emit(consolidatedJSONString); } + private boolean isV2Schema() { + if ("v2".equalsIgnoreCase(schema)) { + return true; + } + if (!"v1".equalsIgnoreCase(schema)) { + // Never silently fall back on an unrecognised flag value — the caller asked for something + // specific and would otherwise process the wrong shape. + throw new ParameterException(spec.commandLine(), + "error: unknown --schema value '" + schema + "'; use v1 or v2"); + } + return false; + } + + /** + * Emit the canonical schema v2 payload. Only the surfaces that exist today are accepted: level 1, + * whole-project, JSON. Anything else is an explicit error rather than a silently different result. + */ + private void analyzeV2() throws Exception { + if (analysisLevel > 1) { + throw new ParameterException(spec.commandLine(), + "error: --schema v2 currently supports --analysis-level 1 only"); + } + if ("neo4j".equalsIgnoreCase(emit)) { + throw new ParameterException(spec.commandLine(), + "error: --schema v2 does not support --emit neo4j yet; the graph projection is still v1"); + } + if (sourceAnalysis != null || targetFiles != null) { + throw new ParameterException(spec.commandLine(), + "error: --schema v2 supports whole-project analysis only " + + "(not --source-analysis or --target-files)"); + } + if (input == null) { + throw new ParameterException(spec.commandLine(), "error: --input is required"); + } + + String application = appName != null && !appName.isBlank() + ? appName + : Paths.get(input).toAbsolutePath().normalize().getFileName().toString(); + Map modules = L1Extractor.extractAll(Paths.get(input), application); + Analysis analysis = V2Emitter.emit(application, 1, modules, analyzerVersion()); + + if (output == null) { + // stdout is the data channel: compact JSON only, so the SDK can parse it directly. + System.out.println(V2Json.compact().toJson(analysis)); + } else { + Path outputPath = Paths.get(output); + if (!Files.exists(outputPath)) { + Files.createDirectories(outputPath); + } + try (FileWriter writer = new FileWriter(new File(output, outputFileName))) { + writer.write(V2Json.pretty().toJson(analysis)); + } + } + } + + private static String analyzerVersion() { + try { + String[] versions = new VersionProvider().getVersion(); + return versions.length > 0 ? versions[0] : "unknown"; + } catch (Exception e) { + return "unknown"; + } + } + private static void emit(String consolidatedJSONString) throws IOException { if (output == null) { System.out.println(consolidatedJSONString); diff --git a/src/main/java/com/ibm/cldk/schema/Analysis.java b/src/main/java/com/ibm/cldk/schema/Analysis.java index 2b73d3c..dd3aaa4 100644 --- a/src/main/java/com/ibm/cldk/schema/Analysis.java +++ b/src/main/java/com/ibm/cldk/schema/Analysis.java @@ -12,5 +12,6 @@ public class Analysis { private String schemaVersion; private String language; private int maxLevel; + private JAnalyzerInfo analyzer; private JApplication application; } diff --git a/src/main/java/com/ibm/cldk/schema/JAnalyzerInfo.java b/src/main/java/com/ibm/cldk/schema/JAnalyzerInfo.java new file mode 100644 index 0000000..7fb3142 --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JAnalyzerInfo.java @@ -0,0 +1,13 @@ +package com.ibm.cldk.schema; + +import lombok.Data; + +/** + * Which analyzer produced a payload, and at which version — part of the v2 envelope manifest so a + * consumer can tell what wrote the file it is reading. + */ +@Data +public class JAnalyzerInfo { + private String name = "codeanalyzer-java"; + private String version; +} diff --git a/src/main/java/com/ibm/cldk/schema/V2Emitter.java b/src/main/java/com/ibm/cldk/schema/V2Emitter.java index d5852f9..6e41643 100644 --- a/src/main/java/com/ibm/cldk/schema/V2Emitter.java +++ b/src/main/java/com/ibm/cldk/schema/V2Emitter.java @@ -16,6 +16,12 @@ private V2Emitter() {} /** Wrap already-built modules (keyed by relative file key) into the v2 envelope. */ public static Analysis emit(String appName, int maxLevel, Map modules) { + return emit(appName, maxLevel, modules, null); + } + + /** As above, stamping the analyzer version into the envelope manifest. */ + public static Analysis emit( + String appName, int maxLevel, Map modules, String analyzerVersion) { JApplication application = new JApplication(); application.setId(CanId.applicationId(appName)); @@ -30,6 +36,11 @@ public static Analysis emit(String appName, int maxLevel, Map m analysis.setSchemaVersion("2.0.0"); analysis.setLanguage("java"); analysis.setMaxLevel(maxLevel); + if (analyzerVersion != null) { + JAnalyzerInfo analyzer = new JAnalyzerInfo(); + analyzer.setVersion(analyzerVersion); + analysis.setAnalyzer(analyzer); + } analysis.setApplication(application); return analysis; } diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java new file mode 100644 index 0000000..777117b --- /dev/null +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -0,0 +1,115 @@ +package com.ibm.cldk; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +/** + * CLI-level tests for the {@code --schema v2} path: the emitted envelope, and the flag-validation + * rules from the CLI contract (an unsupported combination must fail loudly rather than silently + * produce a different shape). + */ +class CodeAnalyzerV2CliTest { + + private static Path project(Path root) throws IOException { + Path pkg = root.resolve("src/main/java/com/example"); + Files.createDirectories(pkg); + Files.writeString(pkg.resolve("Widget.java"), + "package com.example;\npublic class Widget {\n public int size() { return 1; }\n}\n", + StandardCharsets.UTF_8); + return root; + } + + private static int run(String... args) { + return new CommandLine(new CodeAnalyzer()).execute(args); + } + + /** + * The pre-existing CLI options on {@link CodeAnalyzer} are static, so a value set by one test + * would leak into the next. Reset the ones these tests touch so each case starts from defaults. + */ + @BeforeEach + void resetStaticOptions() throws Exception { + set("emit", "json"); + set("analysisLevel", 1); + set("output", null); + set("input", null); + set("targetFiles", null); + set("sourceAnalysis", null); + } + + private static void set(String field, Object value) throws Exception { + Field f = CodeAnalyzer.class.getDeclaredField(field); + f.setAccessible(true); + f.set(null, value); + } + + @Test + void v2SchemaWritesCanonicalEnvelopeToAnalysisJson(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", "--app-name", "widgets")); + + Path analysis = out.resolve("analysis.json"); + assertTrue(Files.exists(analysis), "analysis.json must be written"); + JsonObject root = JsonParser.parseString(Files.readString(analysis)).getAsJsonObject(); + + assertEquals("2.0.0", root.get("schema_version").getAsString()); + assertEquals("java", root.get("language").getAsString()); + assertEquals(1, root.get("max_level").getAsInt()); + assertEquals("codeanalyzer-java", root.getAsJsonObject("analyzer").get("name").getAsString()); + + JsonObject app = root.getAsJsonObject("application"); + assertEquals("can://java/widgets", app.get("id").getAsString()); + JsonObject symbolTable = app.getAsJsonObject("symbol_table"); + assertTrue(symbolTable.has("src/main/java/com/example/Widget.java"), + "keyed by relative path, got: " + symbolTable.keySet()); + } + + @Test + void v2SchemaIsNotTheDefault(@TempDir Path tmp) throws IOException { + // The legacy shape stays the default until the rest of the migration lands, so existing + // consumers are unaffected by this change. + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString())); + JsonObject root = JsonParser.parseString(Files.readString(out.resolve("analysis.json"))).getAsJsonObject(); + assertFalse(root.has("schema_version"), "default output is still the v1 shape"); + assertTrue(root.has("symbol_table"), "v1 keeps symbol_table at the top level"); + } + + @Test + void unknownSchemaValueFailsLoudly(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "--schema", "v3"), + "an unrecognised flag value must not silently fall back"); + } + + @Test + void v2WithAnalysisLevelAboveOneFailsLoudly(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "--schema", "v2", "-a", "2"), + "v2 has no call graph yet; asking for level 2 must be an error, not a level-1 payload"); + } + + @Test + void v2WithNeo4jEmitFailsLoudly(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "--schema", "v2", "--emit", "neo4j"), + "the graph projection is still v1"); + } +} From 3438a76d899105c529e5d7bc5b23bf274d8c43f2 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 10:49:41 -0400 Subject: [PATCH 16/22] test: L1 conformance gate against the canonical v2 JSON Schema (#180) Adds the strict in-repo schema used as the L1 oracle until the SDK's v2 models exist, a gate over the in-repo fixtures in the default suite, and a realWorldConformanceTest task for the submodule applications (too slow for the inner loop, but required). --- .claude/SCHEMA_DECISIONS.md | 18 ++ build.gradle | 26 +- .../cldk/schema/L1ConformanceGateTest.java | 194 +++++++++++ .../resources/schema/analysis.v2.schema.json | 304 ++++++++++++++++++ 4 files changed, 541 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/ibm/cldk/schema/L1ConformanceGateTest.java create mode 100644 src/test/resources/schema/analysis.v2.schema.json diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 34cb1d1..1726717 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -125,6 +125,24 @@ Refinements settled while building L1 (2026-08), each checked against the keysto - **`module.content_hash` is SHA-256 hex of the UTF-8 source** — for incremental caching and the Neo4j writer's per-module diffing; never identity (the `id` is). +### D11 — L1 conformance oracle and gate + +- **Oracle:** emitted output is validated against an in-repo JSON Schema, + `src/test/resources/schema/analysis.v2.schema.json`, because the SDK's v2 models do not exist yet. + The schema is **strict** (`additionalProperties: false`) so a renamed or stray key fails the gate + instead of reaching consumers, and it encodes the structural invariants directly: `can://java/` id + prefixes, `line:col`/`@tag` body keys via `propertyNames`, relative `symbol_table` keys, and + `[from, to)` byte spans. Replace it with the SDK models once they land. +- **The gate runs at two scales.** In-repo fixtures run in the default `test` task on every change. + Whole real-world applications (the git-submodule fixtures) take minutes under full symbol + resolution, so they are tagged `realworld`, excluded from `test`, and run via + `./gradlew realWorldConformanceTest`. They are not optional — scale-dependent problems + (unresolvable dependencies, unusual constructs, memory) only appear there. +- **v2 is opt-in for now.** `--schema v2` emits the canonical envelope; `v1` stays the default until + the rest of the migration lands, so existing consumers are unaffected. Unsupported combinations + (`-a > 1`, `--emit neo4j`, `--source-analysis`, `--target-files`, unknown `--schema`) exit non-zero + with a clear message rather than silently emitting a different shape. + ### D9 — Neo4j namespace: keep the `J_` relationship prefix Existing convention (`J_CALLS`, …); dual-label `JSymbol` merge pattern retained. `SchemaCatalog` takes a major bump (families rename v1→v2). diff --git a/build.gradle b/build.gradle index a5b17b6..468e2ea 100644 --- a/build.gradle +++ b/build.gradle @@ -142,6 +142,9 @@ dependencies { testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.10.1' // SLF4J - for TestContainers logging + // Validates emitted analysis.json against the canonical v2 JSON Schema (the L1 conformance oracle + // until the SDK's v2 models exist). + testImplementation 'com.networknt:json-schema-validator:1.5.1' testImplementation 'org.slf4j:slf4j-api:2.0.9' testImplementation 'org.slf4j:slf4j-simple:2.0.9' implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" @@ -149,11 +152,32 @@ dependencies { } test { - useJUnitPlatform() + useJUnitPlatform { + // Whole-application conformance runs take minutes (full symbol resolution over real projects), + // so they are opt-in via `realWorldConformanceTest` rather than part of the inner loop. + excludeTags 'realworld' + } // Optional: Enable TestContainers reuse to speed up tests systemProperty 'testcontainers.reuse.enable', 'true' } +// The L1 conformance gate over the real-world fixture applications (git submodules). +tasks.register('realWorldConformanceTest', Test) { + description = 'Runs the L1 conformance gate over the real-world fixture applications.' + group = 'verification' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + useJUnitPlatform { + includeTags 'realworld' + } + // These projects are large; give the JVM room and do not let a slow app fail the run spuriously. + maxHeapSize = '4g' + testLogging { + events 'passed', 'failed', 'skipped' + showStandardStreams = false + } +} + spotless { java { // Format only the analyzer's own sources. Test-application fixtures under diff --git a/src/test/java/com/ibm/cldk/schema/L1ConformanceGateTest.java b/src/test/java/com/ibm/cldk/schema/L1ConformanceGateTest.java new file mode 100644 index 0000000..fe19683 --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/L1ConformanceGateTest.java @@ -0,0 +1,194 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ibm.cldk.syntactic_analysis.L1Extractor; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The L1 conformance gate: emitted v2 output must validate against the canonical schema, and the + * structural invariants the design spec names must hold. + * + *

The real-world cases are tagged {@code realworld} and excluded from the default {@code test} + * task: analysing whole applications with full symbol resolution takes minutes, which is too slow for + * an inner loop. Run them with {@code ./gradlew realWorldConformanceTest}. The in-repo fixture cases + * stay in the default suite so the gate still guards every change. + * + *

The oracle is the in-repo JSON Schema (the SDK's v2 models do not exist yet); it is strict, so a + * renamed or stray key fails here rather than reaching consumers. The gate runs over the small + * in-repo fixtures and — when the git submodules are checked out — over real-world applications, + * which is where scale-dependent problems (unresolvable dependencies, odd constructs) show up. + */ +class L1ConformanceGateTest { + + private static final Path TEST_APPS = Paths.get("src/test/resources/test-applications"); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static JsonSchema schema() throws IOException { + try (InputStream in = L1ConformanceGateTest.class.getResourceAsStream("/schema/analysis.v2.schema.json")) { + assertNotNull(in, "the canonical v2 schema must be on the test classpath"); + return JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(in); + } + } + + /** Analyse a project and return its emitted payload as JSON. */ + private static JsonNode analyse(Path project) throws IOException { + Map modules = L1Extractor.extractAll(project, project.getFileName().toString()); + Analysis analysis = V2Emitter.emit(project.getFileName().toString(), 1, modules, "test"); + return MAPPER.readTree(V2Json.compact().toJson(analysis)); + } + + private static void assertConformant(JsonNode payload) throws IOException { + Set problems = schema().validate(payload); + assertTrue(problems.isEmpty(), + "output must validate against the canonical v2 schema, but got:\n " + + problems.stream().map(ValidationMessage::getMessage).collect(Collectors.joining("\n "))); + } + + /** Every module's source must be reproducible by slicing itself — the get_method_body contract. */ + private static void assertNodeTextIsSliceable(JsonNode payload) { + JsonNode symbolTable = payload.get("application").get("symbol_table"); + assertTrue(symbolTable.size() > 0, "symbol_table must not be empty"); + symbolTable.fields().forEachRemaining(entry -> { + JsonNode module = entry.getValue(); + byte[] source = module.get("source").asText().getBytes(StandardCharsets.UTF_8); + JsonNode span = module.get("span"); + assertEquals(0, span.get("bytes").get(0).asInt(), entry.getKey()); + assertEquals(source.length, span.get("bytes").get(1).asInt(), + "module span must cover the whole file: " + entry.getKey()); + }); + } + + /** No body-node key may be a bare line — the two-tier identity gate requires the column. */ + private static void assertLocalIdsCarryColumns(JsonNode payload) { + payload.get("application").get("symbol_table").forEach(module -> + module.path("types").forEach(type -> assertTypeLocalIds(type))); + } + + private static void assertTypeLocalIds(JsonNode type) { + type.path("callables").forEach(callable -> + callable.path("body").fieldNames().forEachRemaining(key -> + assertTrue(key.matches("\\d+:\\d+") || key.startsWith("@"), + "body key must be line:col or @tag, got: " + key))); + type.path("types").forEach(L1ConformanceGateTest::assertTypeLocalIds); + } + + @ParameterizedTest(name = "L1 gate on in-repo fixture: {0}") + @ValueSource(strings = {"record-class-test", "init-blocks-test", "call-graph-test"}) + void inRepoFixturesConformToTheCanonicalSchema(String fixture) throws IOException { + Path project = TEST_APPS.resolve(fixture); + JsonNode payload = analyse(project); + assertConformant(payload); + assertNodeTextIsSliceable(payload); + assertLocalIdsCarryColumns(payload); + } + + @Test + void idsAreStableAndOutputDeterministicAcrossRuns() throws IOException { + Path project = TEST_APPS.resolve("record-class-test"); + assertEquals(analyse(project).toString(), analyse(project).toString(), + "two runs over unchanged source must be byte-identical"); + } + + @Test + void recordFixtureExercisesRecordComponents() throws IOException { + // A field with no test is a silent regression point: assert a concrete value, not just a shape. + JsonNode payload = analyse(TEST_APPS.resolve("record-class-test")); + boolean sawRecordWithComponents = false; + for (JsonNode module : payload.get("application").get("symbol_table")) { + for (JsonNode type : module.path("types")) { + if ("record".equals(type.path("kind").asText()) && type.path("record_components").size() > 0) { + sawRecordWithComponents = true; + } + } + } + assertTrue(sawRecordWithComponents, "the record fixture should yield a record with components"); + } + + @Test + void initBlocksFixtureExercisesInitializerCallables() throws IOException { + JsonNode payload = analyse(TEST_APPS.resolve("init-blocks-test")); + boolean sawInitializer = false; + for (JsonNode module : payload.get("application").get("symbol_table")) { + for (JsonNode type : module.path("types")) { + for (JsonNode callable : type.path("callables")) { + if ("initializer".equals(callable.path("kind").asText())) { + sawInitializer = true; + } + } + } + } + assertTrue(sawInitializer, "initializer blocks must surface as callables"); + } + + static boolean submodulesCheckedOut() { + return Files.isDirectory(TEST_APPS.resolve("spring-petclinic/src")); + } + + @Tag("realworld") + @ParameterizedTest(name = "L1 gate on real-world app: {0}") + @EnabledIf("submodulesCheckedOut") + @ValueSource(strings = { + "spring-petclinic", + "quarkuscoffeeshop-counter", + "quarkuscoffeeshop-domain", + "commons-lang" + }) + void realWorldApplicationsConformToTheCanonicalSchema(String app) throws IOException { + Path project = TEST_APPS.resolve(app); + JsonNode payload = analyse(project); + assertConformant(payload); + assertNodeTextIsSliceable(payload); + assertLocalIdsCarryColumns(payload); + assertFalse(payload.get("application").get("symbol_table").isEmpty(), + "a real application must yield modules"); + } + + @Test + @Tag("realworld") + @EnabledIf("submodulesCheckedOut") + void springPetclinicResolvesFrameworkAnnotationsAndEntrypoints() throws IOException { + // Spring controllers are the canonical entrypoint case, and structured decorators are what make + // annotation arguments (routes) machine-readable. + JsonNode payload = analyse(TEST_APPS.resolve("spring-petclinic")); + boolean sawEntrypointClass = false; + boolean sawDecoratorWithArgs = false; + for (JsonNode module : payload.get("application").get("symbol_table")) { + for (JsonNode type : module.path("types")) { + sawEntrypointClass |= type.path("is_entrypoint_class").asBoolean(false); + for (JsonNode decorator : type.path("decorators")) { + sawDecoratorWithArgs |= decorator.path("args").size() > 0; + } + for (JsonNode callable : type.path("callables")) { + for (JsonNode decorator : callable.path("decorators")) { + sawDecoratorWithArgs |= decorator.path("args").size() > 0; + } + } + } + } + assertTrue(sawEntrypointClass, "petclinic has Spring controllers, so some type is an entrypoint class"); + assertTrue(sawDecoratorWithArgs, "structured decorators must retain annotation arguments"); + } +} diff --git a/src/test/resources/schema/analysis.v2.schema.json b/src/test/resources/schema/analysis.v2.schema.json new file mode 100644 index 0000000..c467ecf --- /dev/null +++ b/src/test/resources/schema/analysis.v2.schema.json @@ -0,0 +1,304 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codellm-devkit.info/schema/java/analysis.v2.schema.json", + "title": "CLDK canonical analysis schema v2 (Java, level 1)", + "description": "Conformance oracle for codeanalyzer-java's v2 output. Encodes the canonical CPG shape: an envelope carrying one application node, whose containment tree is named maps down to callable depth, with body nodes keyed by local id. Strict (additionalProperties: false) so an accidentally renamed or stray key fails the gate rather than silently reaching consumers. Replace with the SDK's v2 models once they exist.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "language", "max_level", "application"], + "properties": { + "schema_version": { "const": "2.0.0" }, + "language": { "const": "java" }, + "max_level": { "type": "integer", "minimum": 1, "maximum": 4 }, + "k_limit": { "type": "integer", "minimum": 1 }, + "analyzer": { "$ref": "#/$defs/analyzer" }, + "application": { "$ref": "#/$defs/application" } + }, + "$defs": { + "analyzer": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string" } + } + }, + + "canId": { + "type": "string", + "pattern": "^can://java/", + "description": "Durable id for nodes at or above callable depth." + }, + + "localId": { + "type": "string", + "pattern": "^(\\d+:\\d+|@[A-Za-z0-9_:.$/-]+)$", + "description": "Body-node id within a callable: a line:col position, or an @tag for a synthetic vertex." + }, + + "span": { + "type": "object", + "additionalProperties": false, + "required": ["start", "end", "bytes"], + "properties": { + "start": { "$ref": "#/$defs/position" }, + "end": { "$ref": "#/$defs/position" }, + "bytes": { + "type": "array", + "items": { "type": "integer", "minimum": 0 }, + "minItems": 2, + "maxItems": 2, + "description": "[from, to) UTF-8 offsets into module.source, so node text is an O(1) slice." + } + } + }, + "position": { + "type": "array", + "items": { "type": "integer", "minimum": 0 }, + "minItems": 2, + "maxItems": 2 + }, + + "comment": { + "type": "object", + "additionalProperties": false, + "required": ["content"], + "properties": { + "content": { "type": "string" }, + "span": { "$ref": "#/$defs/span" }, + "is_javadoc": { "type": "boolean" } + } + }, + + "decorator": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "args": { "type": "array", "items": { "type": "string" } }, + "span": { "$ref": "#/$defs/span" } + } + }, + + "import": { + "type": "object", + "additionalProperties": false, + "required": ["path"], + "properties": { + "name": { "type": "string" }, + "path": { "type": "string", "minLength": 1 }, + "span": { "$ref": "#/$defs/span" }, + "is_static": { "type": "boolean" }, + "is_wildcard": { "type": "boolean" } + } + }, + + "stringList": { "type": "array", "items": { "type": "string" } }, + + "application": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "symbol_table"], + "properties": { + "id": { "$ref": "#/$defs/canId" }, + "kind": { "const": "application" }, + "symbol_table": { + "type": "object", + "propertyNames": { + "pattern": "^(?!/)(?!.*\\.\\.).*$", + "description": "Keys are project-relative paths: never absolute, never escaping the root." + }, + "additionalProperties": { "$ref": "#/$defs/module" } + }, + "call_graph": { "type": "array" }, + "param_in": { "type": "array" }, + "param_out": { "type": "array" } + } + }, + + "module": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "source"], + "properties": { + "id": { "$ref": "#/$defs/canId" }, + "kind": { "const": "module" }, + "span": { "$ref": "#/$defs/span" }, + "package": { "type": "string" }, + "source": { "type": "string" }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } }, + "imports": { "type": "array", "items": { "$ref": "#/$defs/import" } }, + "types": { "type": "object", "additionalProperties": { "$ref": "#/$defs/type" } }, + "content_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, + + "type": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind"], + "properties": { + "id": { "$ref": "#/$defs/canId" }, + "kind": { "enum": ["class", "interface", "enum", "record", "annotation"] }, + "span": { "$ref": "#/$defs/span" }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } }, + "modifiers": { "$ref": "#/$defs/stringList" }, + "base_types": { "$ref": "#/$defs/stringList" }, + "interfaces": { "$ref": "#/$defs/stringList" }, + "decorators": { "type": "array", "items": { "$ref": "#/$defs/decorator" } }, + "is_entrypoint_class": { "type": "boolean" }, + "enum_constants": { "type": "array", "items": { "$ref": "#/$defs/enumConstant" } }, + "record_components": { "type": "array", "items": { "$ref": "#/$defs/recordComponent" } }, + "fields": { "type": "object", "additionalProperties": { "$ref": "#/$defs/field" } }, + "callables": { "type": "object", "additionalProperties": { "$ref": "#/$defs/callable" } }, + "types": { "type": "object", "additionalProperties": { "$ref": "#/$defs/type" } } + } + }, + + "enumConstant": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "arguments": { "$ref": "#/$defs/stringList" }, + "span": { "$ref": "#/$defs/span" }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } } + } + }, + + "recordComponent": { + "type": "object", + "additionalProperties": false, + "required": ["name", "type"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "type": { "type": "string" }, + "span": { "$ref": "#/$defs/span" }, + "modifiers": { "$ref": "#/$defs/stringList" }, + "decorators": { "type": "array", "items": { "$ref": "#/$defs/decorator" } }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } }, + "is_variadic": { "type": "boolean" } + } + }, + + "field": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "name", "type"], + "properties": { + "id": { "$ref": "#/$defs/canId" }, + "kind": { "const": "field" }, + "name": { "type": "string", "minLength": 1 }, + "type": { "type": "string" }, + "span": { "$ref": "#/$defs/span" }, + "modifiers": { "$ref": "#/$defs/stringList" }, + "decorators": { "type": "array", "items": { "$ref": "#/$defs/decorator" } }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } }, + "initializer": { "type": "string" } + } + }, + + "parameter": { + "type": "object", + "additionalProperties": false, + "required": ["name", "type"], + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "span": { "$ref": "#/$defs/span" }, + "modifiers": { "$ref": "#/$defs/stringList" }, + "decorators": { "type": "array", "items": { "$ref": "#/$defs/decorator" } }, + "is_variadic": { "type": "boolean" } + } + }, + + "callable": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "signature"], + "properties": { + "id": { "$ref": "#/$defs/canId" }, + "kind": { "enum": ["method", "constructor", "initializer", "lambda"] }, + "signature": { "type": "string", "minLength": 1 }, + "span": { "$ref": "#/$defs/span" }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } }, + "parameters": { "type": "array", "items": { "$ref": "#/$defs/parameter" } }, + "return_type": { "type": "string" }, + "error_channel": { "$ref": "#/$defs/stringList" }, + "modifiers": { "$ref": "#/$defs/stringList" }, + "decorators": { "type": "array", "items": { "$ref": "#/$defs/decorator" } }, + "declaration": { "type": "string" }, + "code_start_line": { "type": "integer" }, + "is_implicit": { "type": "boolean" }, + "is_entrypoint": { "type": "boolean" }, + "metrics": { + "type": "object", + "additionalProperties": false, + "properties": { "cyclomatic": { "type": "integer", "minimum": 1 } } + }, + "refs": { + "type": "object", + "additionalProperties": false, + "properties": { + "types": { "$ref": "#/$defs/stringList" }, + "fields": { "$ref": "#/$defs/stringList" } + } + }, + "local_variables": { "type": "array", "items": { "$ref": "#/$defs/variable" } }, + "body": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/localId" }, + "additionalProperties": { "$ref": "#/$defs/bodyNode" } + }, + "cfg": { "type": "array" }, + "cdg": { "type": "array" }, + "ddg": { "type": "array" }, + "summary": { "type": "array" }, + "types": { "type": "object", "additionalProperties": { "$ref": "#/$defs/type" } } + } + }, + + "variable": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "type": { "type": "string" }, + "initializer": { "type": "string" }, + "span": { "$ref": "#/$defs/span" }, + "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } } + } + }, + + "bodyNode": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { + "enum": [ + "call", "statement", "return", "branch", "loop", "switch", + "entry", "exit", "formal_in", "formal_out", "actual_in", "actual_out", + "expression", "block" + ] + }, + "span": { "$ref": "#/$defs/span" }, + "parent": { "$ref": "#/$defs/localId" }, + "of": { "type": "string" }, + "callee": { "$ref": "#/$defs/canId" }, + "arguments": { "type": "array", "items": { "$ref": "#/$defs/localId" } }, + "receiver_expr": { "type": "string" }, + "receiver_type": { "type": "string" }, + "argument_types": { "$ref": "#/$defs/stringList" }, + "argument_expr": { "$ref": "#/$defs/stringList" }, + "callee_signature": { "type": "string" }, + "is_static_call": { "type": "boolean" }, + "is_constructor_call": { "type": "boolean" } + } + } + } +} From 9cb46012ffe40852fd3373016d5fbe3e7094b718 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 11:30:42 -0400 Subject: [PATCH 17/22] fix(schema): resolve library types at L1; omit unknown static-call flag (#180) L1 now downloads the project's dependencies and puts a JarTypeSolver on the solver path, so third-party types resolve to qualified names as they do in v1 (verified on spring-petclinic: Model, Pageable, Page, and callee_signature on 99% of call sites). Reflection is restricted to the JRE so the analyzer's own dependencies can no longer be resolved as if the analysed project depended on them. is_static_call becomes a Boolean that is omitted when the callee is unresolved rather than reported as false. --- .claude/SCHEMA_DECISIONS.md | 16 +++ src/main/java/com/ibm/cldk/CodeAnalyzer.java | 25 ++++- .../java/com/ibm/cldk/schema/JBodyNode.java | 9 +- .../syntactic_analysis/CallSiteBuilder.java | 2 +- .../cldk/syntactic_analysis/L1Extractor.java | 100 +++++++++++++++--- .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 24 +++++ .../CallSiteBuilderTest.java | 6 +- .../syntactic_analysis/L1ExtractorTest.java | 41 +++++++ 8 files changed, 201 insertions(+), 22 deletions(-) diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 1726717..37460a5 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -125,6 +125,22 @@ Refinements settled while building L1 (2026-08), each checked against the keysto - **`module.content_hash` is SHA-256 hex of the UTF-8 source** — for incremental caching and the Neo4j writer's per-module diffing; never identity (the `id` is). +### D12 — L1 type resolution: library dependencies are always attempted + +- **Dependency jars go on the solver's path.** L1 downloads the project's library dependencies before + parsing and adds a `JarTypeSolver` per jar, so third-party types resolve to qualified names + (`org.springframework.ui.Model`, `org.springframework.data.domain.Page<…Owner>`) instead of bare + spellings. Skipping this made v2 resolution strictly worse than v1's; it is now verified on a real + Spring application. A download failure only thins resolution — it warns, never fails the analysis. +- **Reflection is JRE-only.** A classpath-wide `ReflectionTypeSolver` resolves the *analyzer's own* + dependencies (WALA, Guava, JavaParser, …) as if the analysed project depended on them, inventing + qualified names that are simply wrong. Project types come from source roots, library types from the + dependency jars, and reflection covers only the JDK. +- **Resolution-derived flags are absent when unknown.** `is_static_call` is a `Boolean`: when the + callee cannot be resolved, staticness is genuinely unknown and the key is omitted rather than + emitted as `false`, which would assert "not static". Syntactically evident flags + (`is_constructor_call`) stay primitive. + ### D11 — L1 conformance oracle and gate - **Oracle:** emitted output is validated against an in-repo JSON Schema, diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index 7f0b7e5..775f76e 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -352,7 +352,30 @@ private void analyzeV2() throws Exception { String application = appName != null && !appName.isBlank() ? appName : Paths.get(input).toAbsolutePath().normalize().getFileName().toString(); - Map modules = L1Extractor.extractAll(Paths.get(input), application); + + // Always attempt library type resolution: without the dependency jars on the solver's path, + // third-party types degrade to bare spellings (`Model` rather than `org.springframework.ui.Model`) + // and consumers lose the qualified names they join on. A failure here only thins resolution, so + // it is a warning rather than a fatal error. + projectRootPom = projectRootPom == null ? input : projectRootPom; + Path dependencyDir = null; + try { + if (BuildProject.downloadLibraryDependencies(input, projectRootPom)) { + dependencyDir = BuildProject.libDownloadPath; + } else { + Log.warn("Failed to download library dependencies; third-party types may not resolve"); + } + } catch (Exception e) { + Log.warn("Failed to download library dependencies (" + e.getMessage() + + "); third-party types may not resolve"); + } + + Map modules; + try { + modules = L1Extractor.extractAll(Paths.get(input), application, dependencyDir); + } finally { + BuildProject.cleanLibraryDependencies(); + } Analysis analysis = V2Emitter.emit(application, 1, modules, analyzerVersion()); if (output == null) { diff --git a/src/main/java/com/ibm/cldk/schema/JBodyNode.java b/src/main/java/com/ibm/cldk/schema/JBodyNode.java index 05b87d3..5de65e0 100644 --- a/src/main/java/com/ibm/cldk/schema/JBodyNode.java +++ b/src/main/java/com/ibm/cldk/schema/JBodyNode.java @@ -39,6 +39,13 @@ public class JBodyNode { /** Erased signature of the resolved callee ({@code substring(int)}); absent when unresolvable. */ private String calleeSignature; - private boolean isStaticCall; + /** + * Whether the callee is static. A {@code Boolean} rather than a primitive: when the callee cannot + * be resolved this is genuinely unknown, and absence says that honestly where {@code false} + * would assert "not static". + */ + private Boolean isStaticCall; + + /** Syntactically evident (a {@code new} expression or {@code this(...)}/{@code super(...)}). */ private boolean isConstructorCall; } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java index e0bd621..32f33b7 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java @@ -92,7 +92,7 @@ private void enrich(JBodyNode node, Node site) { try { ResolvedMethodDeclaration resolved = call.resolve(); node.setCalleeSignature(Signatures.typeErasure(resolved)); - node.setStaticCall(resolved.isStatic()); + node.setIsStaticCall(resolved.isStatic()); } catch (Throwable e) { Log.debug("Could not resolve call: " + call + ": " + e.getMessage()); } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java index 88748fb..107005c 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java @@ -3,7 +3,12 @@ import com.github.javaparser.ParseResult; import com.github.javaparser.ParserConfiguration; import com.github.javaparser.ast.CompilationUnit; -import com.github.javaparser.symbolsolver.utils.SymbolSolverCollectionStrategy; +import com.github.javaparser.symbolsolver.JavaSymbolSolver; +import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver; +import com.github.javaparser.symbolsolver.resolution.typesolvers.JarTypeSolver; +import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver; +import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver; +import com.github.javaparser.utils.ParserCollectionStrategy; import com.github.javaparser.utils.ProjectRoot; import com.github.javaparser.utils.SourceRoot; import com.ibm.cldk.schema.CanId; @@ -14,20 +19,27 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.regex.Pattern; +import java.util.stream.Stream; /** * Orchestrates L1: discovers a project's source roots, parses each file with a symbol solver * (so type resolution and erased signatures work), and builds one canonical schema v2 {@code module} * per file via {@link ModuleBuilder}. * - *

The symbol solver is configured from the project itself, which is what lets types declared in - * other files of the same project resolve to qualified names. Modules are keyed by the file's path - * relative to the project root, normalised to {@code /} — the key must be stable across runs - * and machines for caching and SDK lookups to work. + *

The type solver is assembled explicitly from three sources, because resolution quality is what + * makes L1 type fields useful: the JDK (reflection), the project's own source roots, and — crucially — + * the project's library dependencies. Without the dependency jars, third-party types degrade to + * bare spellings ({@code Model} instead of {@code org.springframework.ui.Model}), which loses exactly + * the qualified names downstream consumers join on. + * + *

Modules are keyed by the file's path relative to the project root, normalised to + * {@code /} — the key must be stable across runs and machines for caching and SDK lookups to work. */ public final class L1Extractor { @@ -40,29 +52,40 @@ private L1Extractor() {} Paths.get("src", "xdocs-examples").toString() }; + /** Analyse a project with no library dependencies available (JDK + project sources only). */ + public static Map extractAll(Path projectRoot, String appName) throws IOException { + return extractAll(projectRoot, appName, null); + } + /** * Build the v2 symbol table for a project. * * @param projectRoot the project's root directory * @param appName the application name — the {@code can://java/} segment of every id + * @param dependencyDir directory of dependency jars to put on the solver's path, or {@code null}; + * missing or unreadable jars are skipped rather than failing the analysis * @return modules keyed by relative file path, iterated in sorted key order for determinism */ - public static Map extractAll(Path projectRoot, String appName) throws IOException { - ParserConfiguration config = new ParserConfiguration() - .setStoreTokens(true) - .setAttributeComments(true) - .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21); - SymbolSolverCollectionStrategy strategy = new SymbolSolverCollectionStrategy(config); - ProjectRoot root = strategy.collect(projectRoot); - String applicationId = CanId.applicationId(appName); + public static Map extractAll(Path projectRoot, String appName, Path dependencyDir) + throws IOException { + ParserConfiguration discovery = parserConfiguration(); + ProjectRoot root = new ParserCollectionStrategy(discovery).collect(projectRoot); + + List sourceRoots = new ArrayList<>(); + for (SourceRoot sourceRoot : root.getSourceRoots()) { + if (!isExcluded(sourceRoot.getRoot(), projectRoot)) { + sourceRoots.add(sourceRoot); + } + } + + ParserConfiguration config = parserConfiguration() + .setSymbolResolver(new JavaSymbolSolver(typeSolver(sourceRoots, dependencyDir, discovery))); // Collect into a sorted map first: source roots and directory listings are not ordered, and // `-j N` output must be byte-identical to `-j 1`. Map modules = new TreeMap<>(); - for (SourceRoot sourceRoot : root.getSourceRoots()) { - if (isExcluded(sourceRoot.getRoot(), projectRoot)) { - continue; - } + String applicationId = CanId.applicationId(appName); + for (SourceRoot sourceRoot : sourceRoots) { sourceRoot.setParserConfiguration(config); for (ParseResult parseResult : sourceRoot.tryToParse()) { if (parseResult.getResult().isEmpty()) { @@ -85,6 +108,49 @@ public static Map extractAll(Path projectRoot, String appName) return new LinkedHashMap<>(modules); } + private static ParserConfiguration parserConfiguration() { + return new ParserConfiguration() + .setStoreTokens(true) + .setAttributeComments(true) + .setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_21); + } + + /** JDK + project sources + dependency jars, in that resolution order. */ + private static CombinedTypeSolver typeSolver( + List sourceRoots, Path dependencyDir, ParserConfiguration config) { + CombinedTypeSolver solver = new CombinedTypeSolver(); + // JRE types only. A classpath-wide ReflectionTypeSolver would resolve the *analyzer's* own + // dependencies (WALA, Guava, JavaParser, ...) as if the analysed project depended on them, + // silently inventing qualified names. Project types come from the source roots below, and + // library types from the dependency jars. + solver.add(new ReflectionTypeSolver()); + for (SourceRoot sourceRoot : sourceRoots) { + solver.add(new JavaParserTypeSolver(sourceRoot.getRoot(), config)); + } + int jars = 0; + if (dependencyDir != null && Files.isDirectory(dependencyDir)) { + try (Stream entries = Files.walk(dependencyDir)) { + List jarFiles = entries + .filter(p -> p.toString().endsWith(".jar")) + .sorted() + .collect(java.util.stream.Collectors.toList()); + for (Path jar : jarFiles) { + try { + solver.add(new JarTypeSolver(jar)); + jars++; + } catch (IOException e) { + // A corrupt or unreadable jar degrades resolution; it must not fail analysis. + Log.debug("Skipping unreadable dependency jar " + jar + ": " + e.getMessage()); + } + } + } catch (IOException e) { + Log.warn("Could not scan dependency directory " + dependencyDir + ": " + e.getMessage()); + } + } + Log.debug("Type solver: " + sourceRoots.size() + " source root(s), " + jars + " dependency jar(s)"); + return solver; + } + /** The {@code symbol_table} key: path relative to the project root, always {@code /}-separated. */ private static String fileKey(Path projectRoot, Path file) { Path relative = projectRoot.toAbsolutePath().normalize().relativize(file.toAbsolutePath().normalize()); diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index 777117b..0c3c293 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -12,6 +12,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -57,6 +58,29 @@ private static void set(String field, Object value) throws Exception { f.set(null, value); } + /** + * These tests drive the whole CLI, which runs the v1 symbol table and leaves its static + * state populated ({@code javaSymbolSolver}, the resolution caches, the declared-callables table). + * {@link SymbolTable#extractSingle} does not assign that solver field, so it behaves differently + * depending on whether something else ran first — restore the initial state so this class cannot + * change the outcome of tests that run after it. + */ + @AfterEach + void restoreSymbolTableStatics() throws Exception { + Field solver = SymbolTable.class.getDeclaredField("javaSymbolSolver"); + solver.setAccessible(true); + solver.set(null, null); + clearCollection("unresolvedTypes"); + clearCollection("unresolvedExpressions"); + SymbolTable.declaredMethodsAndConstructors.clear(); + } + + private static void clearCollection(String field) throws Exception { + Field f = SymbolTable.class.getDeclaredField(field); + f.setAccessible(true); + ((java.util.Collection) f.get(null)).clear(); + } + @Test void v2SchemaWritesCanonicalEnvelopeToAnalysisJson(@TempDir Path tmp) throws IOException { Path in = project(tmp.resolve("app")); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java index eab90e2..ec850e2 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java @@ -125,14 +125,14 @@ void build_callNodeCapturesReceiverAndResolvedTypes() { assertEquals(List.of("int"), node.getArgumentTypes()); assertEquals(List.of("1"), node.getArgumentExpr()); assertEquals("substring(int)", node.getCalleeSignature()); - assertFalse(node.isStaticCall()); + assertEquals(Boolean.FALSE, node.getIsStaticCall(), "String.substring is an instance method"); assertFalse(node.isConstructorCall()); } @Test void build_flagsStaticCall() { JBodyNode node = build("package p;\nclass Foo {\n void m() {\n Math.max(1, 2);\n }\n}\n").get("4:10"); - assertTrue(node.isStaticCall(), "Math.max is static"); + assertEquals(Boolean.TRUE, node.getIsStaticCall(), "Math.max is static"); assertEquals("java.lang.Math", node.getReceiverType()); } @@ -149,6 +149,8 @@ void build_unresolvableCallStillEmitsNodeWithoutResolvedFacts() { JBodyNode node = build("package p;\nclass Foo {\n void m() {\n mystery(x);\n }\n}\n").get("4:5"); assertEquals("call", node.getKind()); assertNull(node.getCalleeSignature()); + assertNull(node.getIsStaticCall(), + "staticness is unknown for an unresolved callee — absent, not a false claim"); } @Test diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java index 0e6134e..e6acc73 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/L1ExtractorTest.java @@ -12,6 +12,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -95,6 +96,46 @@ void extractAll_resolvesAcrossFilesInTheProject(@TempDir Path tmp) throws IOExce "the greet(...) call's receiver type should resolve across files"); } + /** Copy one jar off the test classpath into {@code dir}, standing in for a downloaded dependency. */ + private static Path stageDependencyJar(Path dir, String jarNameFragment) throws IOException { + Files.createDirectories(dir); + for (String entry : System.getProperty("java.class.path").split(java.io.File.pathSeparator)) { + if (entry.endsWith(".jar") && entry.contains(jarNameFragment)) { + Path target = dir.resolve(Paths.get(entry).getFileName()); + Files.copy(Paths.get(entry), target); + return target; + } + } + throw new IllegalStateException("no jar matching '" + jarNameFragment + "' on the test classpath"); + } + + @Test + void extractAll_resolvesLibraryTypesFromDependencyJars(@TempDir Path tmp) throws IOException { + // Without the dependency jars a third-party type degrades to its bare spelling, losing the + // qualified name consumers join on — so library resolution is part of L1's contract. + Path project = tmp.resolve("app"); + Path pkg = project.resolve("src/main/java/com/example"); + Files.createDirectories(pkg); + Files.writeString(pkg.resolve("Holder.java"), + "package com.example;\nimport com.google.gson.Gson;\n" + + "public class Holder {\n Gson gson;\n Gson make() { return new Gson(); }\n}\n", + StandardCharsets.UTF_8); + Path deps = stageDependencyJar(tmp.resolve("deps"), "gson").getParent(); + + JModule withJars = L1Extractor.extractAll(project, "app", deps) + .get("src/main/java/com/example/Holder.java"); + JType holder = withJars.getTypes().get("Holder"); + assertEquals("com.google.gson.Gson", holder.getFields().get("gson").getType(), + "library field type must resolve to its qualified name"); + assertTrue(holder.getCallables().containsKey("make()")); + assertEquals("com.google.gson.Gson", holder.getCallables().get("make()").getReturnType()); + + JModule withoutJars = L1Extractor.extractAll(project, "app", null) + .get("src/main/java/com/example/Holder.java"); + assertEquals("Gson", withoutJars.getTypes().get("Holder").getFields().get("gson").getType(), + "with no jars on the path it degrades to the AST spelling rather than failing"); + } + @Test void extractAll_producesStableIdsAndDeterministicOutputAcrossRuns(@TempDir Path tmp) throws IOException { Path root = writeProject(tmp); From 139a39fe5a94de5ebed3392de6bcf584b449768c Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 11:58:46 -0400 Subject: [PATCH 18/22] test: drop the daytrader-microservices fixture (contains no Java source) The repo is deployment tooling (Makefile, docker-compose, helm charts) with zero .java files, so it cannot exercise the analyzer. Also ignores output/, used for ad-hoc v1/v2 comparison runs. --- .gitignore | 3 +++ .gitmodules | 3 --- src/test/resources/test-applications/daytrader-microservices | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) delete mode 160000 src/test/resources/test-applications/daytrader-microservices diff --git a/.gitignore b/.gitignore index c4e7b66..4116aec 100644 --- a/.gitignore +++ b/.gitignore @@ -196,3 +196,6 @@ gradle-app.setting bin/ etc/ /src/test/resources/sample_apps/daytrader8/output/ + +# Ad-hoc analysis output from manual v1/v2 comparison runs +output/ diff --git a/.gitmodules b/.gitmodules index b78ff42..9f967d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,9 +7,6 @@ [submodule "src/test/resources/test-applications/commons-lang"] path = src/test/resources/test-applications/commons-lang url = https://github.com/apache/commons-lang.git -[submodule "src/test/resources/test-applications/daytrader-microservices"] - path = src/test/resources/test-applications/daytrader-microservices - url = https://github.com/sample-daytrader/sample.daytrader.microservices.git [submodule "src/test/resources/test-applications/quarkuscoffeeshop-counter"] path = src/test/resources/test-applications/quarkuscoffeeshop-counter url = https://github.com/quarkuscoffeeshop/quarkuscoffeeshop-counter.git diff --git a/src/test/resources/test-applications/daytrader-microservices b/src/test/resources/test-applications/daytrader-microservices deleted file mode 160000 index 8a68b59..0000000 --- a/src/test/resources/test-applications/daytrader-microservices +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8a68b59430a94a242c54384763da9eb7682728b4 From 57f7461e773a2389442b7ae171d476eafb327f15 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 12:50:52 -0400 Subject: [PATCH 19/22] feat(schema): model anonymous classes; add callable body_span (#180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anonymous class bodies now get their own type node ($anon$N) under the callable that declares them, like named local classes — v1 mis-attributed their initializers and locals to the enclosing type and the first v2 attempt dropped them. Re-measured, initializer blocks and local variables are back at parity with v1. body_span delimits the { ... } block so source[body_span.bytes] reproduces v1's per-callable code byte for byte without duplicating the text; the callable's own span covers the whole declaration. Pinned by a test that compares against the v1 emitter directly. --- .claude/SCHEMA_DECISIONS.md | 19 +++ .../java/com/ibm/cldk/schema/JCallable.java | 7 + .../syntactic_analysis/CallableBuilder.java | 22 ++- .../cldk/syntactic_analysis/TypeBuilder.java | 143 ++++++++++++------ .../BodyTextParityTest.java | 123 +++++++++++++++ .../CallableBuilderTest.java | 46 ++++++ .../resources/schema/analysis.v2.schema.json | 1 + 7 files changed, 316 insertions(+), 45 deletions(-) create mode 100644 src/test/java/com/ibm/cldk/syntactic_analysis/BodyTextParityTest.java diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 37460a5..551c3b2 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -125,6 +125,25 @@ Refinements settled while building L1 (2026-08), each checked against the keysto - **`module.content_hash` is SHA-256 hex of the UTF-8 source** — for incremental caching and the Neo4j writer's per-module diffing; never identity (the `id` is). +### D13 — Anonymous classes are modelled; body text is recovered via `body_span` + +Both refinements came out of a field-by-field v1-vs-v2 comparison over ten real-world applications +(`docs/design/notes/l1-v1-v2-comparison.md`). + +- **Anonymous inner classes get their own `type` node**, keyed positionally (`$anon$0`, `$anon$1`, … in + declaration order) under the callable that declares them, exactly as named local classes are. v1 + recursed into anonymous bodies and mis-attributed their initializers and locals to the *enclosing + type*; simply excluding them (the first v2 attempt) lost those facts instead. Modelling them closed + the measured gap exactly: initializer blocks and local variables went from -10/-20 to parity. +- **`callable.body_span` delimits the body block.** v2 drops v1's per-callable `code` string (D1) on + the basis that body text is a slice of `module.source` — but the callable's own `span` covers the + *whole declaration*, so slicing it yields signature + body, not v1's body-only `code`. `body_span` + is the span of the `{ … }` block, so `source[body_span.bytes]` reproduces v1's `code` byte for byte + (pinned by `BodyTextParityTest`, which compares against the v1 emitter directly) without + reintroducing duplicated text. Absent when there is no body (abstract/interface methods). + **Canonical note:** the keystone defines `get_method_body(sig)` as `module.source[callable.span.bytes]`, + which is *not* v1's `code` semantics; the discrepancy is worth resolving in the canonical schema. + ### D12 — L1 type resolution: library dependencies are always attempted - **Dependency jars go on the solver's path.** L1 downloads the project's library dependencies before diff --git a/src/main/java/com/ibm/cldk/schema/JCallable.java b/src/main/java/com/ibm/cldk/schema/JCallable.java index ca19992..e29ff65 100644 --- a/src/main/java/com/ibm/cldk/schema/JCallable.java +++ b/src/main/java/com/ibm/cldk/schema/JCallable.java @@ -24,6 +24,13 @@ public class JCallable { private List errorChannel = new ArrayList<>(); private List modifiers = new ArrayList<>(); private List decorators = new ArrayList<>(); + /** + * Span of the body block ({@code { ... }}) alone, absent when there is no body. The callable's own + * {@code span} covers the whole declaration, so this is what a consumer slices to obtain just the + * method body — the text v1 carried in its per-callable {@code code} field, without duplicating it. + */ + private Span bodySpan; + /** Signature-with-parameter-names text (not recoverable from span.bytes, which covers the body). */ private String declaration; diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index 163a5a0..3dd4db8 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -28,6 +28,7 @@ import com.ibm.cldk.schema.JType; import com.ibm.cldk.schema.JVariableDeclaration; import java.util.ArrayList; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -96,6 +97,7 @@ public JCallable build( Optional body = bodyOf(cd); body.flatMap(b -> b.getRange().map(r -> r.begin.line)).ifPresent(callable::setCodeStartLine); + body.ifPresent(b -> callable.setBodySpan(ctx.spanOf(b))); callable.setRefs(refs(body, typeFqn, classFieldNames)); body.ifPresent(b -> callable.setLocalVariables(localVariables(b))); body.ifPresent(b -> callable.setBody(callSiteBuilder.build(b))); @@ -131,6 +133,7 @@ public JCallable buildInitializer( BlockStmt body = id.getBody(); body.getRange().map(r -> r.begin.line).ifPresent(callable::setCodeStartLine); + callable.setBodySpan(ctx.spanOf(body)); callable.setRefs(refs(Optional.of(body), typeFqn, classFieldNames)); callable.setLocalVariables(localVariables(body)); callable.setBody(callSiteBuilder.build(body)); @@ -163,13 +166,30 @@ private List localVariables(BlockStmt body) { return locals; } - /** Local (method-body) classes declared directly in the body, keyed by simple name (sorted). */ + /** + * Types declared inside this callable's body: named local classes, plus anonymous class bodies. + * Both are attributed here rather than to the enclosing type, so their members, locals and call + * sites belong to the code that actually declares them (D4 containment). + */ private Map localClasses(BlockStmt body, String callableId) { TypeBuilder typeBuilder = new TypeBuilder(ctx); Map locals = new TreeMap<>(); body.findAll(TypeDeclaration.class).stream() .filter(td -> AstScopes.belongsDirectlyTo(td, body)) .forEach(td -> locals.put(td.getNameAsString(), typeBuilder.build(td, callableId))); + + // Anonymous classes have no name, so they are numbered in declaration order. + List anonymous = body.findAll(ObjectCreationExpr.class).stream() + .filter(oce -> oce.getAnonymousClassBody().isPresent()) + .filter(oce -> AstScopes.belongsDirectlyTo(oce, body)) + .sorted(Comparator + .comparingInt((ObjectCreationExpr oce) -> oce.getBegin().map(pos -> pos.line).orElse(0)) + .thenComparingInt(oce -> oce.getBegin().map(pos -> pos.column).orElse(0))) + .collect(Collectors.toList()); + for (int i = 0; i < anonymous.size(); i++) { + String name = "$anon$" + i; + locals.put(name, typeBuilder.buildAnonymous(anonymous.get(i), callableId, name)); + } return new LinkedHashMap<>(locals); } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index 0f37841..b895184 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -1,6 +1,8 @@ package com.ibm.cldk.syntactic_analysis; +import com.github.javaparser.ast.NodeList; import com.github.javaparser.ast.body.AnnotationDeclaration; +import com.github.javaparser.ast.body.BodyDeclaration; import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.EnumConstantDeclaration; @@ -10,6 +12,7 @@ import com.github.javaparser.ast.body.Parameter; import com.github.javaparser.ast.body.RecordDeclaration; import com.github.javaparser.ast.body.TypeDeclaration; +import com.github.javaparser.ast.expr.ObjectCreationExpr; import com.ibm.cldk.javaee.EntrypointsFinderFactory; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; @@ -106,50 +109,7 @@ public JType build(TypeDeclaration td, String parentId) { type.setRecordComponents(components); } - // Fields, keyed by simple name — one entry per declared variable (int a, b; -> a, b). - Map fields = new LinkedHashMap<>(); - for (FieldDeclaration fd : td.getFields()) { - fieldBuilder.build(fd, type.getId()).forEach(f -> fields.put(f.getName(), f)); - } - type.setFields(fields); - - // Callables (methods + constructors) declared directly in this type — getMethods()/ - // getConstructors() return only direct members, so nested-type methods are not swept in. - // Keyed by type-erasure signature. Field names are handed down so each callable's - // refs.fields can recognize accesses to this type's fields. - List fieldNames = new ArrayList<>(fields.keySet()); - String typeFqn = td.getFullyQualifiedName().orElse(td.getNameAsString()); - List> declared = new ArrayList<>(); - declared.addAll(td.getConstructors()); - declared.addAll(td.getMethods()); - Map callables = new TreeMap<>(); - for (CallableDeclaration cd : declared) { - JCallable callable = callableBuilder.build(cd, type.getId(), typeFqn, fieldNames); - callables.put(callable.getSignature(), callable); - } - // Initializer blocks are callables too (keystone kind `initializer`) — L3 gives them their own - // CFGs. Numbered per kind so the id survives line edits; `$` marks the synthetic member. - int staticIndex = 0; - int instanceIndex = 0; - for (InitializerDeclaration id : td.getMembers().stream() - .filter(m -> m instanceof InitializerDeclaration) - .map(m -> (InitializerDeclaration) m) - .collect(Collectors.toList())) { - String signature = id.isStatic() - ? "$" + staticIndex++ + "()" - : "$" + instanceIndex++ + "()"; - callables.put(signature, callableBuilder.buildInitializer(id, type.getId(), typeFqn, fieldNames, signature)); - } - type.setCallables(new LinkedHashMap<>(callables)); - - // Recurse into member (inner) types; nesting/parent are encoded by this containment (and the - // id path). Local classes in method bodies are handled later by the callable builder. - Map nested = new TreeMap<>(); - td.getMembers().stream() - .filter(m -> m instanceof TypeDeclaration) - .map(m -> (TypeDeclaration) m) - .forEach(member -> nested.put(member.getNameAsString(), build(member, type.getId()))); - type.setTypes(new LinkedHashMap<>(nested)); + populateMembers(type, td.getMembers(), typeFqnOf(td)); return type; } @@ -170,4 +130,99 @@ private static String kindOf(TypeDeclaration td) { } return "class"; } + + /** The fully-qualified name used to qualify field references, falling back to the simple name. */ + private static String typeFqnOf(TypeDeclaration td) { + return td.getFullyQualifiedName().orElse(td.getNameAsString()); + } + + /** + * Build a {@code type} node for an anonymous class body ({@code new Runnable() { ... }}). + * + *

An anonymous class has no name, so it is keyed positionally ({@code $anon$0}, {@code $anon$1}, + * ... in declaration order within the callable) — stable across line edits, and {@code $} marks it + * synthetic. Modelling it as its own type is what keeps its methods, initializers, locals and call + * sites attributed to it rather than mis-attributed to the enclosing callable or dropped. + */ + public JType buildAnonymous(ObjectCreationExpr creation, String parentId, String name) { + JType type = new JType(); + type.setId(CanId.childId(parentId, name)); + type.setKind("class"); + type.setSpan(ctx.spanOf(creation)); + + // The instantiated type is a supertype: an interface if it resolves to one, else a base class. + String supertype = ctx.resolveType(creation.getType()); + if (resolvesToInterface(creation)) { + type.setInterfaces(List.of(supertype)); + } else { + type.setBaseTypes(List.of(supertype)); + } + + populateMembers(type, creation.getAnonymousClassBody().orElseGet(NodeList::new), supertype); + return type; + } + + private static boolean resolvesToInterface(ObjectCreationExpr creation) { + try { + return creation.getType().resolve().asReferenceType().getTypeDeclaration() + .map(d -> d.isInterface()) + .orElse(false); + } catch (Throwable e) { + // Unresolvable supertype: treat it as a base class rather than guessing. + return false; + } + } + + /** + * Populate a type's fields, callables (methods, constructors and initializer blocks) and member + * types from its declared members. Shared by named types and anonymous class bodies so both get the + * same treatment. + */ + private void populateMembers(JType type, List> members, String typeFqn) { + // Fields, keyed by simple name — one entry per declared variable (int a, b; -> a, b). + Map fields = new LinkedHashMap<>(); + for (BodyDeclaration member : members) { + if (member instanceof FieldDeclaration) { + fieldBuilder.build((FieldDeclaration) member, type.getId()) + .forEach(f -> fields.put(f.getName(), f)); + } + } + type.setFields(fields); + + // Field names are handed down so each callable's refs.fields can recognise accesses to them. + List fieldNames = new ArrayList<>(fields.keySet()); + Map callables = new TreeMap<>(); + for (BodyDeclaration member : members) { + if (member instanceof CallableDeclaration) { + JCallable callable = + callableBuilder.build((CallableDeclaration) member, type.getId(), typeFqn, fieldNames); + callables.put(callable.getSignature(), callable); + } + } + // Initializer blocks are callables too (keystone kind `initializer`) — L3 gives them their own + // CFGs. Numbered per kind so the id survives line edits; `$` marks the synthetic member. + int staticIndex = 0; + int instanceIndex = 0; + for (BodyDeclaration member : members) { + if (member instanceof InitializerDeclaration) { + InitializerDeclaration id = (InitializerDeclaration) member; + String signature = id.isStatic() + ? "$" + staticIndex++ + "()" + : "$" + instanceIndex++ + "()"; + callables.put(signature, + callableBuilder.buildInitializer(id, type.getId(), typeFqn, fieldNames, signature)); + } + } + type.setCallables(new LinkedHashMap<>(callables)); + + // Member types; nesting/parent are encoded by this containment (and the id path). + Map nested = new TreeMap<>(); + for (BodyDeclaration member : members) { + if (member instanceof TypeDeclaration) { + TypeDeclaration nestedType = (TypeDeclaration) member; + nested.put(nestedType.getNameAsString(), build(nestedType, type.getId())); + } + } + type.setTypes(new LinkedHashMap<>(nested)); + } } diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/BodyTextParityTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/BodyTextParityTest.java new file mode 100644 index 0000000..0104815 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/BodyTextParityTest.java @@ -0,0 +1,123 @@ +package com.ibm.cldk.syntactic_analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.github.javaparser.ast.CompilationUnit; +import com.ibm.cldk.SymbolTable; +import com.ibm.cldk.entities.Callable; +import com.ibm.cldk.entities.JavaCompilationUnit; +import com.ibm.cldk.schema.CanId; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.Span; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * v2 drops the per-callable {@code code} string that v1 carried, on the basis that body text is a slice + * of {@code module.source}. That only holds if some span actually delimits the body: a callable's own + * span covers its whole declaration (modifiers, signature and body), so slicing it does not + * reproduce v1's {@code code}, which was the {@code { ... }} block alone. + * + *

These tests pin the equivalence directly — for the same source, slicing {@code body_span} out of + * {@code module.source} must yield exactly what v1 put in {@code code} — so the migration cannot + * silently change what downstream consumers get from {@code get_method_body}. + */ +class BodyTextParityTest { + + @AfterEach + void clearV1StaticState() { + // The v1 symbol table accumulates into a static table; keep it from leaking into other tests. + SymbolTable.declaredMethodsAndConstructors.clear(); + } + + /** Slice a span out of the module source the way a consumer would. */ + private static String slice(JModule module, Span span) { + byte[] source = module.getSource().getBytes(StandardCharsets.UTF_8); + int[] bytes = span.getBytes(); + return new String(source, bytes[0], bytes[1] - bytes[0], StandardCharsets.UTF_8); + } + + private static JModule buildV2(String source) { + CompilationUnit cu = TestParsers.parseResolved(source); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("app"), "src/Foo.java", source); + return new ModuleBuilder(ctx).build(cu); + } + + private static Callable v1Callable(String source, String namePrefix) throws IOException { + Map table = SymbolTable.extractSingle(source).getLeft(); + for (JavaCompilationUnit cu : table.values()) { + for (com.ibm.cldk.entities.Type type : cu.getTypeDeclarations().values()) { + for (Map.Entry e : type.getCallableDeclarations().entrySet()) { + if (e.getKey().startsWith(namePrefix)) { + return e.getValue(); + } + } + } + } + throw new IllegalStateException("no v1 callable starting with " + namePrefix); + } + + private static JCallable v2Callable(JModule module, String namePrefix) { + for (Map.Entry e : module.getTypes().get("Foo").getCallables().entrySet()) { + if (e.getKey().startsWith(namePrefix)) { + return e.getValue(); + } + } + throw new IllegalStateException("no v2 callable starting with " + namePrefix); + } + + @Test + void bodySpanSliceEqualsV1CodeForAMethod() throws IOException { + String source = "package p;\n" + + "class Foo {\n" + + " int add(int a, int b) {\n" + + " int sum = a + b;\n" + + " return sum;\n" + + " }\n" + + "}\n"; + JModule module = buildV2(source); + JCallable v2 = v2Callable(module, "add("); + assertNotNull(v2.getBodySpan(), "a method with a body must carry body_span"); + assertEquals(v1Callable(source, "add(").getCode(), slice(module, v2.getBodySpan())); + } + + @Test + void bodySpanSliceEqualsV1CodeForAConstructor() throws IOException { + String source = "package p;\nclass Foo {\n Foo(int x) {\n this.x = x;\n }\n int x;\n}\n"; + JModule module = buildV2(source); + JCallable v2 = v2Callable(module, ""); + assertEquals(v1Callable(source, "").getCode(), slice(module, v2.getBodySpan())); + } + + @Test + void callableSpanIsWiderThanBodySpan() throws IOException { + // The distinction that makes body_span necessary: the callable's own span includes the signature. + String source = "package p;\nclass Foo {\n public int add(int a) { return a; }\n}\n"; + JModule module = buildV2(source); + JCallable v2 = v2Callable(module, "add("); + assertEquals("{ return a; }", slice(module, v2.getBodySpan())); + assertEquals("public int add(int a) { return a; }", slice(module, v2.getSpan())); + } + + @Test + void abstractMethodHasNoBodySpan() { + String source = "package p;\nabstract class Foo {\n abstract int f();\n}\n"; + JModule module = buildV2(source); + assertNull(v2Callable(module, "f(").getBodySpan(), "no body -> no body_span (absent = no fact)"); + } + + @Test + void initializerBlockCarriesBodySpan() { + String source = "package p;\nclass Foo {\n static {\n setUp();\n }\n}\n"; + JModule module = buildV2(source); + JCallable init = module.getTypes().get("Foo").getCallables().get("$0()"); + assertNotNull(init.getBodySpan()); + assertEquals("{\n setUp();\n }", slice(module, init.getBodySpan())); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java index e421b55..65b83cb 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -12,6 +12,7 @@ import com.github.javaparser.ast.body.CallableDeclaration; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JType; import com.ibm.cldk.schema.JVariableDeclaration; import java.util.List; import java.util.stream.Collectors; @@ -176,6 +177,51 @@ void build_capturesLocalClassUnderCallableTypesViaContainment() { assertEquals(c.getId() + "/Local", c.getTypes().get("Local").getId()); } + @Test + void build_modelsAnonymousClassAsTypeUnderTheCallable() { + // v1 recursed into anonymous bodies and mis-attributed their members to the enclosing type; + // dropping them instead loses real facts, so they get their own node like a local class does. + JCallable c = build("void m() { Runnable r = new Runnable() { public void run() { log(); } }; }"); + JType anon = c.getTypes().get("$anon$0"); + assertNotNull(anon, "expected an anonymous-class type node, got: " + c.getTypes().keySet()); + assertEquals("class", anon.getKind()); + assertEquals(c.getId() + "/$anon$0", anon.getId()); + assertEquals(List.of("java.lang.Runnable"), anon.getInterfaces(), + "an anonymous class implementing an interface records it under interfaces"); + assertTrue(anon.getCallables().containsKey("run()"), "its methods are its own callables"); + assertEquals(1, anon.getCallables().get("run()").getBody().size(), + "log() belongs to the anonymous class's run(), not to m()"); + } + + @Test + void build_anonymousClassCallsAreNotAttributedToTheEnclosingCallable() { + JCallable c = build("void m() { outer(); Runnable r = new Runnable() { public void run() { hidden(); } }; }"); + // m()'s own body holds outer() and the `new Runnable()` constructor call, but never hidden(). + assertEquals(2, c.getBody().size(), "got: " + c.getBody().keySet()); + assertTrue(c.getBody().values().stream().noneMatch(n -> "hidden".equals(n.getCalleeSignature()))); + } + + @Test + void build_capturesAnonymousInstanceInitializerDoubleBraceIdiom() { + // The idiom spring-petclinic uses: new PetType() {{ setName("Dog"); }} + JCallable c = build("void m() { Object o = new Object() { { setUp(); } }; }"); + JType anon = c.getTypes().get("$anon$0"); + assertNotNull(anon); + JCallable init = anon.getCallables().get("$0()"); + assertNotNull(init, "the double-brace initializer must survive as a callable, got: " + + anon.getCallables().keySet()); + assertEquals("initializer", init.getKind()); + assertEquals(1, init.getBody().size(), "setUp() belongs to the anonymous initializer"); + } + + @Test + void build_numbersMultipleAnonymousClassesInDeclarationOrder() { + JCallable c = build("void m() { r(new Runnable() { public void run() {} });" + + " r(new Runnable() { public void run() {} }); }"); + assertTrue(c.getTypes().containsKey("$anon$0")); + assertTrue(c.getTypes().containsKey("$anon$1")); + } + @Test void build_abstractMethodHasEmptyBody() { JCallable c = build("abstract void m();"); diff --git a/src/test/resources/schema/analysis.v2.schema.json b/src/test/resources/schema/analysis.v2.schema.json index c467ecf..2683bd5 100644 --- a/src/test/resources/schema/analysis.v2.schema.json +++ b/src/test/resources/schema/analysis.v2.schema.json @@ -224,6 +224,7 @@ "kind": { "enum": ["method", "constructor", "initializer", "lambda"] }, "signature": { "type": "string", "minLength": 1 }, "span": { "$ref": "#/$defs/span" }, + "body_span": { "$ref": "#/$defs/span" }, "comments": { "type": "array", "items": { "$ref": "#/$defs/comment" } }, "parameters": { "type": "array", "items": { "$ref": "#/$defs/parameter" } }, "return_type": { "type": "string" }, From a35b8116346851f6a664659180275cc3cd8a5fe0 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 14:11:14 -0400 Subject: [PATCH 20/22] feat(schema): model anonymous classes in field initializers; add comparison report Anonymous classes occur in two places: inside a callable body and inside a field initializer, which belongs to no callable. The latter was missed, so commons-lang's AnnotationUtils lost the double-brace initializer configuring its ToStringStyle. Adds docs/design/notes/l1-v1-v2-comparison.md, generated from twenty runs (ten applications x both schemas). v2 matches or exceeds v1 on every structural metric; the two remaining negative deltas are v1 counting bugs (anonymous-class fields reported as method locals, nested initializer blocks counted twice) that v2 does not reproduce. --- .claude/SCHEMA_DECISIONS.md | 6 + docs/design/notes/l1-v1-v2-comparison.md | 175 ++++++++++++++++++ .../cldk/syntactic_analysis/TypeBuilder.java | 18 ++ .../syntactic_analysis/TypeBuilderTest.java | 28 +++ 4 files changed, 227 insertions(+) create mode 100644 docs/design/notes/l1-v1-v2-comparison.md diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 551c3b2..5d99e8c 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -143,6 +143,12 @@ Both refinements came out of a field-by-field v1-vs-v2 comparison over ten real- reintroducing duplicated text. Absent when there is no body (abstract/interface methods). **Canonical note:** the keystone defines `get_method_body(sig)` as `module.source[callable.span.bytes]`, which is *not* v1's `code` semantics; the discrepancy is worth resolving in the canonical schema. +- **Two v1 counting bugs surfaced by the comparison, which v2 deliberately does not reproduce.** v1 + collected a callable's locals with a recursive `findAll(VariableDeclarator)`, so a **field declared in an + anonymous class** was reported as a local of the enclosing method; v2 records it as a field of the + anonymous class. And v1 filled a type's `initialization_blocks` recursively, counting a nested class's + `static { … }` block **twice** — once on the nested class and once on its enclosing type; v2 counts it + once. Where v2's totals are lower than v1's for these two metrics, v2 is the more accurate. ### D12 — L1 type resolution: library dependencies are always attempted diff --git a/docs/design/notes/l1-v1-v2-comparison.md b/docs/design/notes/l1-v1-v2-comparison.md new file mode 100644 index 0000000..f567c60 --- /dev/null +++ b/docs/design/notes/l1-v1-v2-comparison.md @@ -0,0 +1,175 @@ +# L1 output comparison: legacy v1 schema vs canonical schema v2 + +Generated 2026-08-19 from `codeanalyzer-2.4.1`. Each of ten real-world fixture applications was analysed +twice — once with the default (v1) emitter, once with `--schema v2` — and the payloads diffed field by +field. The purpose is to catch silent information loss in the migration: every metric where v2 records +less than v1 is either explained or fixed. + +## How to reproduce + +```bash +./gradlew fatJar +JAR=build/libs/codeanalyzer-2.4.1.jar +APP=src/test/resources/test-applications/spring-petclinic +java -jar $JAR -i $APP -o output/spring-petclinic/v1 -a 1 # legacy +java -jar $JAR -i $APP -o output/spring-petclinic/v2 --schema v2 # canonical +``` + +Payloads land in `output///analysis.json` (`output/` is git-ignored). The figures in this +document are generated from those files, so it cannot drift from the data. + +## Runs + +All twenty runs exited 0 and left the fixture submodules clean. + +| Application | v1 time | v2 time | v1 size | v2 size | +| --- | --- | --- | --- | --- | +| `spring-petclinic` | 5s | 4s | 2.4M | 2.9M | +| `cargotracker` | 17s | 4s | 4.7M | 5.8M | +| `commons-lang` | 623s | 148s | 128M | 142M | +| `quarkuscoffeeshop-counter` | 3s | 3s | 1.5M | 1.9M | +| `quarkuscoffeeshop-barista` | 3s | 2s | 425K | 575K | +| `quarkuscoffeeshop-kitchen` | 2s | 2s | 342K | 460K | +| `quarkuscoffeeshop-inventory` | 3s | 3s | 415K | 581K | +| `quarkuscoffeeshop-domain` | 2s | 2s | 343K | 491K | +| `daytrader8` | 5s | 4s | 8.5M | 10M | +| `plantsbywebsphere` | 3s | 2s | 3.0M | 3.7M | + +**v2 is consistently faster.** It never builds per-callable `code` strings, so it never invokes +JavaParser's `LexicalPreservingPrinter` — the dominant cost on large projects. (A second-order effect: +each v2 run reused dependency jars the preceding v1 run had already downloaded.) + +**v2 payloads are somewhat larger** even though per-callable `code` is gone: source text is stored once +per module rather than duplicated per callable, but that saving is outweighed by spans on every node +(`start`/`end`/`bytes`), per-node comments, local variables, and the resolved call-site facts. + +## Totals across all ten applications + +| Metric | v1 | v2 | Delta | +| --- | --- | --- | --- | +| modules | 1081 | 1081 | +0 | +| types | 1581 | 1814 | +233 | +| callables | 13594 | 13850 | +256 | +| fields | 3727 | 3761 | +34 | +| parameters | 9877 | 10077 | +200 | +| call sites | 94501 | 94917 | +416 | +| local variables | 11650 | 11639 | -11 * | +| comment entries | 39170 | 9710 | -29460 * | +| enum constants | 338 | 338 | +0 | +| record components | 2 | 2 | +0 | +| initializer blocks | 30 | 29 | -1 * | +| entrypoint types | 95 | 95 | +0 | +| entrypoint callables | 258 | 258 | +0 | +| CRUD facts | 107 | 0 | -107 * | + +\* explained below. Of the four, two (`local variables`, `initializer blocks`) turn out to be v1 +over-counting rather than v2 losses; one (`comment entries`) is mostly v1 duplication with a small real +gap; and one (`CRUD facts`) is deliberately deferred. + +**Type resolution is at parity:** 95.7% of v1 parameter types and 95.8% of v2 parameter types are +fully qualified. v2 additionally resolves a callee signature on 94152 of 94917 call sites (99%), which v1 +recorded only on its separate `call_sites` entries. + +**Identity:** all 1081 v1 `symbol_table` keys are absolute filesystem paths; v2 has 0 absolute keys — +every key is project-relative, which the canonical schema requires for stable caching and SDK lookups. + +**Anonymous classes:** 215 are modelled as their own `type` nodes across the ten applications. +**Body text:** 13589 callables carry a `body_span`. + +## Per-application detail + +Metrics where the two schemas differ, per application. Blank means exact parity. + +| Application | types | callables | call sites | locals | initializers | comments | CRUD | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `spring-petclinic` | +3 | +2 | | | | -225 | | +| `cargotracker` | +2 | +4 | +78 | +3 | | -501 | -77 | +| `commons-lang` | +218 | +237 | +296 | -14 | -1 | -25342 | | +| `quarkuscoffeeshop-counter` | +3 | +3 | +3 | | | -71 | | +| `quarkuscoffeeshop-barista` | +1 | +1 | +2 | | | -16 | | +| `quarkuscoffeeshop-kitchen` | +1 | +1 | +3 | | | -65 | | +| `quarkuscoffeeshop-inventory` | +1 | +1 | +3 | | | -23 | | +| `quarkuscoffeeshop-domain` | | | +12 | | | -6 | | +| `daytrader8` | +4 | +7 | +17 | | | -2244 | -30 | +| `plantsbywebsphere` | | | +2 | | | -967 | | + +## Where v2 recovers more than v1 + +- **Types, callables and call sites.** v1 keyed its flat type map by fully-qualified name and skipped + declarations without one, so **local classes declared inside method bodies were dropped entirely**; + v2 nests them under the enclosing callable. v1's call-site scan also missed **explicit constructor + chaining** (`this(...)` / `super(...)`), which v2 emits as `call` nodes so L2 can resolve those edges. +- **Anonymous inner classes** are modelled as `type` nodes (`$anon$0`, `$anon$1`, … in declaration + order) under the callable that declares them, so their methods, initializers, locals and call sites + are attributed to them. v1 recursed into anonymous bodies and mis-attributed those facts to the + *enclosing type*. +- **Resolved call-site facts** — callee signature, receiver expression and type, argument types — sit on + the body `call` nodes. +- **Structured annotation arguments.** v1 stored annotations as flat strings (`@RequestMapping("/x")`); + v2 records `{name, args[], span}`, so routes and column names are machine-readable without re-parsing. + +## Where v2 records less, and why + +### Comment entries (-29460): v1 double-counting, plus one real gap + +v1 filled every node's `comments` with `getAllContainedComments()`, so a comment inside a method was +also listed on that method's type and on the compilation unit. On `spring-petclinic` v1 emits 341 +comment entries of which only **163 are distinct** (a 2.09x duplication factor); v2 emits 116, each +attached to exactly one node. + +The remaining ~47 distinct comments v2 does not carry are **comments inside method bodies**, which +have no declaration to attach to. They stay recoverable from `module.source`, and they belong on the +statement nodes that arrive at L3 — but today they are absent from the tree. This is the one +outstanding information gap. + +### Local variables (-11) and initializer blocks (-1): v1 over-counting + +Both remaining deltas are **v1 defects**, not v2 losses — v2 is the more accurate of the two. + +*Locals.* v1 collected a callable's locals with a recursive `findAll(VariableDeclarator)`, which also +matches **field declarations inside anonymous classes**. In `AtomicInitializerObjectTest`: + +```java +final AtomicInitializer initializer = new AtomicInitializer() { + final AtomicBoolean firstRun = new AtomicBoolean(true); // a field of the anonymous class + ... +}; +``` + +v1 reports the enclosing method's locals as `[initializer, firstRun]`, promoting the anonymous class's +field to a method local. v2 reports `[initializer]` and records `firstRun` under +`$anon$0.fields`, where it belongs. Every one of the remaining local-variable differences is this +pattern. + +*Initializer blocks.* v1 populated a type's `initialization_blocks` with a recursive `findAll`, so a +`static { ... }` block in a nested class was counted **twice**: once on the nested class and again on +its enclosing type. `LocaleUtils` shows this — v1 reports one block on `LocaleUtils` and one on +`LocaleUtils.SyncAvoid`, though only `SyncAvoid` has a block. v2 counts it once, on `SyncAvoid`. + +### CRUD facts (-107): tracked separately + +v2 carries no CRUD data yet. This is deliberate and tracked in codeanalyzer-java issue 187, which also +covers the Neo4j `JCrudOperation`/`JCrudQuery` families that the graph projection needs. + +## Body text: v1 `code` versus a v2 slice + +v2 has no per-callable `code` string — body text is a slice of `module.source`. That equivalence needs +care, because a callable's own `span` covers the **whole declaration** (modifiers, signature and body), +whereas v1's `code` was the `{ … }` **block alone**. `callable.body_span` delimits the block, so: + +``` +source[body_span.bytes] == v1 callable.code (byte for byte) +source[span.bytes] == declaration + body +``` + +A test compares the two emitters directly on the same source for methods, constructors and initializer +blocks, so this cannot regress silently. Note that the canonical schema defines `get_method_body(sig)` +as `module.source[callable.span.bytes]`, which is *not* v1's `code` semantics — a discrepancy worth +resolving upstream. + +## Outstanding follow-ups + +1. **Attach body-internal comments** to the statement nodes introduced at L3. +2. **CRUD enrichment** — codeanalyzer-java issue 187. +3. Consider a more compact span encoding if payload size becomes a concern. + diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java index b895184..b8ac9e6 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/TypeBuilder.java @@ -21,6 +21,7 @@ import com.ibm.cldk.schema.JRecordComponent; import com.ibm.cldk.schema.JType; import java.util.ArrayList; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -223,6 +224,23 @@ private void populateMembers(JType type, List> members, Strin nested.put(nestedType.getNameAsString(), build(nestedType, type.getId())); } } + // Anonymous classes in field initializers are lexically members of this type, not of any + // callable, so they are attributed here (e.g. `static final X F = new X() { { ... } };`). + List anonymous = new ArrayList<>(); + for (BodyDeclaration member : members) { + if (member instanceof FieldDeclaration) { + member.findAll(ObjectCreationExpr.class).stream() + .filter(oce -> oce.getAnonymousClassBody().isPresent()) + .forEach(anonymous::add); + } + } + anonymous.sort(Comparator + .comparingInt((ObjectCreationExpr oce) -> oce.getBegin().map(pos -> pos.line).orElse(0)) + .thenComparingInt(oce -> oce.getBegin().map(pos -> pos.column).orElse(0))); + for (int i = 0; i < anonymous.size(); i++) { + String name = "$anon$" + i; + nested.put(name, buildAnonymous(anonymous.get(i), type.getId(), name)); + } type.setTypes(new LinkedHashMap<>(nested)); } } diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java index 923a7d6..504d07d 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/TypeBuilderTest.java @@ -190,6 +190,34 @@ void build_numbersMultipleInitializersOfTheSameKind() { assertTrue(t.getCallables().containsKey("$1()")); } + @Test + void build_modelsAnonymousClassInAFieldInitializer() { + // commons-lang's AnnotationUtils does exactly this: an anonymous subclass configured by a + // double-brace initializer, in a field initializer — outside any callable body. + JType t = buildFirstType("package p;\nclass Foo {\n" + + " static final Runnable R = new Runnable() {\n" + + " { setUp(); }\n" + + " public void run() { go(); }\n" + + " };\n}\n"); + JType anon = t.getTypes().get("$anon$0"); + assertNotNull(anon, "expected the field-initializer anonymous class, got: " + t.getTypes().keySet()); + assertEquals(t.getId() + "/$anon$0", anon.getId()); + assertTrue(anon.getCallables().containsKey("run()"), "its methods belong to it"); + assertNotNull(anon.getCallables().get("$0()"), + "its double-brace initializer must survive, got: " + anon.getCallables().keySet()); + } + + @Test + void build_numbersFieldInitializerAnonymousClassesSeparatelyFromNestedTypes() { + JType t = buildFirstType("package p;\nclass Foo {\n" + + " static final Runnable A = new Runnable() { public void run() {} };\n" + + " static final Runnable B = new Runnable() { public void run() {} };\n" + + " static class Named {}\n}\n"); + assertTrue(t.getTypes().containsKey("$anon$0")); + assertTrue(t.getTypes().containsKey("$anon$1")); + assertTrue(t.getTypes().containsKey("Named"), "named nested types are unaffected"); + } + @Test void build_capturesStructuredDecoratorWithArgs() { JType t = buildFirstType("package p;\n@SuppressWarnings(\"unchecked\")\nclass Foo {}\n"); From 5c29f7745e4807f2f26f55af3d59f00c1b0a1a16 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 15:23:38 -0400 Subject: [PATCH 21/22] feat(cli): incremental L1 cache keyed on content_hash (-c/--cache-dir, --eager) Reuses modules whose files are byte-for-byte unchanged, skipping the parse as well as the build: commons-lang goes from 130s cold to 4s warm. Caching is opt-in, and the cache is discarded wholesale when the app name or analyzer version changes since both are baked into every can:// id. The extractor now enumerates and hashes files itself instead of parsing whole source roots up front; module discovery is unchanged (commons-lang still yields 625 modules, matching v1). --- .claude/SCHEMA_DECISIONS.md | 15 +++ docs/design/notes/l1-v1-v2-comparison.md | 4 + src/main/java/com/ibm/cldk/CodeAnalyzer.java | 23 ++++- .../ibm/cldk/syntactic_analysis/L1Cache.java | 99 +++++++++++++++++++ .../cldk/syntactic_analysis/L1Extractor.java | 61 +++++++++--- .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 72 ++++++++++++++ 6 files changed, 257 insertions(+), 17 deletions(-) create mode 100644 src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 5d99e8c..c5b340d 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -125,6 +125,21 @@ Refinements settled while building L1 (2026-08), each checked against the keysto - **`module.content_hash` is SHA-256 hex of the UTF-8 source** — for incremental caching and the Neo4j writer's per-module diffing; never identity (the `id` is). +### D14 — Incremental caching keyed on `content_hash` + +`module.content_hash` exists so an unchanged file need not be re-analysed, and the v2 path now uses it: +with `-c/--cache-dir`, modules are persisted to `analysis_cache.json` and reused when the file on disk +still hashes to the same value. The reuse skips **parsing** as well as building — the extractor +enumerates and hashes files itself rather than parsing a whole source root up front — which is where the +cost actually is: `commons-lang` (625 files) goes from 130s cold to 4s warm. + +- **Caching is opt-in.** No `--cache-dir`, no cache file; the analyzer never writes into a project + uninvited. `--eager` ignores an existing cache, which is also how a caller recovers from one they + distrust. +- **The cache is invalidated wholesale when the application name or analyzer version changes**, because + both are baked into every `can://` id — a module cached under different settings would carry wrong + ids. A missing, corrupt or mismatched cache degrades to a full rebuild and is never fatal. + ### D13 — Anonymous classes are modelled; body text is recovered via `body_span` Both refinements came out of a field-by-field v1-vs-v2 comparison over ten real-world applications diff --git a/docs/design/notes/l1-v1-v2-comparison.md b/docs/design/notes/l1-v1-v2-comparison.md index f567c60..3a50320 100644 --- a/docs/design/notes/l1-v1-v2-comparison.md +++ b/docs/design/notes/l1-v1-v2-comparison.md @@ -39,6 +39,10 @@ All twenty runs exited 0 and left the fixture submodules clean. JavaParser's `LexicalPreservingPrinter` — the dominant cost on large projects. (A second-order effect: each v2 run reused dependency jars the preceding v1 run had already downloaded.) +**Incremental caching** (`-c/--cache-dir`) reuses modules whose files are byte-for-byte unchanged, +skipping both the parse and the build: a second `commons-lang` run drops from 130s to 4s. The timings +above are all cold runs, so they measure the emitters rather than the cache. + **v2 payloads are somewhat larger** even though per-callable `code` is gone: source text is stored once per module rather than duplicated per callable, but that saving is outweighed by spans on every node (`start`/`end`/`bytes`), per-node comments, local variables, and the resolved call-site facts. diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index 775f76e..b6d342e 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -28,6 +28,7 @@ import com.ibm.cldk.schema.JModule; import com.ibm.cldk.schema.V2Emitter; import com.ibm.cldk.schema.V2Json; +import com.ibm.cldk.syntactic_analysis.L1Cache; import com.ibm.cldk.syntactic_analysis.L1Extractor; import com.ibm.cldk.utils.BuildProject; import com.ibm.cldk.utils.Log; @@ -131,6 +132,15 @@ public class CodeAnalyzer implements Runnable { // values between CommandLine instances in the same JVM. New flags do not add to that. private String schema = "v1"; + @Option(names = {"-c", + "--cache-dir" }, description = "Directory holding the incremental analysis cache. When set, " + + "unchanged files are reused from analysis_cache.json instead of being reparsed.") + private String cacheDir; + + @Option(names = { + "--eager" }, description = "Ignore any cached modules and rebuild everything (default: lazy).") + private boolean eager = false; + /** Handle used to report flag-validation errors as clean, non-zero picocli failures. */ @Spec private CommandSpec spec; @@ -370,13 +380,22 @@ private void analyzeV2() throws Exception { + "); third-party types may not resolve"); } + // Lazy by default: reuse modules whose files are byte-for-byte unchanged. `--eager` forces a + // full rebuild, which is also how a caller recovers from a cache they distrust. + Path cache = cacheDir == null ? null : Paths.get(cacheDir); + String version = analyzerVersion(); + Map cached = eager + ? new java.util.LinkedHashMap<>() + : L1Cache.load(cache, application, version); + Map modules; try { - modules = L1Extractor.extractAll(Paths.get(input), application, dependencyDir); + modules = L1Extractor.extractAll(Paths.get(input), application, dependencyDir, cached); } finally { BuildProject.cleanLibraryDependencies(); } - Analysis analysis = V2Emitter.emit(application, 1, modules, analyzerVersion()); + L1Cache.save(cache, application, version, modules); + Analysis analysis = V2Emitter.emit(application, 1, modules, version); if (output == null) { // stdout is the data channel: compact JSON only, so the SDK can parse it directly. diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java new file mode 100644 index 0000000..dbca265 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Cache.java @@ -0,0 +1,99 @@ +package com.ibm.cldk.syntactic_analysis; + +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.V2Json; +import com.ibm.cldk.utils.Log; +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Data; + +/** + * On-disk cache of built L1 modules, so an unchanged file is neither reparsed nor rebuilt on a + * subsequent run. A module is reusable when its {@code content_hash} still matches the file on disk — + * which is what that field exists for. + * + *

The cache is keyed by the same relative file key as {@code symbol_table}, and the whole file is + * discarded when the analyzer version or the application name changes: both are baked into every + * {@code can://} id, so a cached module built under different settings would contain wrong ids. A + * missing, unreadable or stale cache is never fatal — it just means everything is rebuilt. + */ +public final class L1Cache { + + private L1Cache() {} + + private static final String FILE_NAME = "analysis_cache.json"; + + /** What is persisted: the modules plus the settings they were built under. */ + @Data + static class Envelope { + private String schemaVersion; + private String analyzerVersion; + private String appName; + private Map modules = new LinkedHashMap<>(); + } + + public static Path fileIn(Path cacheDir) { + return cacheDir.resolve(FILE_NAME); + } + + /** + * Load reusable modules, or an empty map when there is nothing usable. Never throws: a corrupt or + * mismatched cache degrades to a full rebuild rather than failing the analysis. + */ + public static Map load(Path cacheDir, String appName, String analyzerVersion) { + if (cacheDir == null) { + return new LinkedHashMap<>(); + } + Path path = fileIn(cacheDir); + if (!Files.isRegularFile(path)) { + return new LinkedHashMap<>(); + } + try (Reader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + Envelope envelope = + V2Json.compact().fromJson(reader, new TypeToken() {}.getType()); + if (envelope == null || envelope.getModules() == null) { + return new LinkedHashMap<>(); + } + boolean sameSettings = "2.0.0".equals(envelope.getSchemaVersion()) + && java.util.Objects.equals(appName, envelope.getAppName()) + && java.util.Objects.equals(analyzerVersion, envelope.getAnalyzerVersion()); + if (!sameSettings) { + Log.debug("Ignoring cache built under different settings: " + path); + return new LinkedHashMap<>(); + } + return envelope.getModules(); + } catch (IOException | JsonSyntaxException e) { + Log.debug("Ignoring unreadable cache " + path + ": " + e.getMessage()); + return new LinkedHashMap<>(); + } + } + + /** Persist the built modules. A write failure is reported but does not fail the analysis. */ + public static void save( + Path cacheDir, String appName, String analyzerVersion, Map modules) { + if (cacheDir == null) { + return; + } + Envelope envelope = new Envelope(); + envelope.setSchemaVersion("2.0.0"); + envelope.setAnalyzerVersion(analyzerVersion); + envelope.setAppName(appName); + envelope.setModules(modules); + try { + Files.createDirectories(cacheDir); + try (Writer writer = Files.newBufferedWriter(fileIn(cacheDir), StandardCharsets.UTF_8)) { + V2Json.compact().toJson(envelope, writer); + } + } catch (IOException e) { + Log.warn("Could not write analysis cache to " + cacheDir + ": " + e.getMessage()); + } + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java index 107005c..0047252 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1Extractor.java @@ -1,5 +1,6 @@ package com.ibm.cldk.syntactic_analysis; +import com.github.javaparser.JavaParser; import com.github.javaparser.ParseResult; import com.github.javaparser.ParserConfiguration; import com.github.javaparser.ast.CompilationUnit; @@ -57,6 +58,12 @@ public static Map extractAll(Path projectRoot, String appName) return extractAll(projectRoot, appName, null); } + /** Analyse a project without reusing any cached modules. */ + public static Map extractAll(Path projectRoot, String appName, Path dependencyDir) + throws IOException { + return extractAll(projectRoot, appName, dependencyDir, new LinkedHashMap<>()); + } + /** * Build the v2 symbol table for a project. * @@ -66,7 +73,8 @@ public static Map extractAll(Path projectRoot, String appName) * missing or unreadable jars are skipped rather than failing the analysis * @return modules keyed by relative file path, iterated in sorted key order for determinism */ - public static Map extractAll(Path projectRoot, String appName, Path dependencyDir) + public static Map extractAll( + Path projectRoot, String appName, Path dependencyDir, Map cached) throws IOException { ParserConfiguration discovery = parserConfiguration(); ProjectRoot root = new ParserCollectionStrategy(discovery).collect(projectRoot); @@ -81,33 +89,56 @@ public static Map extractAll(Path projectRoot, String appName, ParserConfiguration config = parserConfiguration() .setSymbolResolver(new JavaSymbolSolver(typeSolver(sourceRoots, dependencyDir, discovery))); - // Collect into a sorted map first: source roots and directory listings are not ordered, and - // `-j N` output must be byte-identical to `-j 1`. + // Collect into a sorted map first: directory listings are not ordered, and output must not + // depend on traversal order. Map modules = new TreeMap<>(); String applicationId = CanId.applicationId(appName); + JavaParser parser = new JavaParser(config); + int reused = 0; for (SourceRoot sourceRoot : sourceRoots) { - sourceRoot.setParserConfiguration(config); - for (ParseResult parseResult : sourceRoot.tryToParse()) { - if (parseResult.getResult().isEmpty()) { - Log.debug("Skipping unparsable file: " + parseResult.getProblems()); - continue; - } - CompilationUnit cu = parseResult.getResult().get(); - if (cu.getStorage().isEmpty()) { - continue; - } - Path path = cu.getStorage().get().getPath(); + for (Path path : javaFilesUnder(sourceRoot.getRoot())) { String fileKey = fileKey(projectRoot, path); // Read the file's own text rather than printing the AST: `span.bytes` must index the // real file, byte for byte. String source = Files.readString(path, StandardCharsets.UTF_8); L1BuildContext ctx = new L1BuildContext(applicationId, fileKey, source); - modules.put(fileKey, new ModuleBuilder(ctx).build(cu)); + + // Reuse the cached module when the file is byte-for-byte what it was last time. This + // skips the parse as well as the build, which is where the cost is. + JModule cachedModule = cached.get(fileKey); + if (cachedModule != null && ctx.contentHash().equals(cachedModule.getContentHash())) { + modules.put(fileKey, cachedModule); + reused++; + continue; + } + + ParseResult parseResult = parser.parse(path); + if (parseResult.getResult().isEmpty()) { + Log.debug("Skipping unparsable file " + path + ": " + parseResult.getProblems()); + continue; + } + modules.put(fileKey, new ModuleBuilder(ctx).build(parseResult.getResult().get())); } } + if (!cached.isEmpty()) { + Log.debug("Reused " + reused + " of " + modules.size() + " modules from cache"); + } return new LinkedHashMap<>(modules); } + /** Java sources under a source root, in sorted order so traversal cannot affect output. */ + private static List javaFilesUnder(Path root) throws IOException { + if (!Files.isDirectory(root)) { + return List.of(); + } + try (Stream paths = Files.walk(root)) { + return paths.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(".java")) + .sorted() + .collect(java.util.stream.Collectors.toList()); + } + } + private static ParserConfiguration parserConfiguration() { return new ParserConfiguration() .setStoreTokens(true) diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index 0c3c293..8014ee1 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -116,6 +116,78 @@ void v2SchemaIsNotTheDefault(@TempDir Path tmp) throws IOException { assertTrue(root.has("symbol_table"), "v1 keeps symbol_table at the top level"); } + @Test + void cacheFileIsWrittenAndReusedOnASecondRun(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + Path cache = tmp.resolve("cache"); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", + "--app-name", "widgets", "-c", cache.toString())); + Path cacheFile = cache.resolve("analysis_cache.json"); + assertTrue(Files.exists(cacheFile), "a run with --cache-dir must write analysis_cache.json"); + + // Prove reuse rather than timing it: plant a sentinel in the cached module. If the second run + // reuses the cache the sentinel survives into the output; if it rebuilds, it cannot. + String doctored = Files.readString(cacheFile).replace("\"package\":\"com.example\"", + "\"package\":\"SENTINEL\""); + assertTrue(doctored.contains("SENTINEL"), "precondition: the cache holds the package name"); + Files.writeString(cacheFile, doctored); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", + "--app-name", "widgets", "-c", cache.toString())); + assertTrue(Files.readString(out.resolve("analysis.json")).contains("SENTINEL"), + "the second run should have reused the cached module"); + } + + @Test + void eagerIgnoresTheCache(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + Path cache = tmp.resolve("cache"); + run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", "-c", cache.toString()); + Path cacheFile = cache.resolve("analysis_cache.json"); + Files.writeString(cacheFile, + Files.readString(cacheFile).replace("\"package\":\"com.example\"", "\"package\":\"SENTINEL\"")); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", + "-c", cache.toString(), "--eager")); + assertFalse(Files.readString(out.resolve("analysis.json")).contains("SENTINEL"), + "--eager must rebuild instead of trusting the cache"); + } + + @Test + void changedFileIsRebuiltWhileOthersAreReused(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + Path cache = tmp.resolve("cache"); + Path second = in.resolve("src/main/java/com/example/Other.java"); + Files.writeString(second, "package com.example;\npublic class Other { int n() { return 2; } }\n", + StandardCharsets.UTF_8); + run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", "-c", cache.toString()); + + // Sentinel both cached modules, then edit only one file on disk. + Path cacheFile = cache.resolve("analysis_cache.json"); + Files.writeString(cacheFile, Files.readString(cacheFile).replace("int n()", "int SENTINEL()")); + Files.writeString(second, "package com.example;\npublic class Other { int n() { return 3; } }\n", + StandardCharsets.UTF_8); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v2", + "-c", cache.toString())); + String analysis = Files.readString(out.resolve("analysis.json")); + assertFalse(analysis.contains("SENTINEL"), "the edited file must be rebuilt, not reused"); + assertTrue(analysis.contains("return 3"), "and the new content must be present"); + } + + @Test + void noCacheDirMeansNoCacheFile(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v2")); + assertFalse(Files.exists(out.resolve("analysis_cache.json")), + "caching is opt-in: no --cache-dir, no cache file"); + } + @Test void unknownSchemaValueFailsLoudly(@TempDir Path tmp) throws IOException { Path in = project(tmp.resolve("app")); From 16f77f0ed418a06ecc91ae858070c8837157ba0b Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Wed, 19 Aug 2026 17:16:28 -0400 Subject: [PATCH 22/22] fix(schema): six correctness fixes from code review - Constructor callee_signature normalises to so it joins against the target callable's signature; otherwise L2 drops every constructor edge (88 of petclinic's call sites). - Expression-type resolution no longer memoises failures by expression text: the same text can denote different types in different scopes of one file, so a failure blanked later resolvable occurrences. - metrics.cyclomatic is scope-filtered like every other callable fact, so branches inside a nested or anonymous class are no longer double-counted. - Call sites with no source range are skipped rather than colliding on 0:0 and silently overwriting one another. - Module span end position is computed for universal newlines and for files with no trailing newline. - Corrected the byteOffsets javadoc: the range is end-exclusive. --- .claude/SCHEMA_DECISIONS.md | 10 +++++ src/main/java/com/ibm/cldk/schema/Spans.java | 6 +-- .../syntactic_analysis/CallSiteBuilder.java | 27 ++++++++--- .../syntactic_analysis/CallableBuilder.java | 33 +++++++++----- .../syntactic_analysis/L1BuildContext.java | 30 ++++++++----- .../cldk/syntactic_analysis/Signatures.java | 7 ++- .../CallSiteBuilderTest.java | 45 +++++++++++++++++++ .../CallableBuilderTest.java | 11 +++++ .../syntactic_analysis/ModuleBuilderTest.java | 19 ++++++++ 9 files changed, 156 insertions(+), 32 deletions(-) diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index c5b340d..cb179be 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -122,6 +122,16 @@ Refinements settled while building L1 (2026-08), each checked against the keysto - **`module.span` covers the whole file**, computed from the source rather than the compilation unit's AST range (which ends inconsistently around trailing whitespace), so `module.source[span.bytes] == module.source` always holds. +- **A call site's `callee_signature` must be joinable against the target callable's `signature`.** A + resolved constructor's name is its *class* name, while the declaration side emits ``, so the + callee side normalises to `` too. Without this every constructor edge would be unjoinable and + L2 would silently drop it (88 of petclinic's call sites). +- **Call sites with no source range are skipped.** They cannot be addressed by a `line:col` id, and + fabricating one would both invent a location and collide with every other rangeless node, silently + overwriting entries in `body`. +- **Metrics are scope-filtered like every other callable fact.** `metrics.cyclomatic` counts only branch + points belonging to the callable itself; those inside a nested or anonymous class belong to that + class's callables and would otherwise be counted twice. - **`module.content_hash` is SHA-256 hex of the UTF-8 source** — for incremental caching and the Neo4j writer's per-module diffing; never identity (the `id` is). diff --git a/src/main/java/com/ibm/cldk/schema/Spans.java b/src/main/java/com/ibm/cldk/schema/Spans.java index 15fb913..968f768 100644 --- a/src/main/java/com/ibm/cldk/schema/Spans.java +++ b/src/main/java/com/ibm/cldk/schema/Spans.java @@ -19,7 +19,7 @@ private Spans() {} /** Byte offset into {@code source} of the position at (1-based {@code line}, 0-based {@code col}). */ public static int byteOffset(String source, int line, int col) { - List lines = splitKeepEnds(source); + List lines = splitLinesKeepingTerminators(source); int prefixBytes = 0; for (int k = 0; k < line - 1 && k < lines.size(); k++) { prefixBytes += utf8Length(lines.get(k)); @@ -29,7 +29,7 @@ public static int byteOffset(String source, int line, int col) { return prefixBytes + utf8Length(current.substring(0, c)); } - /** {@code [from, to]} byte offsets for a span from (startLine,startCol) to (endLine,endCol). */ + /** {@code [from, to)} byte offsets (end exclusive) for a span from (startLine,startCol) to (endLine,endCol). */ public static int[] byteOffsets(String source, int startLine, int startCol, int endLine, int endCol) { return new int[] {byteOffset(source, startLine, startCol), byteOffset(source, endLine, endCol)}; } @@ -43,7 +43,7 @@ private static int utf8Length(String s) { * {@code \r\n}, {@code \r}), mirroring Python's {@code splitlines(keepends=True)}. A final line * without a terminator is included. */ - private static List splitKeepEnds(String s) { + public static List splitLinesKeepingTerminators(String s) { List out = new ArrayList<>(); int n = s.length(); int start = 0; diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java index 32f33b7..cbcb6b2 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilder.java @@ -55,6 +55,11 @@ public Map build(BlockStmt body) { .filter(n -> AstScopes.belongsDirectlyTo(n, body)) .forEach(sites::add); + // A node with no source range cannot be addressed by a line:col id. Inventing one would both + // fabricate a location and collide with every other rangeless node, silently overwriting call + // sites; skipping is the honest degradation. + sites.removeIf(site -> !hasPosition(site)); + sites.sort(Comparator.comparingInt(n -> anchorPosition(n)[0]) .thenComparingInt(n -> anchorPosition(n)[1])); @@ -128,6 +133,12 @@ private static List argumentsOf(Node site) { return new ArrayList<>(args); } + /** Whether a call site has a usable source position (its anchor's, or its own). */ + private static boolean hasPosition(Node site) { + Node anchor = anchorOf(site); + return anchor.getRange().isPresent() || site.getRange().isPresent(); + } + /** The local id {@code line:col} of a node's addressing anchor. */ private static String localId(Node node) { int[] pos = anchorPosition(node); @@ -139,14 +150,18 @@ private static String localId(Node node) { * {@code new} expression, and the statement itself for {@code this(...)}/{@code super(...)} — * so sites nested in one expression stay distinct. Falls back to the node's own begin. */ - private static int[] anchorPosition(Node node) { - Node anchor = node; + private static Node anchorOf(Node node) { if (node instanceof MethodCallExpr) { - anchor = ((MethodCallExpr) node).getName(); - } else if (node instanceof ObjectCreationExpr) { - anchor = ((ObjectCreationExpr) node).getType(); + return ((MethodCallExpr) node).getName(); } - return anchor.getRange() + if (node instanceof ObjectCreationExpr) { + return ((ObjectCreationExpr) node).getType(); + } + return node; + } + + private static int[] anchorPosition(Node node) { + return anchorOf(node).getRange() .map(r -> new int[] {r.begin.line, r.begin.column}) .orElseGet(() -> node.getRange() .map(r -> new int[] {r.begin.line, r.begin.column}) diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java index 3dd4db8..a99b175 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/CallableBuilder.java @@ -241,22 +241,33 @@ private JRefs refs(Optional body, String typeFqn, List classF * in the callable (mirrors the v1 symbol-table metric). */ private static int cyclomaticComplexity(InitializerDeclaration id) { - return branchPoints(id) + 1; + return branchPoints(id.getBody()) + 1; } private static int cyclomaticComplexity(CallableDeclaration cd) { - return branchPoints(cd) + 1; + return bodyOf(cd).map(CallableBuilder::branchPoints).orElse(0) + 1; } - /** Branch points (if / loop / switch-case / ternary / catch) inside any node. */ - private static int branchPoints(com.github.javaparser.ast.Node node) { - int ifCount = node.findAll(IfStmt.class).size(); - int loopCount = node.findAll(DoStmt.class).size() + node.findAll(ForStmt.class).size() - + node.findAll(ForEachStmt.class).size() + node.findAll(WhileStmt.class).size(); - int switchCaseCount = - node.findAll(SwitchStmt.class).stream().mapToInt(s -> s.getEntries().size()).sum(); - int ternaryCount = node.findAll(ConditionalExpr.class).size(); - int catchCount = node.findAll(CatchClause.class).size(); + /** + * Branch points (if / loop / switch-case / ternary / catch) belonging to this body itself. Branches + * inside a nested type or anonymous class belong to its callables — counting them here too + * would inflate the enclosing callable and double-count them, and every other metric on the callable + * is scope-filtered the same way. + */ + private static int branchPoints(BlockStmt node) { + int ifCount = own(node, IfStmt.class).size(); + int loopCount = own(node, DoStmt.class).size() + own(node, ForStmt.class).size() + + own(node, ForEachStmt.class).size() + own(node, WhileStmt.class).size(); + int switchCaseCount = own(node, SwitchStmt.class).stream().mapToInt(s -> s.getEntries().size()).sum(); + int ternaryCount = own(node, ConditionalExpr.class).size(); + int catchCount = own(node, CatchClause.class).size(); return ifCount + loopCount + switchCaseCount + ternaryCount + catchCount; } + + /** Nodes of a kind that belong to {@code body} itself, not to a type nested within it. */ + private static List own(BlockStmt body, Class kind) { + return body.findAll(kind).stream() + .filter(n -> AstScopes.belongsDirectlyTo(n, body)) + .collect(Collectors.toList()); + } } diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java index 0314f65..fa83722 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/L1BuildContext.java @@ -31,9 +31,13 @@ public final class L1BuildContext { private final String fileKey; private final String source; - /** Memoized resolution failures — retrying them is expensive and they recur across a project. */ + /** + * Memoized *type* resolution failures. Safe because a declared type spelling resolves consistently + * within one file (this context is per-file), and retrying an unresolvable spelling is expensive. + * Expression results are deliberately NOT memoized: the same text (`x`) can denote different types + * in different scopes of one file, so caching a failure would blank later resolvable occurrences. + */ private final Set unresolvedTypes = new HashSet<>(); - private final Set unresolvedExpressions = new HashSet<>(); public L1BuildContext(String applicationId, String fileKey, String source) { this.applicationId = applicationId; @@ -75,15 +79,10 @@ public String resolveType(Type type) { * (mirrors the v1 behaviour, where an unresolved expression contributes no type fact). */ public String resolveExpressionType(Expression expression) { - String spelling = expression.toString(); - if (unresolvedExpressions.contains(spelling)) { - return ""; - } try { return expression.calculateResolvedType().describe(); } catch (Throwable e) { - Log.debug("Could not resolve expression: " + spelling + ": " + e.getMessage()); - unresolvedExpressions.add(spelling); + Log.debug("Could not resolve expression: " + expression + ": " + e.getMessage()); return ""; } } @@ -114,9 +113,18 @@ public JComment comment(Comment c) { * invariant {@code module.source[span.bytes] == module.source} always holds. */ public Span wholeFileSpan() { - String[] lines = source.split("\n", -1); - int lastLine = Math.max(1, lines.length); - int lastCol = lines[lines.length - 1].length() + 1; + // The end is the position one past the last character. Lines are split on universal newlines + // (shared with Spans, so \r\n and lone \r count like \n) and keep their terminators, so when the + // file ends with one the position is the start of the following line. + List lines = Spans.splitLinesKeepingTerminators(source); + int lastLine = 1; + int lastCol = 1; + if (!lines.isEmpty()) { + String last = lines.get(lines.size() - 1); + boolean endsWithTerminator = last.endsWith("\n") || last.endsWith("\r"); + lastLine = endsWithTerminator ? lines.size() + 1 : lines.size(); + lastCol = endsWithTerminator ? 1 : last.length() + 1; + } Span span = new Span(); span.setStart(new int[] {1, 1}); span.setEnd(new int[] {lastLine, lastCol}); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java b/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java index 79b69d4..0d54d38 100644 --- a/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/Signatures.java @@ -3,6 +3,7 @@ import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.body.Parameter; +import com.github.javaparser.resolution.declarations.ResolvedConstructorDeclaration; import com.github.javaparser.resolution.declarations.ResolvedMethodLikeDeclaration; import com.github.javaparser.resolution.types.ResolvedType; import com.ibm.cldk.utils.Log; @@ -25,7 +26,11 @@ private Signatures() {} * {@code callee_signature} matches the target callable's {@code signature}. */ public static String typeErasure(ResolvedMethodLikeDeclaration methodDecl) { - StringBuilder signature = new StringBuilder(methodDecl.getName()); + // A ResolvedConstructorDeclaration's name is its *class* name; the declaration side emits + // ``. Using the class name here would make a call site's callee_signature unjoinable + // against the constructor's own signature, so every constructor edge would be missed. + String name = methodDecl instanceof ResolvedConstructorDeclaration ? "" : methodDecl.getName(); + StringBuilder signature = new StringBuilder(name); List erasureParameterTypes = new ArrayList<>(); for (int i = 0; i < methodDecl.getNumberOfParams(); i++) { erasureParameterTypes.add(methodDecl.getParam(i).getType().erasure().describe()); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java index ec850e2..12c5461 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallSiteBuilderTest.java @@ -12,7 +12,9 @@ import com.github.javaparser.ast.body.CallableDeclaration; import com.github.javaparser.ast.body.ConstructorDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.expr.MethodCallExpr; import com.github.javaparser.ast.stmt.BlockStmt; +import com.github.javaparser.ast.stmt.ExpressionStmt; import com.ibm.cldk.schema.CanId; import com.ibm.cldk.schema.JBodyNode; import java.util.ArrayList; @@ -143,6 +145,49 @@ void build_flagsConstructorCall() { assertEquals("java.lang.String", node.getReceiverType(), "the instantiated type"); } + @Test + void build_constructorCalleeSignatureMatchesTheDeclarationSideSignature() { + // The callee signature must be joinable against the target callable's `signature`, which uses + // `` for constructors. A class-named signature would never match, so L2 would silently + // drop every constructor edge. + JBodyNode node = build("package p;\nclass Foo {\n void m() {\n new String(\"x\");\n }\n}\n") + .get("4:9"); + assertEquals("(java.lang.String)", node.getCalleeSignature()); + } + + @Test + void build_resolutionFailureForOneExpressionDoesNotPoisonAnother() { + // Two receivers spelled `x` in different methods: one unresolvable, one not. Memoizing the + // failure by expression text would wrongly blank the second. + String source = "package p;\nclass Foo {\n" + + " void a(Mystery x) { x.f(); }\n" + + " void b(String x) { x.length(); }\n}\n"; + CompilationUnit cu = TestParsers.parseResolved(source); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, source); + CallSiteBuilder builder = new CallSiteBuilder(ctx); + MethodDeclaration a = cu.getType(0).getMethodsByName("a").get(0); + MethodDeclaration b = cu.getType(0).getMethodsByName("b").get(0); + + builder.build(a.getBody().orElseThrow()); // fails to resolve `x` + Map second = builder.build(b.getBody().orElseThrow()); + + assertEquals("java.lang.String", second.values().iterator().next().getReceiverType(), + "the resolvable `x` must still resolve after the unresolvable one"); + } + + @Test + void build_skipsCallSitesWithoutASourceRange() { + // Programmatically constructed nodes carry no range. They cannot be addressed by a line:col id, + // and inventing one would both fabricate a location and collide with any other rangeless node. + BlockStmt body = new BlockStmt(); + body.addStatement(new ExpressionStmt(new MethodCallExpr("foo"))); + body.addStatement(new ExpressionStmt(new MethodCallExpr("bar"))); + L1BuildContext ctx = new L1BuildContext(CanId.applicationId("myapp"), FILE_KEY, "class X {}\n"); + + assertTrue(new CallSiteBuilder(ctx).build(body).isEmpty(), + "rangeless call sites are skipped rather than silently overwriting each other"); + } + @Test void build_unresolvableCallStillEmitsNodeWithoutResolvedFacts() { // Honest degradation: an unresolvable callee must not drop the call node or crash. diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java index 65b83cb..4ad53d3 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/CallableBuilderTest.java @@ -155,6 +155,17 @@ void build_refsTypesIncludeCastsInstanceofAndCatchTypes() { assertTrue(c.getRefs().getTypes().contains("java.lang.IllegalStateException"), "catch type"); } + @Test + void build_cyclomaticMetricExcludesNestedAnonymousClassBranches() { + // Every other metric is scope-filtered; complexity must be too, or the branches of a nested + // class are counted twice — once on it and once on the method that merely declares it. + JCallable c = build("void m(boolean p) { Runnable r = new Runnable() {" + + " public void run() { if (p) {} if (!p) {} } }; }"); + assertEquals(1, c.getMetrics().getCyclomatic(), "m() itself branches nowhere"); + assertEquals(3, c.getTypes().get("$anon$0").getCallables().get("run()") + .getMetrics().getCyclomatic(), "the two ifs belong to run()"); + } + @Test void build_capturesBodyCallNodes() { JCallable c = build("void m() { foo(); }"); diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java index 24c5410..ca36abb 100644 --- a/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/ModuleBuilderTest.java @@ -106,6 +106,25 @@ void build_moduleSpanCoversTheWholeFile() { assertEquals(source.getBytes(StandardCharsets.UTF_8).length, bytes[1]); } + @Test + void build_moduleSpanEndIsCorrectForCrlfSources() { + // Splitting on "\n" alone leaves the "\r" attached, inflating the line count on + // Windows-authored files. + String source = "package p;\r\nclass Foo {}\r\n"; + JModule module = build(source); + assertEquals(3, module.getSpan().getEnd()[0], "two terminated lines -> end on line 3, col 1"); + assertEquals(source.getBytes(StandardCharsets.UTF_8).length, module.getSpan().getBytes()[1]); + } + + @Test + void build_moduleSpanEndHandlesMissingTrailingNewline() { + String source = "package p;\nclass Foo {}"; + JModule module = build(source); + assertEquals(2, module.getSpan().getEnd()[0]); + assertEquals("class Foo {}".length() + 1, module.getSpan().getEnd()[1], + "with no trailing newline the end sits just past the last character"); + } + @Test void build_defaultsPackageToEmptyWhenAbsent() { String source = "public class Foo {}\n";