From 52bd01669ef6b05df4c9e9f62296165ff655b18b Mon Sep 17 00:00:00 2001 From: Simon Massey <322608+simbo1905@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:37:37 +0100 Subject: [PATCH 1/2] Issue #153 treat upstream fetch failure as drift in the API tracker hasDifferences now returns true when missingUpstream > 0, the fingerprint includes UPSTREAM_ERROR classes so issue dedup works for fetch-failure drift, and the summary renders a Missing Upstream section listing each affected class and error. Previously a dead upstream fetch path reported all-clear, which is what blinded the daily tracker after the upstream incubator move (see #145). Verify: mvnd -pl json-java21-api-tracker -am clean test (14 tracker tests green, 3 new); full reactor clean verify = 1679 tests 0 skipped; ci.yml exp_tests 1676 -> 1679. --- .github/workflows/ci.yml | 2 +- .../github/simbo1905/tracker/ApiTracker.java | 45 +++++++- .../simbo1905/tracker/ApiTrackerTest.java | 106 ++++++++++++++++++ 3 files changed, 149 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da35e7c1..bc3a45af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: for k in totals: totals[k]+=int(r.get(k,'0')) except Exception: pass - exp_tests=1676 + exp_tests=1679 exp_skipped=0 if totals['tests']!=exp_tests or totals['skipped']!=exp_skipped: print(f"Unexpected test totals: {totals} != expected tests={exp_tests}, skipped={exp_skipped}") diff --git a/json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java b/json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java index 0266f08f..305668f3 100644 --- a/json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java +++ b/json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java @@ -947,7 +947,7 @@ static String fetchUpstreamSource(String className) { /// @param report the full comparison report /// @return 7-character fingerprint or "0000000" if no differences static String generateFingerprint(JsonObject report) { - if (getDifferentApiCount(report) == 0) { + if (!hasDifferences(report)) { return "0000000"; } @@ -959,6 +959,13 @@ static String generateFingerprint(JsonObject report) { final var diffObj = (JsonObject) diff; final var status = ((JsonString) diffObj.asMap().get("status")).asString(); + if ("UPSTREAM_ERROR".equals(status)) { + // Fetch failures are drift too: include them so the issue + // dedup fingerprint is distinct per affected class set. + stableLines.add(((JsonString) diffObj.asMap().get("className")).asString() + ":UPSTREAM_ERROR"); + continue; + } + if (!"DIFFERENT".equals(status)) continue; final var className = ((JsonString) diffObj.asMap().get("className")).asString(); @@ -1009,6 +1016,21 @@ private static long getDifferentApiCount(JsonObject report) { return 0; } + /// Extracts the missingUpstream count from a report summary + /// @param report the comparison report + /// @return the count of classes whose upstream fetch or parse failed + private static long getMissingUpstreamCount(JsonObject report) { + final var summary = (JsonObject) report.asMap().get("summary"); + if (summary == null) { + return 0; + } + final var missingUpstreamValue = summary.asMap().get("missingUpstream"); + if (missingUpstreamValue instanceof JsonNumber num) { + return num.asLong(); + } + return 0; + } + /// Generates a terse human-readable summary of the API differences /// Suitable for GitHub issue body /// @param report the full comparison report @@ -1068,6 +1090,23 @@ static String generateSummary(JsonObject report) { } } + if (missingUpstream > 0) { + sb.append("## Missing Upstream\n\n"); + sb.append("These classes could not be fetched or parsed upstream. A fetch failure is drift: the tracker cannot confirm we are current.\n\n"); + for (final var diff : differences.asList()) { + final var diffObj = (JsonObject) diff; + final var status = ((JsonString) diffObj.asMap().get("status")).asString(); + + if (!"UPSTREAM_ERROR".equals(status)) continue; + + final var className = ((JsonString) diffObj.asMap().get("className")).asString(); + final var errorValue = diffObj.asMap().get("error"); + final var error = errorValue instanceof JsonString js ? js.asString() : "unknown error"; + sb.append("- ⚠️ **").append(className).append("**: `").append(error).append("`\n"); + } + sb.append("\n"); + } + sb.append("---\n"); final var timestamp = ((JsonString) report.asMap().get("timestamp")).asString(); sb.append("*Generated by API Tracker on ").append(timestamp.split("T")[0]).append("*\n"); @@ -1077,8 +1116,8 @@ static String generateSummary(JsonObject report) { /// Checks if there are any API differences in the report /// @param report the comparison report - /// @return true if differentApi > 0 + /// @return true if differentApi > 0 or missingUpstream > 0 static boolean hasDifferences(JsonObject report) { - return getDifferentApiCount(report) > 0; + return getDifferentApiCount(report) > 0 || getMissingUpstreamCount(report) > 0; } } \ No newline at end of file diff --git a/json-java21-api-tracker/src/test/java/io/github/simbo1905/tracker/ApiTrackerTest.java b/json-java21-api-tracker/src/test/java/io/github/simbo1905/tracker/ApiTrackerTest.java index 3a4501ca..2d46dc95 100644 --- a/json-java21-api-tracker/src/test/java/io/github/simbo1905/tracker/ApiTrackerTest.java +++ b/json-java21-api-tracker/src/test/java/io/github/simbo1905/tracker/ApiTrackerTest.java @@ -9,8 +9,10 @@ import jdk.incubator.java.util.json.JsonBoolean; import jdk.incubator.java.util.json.JsonArray; +import jdk.incubator.java.util.json.JsonNumber; import jdk.incubator.java.util.json.JsonObject; import jdk.incubator.java.util.json.JsonString; +import jdk.incubator.java.util.json.Json; import java.util.Set; import java.util.Map; @@ -234,4 +236,108 @@ void testNormalizeTypeName() { .isEqualTo("String"); } } + + @Nested + @DisplayName("Drift Detection Gates") + class DriftDetectionGateTests { + + private static JsonObject upstreamErrorDiff(String className, String error) { + return JsonObject.of(Map.of( + "className", JsonString.of(className), + "status", JsonString.of("UPSTREAM_ERROR"), + "error", JsonString.of(error) + )); + } + + private static JsonObject report(long differentApi, long missingUpstream, JsonArray differences) { + return JsonObject.of(Map.of( + "timestamp", JsonString.of("2026-08-30T00:00:00Z"), + "summary", JsonObject.of(Map.of( + "totalClasses", JsonNumber.of(differentApi + missingUpstream), + "matchingClasses", JsonNumber.of(0), + "differentApi", JsonNumber.of(differentApi), + "missingUpstream", JsonNumber.of(missingUpstream) + )), + "differences", differences + )); + } + + @Test + @DisplayName("Upstream fetch failures must count as drift, not all-clear") + void testHasDifferencesTreatsUpstreamErrorAsDrift() { + LoggingControl.setupCleanLogging(); + java.util.logging.Logger.getLogger(getClass().getName()) + .info(() -> "TEST: testHasDifferencesTreatsUpstreamErrorAsDrift"); + final var allErrors = report(0, 2, JsonArray.of(java.util.List.of( + upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found"), + upstreamErrorDiff("JsonObject", "HTTP_ERROR: Status 500") + ))); + + assertThat(ApiTracker.hasDifferences(allErrors)) + .as("a fully-blind detector run (every class UPSTREAM_ERROR) must report drift") + .isTrue(); + + final var mixed = report(1, 1, JsonArray.of(java.util.List.of( + upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found"), + JsonObject.of(Map.of( + "className", JsonString.of("JsonValue"), + "status", JsonString.of("DIFFERENT"), + "differences", JsonArray.of(java.util.List.of()) + )) + ))); + assertThat(ApiTracker.hasDifferences(mixed)).isTrue(); + } + + @Test + @DisplayName("Fingerprint covers UPSTREAM_ERROR classes and is stable and distinct") + void testFingerprintCoversUpstreamErrors() { + LoggingControl.setupCleanLogging(); + java.util.logging.Logger.getLogger(getClass().getName()) + .info(() -> "TEST: testFingerprintCoversUpstreamErrors"); + final var none = report(0, 0, JsonArray.of(java.util.List.of())); + assertThat(ApiTracker.generateFingerprint(none)).isEqualTo("0000000"); + + final var errorsA = report(0, 2, JsonArray.of(java.util.List.of( + upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found"), + upstreamErrorDiff("JsonObject", "NOT_FOUND: Upstream file not found") + ))); + final var errorsAgain = report(0, 2, JsonArray.of(java.util.List.of( + upstreamErrorDiff("JsonObject", "NOT_FOUND: Upstream file not found"), + upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found") + ))); + final var errorsB = report(0, 1, JsonArray.of(java.util.List.of( + upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found") + ))); + + final var fpA = ApiTracker.generateFingerprint(errorsA); + assertThat(fpA) + .as("fetch-failure drift must not hash to the no-differences sentinel") + .isNotEqualTo("0000000"); + assertThat(ApiTracker.generateFingerprint(errorsAgain)) + .as("same error set in different order must fingerprint identically") + .isEqualTo(fpA); + assertThat(ApiTracker.generateFingerprint(errorsB)) + .as("different error sets must fingerprint differently") + .isNotEqualTo(fpA); + } + + @Test + @DisplayName("Summary renders a Missing Upstream section for fetch failures") + void testSummaryRendersMissingUpstreamSection() { + LoggingControl.setupCleanLogging(); + java.util.logging.Logger.getLogger(getClass().getName()) + .info(() -> "TEST: testSummaryRendersMissingUpstreamSection"); + final var allErrors = report(0, 2, JsonArray.of(java.util.List.of( + upstreamErrorDiff("JsonNumber", "NOT_FOUND: Upstream file not found (possibly deleted or renamed)"), + upstreamErrorDiff("JsonObject", "HTTP_ERROR: Status 500") + ))); + + final var summary = ApiTracker.generateSummary(allErrors); + + assertThat(summary).contains("Missing Upstream"); + assertThat(summary).contains("JsonNumber"); + assertThat(summary).contains("JsonObject"); + assertThat(summary).contains("NOT_FOUND: Upstream file not found (possibly deleted or renamed)"); + } + } } \ No newline at end of file From 1d31a918dd137600e49bee8529ec684c973dcb0e Mon Sep 17 00:00:00 2001 From: Simon Massey <322608+simbo1905@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:37:37 +0100 Subject: [PATCH 2/2] Issue #154 refresh sync tooling for the incubator module layout RefreshFromUpstream.java now fetches the impl package from src/jdk.incubator.json/.../jdk/incubator/json/impl/, snapshots it under upstream/jdk.incubator.json.impl and skips the local-only LazyConstant polyfill. transform_upstream.py maps the upstream incubator packages to our jdk.incubator.* packages, drops the obsolete StableValue skip and reminds the operator to re-append the Utils.powExact polyfill. Transform regexes dry-run verified against fetched upstream sources. Verify: python3 dry-run of transform() on JsonStringImpl.java (5/5 mapping checks pass); no source-tree changes made by the dry run. --- updates/2025-09-04/RefreshFromUpstream.java | 10 ++++++---- updates/2025-09-04/transform_upstream.py | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/updates/2025-09-04/RefreshFromUpstream.java b/updates/2025-09-04/RefreshFromUpstream.java index 917d1ec0..1efdfcf4 100644 --- a/updates/2025-09-04/RefreshFromUpstream.java +++ b/updates/2025-09-04/RefreshFromUpstream.java @@ -1,5 +1,6 @@ // Compact single-file Java 25 script to refresh and compare impl sources // Run: java RefreshFromUpstream.java +// Refreshed 2026-08-30 for the upstream jdk.incubator.json module layout (issue #154). import java.io.*; import java.net.URI; @@ -37,19 +38,20 @@ void main() throws Exception { // Work dirs (not checked in) Path updatesDir = repoRoot.resolve("updates/2025-09-04"); - Path upstreamDir = updatesDir.resolve("upstream/jdk.internal.util.json"); + Path upstreamDir = updatesDir.resolve("upstream/jdk.incubator.json.impl"); Path reportDir = updatesDir.resolve("reports"); Files.createDirectories(upstreamDir); Files.createDirectories(reportDir); - // Upstream raw base for impl package - String upstreamBase = "https://raw.githubusercontent.com/openjdk/jdk-sandbox/refs/heads/json/src/java.base/share/classes/jdk/internal/util/json/"; + // Upstream raw base for the incubator module impl package + String upstreamBase = "https://raw.githubusercontent.com/openjdk/jdk-sandbox/refs/heads/json/src/jdk.incubator.json/share/classes/jdk/incubator/json/impl/"; - // Discover local impl files + // Discover local impl files (LazyConstant.java is a local-only polyfill with no upstream counterpart) List localFiles; try (var stream = Files.list(localImplDir)) { localFiles = stream .filter(p -> p.getFileName().toString().endsWith(".java")) + .filter(p -> !p.getFileName().toString().equals("LazyConstant.java")) .sorted() .toList(); } diff --git a/updates/2025-09-04/transform_upstream.py b/updates/2025-09-04/transform_upstream.py index 08b29293..5a6c1e73 100644 --- a/updates/2025-09-04/transform_upstream.py +++ b/updates/2025-09-04/transform_upstream.py @@ -1,6 +1,9 @@ import os, sys, re, shutil -SRC = 'updates/2025-09-04/upstream/jdk.internal.util.json' +# Refreshed 2026-08-30 for the upstream jdk.incubator.json module layout (issue #154). +# Scope: impl files fetched into the snapshot dir (see RefreshFromUpstream.java). +# Public API files follow the manual process in json-java21/AGENTS.md. +SRC = 'updates/2025-09-04/upstream/jdk.incubator.json.impl' DST = 'json-java21/src/main/java/jdk/incubator/internal/util/json' def read(path): @@ -27,10 +30,11 @@ def write_safe(path, text): return True def transform(text, name): - # package - text = re.sub(r'^package\s+jdk\.internal\.util\.json;', 'package jdk.incubator.internal.util.json;', text, flags=re.M) - # imports for public API - text = re.sub(r'^(\s*import\s+)java\.util\.json\.', r'\1jdk.incubator.java.util.json.', text, flags=re.M) + # package: upstream impl package -> our internal package + text = re.sub(r'^package\s+jdk\.incubator\.json\.impl;', 'package jdk.incubator.internal.util.json;', text, flags=re.M) + # imports: impl-internal first (defensive; upstream impl rarely imports itself), then public API + text = re.sub(r'^(\s*import\s+)jdk\.incubator\.json\.impl\.', r'\1jdk.incubator.internal.util.json.', text, flags=re.M) + text = re.sub(r'^(\s*import\s+)jdk\.incubator\.json\.', r'\1jdk.incubator.java.util.json.', text, flags=re.M) # annotations (single-line) text = re.sub(r'^\s*@(?:jdk\.internal\..*|ValueBased|StableValue).*\n', '', text, flags=re.M) # remove import of ValueBased if present @@ -57,9 +61,6 @@ def main(): for name in os.listdir(SRC): if not name.endswith('.java'): continue - if name in ('StableValue.java', 'Utils.java'): - # Keep local backport helper and existing Utils for now - continue src_path = os.path.join(SRC, name) dst_path = os.path.join(DST, name) data = read(src_path) @@ -69,6 +70,7 @@ def main(): if not ok: sys.exit(2) print('Transform complete') + print('Reminder: after transforming, re-append the Utils.powExact polyfill (Java 21 lacks Math.powExact) and keep LazyConstant.java untouched (local polyfill).') if __name__ == '__main__': main()