From 1c89903d3731583de048f2700a3cb576a633cf88 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sat, 15 Aug 2026 08:02:25 -0500 Subject: [PATCH] fix: model LangSmith fleet evidence states --- .../maint-80-langsmith-metrics-dashboard.yml | 9 +- .../maint-81-langsmith-fleet-conformance.yml | 10 +- config/langsmith_fleet_allowlist.json | 23 +++ config/langsmith_fleet_registry.json | 11 ++ docs/contracts/langsmith-fleet-v1.md | 45 ++++- docs/ops/DURABLE_TRACKING_ISSUES.md | 7 + scripts/langsmith_fleet.py | 169 ++++++++++++++++-- scripts/langsmith_fleet_conformance.py | 61 ++++++- scripts/metrics_dashboard_generator.py | 14 +- tests/scripts/test_langsmith_fleet.py | 84 ++++++++- .../test_langsmith_fleet_conformance.py | 17 ++ .../test_metrics_dashboard_generator.py | 34 ++-- ...st_langsmith_fleet_conformance_workflow.py | 8 + .../test_langsmith_metrics_dashboard.py | 22 +++ 14 files changed, 454 insertions(+), 60 deletions(-) create mode 100644 config/langsmith_fleet_allowlist.json diff --git a/.github/workflows/maint-80-langsmith-metrics-dashboard.yml b/.github/workflows/maint-80-langsmith-metrics-dashboard.yml index f8971681a..4854709b0 100644 --- a/.github/workflows/maint-80-langsmith-metrics-dashboard.yml +++ b/.github/workflows/maint-80-langsmith-metrics-dashboard.yml @@ -156,8 +156,9 @@ jobs: REGISTRY=config/langsmith_fleet_registry.json mkdir -p .metrics-tmp/fleet trusted_workflow_paths="$(jq -c '.trusted_artifact_workflow_paths // []' "$REGISTRY")" - # Always start from an empty combined file so repos with no artifact - # surface as "missing" (the registry-driven rollup never skips them). + # Always start from an empty combined file so artifact-backed repos + # with no artifact surface as "missing". Direct and not-applicable + # entries remain visible through their explicit status metadata. : > .metrics-tmp/fleet/combined-fleet.ndjson # Enumerate (repo, artifact_name) from the registry. @@ -168,6 +169,8 @@ jobs: with open(sys.argv[1], encoding="utf-8") as fh: registry = json.load(fh) for entry in registry.get("repos", []): + if entry.get("evidence_mode", "artifact") != "artifact": + continue repo = entry.get("repo", "") artifact = entry.get("artifact_name", "langsmith-fleet.ndjson") if repo: @@ -249,7 +252,7 @@ jobs: done < .metrics-tmp/fleet/repos.tsv # Reuse the already-tested rollup. The registry drives the table, so - # every registered repo gets a row even with zero combined records. + # every registered and allowlisted repo gets an explicit status row. python scripts/langsmith_fleet.py .metrics-tmp/fleet/combined-fleet.ndjson \ --registry "$REGISTRY" --summary --format markdown \ > .metrics-tmp/fleet/fleet-status.md diff --git a/.github/workflows/maint-81-langsmith-fleet-conformance.yml b/.github/workflows/maint-81-langsmith-fleet-conformance.yml index 2b31c1114..984c6dfb6 100644 --- a/.github/workflows/maint-81-langsmith-fleet-conformance.yml +++ b/.github/workflows/maint-81-langsmith-fleet-conformance.yml @@ -56,6 +56,9 @@ jobs: fs.mkdirSync('.metrics-tmp/fleet', { recursive: true }); for (const entry of registry.repos) { + if ((entry.evidence_mode || 'artifact') !== 'artifact') { + continue; + } const [owner, repoName] = entry.repo.split('/'); const safeRepo = entry.repo.replace('/', '__'); const outDir = path.join('.metrics-tmp/fleet', safeRepo); @@ -127,7 +130,8 @@ jobs: unzip -q -o "$zip_path" -d "$out_dir" rm "$zip_path" done - jq -c '.repos[]' config/langsmith_fleet_registry.json | while read -r entry; do + jq -c '.repos[] | select((.evidence_mode // "artifact") == "artifact")' \ + config/langsmith_fleet_registry.json | while read -r entry; do repo="$(jq -r '.repo' <<<"$entry")" artifact_name="$(jq -r '.artifact_name' <<<"$entry")" safe_repo="${repo//\//__}" @@ -155,12 +159,12 @@ jobs: > .metrics-tmp/langsmith-fleet-conformance.md cat .metrics-tmp/langsmith-fleet-conformance.md >> "$GITHUB_STEP_SUMMARY" - jq -r '.rows[] | select(.status != "valid") | + jq -r '.rows[] | select(.status == "missing" or .status == "invalid" or .status == "stale") | "::warning title=LangSmith fleet conformance \(.status)::" + "\(.repo)/\(.surface): \(.first_error // "no artifact uploaded")"' \ .metrics-tmp/langsmith-fleet-conformance.json - non_valid_count="$(jq '[.rows[] | select(.status != "valid")] | length' \ + non_valid_count="$(jq '[.rows[] | select(.status == "missing" or .status == "invalid" or .status == "stale")] | length' \ .metrics-tmp/langsmith-fleet-conformance.json)" echo "LANGSMITH_FLEET_NON_VALID_COUNT=${non_valid_count}" >> "$GITHUB_ENV" diff --git a/config/langsmith_fleet_allowlist.json b/config/langsmith_fleet_allowlist.json new file mode 100644 index 000000000..8321f0d40 --- /dev/null +++ b/config/langsmith_fleet_allowlist.json @@ -0,0 +1,23 @@ +{ + "schema_version": "langsmith-fleet-allowlist/v1", + "repos": [ + { + "repo": "stranske/Template", + "status": "not-applicable", + "reason": "Canonical inactive template that receives fleet updates but has no substantive runtime to trace.", + "registry_activation_condition": "Add the substantive repository created from this template to the LangSmith registry when its first LLM-backed runtime or agent-observability surface is implemented." + }, + { + "repo": "stranske/Ready", + "status": "not-applicable", + "reason": "Standby repository maintained for future use; it receives fleet updates but has no active substantive runtime.", + "registry_activation_condition": "Move the activated substantive repository into the LangSmith registry when development begins on an LLM-backed runtime or agent-observability surface." + }, + { + "repo": "stranske/Collab-Admin", + "status": "not-applicable", + "reason": "Inactive administrative/template consumer that receives fleet updates without operating a substantive traced runtime.", + "registry_activation_condition": "Register any substantive repository created from this source when it gains an LLM-backed runtime or agent-observability surface." + } + ] +} diff --git a/config/langsmith_fleet_registry.json b/config/langsmith_fleet_registry.json index 37a5abb5f..c6cf636f0 100644 --- a/config/langsmith_fleet_registry.json +++ b/config/langsmith_fleet_registry.json @@ -15,6 +15,7 @@ "surface": "agent-automation", "operations": ["autopilot", "keepalive", "verifier", "dashboard-ingestion", "durability"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "paused", "required_domain_fields": [ "workflow", @@ -32,6 +33,7 @@ "surface": "planner-runtime", "operations": ["conversation", "tool-call", "itinerary-generation"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented", "required_domain_fields": [ "planning_mode", @@ -48,6 +50,7 @@ "surface": "nl-to-sql", "operations": ["sql-generation", "validation", "execution", "replay"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented", "required_domain_fields": [ "query_category", @@ -64,6 +67,7 @@ "surface": "ai-api", "operations": ["nl-query", "holdings-analysis", "rag"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented", "required_domain_fields": [ "endpoint", @@ -80,6 +84,7 @@ "surface": "risk-reporting", "operations": ["data-quality", "risk-proxy", "limit-monitoring", "report-generation"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented", "required_domain_fields": [ "as_of_date", @@ -96,6 +101,7 @@ "surface": "intake-extraction", "operations": ["package-intake", "extraction", "validation", "review-routing"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented-followup-open", "required_domain_fields": [ "package_id", @@ -112,6 +118,7 @@ "surface": "llm-replay", "operations": ["chain", "replay", "validation", "config-analysis"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented", "required_domain_fields": [ "dataset_id", @@ -128,6 +135,7 @@ "surface": "scenario-analysis", "operations": ["scenario-run", "result-explanation", "run-comparison", "config-patch"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "artifact", "rollout_status": "implemented", "required_domain_fields": [ "scenario_id", @@ -144,6 +152,7 @@ "surface": "agent-automation", "operations": ["autopilot", "keepalive", "verifier"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "langsmith-direct", "rollout_status": "covered-via-langsmith-direct", "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"] }, @@ -155,6 +164,7 @@ "surface": "agent-automation", "operations": ["autopilot", "keepalive", "verifier"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "langsmith-direct", "rollout_status": "covered-via-langsmith-direct", "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"] }, @@ -166,6 +176,7 @@ "surface": "agent-automation", "operations": ["autopilot", "keepalive", "verifier"], "artifact_name": "langsmith-fleet.ndjson", + "evidence_mode": "langsmith-direct", "rollout_status": "covered-via-langsmith-direct", "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"] } diff --git a/docs/contracts/langsmith-fleet-v1.md b/docs/contracts/langsmith-fleet-v1.md index ca9d12d42..d6ecb2e05 100644 --- a/docs/contracts/langsmith-fleet-v1.md +++ b/docs/contracts/langsmith-fleet-v1.md @@ -7,7 +7,8 @@ status rollup. ## Design Decision -The fleet design is contract-first, not package-first: +The fleet design is contract-first, not package-first. It also distinguishes +how coverage is proved from whether LangSmith is applicable at all: - Workflows owns the canonical `langsmith-fleet/v1` record contract, JSON Schema, registry, validator, fixtures, and dashboard status rollup. @@ -23,6 +24,13 @@ The fleet design is contract-first, not package-first: That design would only be worth changing to if the fleet starts seeing repeated schema drift, duplicated validator defects, or cross-repo release coordination failures that outweigh the packaging and version-management overhead. +- `artifact` registry entries prove coverage with a repo-local + `langsmith-fleet.ndjson` artifact. +- `langsmith-direct` registry entries prove agent-automation tracing through the + Workflows-owned direct integration and do not emit a repo-local artifact. +- Repositories in `config/langsmith_fleet_allowlist.json` are explicitly + `not-applicable`: they receive shared maintenance updates, but have no + substantive runtime whose LangSmith coverage could be measured. Closer/verifier agents should consult this section before pausing for a human decision about local validators. If the current repo artifacts conform to the @@ -45,12 +53,16 @@ emitters against the contract: | `stranske/Trend_Model_Project` | `#5311` | `#5328` | Repo-local emitter merged. | | `stranske/Portable-Alpha-Extension-Model` | `#1802` | `#1819` | Repo-local emitter merged. | | `stranske/Manager-Database` | `#1048` | `#1067` | Repo-local emitter merged. | +| `stranske/Travel-Plan-Permission` | `#1238` | `#2487` (registry) | Covered by Workflows-owned direct agent tracing; no repo-local artifact expected. | +| `stranske/learning-management-system` | `#334` | `#2487` (registry) | Covered by Workflows-owned direct agent tracing; no repo-local artifact expected. | +| `stranske/Fine-Art-Archive` | `#114` | `#2487` (registry) | Covered by Workflows-owned direct agent tracing; no repo-local artifact expected. | ## Artifact -Each participating repo emits NDJSON at the artifact name registered in -`config/langsmith_fleet_registry.json` (default `langsmith-fleet.ndjson`). Each -line is one JSON object. +Each `evidence_mode=artifact` repo emits NDJSON at the artifact name registered +in `config/langsmith_fleet_registry.json` (default +`langsmith-fleet.ndjson`). Each line is one JSON object. Direct-evidence and +not-applicable repositories do not emit this artifact. The artifact must be safe to publish in GitHub Actions artifacts and dashboards: raw prompts, personal data, documents, SQL result rows, generated report text, @@ -100,15 +112,34 @@ block or reopen the original merge path. - repo, issue, and issue number, - surface and allowed operation family, - artifact name, +- evidence mode (`artifact` or `langsmith-direct`), - rollout status, - required domain fields. +`config/langsmith_fleet_allowlist.json` covers registered maintenance consumers +where runtime observability is intentionally not applicable. The current +allowlist is Template, Ready, and Collab-Admin. A repository created from one +of those templates must move into the registry when it gains a substantive +runtime; that activation condition is recorded in each allowlist entry and is +validated alongside Maint 68 consumer coverage. + Dashboard status is computed per registry entry: - `missing`: no record was emitted for that repo/surface. - `invalid`: records exist but fail the shared or domain-field contract. - `stale`: latest valid record is older than the registry freshness window. - `valid`: at least one current valid record exists. +- `direct`: coverage uses Workflows-owned direct LangSmith tracing, so no + repo-local artifact is expected. +- `not-applicable`: the repository is an explicitly allowlisted maintenance + consumer without a substantive runtime. + +`missing`, `invalid`, and `stale` apply only to artifact-backed entries. A +missing artifact means the GitHub dashboard cannot prove current coverage; it +does not, by itself, prove that LangSmith tracing failed. Conversely, a direct +entry proves the configured integration path, not live per-repo trace success; +live direct-trace health must be reported by the Workflows automation telemetry +surface. ## Validation @@ -134,6 +165,8 @@ The canonical schema is versioned at ## Repo Responsibilities -Repo-specific issues should add instrumentation and emit compatible records. +Artifact-backed repo issues should add instrumentation and emit compatible records. They must not move domain tracing logic into Workflows. Workflows only validates -the emitted artifact and displays fleet status. +the emitted artifact and displays fleet status. Direct-evidence repos rely on +the Workflows-owned agent tracing path, while allowlisted repos must be promoted +to the registry as soon as substantive runtime behavior is introduced. diff --git a/docs/ops/DURABLE_TRACKING_ISSUES.md b/docs/ops/DURABLE_TRACKING_ISSUES.md index 2848376b3..88d168b98 100644 --- a/docs/ops/DURABLE_TRACKING_ISSUES.md +++ b/docs/ops/DURABLE_TRACKING_ISSUES.md @@ -47,6 +47,13 @@ The signal flow each tracker carries: - **#2211** — health check on the weekly metrics pipeline. Healthy state is `Parse errors: 0` and non-zero terminal disposition records. A regression here usually means a producer is emitting a malformed artifact, not that the dashboard itself is broken. - **#1836** — work queue for items the local Codex watcher should claim. The body holds the live queue state with a sync hash, repo counts, and per-item status. Active campaigns must not be closed; the controller treats a closed campaign as "stop work." - **#2897** — Health 83 dependency/sync maintenance-efficiency advisory evidence. Comments append only when the material-evidence fingerprint changes; do not redirect this signal onto `#1836`. +- **#2415** — LangSmith fleet observability status. Read the evidence mode as + well as the status: `valid`/`missing`/`stale`/`invalid` are artifact-backed; + `direct` means Workflows-owned direct tracing with no repo-local artifact; + and `not-applicable` is an explicit allowlist state for maintenance-only + consumers. Only artifact-backed `missing`, `stale`, or `invalid` rows are + actionable coverage debt. A missing row means the dashboard lacks evidence, + not that the tracker itself should be closed. ### Leased generated delivery attempts diff --git a/scripts/langsmith_fleet.py b/scripts/langsmith_fleet.py index e3d1fcfd3..c0e6f79f9 100644 --- a/scripts/langsmith_fleet.py +++ b/scripts/langsmith_fleet.py @@ -17,8 +17,26 @@ SCHEMA_VERSION = "langsmith-fleet/v1" SCHEMA_PATH = Path("docs/contracts/schemas/langsmith-fleet-v1.schema.json") REGISTRY_SCHEMA_VERSION = "langsmith-fleet-registry/v1" +ALLOWLIST_SCHEMA_VERSION = "langsmith-fleet-allowlist/v1" +ALLOWLIST_PATH = Path("config/langsmith_fleet_allowlist.json") PARENT_WORKFLOWS_ISSUE = "stranske/Workflows#2150" AGENT_CAPACITY_WINDOWS = {"5h", "weekly", "daily"} +VALID_EVIDENCE_MODES = {"artifact", "langsmith-direct"} +MANAGED_CONSUMER_REPOS = { + "stranske/Travel-Plan-Permission", + "stranske/Template", + "stranske/Counter_Risk", + "stranske/Pension-Data", + "stranske/Inv-Man-Intake", + "stranske/Ready", + "stranske/trip-planner", + "stranske/Manager-Database", + "stranske/Portable-Alpha-Extension-Model", + "stranske/Trend_Model_Project", + "stranske/Collab-Admin", + "stranske/learning-management-system", + "stranske/Fine-Art-Archive", +} REQUIRED_ACTIVE_REPO_ISSUES = { "stranske/trip-planner": 1208, "stranske/Pension-Data": 445, @@ -107,12 +125,29 @@ def load_ndjson(path: Path) -> tuple[list[dict[str, Any]], list[ValidationError] return records, errors -def load_registry(path: Path) -> dict[str, Any]: +def load_allowlist(path: Path = ALLOWLIST_PATH) -> dict[str, Any]: + """Load and validate repositories where LangSmith is explicitly not applicable.""" + data = json.loads(path.read_text()) + if not isinstance(data, dict): + raise ValueError("allowlist must be a JSON object") + validate_allowlist(data) + return data + + +def load_registry( + path: Path, + *, + allowlist_path: Path | None = ALLOWLIST_PATH, +) -> dict[str, Any]: """Load the fleet registry JSON.""" data = json.loads(path.read_text()) if not isinstance(data, dict): raise ValueError("registry must be a JSON object") - validate_registry(data) + allowlist = None + if allowlist_path is not None and allowlist_path.exists(): + allowlist = load_allowlist(allowlist_path) + data["_allowlist"] = allowlist + validate_registry(data, allowlist=allowlist) return data @@ -129,7 +164,36 @@ def registry_surfaces(registry: dict[str, Any]) -> dict[tuple[str, str], dict[st return indexed -def validate_registry(registry: dict[str, Any]) -> None: +def validate_allowlist(allowlist: dict[str, Any]) -> None: + """Validate explicit non-applicable repository coverage.""" + if allowlist.get("schema_version") != ALLOWLIST_SCHEMA_VERSION: + raise ValueError(f"allowlist schema_version must be {ALLOWLIST_SCHEMA_VERSION}") + entries = allowlist.get("repos") + if not isinstance(entries, list) or not entries: + raise ValueError("allowlist repos must be a non-empty list") + seen: set[str] = set() + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise ValueError(f"allowlist repos[{index}] must be an object") + repo = entry.get("repo") + if not isinstance(repo, str) or not repo.strip(): + raise ValueError(f"allowlist repos[{index}].repo must be a non-empty string") + normalized = repo.strip() + if normalized in seen: + raise ValueError(f"allowlist has duplicate repo {normalized}") + seen.add(normalized) + if entry.get("status") != "not-applicable": + raise ValueError(f"allowlist repos[{index}].status must be not-applicable") + for field in ("reason", "registry_activation_condition"): + if not isinstance(entry.get(field), str) or not str(entry[field]).strip(): + raise ValueError(f"allowlist repos[{index}].{field} must be a non-empty string") + + +def validate_registry( + registry: dict[str, Any], + *, + allowlist: dict[str, Any] | None = None, +) -> None: """Validate registry structure and required mappings.""" if "agents" in registry and "repos" not in registry: validate_agent_registry_capacity(registry) @@ -138,6 +202,11 @@ def validate_registry(registry: dict[str, Any]) -> None: if registry.get("schema_version") != REGISTRY_SCHEMA_VERSION: raise ValueError(f"registry schema_version must be {REGISTRY_SCHEMA_VERSION}") + if allowlist is None: + embedded_allowlist = registry.get("_allowlist") + if isinstance(embedded_allowlist, dict): + allowlist = embedded_allowlist + stale_after_hours = registry.get("stale_after_hours") if isinstance(stale_after_hours, bool) or not isinstance(stale_after_hours, int): raise ValueError("registry stale_after_hours must be an integer") @@ -162,6 +231,7 @@ def validate_registry(registry: dict[str, Any]) -> None: operations = entry.get("operations") required_domain_fields = entry.get("required_domain_fields") artifact_name = entry.get("artifact_name") + evidence_mode = entry.get("evidence_mode") rollout_status = entry.get("rollout_status") if not isinstance(repo, str) or not repo.strip(): @@ -205,6 +275,9 @@ def validate_registry(registry: dict[str, Any]) -> None: if not isinstance(artifact_name, str) or not artifact_name.strip(): raise ValueError(f"registry repos[{index}].artifact_name must be a non-empty string") + if evidence_mode not in VALID_EVIDENCE_MODES: + allowed = ", ".join(sorted(VALID_EVIDENCE_MODES)) + raise ValueError(f"registry repos[{index}].evidence_mode must be one of {allowed}") if not isinstance(rollout_status, str) or not rollout_status.strip(): raise ValueError(f"registry repos[{index}].rollout_status must be a non-empty string") @@ -233,6 +306,25 @@ def validate_registry(registry: dict[str, Any]) -> None: + ", ".join(sorted(missing_required)) ) + allowlist_entries = (allowlist or {}).get("repos", []) + allowlisted_repos = { + str(entry.get("repo", "")).strip() + for entry in allowlist_entries + if isinstance(entry, dict) and str(entry.get("repo", "")).strip() + } + overlap = sorted(set(seen_repo_issues) & allowlisted_repos) + if overlap: + raise ValueError( + "repositories cannot be both registered and allowlisted: " + ", ".join(overlap) + ) + covered_consumers = set(seen_repo_issues) | allowlisted_repos + unclassified = sorted(MANAGED_CONSUMER_REPOS - covered_consumers) + if unclassified: + raise ValueError( + "managed consumer repos missing registry or allowlist status: " + + ", ".join(unclassified) + ) + def validate_agent_registry_capacity(registry: dict[str, Any]) -> None: """Validate capacity blocks in the YAML agent registry.""" @@ -417,19 +509,30 @@ def summarize_fleet_records( for key, entry in sorted(registry_entries.items()): repo, surface = key matching = by_repo_surface.get(key, []) - validation_errors = validate_records(matching, registry=registry) if matching else [] - latest_recorded_at = _latest_recorded_at(matching) - if not matching: - status = "missing" - elif validation_errors: - status = "invalid" - elif ( - latest_recorded_at - and (now - latest_recorded_at).total_seconds() > stale_after_hours * 3600 - ): - status = "stale" + evidence_mode = str(entry.get("evidence_mode") or "artifact") + if evidence_mode == "langsmith-direct": + validation_errors: list[ValidationError] = [] + latest_recorded_at = None + status = "direct" + reason = ( + "Evidence is ingested from the central LangSmith project; " + "a repo-local fleet artifact is not required or validated here." + ) else: - status = "valid" + validation_errors = validate_records(matching, registry=registry) if matching else [] + latest_recorded_at = _latest_recorded_at(matching) + reason = None + if not matching: + status = "missing" + elif validation_errors: + status = "invalid" + elif ( + latest_recorded_at + and (now - latest_recorded_at).total_seconds() > stale_after_hours * 3600 + ): + status = "stale" + else: + status = "valid" statuses[status] += 1 rows.append( { @@ -437,6 +540,7 @@ def summarize_fleet_records( "surface": surface, "issue": entry.get("issue"), "artifact_name": entry.get("artifact_name"), + "evidence_mode": evidence_mode, "rollout_status": entry.get("rollout_status"), "record_count": len(matching), "latest_recorded_at": ( @@ -444,12 +548,35 @@ def summarize_fleet_records( ), "status": status, "first_error": validation_errors[0].message if validation_errors else None, + "reason": reason, + } + ) + + allowlist = registry.get("_allowlist") or {} + allowlist_entries = allowlist.get("repos", []) if isinstance(allowlist, dict) else [] + for entry in sorted(allowlist_entries, key=lambda item: str(item.get("repo", ""))): + statuses["not-applicable"] += 1 + rows.append( + { + "repo": entry["repo"], + "surface": "", + "issue": None, + "artifact_name": None, + "evidence_mode": "none", + "rollout_status": "not-applicable", + "record_count": 0, + "latest_recorded_at": None, + "status": "not-applicable", + "first_error": None, + "reason": entry["reason"], + "registry_activation_condition": entry["registry_activation_condition"], } ) return { "schema_version": SCHEMA_VERSION, "total_registry_entries": len(registry_entries), + "total_allowlisted_repos": len(allowlist_entries), "status_counts": dict(sorted(statuses.items())), "rows": rows, } @@ -477,23 +604,27 @@ def format_fleet_summary(summary: dict[str, Any]) -> str: lines = ["# LangSmith Fleet Artifact Status", ""] counts = summary.get("status_counts", {}) lines.append(f"- Registry entries: {summary.get('total_registry_entries', 0)}") + lines.append(f"- Allowlisted repositories: {summary.get('total_allowlisted_repos', 0)}") lines.append(f"- Valid: {counts.get('valid', 0)}") lines.append(f"- Missing: {counts.get('missing', 0)}") lines.append(f"- Stale: {counts.get('stale', 0)}") lines.append(f"- Invalid: {counts.get('invalid', 0)}") + lines.append(f"- Direct evidence: {counts.get('direct', 0)}") + lines.append(f"- Not applicable: {counts.get('not-applicable', 0)}") lines.append("") - lines.append("| Repo | Surface | Issue | Status | Records | Latest | First Error |") - lines.append("|------|---------|-------|--------|---------|--------|-------------|") + lines.append("| Repo | Surface | Evidence | Issue | Status | Records | Latest | Detail |") + lines.append("|------|---------|----------|-------|--------|---------|--------|--------|") for row in summary.get("rows", []): lines.append( - "| {repo} | {surface} | {issue} | {status} | {record_count} | {latest} | {error} |".format( + "| {repo} | {surface} | {evidence} | {issue} | {status} | {record_count} | {latest} | {detail} |".format( repo=row["repo"], surface=row["surface"], + evidence=row.get("evidence_mode") or "", issue=row.get("issue") or "", status=row["status"], record_count=row["record_count"], latest=row.get("latest_recorded_at") or "", - error=row.get("first_error") or "", + detail=row.get("first_error") or row.get("reason") or "", ) ) return "\n".join(lines) + "\n" diff --git a/scripts/langsmith_fleet_conformance.py b/scripts/langsmith_fleet_conformance.py index 3e63c7949..94b4e1418 100644 --- a/scripts/langsmith_fleet_conformance.py +++ b/scripts/langsmith_fleet_conformance.py @@ -54,18 +54,30 @@ def build_conformance_report( now = now or datetime.now(UTC) schema = langsmith_fleet.load_record_schema() rows: list[dict[str, Any]] = [] - counts: dict[str, int] = {"missing": 0, "invalid": 0, "stale": 0, "valid": 0} + counts: dict[str, int] = { + "missing": 0, + "invalid": 0, + "stale": 0, + "valid": 0, + "direct": 0, + "not-applicable": 0, + } for entry in sorted(registry["repos"], key=lambda item: (item["repo"], item["surface"])): repo = entry["repo"] surface = entry["surface"] artifact_name = entry["artifact_name"] + evidence_mode = str(entry.get("evidence_mode") or "artifact") artifact_path = artifact_paths.get(repo) first_error: str | None = None latest_recorded_at: datetime | None = None record_count = 0 - if artifact_path is None or not artifact_path.exists(): + if evidence_mode == "langsmith-direct": + status = "direct" + first_error = None + artifact_path = None + elif artifact_path is None or not artifact_path.exists(): status = "missing" else: records, parse_errors = langsmith_fleet.load_ndjson(artifact_path) @@ -111,6 +123,7 @@ def build_conformance_report( "surface": surface, "issue": entry["issue"], "artifact_name": artifact_name, + "evidence_mode": evidence_mode, "artifact_path": str(artifact_path) if artifact_path else None, "record_count": record_count, "latest_recorded_at": ( @@ -118,6 +131,33 @@ def build_conformance_report( ), "status": status, "first_error": first_error, + "reason": ( + "Evidence is ingested from the central LangSmith project; " + "this artifact report does not validate that external path." + if status == "direct" + else None + ), + } + ) + + allowlist = registry.get("_allowlist") or {} + allowlist_entries = allowlist.get("repos", []) if isinstance(allowlist, dict) else [] + for entry in sorted(allowlist_entries, key=lambda item: str(item.get("repo", ""))): + counts["not-applicable"] += 1 + rows.append( + { + "repo": entry["repo"], + "surface": "", + "issue": None, + "artifact_name": None, + "evidence_mode": "none", + "artifact_path": None, + "record_count": 0, + "latest_recorded_at": None, + "status": "not-applicable", + "first_error": None, + "reason": entry["reason"], + "registry_activation_condition": entry["registry_activation_condition"], } ) @@ -125,7 +165,8 @@ def build_conformance_report( "schema_version": REPORT_SCHEMA_VERSION, "generated_at": now.isoformat(), "status_counts": counts, - "total_registry_entries": len(rows), + "total_registry_entries": len(registry["repos"]), + "total_allowlisted_repos": len(allowlist_entries), "rows": rows, } @@ -137,23 +178,27 @@ def format_conformance_markdown(report: dict[str, Any]) -> str: "# LangSmith Fleet Conformance", "", f"- Registry entries: {report.get('total_registry_entries', 0)}", + f"- Allowlisted repositories: {report.get('total_allowlisted_repos', 0)}", f"- Valid: {counts.get('valid', 0)}", f"- Missing: {counts.get('missing', 0)}", f"- Stale: {counts.get('stale', 0)}", f"- Invalid: {counts.get('invalid', 0)}", + f"- Direct evidence: {counts.get('direct', 0)}", + f"- Not applicable: {counts.get('not-applicable', 0)}", "", - "| Repo | Surface | Status | Records | Latest | First Error |", - "|------|---------|--------|---------|--------|-------------|", + "| Repo | Surface | Evidence | Status | Records | Latest | Detail |", + "|------|---------|----------|--------|---------|--------|--------|", ] for row in report.get("rows", []): lines.append( - "| {repo} | {surface} | {status} | {record_count} | {latest} | {error} |".format( + "| {repo} | {surface} | {evidence} | {status} | {record_count} | {latest} | {detail} |".format( repo=row["repo"], surface=row["surface"], + evidence=row.get("evidence_mode") or "", status=row["status"], record_count=row["record_count"], latest=row.get("latest_recorded_at") or "", - error=row.get("first_error") or "", + detail=row.get("first_error") or row.get("reason") or "", ) ) return "\n".join(lines) + "\n" @@ -163,6 +208,8 @@ def artifact_paths_from_root(records_root: Path, registry: dict[str, Any]) -> di """Build repo -> artifact path mapping from the conventional records root.""" paths: dict[str, Path] = {} for entry in registry["repos"]: + if entry.get("evidence_mode") == "langsmith-direct": + continue paths[entry["repo"]] = artifact_path_for_repo( records_root, entry["repo"], diff --git a/scripts/metrics_dashboard_generator.py b/scripts/metrics_dashboard_generator.py index 95b06396e..2f644523f 100755 --- a/scripts/metrics_dashboard_generator.py +++ b/scripts/metrics_dashboard_generator.py @@ -314,23 +314,29 @@ def build_dashboard( lines.append("") counts = fleet_summary.get("status_counts", {}) lines.append(f"- Registry entries: {fleet_summary.get('total_registry_entries', 0)}") + lines.append( + f"- Allowlisted repositories: {fleet_summary.get('total_allowlisted_repos', 0)}" + ) lines.append(f"- Valid: {counts.get('valid', 0)}") lines.append(f"- Missing: {counts.get('missing', 0)}") lines.append(f"- Stale: {counts.get('stale', 0)}") lines.append(f"- Invalid: {counts.get('invalid', 0)}") + lines.append(f"- Direct evidence: {counts.get('direct', 0)}") + lines.append(f"- Not applicable: {counts.get('not-applicable', 0)}") lines.append("") - lines.append("| Repo | Surface | Issue | Status | Records | Latest | First Error |") - lines.append("|------|---------|-------|--------|---------|--------|-------------|") + lines.append("| Repo | Surface | Evidence | Issue | Status | Records | Latest | Detail |") + lines.append("|------|---------|----------|-------|--------|---------|--------|--------|") for row in fleet_summary.get("rows", []): lines.append( - "| {repo} | {surface} | {issue} | {status} | {record_count} | {latest} | {error} |".format( + "| {repo} | {surface} | {evidence} | {issue} | {status} | {record_count} | {latest} | {detail} |".format( repo=row["repo"], surface=row["surface"], + evidence=row.get("evidence_mode") or "", issue=row.get("issue") or "", status=row["status"], record_count=row["record_count"], latest=row.get("latest_recorded_at") or "", - error=row.get("first_error") or "", + detail=row.get("first_error") or row.get("reason") or "", ) ) lines.append("") diff --git a/tests/scripts/test_langsmith_fleet.py b/tests/scripts/test_langsmith_fleet.py index 21552921a..91bfd7ddd 100644 --- a/tests/scripts/test_langsmith_fleet.py +++ b/tests/scripts/test_langsmith_fleet.py @@ -9,6 +9,8 @@ ROOT = Path(__file__).resolve().parents[2] REGISTRY = ROOT / "config" / "langsmith_fleet_registry.json" +ALLOWLIST = ROOT / "config" / "langsmith_fleet_allowlist.json" +MAINT_68 = ROOT / ".github" / "workflows" / "maint-68-sync-consumer-repos.yml" AGENT_REGISTRY = ROOT / ".github" / "agents" / "registry.yml" FIXTURES = ROOT / "tests" / "fixtures" / "langsmith_fleet" @@ -103,6 +105,8 @@ def test_summary_distinguishes_valid_missing_and_invalid() -> None: assert rows[("stranske/trip-planner", "planner-runtime")]["status"] == "valid" assert rows[("stranske/Pension-Data", "nl-to-sql")]["status"] == "invalid" assert rows[("stranske/Counter_Risk", "risk-reporting")]["status"] == "missing" + assert rows[("stranske/Travel-Plan-Permission", "agent-automation")]["status"] == "direct" + assert rows[("stranske/Ready", "")]["status"] == "not-applicable" def test_summary_distinguishes_stale_records() -> None: @@ -127,7 +131,11 @@ def test_markdown_summary_contains_dashboard_status_table() -> None: markdown = langsmith_fleet.format_fleet_summary(summary) assert "# LangSmith Fleet Artifact Status" in markdown - assert "| stranske/Workflows | agent-automation | stranske/Workflows#2150 |" in markdown + assert ( + "| stranske/Workflows | agent-automation | artifact | " + "stranske/Workflows#2150 |" in markdown + ) + assert "| stranske/Ready | | none | | not-applicable |" in markdown def test_markdown_summary_renders_mixed_valid_invalid_missing_rows() -> None: @@ -171,13 +179,21 @@ def test_markdown_summary_renders_mixed_valid_invalid_missing_rows() -> None: assert f"- Registry entries: {len(registry['repos'])}" in markdown assert "- Invalid: 1" in markdown assert "- Missing: 1" in markdown + assert "- Direct evidence: 3" in markdown + assert "- Not applicable: 3" in markdown # Per-repo status rows, one per status flavor. - assert "| stranske/Workflows | agent-automation | stranske/Workflows#2150 | valid |" in markdown - assert "| stranske/Pension-Data | nl-to-sql | stranske/Pension-Data#445 | invalid |" in markdown assert ( - "| stranske/Counter_Risk | risk-reporting | stranske/Counter_Risk#610 | missing |" - in markdown + "| stranske/Workflows | agent-automation | artifact | " + "stranske/Workflows#2150 | valid |" in markdown + ) + assert ( + "| stranske/Pension-Data | nl-to-sql | artifact | " + "stranske/Pension-Data#445 | invalid |" in markdown + ) + assert ( + "| stranske/Counter_Risk | risk-reporting | artifact | " + "stranske/Counter_Risk#610 | missing |" in markdown ) @@ -322,6 +338,64 @@ def test_registry_contains_active_repo_issue_mappings() -> None: assert by_repo[repo]["parent_issue"] == langsmith_fleet.PARENT_WORKFLOWS_ISSUE +def test_managed_consumers_are_registered_or_explicitly_allowlisted() -> None: + registry = langsmith_fleet.load_registry(REGISTRY) + allowlist = langsmith_fleet.load_allowlist(ALLOWLIST) + registered = { + entry["repo"] for entry in registry["repos"] if entry["repo"] != "stranske/Workflows" + } + allowlisted = {entry["repo"] for entry in allowlist["repos"]} + + source = MAINT_68.read_text(encoding="utf-8") + block = source.split("REGISTERED_CONSUMER_REPOS: |", 1)[1].split("\n\n", 1)[0] + maintained = {line.strip() for line in block.splitlines() if line.strip()} + + assert maintained == langsmith_fleet.MANAGED_CONSUMER_REPOS + assert maintained == registered | allowlisted + assert not registered & allowlisted + + +def test_direct_evidence_repos_do_not_require_artifacts() -> None: + registry = langsmith_fleet.load_registry(REGISTRY) + summary = langsmith_fleet.summarize_fleet_records([], registry=registry) + rows = {(row["repo"], row["surface"]): row for row in summary["rows"]} + + for repo in ( + "stranske/Travel-Plan-Permission", + "stranske/learning-management-system", + "stranske/Fine-Art-Archive", + ): + assert rows[(repo, "agent-automation")]["status"] == "direct" + assert rows[(repo, "agent-automation")]["record_count"] == 0 + + +def test_registry_rejects_repo_that_is_also_allowlisted() -> None: + registry = langsmith_fleet.load_registry(REGISTRY) + allowlist = langsmith_fleet.load_allowlist(ALLOWLIST) + allowlist["repos"].append( + { + "repo": "stranske/trip-planner", + "status": "not-applicable", + "reason": "deliberate overlap", + "registry_activation_condition": "already active", + } + ) + + with pytest.raises(ValueError, match="both registered and allowlisted"): + langsmith_fleet.validate_registry(registry, allowlist=allowlist) + + +def test_registry_rejects_unclassified_managed_consumer() -> None: + registry = langsmith_fleet.load_registry(REGISTRY) + allowlist = langsmith_fleet.load_allowlist(ALLOWLIST) + allowlist["repos"] = [ + entry for entry in allowlist["repos"] if entry["repo"] != "stranske/Ready" + ] + + with pytest.raises(ValueError, match="stranske/Ready"): + langsmith_fleet.validate_registry(registry, allowlist=allowlist) + + def test_registry_rejects_missing_parent_issue(tmp_path: Path) -> None: registry = langsmith_fleet.load_registry(REGISTRY) registry["repos"][1].pop("parent_issue") diff --git a/tests/scripts/test_langsmith_fleet_conformance.py b/tests/scripts/test_langsmith_fleet_conformance.py index 1ebcf7745..1cbd0a0c1 100644 --- a/tests/scripts/test_langsmith_fleet_conformance.py +++ b/tests/scripts/test_langsmith_fleet_conformance.py @@ -85,3 +85,20 @@ def test_artifact_paths_from_root_uses_registry_artifact_names(tmp_path: Path) - assert paths["stranske/Portable-Alpha-Extension-Model"] == ( tmp_path / "stranske__Portable-Alpha-Extension-Model" / "langsmith-fleet.ndjson" ) + assert "stranske/Travel-Plan-Permission" not in paths + + +def test_conformance_reports_direct_and_not_applicable_without_artifacts(tmp_path: Path) -> None: + registry = langsmith_fleet.load_registry(REGISTRY) + + report = langsmith_fleet_conformance.build_conformance_report( + {}, + registry=registry, + now=datetime(2026, 5, 30, 1, 0, tzinfo=UTC), + ) + rows = {(row["repo"], row["surface"]): row for row in report["rows"]} + + assert rows[("stranske/Travel-Plan-Permission", "agent-automation")]["status"] == "direct" + assert rows[("stranske/Ready", "")]["status"] == "not-applicable" + assert report["status_counts"]["direct"] == 3 + assert report["status_counts"]["not-applicable"] == 3 diff --git a/tests/scripts/test_metrics_dashboard_generator.py b/tests/scripts/test_metrics_dashboard_generator.py index 76ca84848..d479f4156 100644 --- a/tests/scripts/test_metrics_dashboard_generator.py +++ b/tests/scripts/test_metrics_dashboard_generator.py @@ -252,13 +252,15 @@ def test_build_dashboard_from_path_includes_langsmith_fleet_status(tmp_path: Pat assert errors == 0 assert "## LangSmith Fleet Artifact Status" in dashboard - assert "- Valid: 11" in dashboard + assert "- Valid: 8" in dashboard assert "- Missing: 0" in dashboard assert "- Stale: 0" in dashboard assert "- Invalid: 0" in dashboard + assert "- Direct evidence: 3" in dashboard + assert "- Not applicable: 3" in dashboard assert ( - "| stranske/trip-planner | planner-runtime | stranske/trip-planner#1208 | valid |" - in dashboard + "| stranske/trip-planner | planner-runtime | artifact | " + "stranske/trip-planner#1208 | valid |" in dashboard ) @@ -284,8 +286,8 @@ def test_build_dashboard_from_path_marks_invalid_langsmith_fleet_records(tmp_pat assert "## LangSmith Fleet Artifact Status" in dashboard assert "- Invalid: 1" in dashboard assert ( - "| stranske/trip-planner | planner-runtime | stranske/trip-planner#1208 | invalid |" - in dashboard + "| stranske/trip-planner | planner-runtime | artifact | " + "stranske/trip-planner#1208 | invalid |" in dashboard ) @@ -333,11 +335,13 @@ def summarize_with_fixed_now(*args: object, **kwargs: object) -> dict[str, objec assert exit_code == 0 content = output_path.read_text(encoding="utf-8") assert "## LangSmith Fleet Artifact Status" in content - assert "- Valid: 11" in content + assert "- Valid: 8" in content + assert "- Direct evidence: 3" in content + assert "- Not applicable: 3" in content def test_build_dashboard_from_path_mixed_fleet_status(tmp_path: Path) -> None: - """mixed.ndjson: 1 valid (Workflows) + 1 invalid (trip-planner) + 9 missing repos.""" + """Mixed records keep artifact, direct, and not-applicable states separate.""" metrics_path = tmp_path / "metrics.ndjson" metrics_path.write_text( '{"repo": "octo/alpha", "duration_ms": 10, "timestamp": "2024-01-01T00:00:00Z"}\n', @@ -360,16 +364,20 @@ def test_build_dashboard_from_path_mixed_fleet_status(tmp_path: Path) -> None: assert "## LangSmith Fleet Artifact Status" in dashboard assert "- Valid: 1" in dashboard assert "- Invalid: 1" in dashboard - assert "- Missing: 9" in dashboard + assert "- Missing: 6" in dashboard + assert "- Direct evidence: 3" in dashboard + assert "- Not applicable: 3" in dashboard assert ( - "| stranske/Workflows | agent-automation | stranske/Workflows#2150 | valid |" in dashboard + "| stranske/Workflows | agent-automation | artifact | " + "stranske/Workflows#2150 | valid |" in dashboard ) assert ( - "| stranske/trip-planner | planner-runtime | stranske/trip-planner#1208 | invalid |" - in dashboard + "| stranske/trip-planner | planner-runtime | artifact | " + "stranske/trip-planner#1208 | invalid |" in dashboard ) assert ( - "| stranske/Pension-Data | nl-to-sql | stranske/Pension-Data#445 | missing |" in dashboard + "| stranske/Pension-Data | nl-to-sql | artifact | " + "stranske/Pension-Data#445 | missing |" in dashboard ) fleet_section = dashboard.split("## LangSmith Fleet Artifact Status", maxsplit=1)[1] status_rows = [ @@ -377,4 +385,4 @@ def test_build_dashboard_from_path_mixed_fleet_status(tmp_path: Path) -> None: for line in fleet_section.splitlines() if line.startswith("| stranske/") and line.endswith(" |") ] - assert len(status_rows) == 11 + assert len(status_rows) == 14 diff --git a/tests/workflows/test_langsmith_fleet_conformance_workflow.py b/tests/workflows/test_langsmith_fleet_conformance_workflow.py index bb1e80385..170875fda 100644 --- a/tests/workflows/test_langsmith_fleet_conformance_workflow.py +++ b/tests/workflows/test_langsmith_fleet_conformance_workflow.py @@ -17,3 +17,11 @@ def test_conformance_download_accepts_prefixed_fleet_artifacts() -> None: assert source.count("trusted_artifact_workflow_paths") == 1 assert source.count("github.rest.actions.getWorkflowRun") == 1 assert source.count("trustedWorkflowPaths.has(run.data.path)") == 1 + + +def test_conformance_skips_direct_evidence_entries() -> None: + source = WORKFLOW.read_text(encoding="utf-8") + + assert "(entry.evidence_mode || 'artifact') !== 'artifact'" in source + assert 'select((.evidence_mode // "artifact") == "artifact")' in source + assert 'select(.status == "missing" or .status == "invalid" or .status == "stale")' in source diff --git a/tests/workflows/test_langsmith_metrics_dashboard.py b/tests/workflows/test_langsmith_metrics_dashboard.py index 4be1a078b..e106863b0 100644 --- a/tests/workflows/test_langsmith_metrics_dashboard.py +++ b/tests/workflows/test_langsmith_metrics_dashboard.py @@ -3,6 +3,7 @@ WORKFLOW = Path(".github/workflows/maint-80-langsmith-metrics-dashboard.yml") FLEET_REGISTRY = Path("config/langsmith_fleet_registry.json") +FLEET_ALLOWLIST = Path("config/langsmith_fleet_allowlist.json") def test_fleet_artifact_lookup_does_not_mix_slurp_with_jq() -> None: @@ -29,6 +30,27 @@ def test_fleet_registry_declares_trusted_artifact_workflows() -> None: ] +def test_every_maintained_consumer_has_an_observability_state() -> None: + registry = json.loads(FLEET_REGISTRY.read_text(encoding="utf-8")) + allowlist = json.loads(FLEET_ALLOWLIST.read_text(encoding="utf-8")) + + evidence_modes = {entry["evidence_mode"] for entry in registry["repos"]} + assert evidence_modes == {"artifact", "langsmith-direct"} + assert {entry["repo"] for entry in allowlist["repos"]} == { + "stranske/Template", + "stranske/Ready", + "stranske/Collab-Admin", + } + assert {entry["status"] for entry in allowlist["repos"]} == {"not-applicable"} + + +def test_dashboard_skips_direct_evidence_artifact_downloads() -> None: + source = WORKFLOW.read_text(encoding="utf-8") + + assert 'entry.get("evidence_mode", "artifact") != "artifact"' in source + assert "continue" in source + + def test_dashboard_issue_uses_durable_tracker_labels() -> None: source = WORKFLOW.read_text(encoding="utf-8")