stranske · stranske · Aug 15, 2026 · Aug 15, 2026
@@ -156,8 +156,9 @@ jobs:
           REGISTRY=config/langsmith_fleet_registry.json
           mkdir -p .metrics-tmp/fleet
           trusted_workflow_paths="$(jq -c '.trusted_artifact_workflow_paths // []' "$REGISTRY")"
-          # Always start from an empty combined file so repos with no artifact
-          # surface as "missing" (the registry-driven rollup never skips them).
+          # Always start from an empty combined file so artifact-backed repos
+          # with no artifact surface as "missing". Direct and not-applicable
+          # entries remain visible through their explicit status metadata.
           : > .metrics-tmp/fleet/combined-fleet.ndjson
 
           # Enumerate (repo, artifact_name) from the registry.
@@ -168,6 +169,8 @@ jobs:
           with open(sys.argv[1], encoding="utf-8") as fh:
               registry = json.load(fh)
           for entry in registry.get("repos", []):
+              if entry.get("evidence_mode", "artifact") != "artifact":
+                  continue
               repo = entry.get("repo", "")
               artifact = entry.get("artifact_name", "langsmith-fleet.ndjson")
               if repo:
@@ -249,7 +252,7 @@ jobs:
           done < .metrics-tmp/fleet/repos.tsv
 
           # Reuse the already-tested rollup. The registry drives the table, so
-          # every registered repo gets a row even with zero combined records.
+          # every registered and allowlisted repo gets an explicit status row.
           python scripts/langsmith_fleet.py .metrics-tmp/fleet/combined-fleet.ndjson \
             --registry "$REGISTRY" --summary --format markdown \
             > .metrics-tmp/fleet/fleet-status.md

@@ -56,6 +56,9 @@ jobs:
             fs.mkdirSync('.metrics-tmp/fleet', { recursive: true });
 
             for (const entry of registry.repos) {
+              if ((entry.evidence_mode || 'artifact') !== 'artifact') {
+                continue;
+              }
               const [owner, repoName] = entry.repo.split('/');
               const safeRepo = entry.repo.replace('/', '__');
               const outDir = path.join('.metrics-tmp/fleet', safeRepo);
@@ -127,7 +130,8 @@ jobs:
             unzip -q -o "$zip_path" -d "$out_dir"
             rm "$zip_path"
           done
-          jq -c '.repos[]' config/langsmith_fleet_registry.json | while read -r entry; do
+          jq -c '.repos[] | select((.evidence_mode // "artifact") == "artifact")' \
+            config/langsmith_fleet_registry.json | while read -r entry; do
             repo="$(jq -r '.repo' <<<"$entry")"
             artifact_name="$(jq -r '.artifact_name' <<<"$entry")"
             safe_repo="${repo//\//__}"
@@ -155,12 +159,12 @@ jobs:
             > .metrics-tmp/langsmith-fleet-conformance.md
           cat .metrics-tmp/langsmith-fleet-conformance.md >> "$GITHUB_STEP_SUMMARY"
 
-          jq -r '.rows[] | select(.status != "valid") |
+          jq -r '.rows[] | select(.status == "missing" or .status == "invalid" or .status == "stale") |
             "::warning title=LangSmith fleet conformance \(.status)::" +
             "\(.repo)/\(.surface): \(.first_error // "no artifact uploaded")"' \
             .metrics-tmp/langsmith-fleet-conformance.json
 
-          non_valid_count="$(jq '[.rows[] | select(.status != "valid")] | length' \
+          non_valid_count="$(jq '[.rows[] | select(.status == "missing" or .status == "invalid" or .status == "stale")] | length' \
             .metrics-tmp/langsmith-fleet-conformance.json)"
           echo "LANGSMITH_FLEET_NON_VALID_COUNT=${non_valid_count}" >> "$GITHUB_ENV"
 

diff --git a/config/langsmith_fleet_allowlist.json b/config/langsmith_fleet_allowlist.json
@@ -0,0 +1,23 @@
+{
+  "schema_version": "langsmith-fleet-allowlist/v1",
+  "repos": [
+    {
+      "repo": "stranske/Template",
+      "status": "not-applicable",
+      "reason": "Canonical inactive template that receives fleet updates but has no substantive runtime to trace.",
+      "registry_activation_condition": "Add the substantive repository created from this template to the LangSmith registry when its first LLM-backed runtime or agent-observability surface is implemented."
+    },
+    {
+      "repo": "stranske/Ready",
+      "status": "not-applicable",
+      "reason": "Standby repository maintained for future use; it receives fleet updates but has no active substantive runtime.",
+      "registry_activation_condition": "Move the activated substantive repository into the LangSmith registry when development begins on an LLM-backed runtime or agent-observability surface."
+    },
+    {
+      "repo": "stranske/Collab-Admin",
+      "status": "not-applicable",
+      "reason": "Inactive administrative/template consumer that receives fleet updates without operating a substantive traced runtime.",
+      "registry_activation_condition": "Register any substantive repository created from this source when it gains an LLM-backed runtime or agent-observability surface."
+    }
+  ]
+}
diff --git a/config/langsmith_fleet_registry.json b/config/langsmith_fleet_registry.json
@@ -15,6 +15,7 @@
       "surface": "agent-automation",
       "operations": ["autopilot", "keepalive", "verifier", "dashboard-ingestion", "durability"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "paused",
       "required_domain_fields": [
         "workflow",
@@ -32,6 +33,7 @@
       "surface": "planner-runtime",
       "operations": ["conversation", "tool-call", "itinerary-generation"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented",
       "required_domain_fields": [
         "planning_mode",
@@ -48,6 +50,7 @@
       "surface": "nl-to-sql",
       "operations": ["sql-generation", "validation", "execution", "replay"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented",
       "required_domain_fields": [
         "query_category",
@@ -64,6 +67,7 @@
       "surface": "ai-api",
       "operations": ["nl-query", "holdings-analysis", "rag"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented",
       "required_domain_fields": [
         "endpoint",
@@ -80,6 +84,7 @@
       "surface": "risk-reporting",
       "operations": ["data-quality", "risk-proxy", "limit-monitoring", "report-generation"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented",
       "required_domain_fields": [
         "as_of_date",
@@ -96,6 +101,7 @@
       "surface": "intake-extraction",
       "operations": ["package-intake", "extraction", "validation", "review-routing"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented-followup-open",
       "required_domain_fields": [
         "package_id",
@@ -112,6 +118,7 @@
       "surface": "llm-replay",
       "operations": ["chain", "replay", "validation", "config-analysis"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented",
       "required_domain_fields": [
         "dataset_id",
@@ -128,6 +135,7 @@
       "surface": "scenario-analysis",
       "operations": ["scenario-run", "result-explanation", "run-comparison", "config-patch"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "artifact",
       "rollout_status": "implemented",
       "required_domain_fields": [
         "scenario_id",
@@ -144,6 +152,7 @@
       "surface": "agent-automation",
       "operations": ["autopilot", "keepalive", "verifier"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "langsmith-direct",
       "rollout_status": "covered-via-langsmith-direct",
       "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"]
     },
@@ -155,6 +164,7 @@
       "surface": "agent-automation",
       "operations": ["autopilot", "keepalive", "verifier"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "langsmith-direct",
       "rollout_status": "covered-via-langsmith-direct",
       "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"]
     },
@@ -166,6 +176,7 @@
       "surface": "agent-automation",
       "operations": ["autopilot", "keepalive", "verifier"],
       "artifact_name": "langsmith-fleet.ndjson",
+      "evidence_mode": "langsmith-direct",
       "rollout_status": "covered-via-langsmith-direct",
       "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"]
     }

@@ -7,7 +7,8 @@ status rollup.
 
 ## Design Decision
 
-The fleet design is contract-first, not package-first:
+The fleet design is contract-first, not package-first. It also distinguishes
+how coverage is proved from whether LangSmith is applicable at all:
 
 - Workflows owns the canonical `langsmith-fleet/v1` record contract, JSON Schema,
   registry, validator, fixtures, and dashboard status rollup.
@@ -23,6 +24,13 @@ The fleet design is contract-first, not package-first:
   That design would only be worth changing to if the fleet starts seeing repeated
   schema drift, duplicated validator defects, or cross-repo release coordination
   failures that outweigh the packaging and version-management overhead.
+- `artifact` registry entries prove coverage with a repo-local
+  `langsmith-fleet.ndjson` artifact.
+- `langsmith-direct` registry entries prove agent-automation tracing through the
+  Workflows-owned direct integration and do not emit a repo-local artifact.
+- Repositories in `config/langsmith_fleet_allowlist.json` are explicitly
+  `not-applicable`: they receive shared maintenance updates, but have no
+  substantive runtime whose LangSmith coverage could be measured.
 
 Closer/verifier agents should consult this section before pausing for a human
 decision about local validators. If the current repo artifacts conform to the
@@ -45,12 +53,16 @@ emitters against the contract:
 | `stranske/Trend_Model_Project` | `#5311` | `#5328` | Repo-local emitter merged. |
 | `stranske/Portable-Alpha-Extension-Model` | `#1802` | `#1819` | Repo-local emitter merged. |
 | `stranske/Manager-Database` | `#1048` | `#1067` | Repo-local emitter merged. |
+| `stranske/Travel-Plan-Permission` | `#1238` | `#2487` (registry) | Covered by Workflows-owned direct agent tracing; no repo-local artifact expected. |
+| `stranske/learning-management-system` | `#334` | `#2487` (registry) | Covered by Workflows-owned direct agent tracing; no repo-local artifact expected. |
+| `stranske/Fine-Art-Archive` | `#114` | `#2487` (registry) | Covered by Workflows-owned direct agent tracing; no repo-local artifact expected. |
 
 ## Artifact
 
-Each participating repo emits NDJSON at the artifact name registered in
-`config/langsmith_fleet_registry.json` (default `langsmith-fleet.ndjson`). Each
-line is one JSON object.
+Each `evidence_mode=artifact` repo emits NDJSON at the artifact name registered
+in `config/langsmith_fleet_registry.json` (default
+`langsmith-fleet.ndjson`). Each line is one JSON object. Direct-evidence and
+not-applicable repositories do not emit this artifact.
 
 The artifact must be safe to publish in GitHub Actions artifacts and dashboards:
 raw prompts, personal data, documents, SQL result rows, generated report text,
@@ -100,15 +112,34 @@ block or reopen the original merge path.
 - repo, issue, and issue number,
 - surface and allowed operation family,
 - artifact name,
+- evidence mode (`artifact` or `langsmith-direct`),
 - rollout status,
 - required domain fields.
 
+`config/langsmith_fleet_allowlist.json` covers registered maintenance consumers
+where runtime observability is intentionally not applicable. The current
+allowlist is Template, Ready, and Collab-Admin. A repository created from one
+of those templates must move into the registry when it gains a substantive
+runtime; that activation condition is recorded in each allowlist entry and is
+validated alongside Maint 68 consumer coverage.
+
 Dashboard status is computed per registry entry:
 
 - `missing`: no record was emitted for that repo/surface.
 - `invalid`: records exist but fail the shared or domain-field contract.
 - `stale`: latest valid record is older than the registry freshness window.
 - `valid`: at least one current valid record exists.
+- `direct`: coverage uses Workflows-owned direct LangSmith tracing, so no
+  repo-local artifact is expected.
+- `not-applicable`: the repository is an explicitly allowlisted maintenance
+  consumer without a substantive runtime.
+
+`missing`, `invalid`, and `stale` apply only to artifact-backed entries. A
+missing artifact means the GitHub dashboard cannot prove current coverage; it
+does not, by itself, prove that LangSmith tracing failed. Conversely, a direct
+entry proves the configured integration path, not live per-repo trace success;
+live direct-trace health must be reported by the Workflows automation telemetry
+surface.
 
 ## Validation
 
@@ -134,6 +165,8 @@ The canonical schema is versioned at
 
 ## Repo Responsibilities
 
-Repo-specific issues should add instrumentation and emit compatible records.
+Artifact-backed repo issues should add instrumentation and emit compatible records.
 They must not move domain tracing logic into Workflows. Workflows only validates
-the emitted artifact and displays fleet status.
+the emitted artifact and displays fleet status. Direct-evidence repos rely on
+the Workflows-owned agent tracing path, while allowlisted repos must be promoted
+to the registry as soon as substantive runtime behavior is introduced.
@@ -47,6 +47,13 @@ The signal flow each tracker carries:
 - **#2211** — health check on the weekly metrics pipeline. Healthy state is `Parse errors: 0` and non-zero terminal disposition records. A regression here usually means a producer is emitting a malformed artifact, not that the dashboard itself is broken.
 - **#1836** — work queue for items the local Codex watcher should claim. The body holds the live queue state with a sync hash, repo counts, and per-item status. Active campaigns must not be closed; the controller treats a closed campaign as "stop work."
 - **#2897** — Health 83 dependency/sync maintenance-efficiency advisory evidence. Comments append only when the material-evidence fingerprint changes; do not redirect this signal onto `#1836`.
+- **#2415** — LangSmith fleet observability status. Read the evidence mode as
+  well as the status: `valid`/`missing`/`stale`/`invalid` are artifact-backed;
+  `direct` means Workflows-owned direct tracing with no repo-local artifact;
+  and `not-applicable` is an explicit allowlist state for maintenance-only
+  consumers. Only artifact-backed `missing`, `stale`, or `invalid` rows are
+  actionable coverage debt. A missing row means the dashboard lacks evidence,
+  not that the tracker itself should be closed.
 
 ### Leased generated delivery attempts