From ef0cb508c14399f57364cd1bc26ea9448292e63b Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sat, 20 Jun 2026 09:45:24 -0500 Subject: [PATCH] feat(langsmith): add Travel-Plan-Permission, LMS, Fine-Art-Archive to fleet registry Three actively-developed consumer repos were missing from the LangSmith fleet registry. Their agent-automation telemetry is captured centrally in the workflows-agents LangSmith project and ingested into the Orchestrator Brain via langsmith_direct.py (cost/token joined by repo+PR); coverage verified live. - config/langsmith_fleet_registry.json: 3 entries (surface=agent-automation, rollout_status=covered-via-langsmith-direct), issues #1238/#334/#114 under #2150 - config/repo_review_registry.json: Fine-Art-Archive -> active - tests/fixtures/langsmith_fleet/valid.ndjson: golden records for the 3 new rows - tests update: 11-repo fleet counts (Valid 8->11, Missing 6->9, status rows->11) Excludes Collab-Deliverables per owner. validate_registry OK; 56 fleet-coupled tests pass locally. maint-80 lists the 3 as artifact-"missing" by design. Co-Authored-By: Claude Opus 4.8 (1M context) --- config/langsmith_fleet_registry.json | 33 +++++++++++++++++++ config/repo_review_registry.json | 7 ++++ tests/fixtures/langsmith_fleet/valid.ndjson | 3 ++ tests/scripts/test_langsmith_fleet.py | 2 +- .../test_metrics_dashboard_generator.py | 10 +++--- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/config/langsmith_fleet_registry.json b/config/langsmith_fleet_registry.json index 09126619b..1b488cfe1 100644 --- a/config/langsmith_fleet_registry.json +++ b/config/langsmith_fleet_registry.json @@ -130,6 +130,39 @@ "seed", "metric_delta" ] + }, + { + "repo": "stranske/Travel-Plan-Permission", + "issue": "stranske/Travel-Plan-Permission#1238", + "issue_number": 1238, + "parent_issue": "stranske/Workflows#2150", + "surface": "agent-automation", + "operations": ["autopilot", "keepalive", "verifier"], + "artifact_name": "langsmith-fleet.ndjson", + "rollout_status": "covered-via-langsmith-direct", + "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"] + }, + { + "repo": "stranske/learning-management-system", + "issue": "stranske/learning-management-system#334", + "issue_number": 334, + "parent_issue": "stranske/Workflows#2150", + "surface": "agent-automation", + "operations": ["autopilot", "keepalive", "verifier"], + "artifact_name": "langsmith-fleet.ndjson", + "rollout_status": "covered-via-langsmith-direct", + "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"] + }, + { + "repo": "stranske/Fine-Art-Archive", + "issue": "stranske/Fine-Art-Archive#114", + "issue_number": 114, + "parent_issue": "stranske/Workflows#2150", + "surface": "agent-automation", + "operations": ["autopilot", "keepalive", "verifier"], + "artifact_name": "langsmith-fleet.ndjson", + "rollout_status": "covered-via-langsmith-direct", + "required_domain_fields": ["workflow", "agent", "step", "attempt", "result"] } ] } diff --git a/config/repo_review_registry.json b/config/repo_review_registry.json index fee73ee5c..8b9c78032 100644 --- a/config/repo_review_registry.json +++ b/config/repo_review_registry.json @@ -94,6 +94,13 @@ "cadence": "weekly", "decision_anchor": "Learning-engine v1: evidence-based mastery (verbose EvidenceRecord schema, computed MasteryEstimate view), source-grounded prompts with content-hash drift detection, formative LLM interaction with local-first redaction, FSRS-4.5 scheduler with explicit evidence-adapter rule table, and the Minimum Demo Criterion as the Milestone 4 thesis-validation gate." }, + { + "repo": "stranske/Fine-Art-Archive", + "local_path": "Fine-Art-Archive", + "status": "active", + "cadence": "weekly", + "decision_anchor": "Canonical fine-art reproduction archive; consumer of stranske/Workflows. Archive/catalog integrity, metadata, and agent-automation workflow integration." + }, { "repo": "stranske/tim-bot", "local_path": "tim-bot", diff --git a/tests/fixtures/langsmith_fleet/valid.ndjson b/tests/fixtures/langsmith_fleet/valid.ndjson index 4ee0f0eca..25fac1b74 100644 --- a/tests/fixtures/langsmith_fleet/valid.ndjson +++ b/tests/fixtures/langsmith_fleet/valid.ndjson @@ -6,3 +6,6 @@ {"schema_version":"langsmith-fleet/v1","repo":"stranske/Inv-Man-Intake","surface":"intake-extraction","operation":"extraction","run_id":"intake-batch-12","status":"fallback","provider":"openai","model":"gpt-5.4","latency_ms":1110,"cost_usd":0.01,"input_hash":"sha256:intake-in","output_hash":"sha256:intake-out","github_issue":"stranske/Inv-Man-Intake#438","recorded_at":"2026-05-24T02:05:00Z","domain":{"package_id":"pkg-12","document_type":"statement","redaction_status":"applied","validation_status":"manual-review"}} {"schema_version":"langsmith-fleet/v1","repo":"stranske/Trend_Model_Project","surface":"llm-replay","operation":"replay","run_id":"trend-replay-3","status":"success","provider":"openai","model":"gpt-5.4","latency_ms":1320,"cost_usd":0.013,"input_hash":"sha256:trend-in","output_hash":"sha256:trend-out","github_issue":"stranske/Trend_Model_Project#5311","recorded_at":"2026-05-24T02:06:00Z","domain":{"dataset_id":"daily-v2","config_fingerprint":"cfg-8a7f","replay_status":"matched","match_score":0.97}} {"schema_version":"langsmith-fleet/v1","repo":"stranske/Portable-Alpha-Extension-Model","surface":"scenario-analysis","operation":"scenario-run","run_id":"paem-scenario-5","status":"error","provider":"openai","model":"gpt-5.4","latency_ms":1430,"cost_usd":0.015,"input_hash":"sha256:paem-in","output_hash":"sha256:paem-out","github_issue":"stranske/Portable-Alpha-Extension-Model#1802","recorded_at":"2026-05-24T02:07:00Z","error_category":"upstream-timeout","domain":{"scenario_id":"scn-5","config_hash":"sha256:cfg-paem","seed":42,"metric_delta":-0.12}} +{"schema_version":"langsmith-fleet/v1","repo":"stranske/Travel-Plan-Permission","surface":"agent-automation","operation":"verifier","run_id":"tpp-run-1","trace_id":"trace-tpp-1","trace_url":"https://smith.langchain.com/r/trace-tpp-1","status":"success","provider":"openai","model":"gpt-5.4","latency_ms":1100,"cost_usd":0.011,"input_hash":"sha256:tpp-in","output_hash":"sha256:tpp-out","github_issue":"stranske/Travel-Plan-Permission#1238","github_pr":"stranske/Travel-Plan-Permission#1239","recorded_at":"2026-05-24T02:08:00Z","domain":{"workflow":"agents-verifier","agent":"codex","step":"compare","attempt":1,"result":"pass"}} +{"schema_version":"langsmith-fleet/v1","repo":"stranske/learning-management-system","surface":"agent-automation","operation":"verifier","run_id":"lms-run-1","trace_id":"trace-lms-1","trace_url":"https://smith.langchain.com/r/trace-lms-1","status":"success","provider":"openai","model":"gpt-5.4","latency_ms":1050,"cost_usd":0.0105,"input_hash":"sha256:lms-in","output_hash":"sha256:lms-out","github_issue":"stranske/learning-management-system#334","github_pr":"stranske/learning-management-system#335","recorded_at":"2026-05-24T02:09:00Z","domain":{"workflow":"agents-verifier","agent":"claude","step":"compare","attempt":1,"result":"pass"}} +{"schema_version":"langsmith-fleet/v1","repo":"stranske/Fine-Art-Archive","surface":"agent-automation","operation":"verifier","run_id":"faa-run-1","trace_id":"trace-faa-1","trace_url":"https://smith.langchain.com/r/trace-faa-1","status":"success","provider":"openai","model":"gpt-5.4","latency_ms":1000,"cost_usd":0.01,"input_hash":"sha256:faa-in","output_hash":"sha256:faa-out","github_issue":"stranske/Fine-Art-Archive#114","github_pr":"stranske/Fine-Art-Archive#115","recorded_at":"2026-05-24T02:10:00Z","domain":{"workflow":"agents-verifier","agent":"codex","step":"compare","attempt":1,"result":"pass"}} diff --git a/tests/scripts/test_langsmith_fleet.py b/tests/scripts/test_langsmith_fleet.py index 3263626df..21552921a 100644 --- a/tests/scripts/test_langsmith_fleet.py +++ b/tests/scripts/test_langsmith_fleet.py @@ -166,7 +166,7 @@ def test_markdown_summary_renders_mixed_valid_invalid_missing_rows() -> None: ) markdown = langsmith_fleet.format_fleet_summary(summary) - # Aggregate counts (one registry row per repo; 8 total). + # Aggregate counts (one registry row per repo; 11 total). assert "# LangSmith Fleet Artifact Status" in markdown assert f"- Registry entries: {len(registry['repos'])}" in markdown assert "- Invalid: 1" in markdown diff --git a/tests/scripts/test_metrics_dashboard_generator.py b/tests/scripts/test_metrics_dashboard_generator.py index cc35d7d38..76ca84848 100644 --- a/tests/scripts/test_metrics_dashboard_generator.py +++ b/tests/scripts/test_metrics_dashboard_generator.py @@ -252,7 +252,7 @@ def test_build_dashboard_from_path_includes_langsmith_fleet_status(tmp_path: Pat assert errors == 0 assert "## LangSmith Fleet Artifact Status" in dashboard - assert "- Valid: 8" in dashboard + assert "- Valid: 11" in dashboard assert "- Missing: 0" in dashboard assert "- Stale: 0" in dashboard assert "- Invalid: 0" in dashboard @@ -333,11 +333,11 @@ def summarize_with_fixed_now(*args: object, **kwargs: object) -> dict[str, objec assert exit_code == 0 content = output_path.read_text(encoding="utf-8") assert "## LangSmith Fleet Artifact Status" in content - assert "- Valid: 8" in content + assert "- Valid: 11" in content def test_build_dashboard_from_path_mixed_fleet_status(tmp_path: Path) -> None: - """mixed.ndjson: 1 valid (Workflows) + 1 invalid (trip-planner) + 6 missing repos.""" + """mixed.ndjson: 1 valid (Workflows) + 1 invalid (trip-planner) + 9 missing repos.""" metrics_path = tmp_path / "metrics.ndjson" metrics_path.write_text( '{"repo": "octo/alpha", "duration_ms": 10, "timestamp": "2024-01-01T00:00:00Z"}\n', @@ -360,7 +360,7 @@ def test_build_dashboard_from_path_mixed_fleet_status(tmp_path: Path) -> None: assert "## LangSmith Fleet Artifact Status" in dashboard assert "- Valid: 1" in dashboard assert "- Invalid: 1" in dashboard - assert "- Missing: 6" in dashboard + assert "- Missing: 9" in dashboard assert ( "| stranske/Workflows | agent-automation | stranske/Workflows#2150 | valid |" in dashboard ) @@ -377,4 +377,4 @@ def test_build_dashboard_from_path_mixed_fleet_status(tmp_path: Path) -> None: for line in fleet_section.splitlines() if line.startswith("| stranske/") and line.endswith(" |") ] - assert len(status_rows) == 8 + assert len(status_rows) == 11