diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index d86a84e48..3162db4b8 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -417,10 +417,20 @@ def _safe_int(value: Any) -> int | None: def _langsmith_trace_count(entry: dict[str, Any]) -> int: - count = 1 if entry.get("langsmith_trace_id") else 0 + count = 1 if entry.get("langsmith_trace_id") or entry.get("langsmith_trace_url") else 0 traces = entry.get("langsmith_traces") if isinstance(traces, list): - list_count = sum(1 for item in traces if isinstance(item, dict) and item.get("trace_id")) + list_count = sum( + 1 + for item in traces + if isinstance(item, dict) + and ( + item.get("trace_id") + or item.get("langsmith_trace_id") + or item.get("trace_url") + or item.get("langsmith_trace_url") + ) + ) return max(count, list_count) return count diff --git a/scripts/langchain/issue_pr_context.py b/scripts/langchain/issue_pr_context.py index 803c872f6..884a5a8fa 100644 --- a/scripts/langchain/issue_pr_context.py +++ b/scripts/langchain/issue_pr_context.py @@ -60,9 +60,11 @@ def estimate_tokens(text: str, *, model: str | None = None) -> int: encoding = ( tiktoken.encoding_for_model(model) if model else tiktoken.get_encoding("cl100k_base") ) + return max(len(encoding.encode(value)), chars_estimate) except Exception: - encoding = tiktoken.get_encoding("cl100k_base") - return max(len(encoding.encode(value)), chars_estimate) + # tiktoken may need to download encoding metadata on first use; fall back + # to the chars-based estimate when network access is unavailable. + return chars_estimate def build_issue_context( diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index d86a84e48..3162db4b8 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -417,10 +417,20 @@ def _safe_int(value: Any) -> int | None: def _langsmith_trace_count(entry: dict[str, Any]) -> int: - count = 1 if entry.get("langsmith_trace_id") else 0 + count = 1 if entry.get("langsmith_trace_id") or entry.get("langsmith_trace_url") else 0 traces = entry.get("langsmith_traces") if isinstance(traces, list): - list_count = sum(1 for item in traces if isinstance(item, dict) and item.get("trace_id")) + list_count = sum( + 1 + for item in traces + if isinstance(item, dict) + and ( + item.get("trace_id") + or item.get("langsmith_trace_id") + or item.get("trace_url") + or item.get("langsmith_trace_url") + ) + ) return max(count, list_count) return count diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index 31c690ec6..91da0c65d 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -182,6 +182,12 @@ def test_langsmith_trace_counts_cover_single_ids_and_trace_lists() -> None: {"trace_id": "verifier-trace-3"}, ], }, + { + "metric_type": "verifier", + "run_id": "verify-3", + "verdict": "pass", + "langsmith_trace_url": "https://smith.langchain.com/r/verifier-trace-url", + }, { "metric_type": "step", "issue_number": 11, @@ -201,18 +207,40 @@ def test_langsmith_trace_counts_cover_single_ids_and_trace_lists() -> None: {"trace_id": "autopilot-trace-3"}, ], }, + { + "metric_type": "step", + "issue_number": 12, + "step_name": "format", + "success": True, + "duration_ms": 300, + "langsmith_trace_url": "https://smith.langchain.com/r/autopilot-trace-url", + }, ] summary = aggregate_agent_metrics.build_summary(entries, errors=0) - assert "LangSmith trace coverage: 100.0% (2/2) (3 traces)" in summary - assert summary.count("LangSmith trace coverage: 100.0% (2/2) (3 traces)") == 2 + assert "LangSmith trace coverage: 100.0% (3/3) (4 traces)" in summary + assert summary.count("LangSmith trace coverage: 100.0% (3/3) (4 traces)") == 2 contract = aggregate_agent_metrics.build_summary_contract(entries, []) - assert contract["summaries"]["verifier"]["langsmith_trace_records"] == 2 - assert contract["summaries"]["verifier"]["langsmith_trace_count"] == 3 - assert contract["summaries"]["autopilot"]["langsmith_trace_records"] == 2 - assert contract["summaries"]["autopilot"]["langsmith_trace_count"] == 3 + assert contract["summaries"]["verifier"]["langsmith_trace_records"] == 3 + assert contract["summaries"]["verifier"]["langsmith_trace_count"] == 4 + assert contract["summaries"]["autopilot"]["langsmith_trace_records"] == 3 + assert contract["summaries"]["autopilot"]["langsmith_trace_count"] == 4 + + +def test_langsmith_trace_counts_cover_url_only_trace_lists() -> None: + assert ( + aggregate_agent_metrics._langsmith_trace_count( + { + "langsmith_traces": [ + {"trace_url": "https://smith.langchain.com/r/list-trace-url"}, + {"langsmith_trace_url": "https://smith.langchain.com/r/list-langsmith-url"}, + ] + } + ) + == 2 + ) def test_main_writes_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: