Require telemetry for model-evaluation evidence

tests/tools/test_evaluate_model_benchmark.py

@@ -158,9 +158,17 @@ def test_unpaired_candidate_cases_are_rejected():
         ("total_cost_usd", math.nan),
         ("total_cost_usd", math.inf),
         ("total_cost_usd", -0.01),
+        ("total_cost_usd", True),
+        ("total_cost_usd", False),
+        ("total_cost_usd", "not-a-number"),
+        ("total_cost_usd", 10**400),
         ("latency_ms", math.nan),
         ("latency_ms", math.inf),
         ("latency_ms", -1),
+        ("latency_ms", True),
+        ("latency_ms", False),
+        ("latency_ms", "not-a-number"),
+        ("latency_ms", 10**400),
     ],
 )
 def test_nonfinite_or_negative_metrics_are_rejected(field, value):
@@ -170,6 +178,15 @@ def test_nonfinite_or_negative_metrics_are_rejected(field, value):
         evaluator.evaluate_benchmark(payload, _policy())
 
 
+@pytest.mark.parametrize("field", ["total_cost_usd", "latency_ms"])
+def test_missing_metrics_are_rejected_instead_of_treated_as_zero(field):
+    payload = _payload()
+    del payload["candidates"][1]["cases"][0][field]
+
+    with pytest.raises(ValueError, match=f"missing {field}"):
+        evaluator.evaluate_benchmark(payload, _policy())
+
+
 def test_nonobject_candidate_is_rejected_as_configuration_error():
     payload = _payload()
     payload["candidates"][1] = None

tests/tools/test_llm_registry_selection.py

@@ -146,6 +146,33 @@ def test_registry_decision_update_changes_slot_without_slot_edit(
     assert "model" not in json.loads(slots_path.read_text())["slots"][0]
 
 
+def test_explicit_empty_slot_model_keeps_the_profile_selection(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    registry_path = tmp_path / "registry.json"
+    slots_path = tmp_path / "slots.json"
+    _write_registry(registry_path)
+    slots_path.write_text(
+        json.dumps(
+            {
+                "slots": [
+                    {
+                        "name": "slot1",
+                        "provider": "openai",
+                        "profile": "verifier-balanced",
+                        "model": "",
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    monkeypatch.setenv(registry.ENV_MODEL_REGISTRY_CONFIG, str(registry_path))
+    monkeypatch.setenv(registry.ENV_SLOT_CONFIG, str(slots_path))
+
+    assert registry.load_slot_config()[0].model == "model-balanced"
+
+
 def test_legacy_slot_pin_is_honored_when_current_and_unblocked(
     monkeypatch: pytest.MonkeyPatch, tmp_path: Path
 ) -> None:

tools/evaluate_model_benchmark.py

@@ -44,6 +44,22 @@ def _case_outcome(case: dict[str, Any]) -> tuple[str, str, bool]:
     return expected, actual, schema_valid
 
 
+def _required_nonnegative_metric(case: dict[str, Any], case_id: str, field: str) -> float:
+    """Return required finite telemetry without treating missing data as zero."""
+    if field not in case:
+        raise ValueError(f"case {case_id} is missing {field}")
+    raw = case[field]
+    if isinstance(raw, bool):
+        raise ValueError(f"case {case_id} has invalid {field}")
+    try:
+        value = float(raw)
+    except (TypeError, ValueError, OverflowError) as exc:
+        raise ValueError(f"case {case_id} has invalid {field}") from exc
+    if not math.isfinite(value) or value < 0:
+        raise ValueError(f"case {case_id} has invalid {field}")
+    return value
+
+
 def _metrics(cases: list[dict[str, Any]], *, z: float = Z_95) -> dict[str, Any]:
     success = false_pass = false_fail = schema_error = 0
     expected_pass = expected_non_pass = 0
@@ -68,12 +84,8 @@ def _metrics(cases: list[dict[str, Any]], *, z: float = Z_95) -> dict[str, Any]:
         false_pass += int(expected == "NON_PASS" and actual == "PASS")
         false_fail += int(expected == "PASS" and actual == "NON_PASS")
         categories[category] = categories.get(category, 0) + 1
-        cost = float(case.get("total_cost_usd", 0.0))
-        latency = float(case.get("latency_ms", 0.0))
-        if not math.isfinite(cost) or cost < 0:
-            raise ValueError(f"case {case_id} has invalid total_cost_usd")
-        if not math.isfinite(latency) or latency < 0:
-            raise ValueError(f"case {case_id} has invalid latency_ms")
+        cost = _required_nonnegative_metric(case, case_id, "total_cost_usd")
+        latency = _required_nonnegative_metric(case, case_id, "latency_ms")
         total_cost += cost
         latencies.append(latency)