refactor(eval): share public-sentinel owner default across read/eval scripts; close #347 answer-path gate

docs/process-hardening.md

@@ -236,9 +236,39 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went
  `mrr@10=0.8148`, `content_mrr@10=0.9244`, strategies `{text_fast_path:25, document_lookup_fast_path:1,
 hybrid:10}`, all 10 forced-embedding vector cases passed (`force_embedding_failure_count=0`). A
  direct read-only DB probe confirmed the premise: all **2,065** documents have `owner_id = NULL`;
- the old eval owner `2bac05f1-…` owns 0. **This closes the #347 part-2 eval debt** — the golden
- retrieval eval is the behavior-preservation proof the rag.ts decomposition owed, and it now runs
- green with no manual owner setup.
+ the old eval owner `2bac05f1-…` owns 0. **This closes the retrieval half of the #347 part-2 eval
+ debt** — the golden retrieval eval is the retrieval behavior-preservation proof, and it now runs
+ green with no manual owner setup. (#347 also owed `eval:quality -- --rag-only`; that answer-path
+ half was still owner-blocked until the follow-up below.)
+
+## Eval-owner default hoisted to all read/eval scripts + #347 answer-path gate closed (2026-07-08)
+
+- **Follow-up to the 2026-07-07 fix above.** PR #348 only patched `eval-retrieval.ts`, so
+ `eval:quality` (incl. `--rag-only`), `eval:rag`, `eval:answer-quality`, and `eval:search` still
+ resolved the owner as `args.ownerId ?? emailLookup ?? undefined` and returned 0/N against the
+ all-public corpus — the exact failure #348 fixed for retrieval, still live for the answer path.
+- **Fix (item 1 + 3):** hoisted `DEFAULT_EVAL_OWNER_ID` + `resolveEvalOwnerId(supabase, args)` into
+ the shared `scripts/eval-utils.ts` and applied it at the final owner-resolution point in all five
+ read/eval scripts (`eval-retrieval` refactored onto it; its local duplicate constant removed).
+ Precedence preserved (explicit id → email lookup → sentinel); the helper prints a **one-line
+ warning on the sentinel fallback** so the narrowing to public-only scope is visible, not silent.
+ Write/backfill scripts (`enrich-documents`, `classify-documents`, `backfill-*`) deliberately
+ excluded — defaulting an owner there could write under the wrong owner. First-ever
+ `resolveEvalOwnerId` unit coverage added to `tests/eval-utils.test.ts`.
+- **Gates:** `verify:cheap` green (1277 tests; 0 lint errors). `eval:retrieval:quality` re-ran
+ **36/36, failed_cases=0** on the refactored script (regression check). The fallback warning was
+ observed firing in a real `eval:quality` run.
+- **#347 answer-path gate CLOSED (item 2) — `eval:quality -- --rag-only`, live, no manual owner:**
+ `cases=44`, **unsupported_correct_rate=1.0**, **citation_failure_rate=0.0455**,
+ **numeric_grounding_failure_rate=0.0227** — all three invariants identical to #343's live
+ baseline; `grounded_supported_rate=0.90` (vs #343 `0.9333`). The 5 failing cases are entirely in
+ #343's documented live-variance set: 2 pure latency-threshold failures (this cloud env's
+ Supabase p95 ≈ 49 s — the local→remote latency #343 called out) and 3 route/retrieval flakes
+ (`illegal-substances`, `discharge-documentation`, `community-admission`) that #343 already
+ identified as pre-existing variance outside the touched paths. Because this change is eval-config
+ only (owner resolution) and touches no answer-generation code, the identical invariant rates
+ confirm the part-2 decomposition preserved answer behavior. **#347 part-2 eval debt now fully
+ settled** (retrieval half via #348, answer-path half here).
 
 ## Answer-thread Back button: URL and visible answer can disagree (2026-07-06)
 

docs/retrieval-quality-runbook.md

@@ -51,13 +51,17 @@ The command requires the same live-eval environment as the existing RAG eval scr
 - Owner is **optional** — see below.
 
 Since the 2026-07-06 public promotion the live corpus is entirely `owner_id = NULL`, the eval now
-**defaults its owner to the public-owner sentinel** `00000000-0000-0000-0000-000000000000`
-(`DEFAULT_EVAL_OWNER_ID` in `scripts/eval-retrieval.ts`). `retrieval_owner_matches` maps that
-sentinel to NULL-owner rows, mirroring anonymous production search, so no session has to set
-`RAG_EVAL_OWNER_ID` by hand. An explicit `RAG_EVAL_OWNER_ID`, `LOCAL_NO_AUTH_OWNER_ID`, or
-`RAG_EVAL_OWNER_EMAIL` (or `--owner-id` / `--owner-email`) still overrides the default. Note a real
-owner UUID now scopes retrieval to zero documents and fails every case, so only override when the
-corpus ownership actually changes.
+**defaults its owner to the public-owner sentinel** `00000000-0000-0000-0000-000000000000` via the
+shared `resolveEvalOwnerId()` helper in `scripts/eval-utils.ts`. This default applies across the
+whole read/eval suite — `eval:retrieval:quality`, `eval:quality` (incl. `--rag-only`), `eval:rag`,
+`eval:answer-quality`, and `eval:search` — not just the retrieval eval. `retrieval_owner_matches`
+maps the sentinel to NULL-owner rows, mirroring anonymous production search, so no session has to
+set `RAG_EVAL_OWNER_ID` by hand; the helper prints a one-line warning whenever it falls back to the
+sentinel so the public-only scope is visible. An explicit `RAG_EVAL_OWNER_ID`, `LOCAL_NO_AUTH_OWNER_ID`,
+or `RAG_EVAL_OWNER_EMAIL` (or `--owner-id` / `--owner-email`) still overrides the default. Note a
+real owner UUID now scopes retrieval to zero documents and fails every case, so only override when
+the corpus ownership actually changes. (Write/backfill scripts deliberately do **not** use this
+default — they must target an explicit owner.)
 
 Optional cost fields:
 

scripts/eval-answer-quality.ts

@@ -16,7 +16,7 @@ import {
  type AnswerQualityMetric,
 } from "@/lib/rag-eval-cases";
 import type { RagAnswer } from "@/lib/types";
-import { findOwnerIdByEmail, loadAdminClient, withProviderBackoff } from "./eval-utils";
+import { loadAdminClient, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils";
 
 loadEnvConfig(process.cwd());
 
@@ -64,7 +64,7 @@ async function main() {
  requireServerEnv();
  requireOpenAIEnv();
 
- const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
+ const ownerId = await resolveEvalOwnerId(supabase, args);
  let cases: AnswerQualityEvalCase[] = answerQualityEvalCases;
  if (args.intent) cases = cases.filter((testCase) => testCase.expectedIntent === args.intent);
  if (args.limit) cases = cases.slice(0, args.limit);

scripts/eval-quality.ts

@@ -13,9 +13,9 @@ import {
 } from "./eval-retrieval";
 import {
  estimateCostUsd,
- findOwnerIdByEmail,
  loadAdminClient,
  percentile,
+ resolveEvalOwnerId,
  validateRagAnswer,
  withProviderBackoff,
 } from "./eval-utils";
@@ -941,7 +941,7 @@ async function main() {
  ? await loadSourceMetadataDebtAcceptance(args.sourceMetadataDebt)
  : undefined;
 
- const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
+ const ownerId = await resolveEvalOwnerId(supabase, args);
  const retrievalResults = args.ragOnly ? [] : await runRetrievalQualityCases({ ...args, ownerId, supabase });
  const ragResults = args.retrievalOnly ? [] : await runRagQualityCases({ ...args, ownerId, supabase });
  const report = buildEvalQualityReport({ retrievalResults, ragResults, sourceMetadataDebtAcceptance });

scripts/eval-rag.ts

@@ -3,9 +3,9 @@ import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases";
 import type { RagAnswer } from "@/lib/types";
 import {
  estimateCostUsd,
- findOwnerIdByEmail,
  loadAdminClient,
  percentile,
+ resolveEvalOwnerId,
  validateRagAnswer,
  withProviderBackoff,
 } from "./eval-utils";
@@ -212,8 +212,8 @@ async function main() {
  requireServerEnv();
  requireOpenAIEnv();
 
- const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
- const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public";
+ const ownerId = await resolveEvalOwnerId(supabase, args);
+ const scope = args.ownerId ? "owner:id" : args.ownerEmail ? `owner:${args.ownerEmail}` : "public";
  const cases = selectRagEvalCases({ limit: args.limit, question: args.question });
  const results: EvalResult[] = [];
 

scripts/eval-retrieval.ts

@@ -5,17 +5,10 @@ import { loadEnvConfig } from "@next/env";
 import { z } from "zod";
 import { loadCapturedRagEvalCases, type RagEvalCase, type SupabaseEvalCaseClient } from "@/lib/rag-eval-cases";
 import type { SearchResult } from "@/lib/types";
-import { findOwnerIdByEmail, loadAdminClient, percentile, withProviderBackoff } from "./eval-utils";
+import { loadAdminClient, percentile, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils";
 
 loadEnvConfig(process.cwd());
 
-// Committed default eval owner. Since the 2026-07-06 public promotion the live corpus is
-// entirely owner_id = NULL, so owner-scoped retrieval must run as the public-owner sentinel
-// (retrieval_owner_matches maps it to NULL-owner rows, mirroring anonymous production search).
-// An explicit RAG_EVAL_OWNER_ID / LOCAL_NO_AUTH_OWNER_ID / RAG_EVAL_OWNER_EMAIL (or --owner-id /
-// --owner-email) still overrides this. See docs/retrieval-quality-runbook.md.
-const DEFAULT_EVAL_OWNER_ID = "00000000-0000-0000-0000-000000000000";
-
 const contentExpectationSchema = z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]);
 
 const goldenCaseSchema = z.object({
@@ -819,10 +812,7 @@ async function main() {
  requireServerEnv();
  requireOpenAIEnv();
 
- const ownerId =
- args.ownerId ??
- (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined) ??
- DEFAULT_EVAL_OWNER_ID;
+ const ownerId = await resolveEvalOwnerId(supabase, args);
  const capturedCaseClient = supabase as unknown as SupabaseEvalCaseClient;
  const capturedCases = await loadCapturedRagEvalCases({ supabase: capturedCaseClient, ownerId, limit: args.limit });
  const allCases = [...capturedCases.map(capturedRagCaseToGoldenCase), ...loadGoldenRetrievalCases(args.fixture)];

scripts/eval-search.ts

@@ -12,10 +12,10 @@ import type { SearchResult } from "@/lib/types";
 import {
  expectedFileCoverage,
  expectedFileHit,
- findOwnerIdByEmail,
  hasInvalidVisualEvidence,
  loadAdminClient,
  percentile,
+ resolveEvalOwnerId,
 } from "./eval-utils";
 
 loadEnvConfig(process.cwd());
@@ -201,8 +201,8 @@ async function main() {
  requireServerEnv();
  requireOpenAIEnv();
 
- const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
- const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public";
+ const ownerId = await resolveEvalOwnerId(supabase, args);
+ const scope = args.ownerId ? "owner:id" : args.ownerEmail ? `owner:${args.ownerEmail}` : "public";
  const baseCases = selectRagEvalCases({ question: args.question });
  const capturedCaseClient = supabase as unknown as SupabaseEvalCaseClient;
  const capturedCases = args.question

scripts/eval-utils.ts

@@ -75,6 +75,36 @@ export async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string)
  throw new Error(`No Supabase Auth user found for ${email}. Sign in once before running evals.`);
 }
 
+/**
+ * Committed default eval owner. Since the 2026-07-06 public promotion the live corpus is entirely
+ * `owner_id = NULL`, so owner-scoped retrieval must run as the public-owner sentinel
+ * (`retrieval_owner_matches` maps it to NULL-owner rows, mirroring anonymous production search). A
+ * real owner UUID now scopes retrieval to zero documents. See docs/retrieval-quality-runbook.md.
+ */
+export const DEFAULT_EVAL_OWNER_ID = "00000000-0000-0000-0000-000000000000";
+
+/**
+ * Resolve the owner id for a READ/eval run. Precedence: explicit `--owner-id` / `RAG_EVAL_OWNER_ID`
+ * / `LOCAL_NO_AUTH_OWNER_ID` (already folded into `args.ownerId`) → `--owner-email` /
+ * `RAG_EVAL_OWNER_EMAIL` lookup → the public-owner sentinel. Emits a one-line warning when it falls
+ * back to the sentinel so the narrowing to public-only scope is visible, not silent.
+ *
+ * Do NOT use in write/backfill scripts — defaulting an owner there could write under the wrong owner.
+ */
+export async function resolveEvalOwnerId(
+ supabase: SupabaseAdmin,
+ args: { ownerId?: string; ownerEmail?: string },
+): Promise<string> {
+ const resolved = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined);
+ if (resolved) return resolved;
+ console.warn(
+ `[eval] No eval owner set (RAG_EVAL_OWNER_ID / LOCAL_NO_AUTH_OWNER_ID / --owner-id / --owner-email); ` +
+ `defaulting to the public-owner sentinel ${DEFAULT_EVAL_OWNER_ID}. The live corpus is all-public ` +
+ `(owner_id = NULL); set an explicit owner to scope the eval to a real owner instead.`,
+ );
+ return DEFAULT_EVAL_OWNER_ID;
+}
+
 export function percentile(values: number[], percentileValue: number) {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);

tests/eval-utils.test.ts

@@ -1,9 +1,12 @@
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import {
+ DEFAULT_EVAL_OWNER_ID,
  expectedFileCoverage,
  isProviderRateLimitError,
+ resolveEvalOwnerId,
  validateRagAnswer,
  withProviderBackoff,
+ type SupabaseAdmin,
 } from "../scripts/eval-utils";
 import type { RagEvalCase } from "../src/lib/rag-eval-cases";
 import type { RagAnswer } from "../src/lib/types";
@@ -95,3 +98,52 @@ describe("RAG eval source identity matching", () => {
  expect(isProviderRateLimitError(new Error("429 too many requests"))).toBe(true);
  });
 });
+
+describe("resolveEvalOwnerId", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ function adminClientWithUsers(users: Array<{ id: string; email: string }>): SupabaseAdmin {
+ return {
+ auth: { admin: { listUsers: async () => ({ data: { users }, error: null }) } },
+ } as unknown as SupabaseAdmin;
+ }
+
+ it("prefers an explicit ownerId over email lookup and the sentinel", async () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const listUsers = vi.fn();
+ const supabase = { auth: { admin: { listUsers } } } as unknown as SupabaseAdmin;
+
+ const ownerId = await resolveEvalOwnerId(supabase, {
+ ownerId: "explicit-owner",
+ ownerEmail: "user@example.com",
+ });
+
+ expect(ownerId).toBe("explicit-owner");
+ expect(listUsers).not.toHaveBeenCalled();
+ expect(warn).not.toHaveBeenCalled();
+ });
+
+ it("resolves ownerEmail via Supabase Auth when no ownerId is set", async () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const supabase = adminClientWithUsers([{ id: "user-123", email: "Clinician@Example.com" }]);
+
+ const ownerId = await resolveEvalOwnerId(supabase, { ownerEmail: "clinician@example.com" });
+
+ expect(ownerId).toBe("user-123");
+ expect(warn).not.toHaveBeenCalled();
+ });
+
+ it("falls back to the public-owner sentinel and warns when no owner is provided", async () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const supabase = adminClientWithUsers([]);
+
+ const ownerId = await resolveEvalOwnerId(supabase, {});
+
+ expect(ownerId).toBe(DEFAULT_EVAL_OWNER_ID);
+ expect(DEFAULT_EVAL_OWNER_ID).toBe("00000000-0000-0000-0000-000000000000");
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn.mock.calls[0]?.[0]).toContain(DEFAULT_EVAL_OWNER_ID);
+ });
+});