diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 1d9fb15e6c..41c0cca98c 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -236,9 +236,39 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went `mrr@10=0.8148`, `content_mrr@10=0.9244`, strategies `{text_fast_path:25, document_lookup_fast_path:1, hybrid:10}`, all 10 forced-embedding vector cases passed (`force_embedding_failure_count=0`). A direct read-only DB probe confirmed the premise: all **2,065** documents have `owner_id = NULL`; - the old eval owner `2bac05f1-…` owns 0. **This closes the #347 part-2 eval debt** — the golden - retrieval eval is the behavior-preservation proof the rag.ts decomposition owed, and it now runs - green with no manual owner setup. + the old eval owner `2bac05f1-…` owns 0. **This closes the retrieval half of the #347 part-2 eval + debt** — the golden retrieval eval is the retrieval behavior-preservation proof, and it now runs + green with no manual owner setup. (#347 also owed `eval:quality -- --rag-only`; that answer-path + half was still owner-blocked until the follow-up below.) + +## Eval-owner default hoisted to all read/eval scripts + #347 answer-path gate closed (2026-07-08) + +- **Follow-up to the 2026-07-07 fix above.** PR #348 only patched `eval-retrieval.ts`, so + `eval:quality` (incl. `--rag-only`), `eval:rag`, `eval:answer-quality`, and `eval:search` still + resolved the owner as `args.ownerId ?? emailLookup ?? undefined` and returned 0/N against the + all-public corpus — the exact failure #348 fixed for retrieval, still live for the answer path. +- **Fix (item 1 + 3):** hoisted `DEFAULT_EVAL_OWNER_ID` + `resolveEvalOwnerId(supabase, args)` into + the shared `scripts/eval-utils.ts` and applied it at the final owner-resolution point in all five + read/eval scripts (`eval-retrieval` refactored onto it; its local duplicate constant removed). + Precedence preserved (explicit id → email lookup → sentinel); the helper prints a **one-line + warning on the sentinel fallback** so the narrowing to public-only scope is visible, not silent. + Write/backfill scripts (`enrich-documents`, `classify-documents`, `backfill-*`) deliberately + excluded — defaulting an owner there could write under the wrong owner. First-ever + `resolveEvalOwnerId` unit coverage added to `tests/eval-utils.test.ts`. +- **Gates:** `verify:cheap` green (1277 tests; 0 lint errors). `eval:retrieval:quality` re-ran + **36/36, failed_cases=0** on the refactored script (regression check). The fallback warning was + observed firing in a real `eval:quality` run. +- **#347 answer-path gate CLOSED (item 2) — `eval:quality -- --rag-only`, live, no manual owner:** + `cases=44`, **unsupported_correct_rate=1.0**, **citation_failure_rate=0.0455**, + **numeric_grounding_failure_rate=0.0227** — all three invariants identical to #343's live + baseline; `grounded_supported_rate=0.90` (vs #343 `0.9333`). The 5 failing cases are entirely in + #343's documented live-variance set: 2 pure latency-threshold failures (this cloud env's + Supabase p95 ≈ 49 s — the local→remote latency #343 called out) and 3 route/retrieval flakes + (`illegal-substances`, `discharge-documentation`, `community-admission`) that #343 already + identified as pre-existing variance outside the touched paths. Because this change is eval-config + only (owner resolution) and touches no answer-generation code, the identical invariant rates + confirm the part-2 decomposition preserved answer behavior. **#347 part-2 eval debt now fully + settled** (retrieval half via #348, answer-path half here). ## Answer-thread Back button: URL and visible answer can disagree (2026-07-06) diff --git a/docs/retrieval-quality-runbook.md b/docs/retrieval-quality-runbook.md index 648332aa26..9a62b5449f 100644 --- a/docs/retrieval-quality-runbook.md +++ b/docs/retrieval-quality-runbook.md @@ -51,13 +51,17 @@ The command requires the same live-eval environment as the existing RAG eval scr - Owner is **optional** — see below. Since the 2026-07-06 public promotion the live corpus is entirely `owner_id = NULL`, the eval now -**defaults its owner to the public-owner sentinel** `00000000-0000-0000-0000-000000000000` -(`DEFAULT_EVAL_OWNER_ID` in `scripts/eval-retrieval.ts`). `retrieval_owner_matches` maps that -sentinel to NULL-owner rows, mirroring anonymous production search, so no session has to set -`RAG_EVAL_OWNER_ID` by hand. An explicit `RAG_EVAL_OWNER_ID`, `LOCAL_NO_AUTH_OWNER_ID`, or -`RAG_EVAL_OWNER_EMAIL` (or `--owner-id` / `--owner-email`) still overrides the default. Note a real -owner UUID now scopes retrieval to zero documents and fails every case, so only override when the -corpus ownership actually changes. +**defaults its owner to the public-owner sentinel** `00000000-0000-0000-0000-000000000000` via the +shared `resolveEvalOwnerId()` helper in `scripts/eval-utils.ts`. This default applies across the +whole read/eval suite — `eval:retrieval:quality`, `eval:quality` (incl. `--rag-only`), `eval:rag`, +`eval:answer-quality`, and `eval:search` — not just the retrieval eval. `retrieval_owner_matches` +maps the sentinel to NULL-owner rows, mirroring anonymous production search, so no session has to +set `RAG_EVAL_OWNER_ID` by hand; the helper prints a one-line warning whenever it falls back to the +sentinel so the public-only scope is visible. An explicit `RAG_EVAL_OWNER_ID`, `LOCAL_NO_AUTH_OWNER_ID`, +or `RAG_EVAL_OWNER_EMAIL` (or `--owner-id` / `--owner-email`) still overrides the default. Note a +real owner UUID now scopes retrieval to zero documents and fails every case, so only override when +the corpus ownership actually changes. (Write/backfill scripts deliberately do **not** use this +default — they must target an explicit owner.) Optional cost fields: diff --git a/scripts/eval-answer-quality.ts b/scripts/eval-answer-quality.ts index 957ce0db17..f7419f2387 100644 --- a/scripts/eval-answer-quality.ts +++ b/scripts/eval-answer-quality.ts @@ -16,7 +16,7 @@ import { type AnswerQualityMetric, } from "@/lib/rag-eval-cases"; import type { RagAnswer } from "@/lib/types"; -import { findOwnerIdByEmail, loadAdminClient, withProviderBackoff } from "./eval-utils"; +import { loadAdminClient, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils"; loadEnvConfig(process.cwd()); @@ -64,7 +64,7 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); + const ownerId = await resolveEvalOwnerId(supabase, args); let cases: AnswerQualityEvalCase[] = answerQualityEvalCases; if (args.intent) cases = cases.filter((testCase) => testCase.expectedIntent === args.intent); if (args.limit) cases = cases.slice(0, args.limit); diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 0c1749ceb8..23eb97877a 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -13,9 +13,9 @@ import { } from "./eval-retrieval"; import { estimateCostUsd, - findOwnerIdByEmail, loadAdminClient, percentile, + resolveEvalOwnerId, validateRagAnswer, withProviderBackoff, } from "./eval-utils"; @@ -941,7 +941,7 @@ async function main() { ? await loadSourceMetadataDebtAcceptance(args.sourceMetadataDebt) : undefined; - const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); + const ownerId = await resolveEvalOwnerId(supabase, args); const retrievalResults = args.ragOnly ? [] : await runRetrievalQualityCases({ ...args, ownerId, supabase }); const ragResults = args.retrievalOnly ? [] : await runRagQualityCases({ ...args, ownerId, supabase }); const report = buildEvalQualityReport({ retrievalResults, ragResults, sourceMetadataDebtAcceptance }); diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts index 261f37eed9..f85d2b4eed 100644 --- a/scripts/eval-rag.ts +++ b/scripts/eval-rag.ts @@ -3,9 +3,9 @@ import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases"; import type { RagAnswer } from "@/lib/types"; import { estimateCostUsd, - findOwnerIdByEmail, loadAdminClient, percentile, + resolveEvalOwnerId, validateRagAnswer, withProviderBackoff, } from "./eval-utils"; @@ -212,8 +212,8 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); - const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public"; + const ownerId = await resolveEvalOwnerId(supabase, args); + const scope = args.ownerId ? "owner:id" : args.ownerEmail ? `owner:${args.ownerEmail}` : "public"; const cases = selectRagEvalCases({ limit: args.limit, question: args.question }); const results: EvalResult[] = []; diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 2e6d4de957..1e3ec67f09 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -5,17 +5,10 @@ import { loadEnvConfig } from "@next/env"; import { z } from "zod"; import { loadCapturedRagEvalCases, type RagEvalCase, type SupabaseEvalCaseClient } from "@/lib/rag-eval-cases"; import type { SearchResult } from "@/lib/types"; -import { findOwnerIdByEmail, loadAdminClient, percentile, withProviderBackoff } from "./eval-utils"; +import { loadAdminClient, percentile, resolveEvalOwnerId, withProviderBackoff } from "./eval-utils"; loadEnvConfig(process.cwd()); -// Committed default eval owner. Since the 2026-07-06 public promotion the live corpus is -// entirely owner_id = NULL, so owner-scoped retrieval must run as the public-owner sentinel -// (retrieval_owner_matches maps it to NULL-owner rows, mirroring anonymous production search). -// An explicit RAG_EVAL_OWNER_ID / LOCAL_NO_AUTH_OWNER_ID / RAG_EVAL_OWNER_EMAIL (or --owner-id / -// --owner-email) still overrides this. See docs/retrieval-quality-runbook.md. -const DEFAULT_EVAL_OWNER_ID = "00000000-0000-0000-0000-000000000000"; - const contentExpectationSchema = z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]); const goldenCaseSchema = z.object({ @@ -819,10 +812,7 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = - args.ownerId ?? - (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined) ?? - DEFAULT_EVAL_OWNER_ID; + const ownerId = await resolveEvalOwnerId(supabase, args); const capturedCaseClient = supabase as unknown as SupabaseEvalCaseClient; const capturedCases = await loadCapturedRagEvalCases({ supabase: capturedCaseClient, ownerId, limit: args.limit }); const allCases = [...capturedCases.map(capturedRagCaseToGoldenCase), ...loadGoldenRetrievalCases(args.fixture)]; diff --git a/scripts/eval-search.ts b/scripts/eval-search.ts index dc458bb663..ea902711ac 100644 --- a/scripts/eval-search.ts +++ b/scripts/eval-search.ts @@ -12,10 +12,10 @@ import type { SearchResult } from "@/lib/types"; import { expectedFileCoverage, expectedFileHit, - findOwnerIdByEmail, hasInvalidVisualEvidence, loadAdminClient, percentile, + resolveEvalOwnerId, } from "./eval-utils"; loadEnvConfig(process.cwd()); @@ -201,8 +201,8 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); - const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); - const scope = ownerId ? `owner:${args.ownerId ? "id" : args.ownerEmail}` : "public"; + const ownerId = await resolveEvalOwnerId(supabase, args); + const scope = args.ownerId ? "owner:id" : args.ownerEmail ? `owner:${args.ownerEmail}` : "public"; const baseCases = selectRagEvalCases({ question: args.question }); const capturedCaseClient = supabase as unknown as SupabaseEvalCaseClient; const capturedCases = args.question diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts index 907934e86c..a4cc0ebce8 100644 --- a/scripts/eval-utils.ts +++ b/scripts/eval-utils.ts @@ -75,6 +75,36 @@ export async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string) throw new Error(`No Supabase Auth user found for ${email}. Sign in once before running evals.`); } +/** + * Committed default eval owner. Since the 2026-07-06 public promotion the live corpus is entirely + * `owner_id = NULL`, so owner-scoped retrieval must run as the public-owner sentinel + * (`retrieval_owner_matches` maps it to NULL-owner rows, mirroring anonymous production search). A + * real owner UUID now scopes retrieval to zero documents. See docs/retrieval-quality-runbook.md. + */ +export const DEFAULT_EVAL_OWNER_ID = "00000000-0000-0000-0000-000000000000"; + +/** + * Resolve the owner id for a READ/eval run. Precedence: explicit `--owner-id` / `RAG_EVAL_OWNER_ID` + * / `LOCAL_NO_AUTH_OWNER_ID` (already folded into `args.ownerId`) → `--owner-email` / + * `RAG_EVAL_OWNER_EMAIL` lookup → the public-owner sentinel. Emits a one-line warning when it falls + * back to the sentinel so the narrowing to public-only scope is visible, not silent. + * + * Do NOT use in write/backfill scripts — defaulting an owner there could write under the wrong owner. + */ +export async function resolveEvalOwnerId( + supabase: SupabaseAdmin, + args: { ownerId?: string; ownerEmail?: string }, +): Promise { + const resolved = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); + if (resolved) return resolved; + console.warn( + `[eval] No eval owner set (RAG_EVAL_OWNER_ID / LOCAL_NO_AUTH_OWNER_ID / --owner-id / --owner-email); ` + + `defaulting to the public-owner sentinel ${DEFAULT_EVAL_OWNER_ID}. The live corpus is all-public ` + + `(owner_id = NULL); set an explicit owner to scope the eval to a real owner instead.`, + ); + return DEFAULT_EVAL_OWNER_ID; +} + export function percentile(values: number[], percentileValue: number) { if (values.length === 0) return 0; const sorted = [...values].sort((a, b) => a - b); diff --git a/tests/eval-utils.test.ts b/tests/eval-utils.test.ts index 6fa1aa3f08..961f54e4b1 100644 --- a/tests/eval-utils.test.ts +++ b/tests/eval-utils.test.ts @@ -1,9 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { + DEFAULT_EVAL_OWNER_ID, expectedFileCoverage, isProviderRateLimitError, + resolveEvalOwnerId, validateRagAnswer, withProviderBackoff, + type SupabaseAdmin, } from "../scripts/eval-utils"; import type { RagEvalCase } from "../src/lib/rag-eval-cases"; import type { RagAnswer } from "../src/lib/types"; @@ -95,3 +98,52 @@ describe("RAG eval source identity matching", () => { expect(isProviderRateLimitError(new Error("429 too many requests"))).toBe(true); }); }); + +describe("resolveEvalOwnerId", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + function adminClientWithUsers(users: Array<{ id: string; email: string }>): SupabaseAdmin { + return { + auth: { admin: { listUsers: async () => ({ data: { users }, error: null }) } }, + } as unknown as SupabaseAdmin; + } + + it("prefers an explicit ownerId over email lookup and the sentinel", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const listUsers = vi.fn(); + const supabase = { auth: { admin: { listUsers } } } as unknown as SupabaseAdmin; + + const ownerId = await resolveEvalOwnerId(supabase, { + ownerId: "explicit-owner", + ownerEmail: "user@example.com", + }); + + expect(ownerId).toBe("explicit-owner"); + expect(listUsers).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + }); + + it("resolves ownerEmail via Supabase Auth when no ownerId is set", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const supabase = adminClientWithUsers([{ id: "user-123", email: "Clinician@Example.com" }]); + + const ownerId = await resolveEvalOwnerId(supabase, { ownerEmail: "clinician@example.com" }); + + expect(ownerId).toBe("user-123"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("falls back to the public-owner sentinel and warns when no owner is provided", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const supabase = adminClientWithUsers([]); + + const ownerId = await resolveEvalOwnerId(supabase, {}); + + expect(ownerId).toBe(DEFAULT_EVAL_OWNER_ID); + expect(DEFAULT_EVAL_OWNER_ID).toBe("00000000-0000-0000-0000-000000000000"); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain(DEFAULT_EVAL_OWNER_ID); + }); +});