diff --git a/src/app/api/ingestion/jobs/[id]/retry/route.ts b/src/app/api/ingestion/jobs/[id]/retry/route.ts index be59087f7b..144eccd5b1 100644 --- a/src/app/api/ingestion/jobs/[id]/retry/route.ts +++ b/src/app/api/ingestion/jobs/[id]/retry/route.ts @@ -24,35 +24,15 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: if (jobError) throw new Error(jobError.message); if (!job) return NextResponse.json({ error: "Ingestion job not found." }, { status: 404 }); - // IDX-C3: refuse to retry a job a live worker still holds. The claim RPC uses - // SKIP LOCKED + stale recovery; resetting here while status='processing' with a fresh - // lock would make the row claimable by a second worker, so two runs would insert against - // the same document_id concurrently and interleave mixed-generation clinical chunks. - if (job.status === "processing" && job.locked_at) { - const lockedAtMs = new Date(job.locked_at).getTime(); - const staleAfterMs = env.WORKER_STALE_AFTER_MINUTES * 60_000; - const lockIsFresh = Number.isFinite(lockedAtMs) && Date.now() - lockedAtMs < staleAfterMs; - if (lockIsFresh) { - return NextResponse.json( - { - error: - "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.", - }, - { status: 409 }, - ); - } - } - - // IDX-H1: do NOT reset the document index here. The worker calls resetDocumentIndex at - // job start (worker/main.ts), so resetting before enqueue would leave a previously-good - // clinical document with zero index if the worker never runs or fails permanently. We - // only re-queue; the prior index stays live until the worker commits a fresh one. - const { error: documentError } = await supabase - .from("documents") - .update({ status: "queued", error_message: null }) - .eq("id", job.document_id) - .eq("owner_id", user.id); - if (documentError) throw new Error(documentError.message); + // IDX-C3 / B6: refuse to retry a job a live worker still holds, atomically. + // A SELECT-then-UPDATE was a TOCTOU race: a worker could claim the job + // between the read and the write, and the unguarded UPDATE would silently + // reset the row the worker is processing → two workers ingest the same + // document_id and interleave mixed-generation clinical chunks. We instead + // make the reset a single conditional UPDATE whose WHERE clause refuses to + // touch a row that is freshly 'processing'. The reset only applies when the + // job is NOT processing, OR its lock is already stale, OR it has no lock. + const staleThreshold = new Date(Date.now() - env.WORKER_STALE_AFTER_MINUTES * 60_000).toISOString(); const { data, error } = await supabase .from("ingestion_jobs") @@ -69,10 +49,34 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: completed_at: null, }) .eq("id", id) + .or(`status.neq.processing,locked_at.is.null,locked_at.lt.${staleThreshold}`) .select() - .single(); + .maybeSingle(); if (error) throw new Error(error.message); + // 0 rows affected means the guard rejected the reset: the job is actively + // being processed with a fresh lock. Refuse rather than clobber it. + if (!data) { + return NextResponse.json( + { + error: + "This job is still being processed by a worker. Wait for it to finish or go stale before retrying.", + }, + { status: 409 }, + ); + } + + // IDX-H1: do NOT reset the document index here. The worker calls resetDocumentIndex at + // job start (worker/main.ts), so resetting before enqueue would leave a previously-good + // clinical document with zero index if the worker never runs or fails permanently. We + // only re-queue; the prior index stays live until the worker commits a fresh one. + const { error: documentError } = await supabase + .from("documents") + .update({ status: "queued", error_message: null }) + .eq("id", job.document_id) + .eq("owner_id", user.id); + if (documentError) throw new Error(documentError.message); + return NextResponse.json({ job: data }); } catch (error) { if (error instanceof AuthenticationError) return unauthorizedResponse(); diff --git a/src/lib/answer-verification.ts b/src/lib/answer-verification.ts index 9af1a7073b..bca2fa821b 100644 --- a/src/lib/answer-verification.ts +++ b/src/lib/answer-verification.ts @@ -18,8 +18,13 @@ import type { Citation, DocumentTableFact, SearchResult } from "@/lib/types"; // and unit-bearing figures. // Hardened to include complex clinical units (mg/kg/day, mmol/L, etc.) and // ensure boundary safety to prevent hallucinated decimal bypass. +// The ×10^n / ×10⁹ scientific-notation branch is listed BEFORE the bare unit +// branches so "2.0 ×10⁹/L" is captured whole rather than truncated to "2.0". +// The percentage branch must NOT be followed by \b: "%" is a non-word char, so +// a trailing \b after it can never match (it would require a word char to its +// right), which previously dropped every percentage token. const NUMERIC_TOKEN_PATTERN = - /\b\d+(?:[.,]\d+)?(?:\s*[-–—]\s*\d+(?:[.,]\d+)?)?\s*(?:%|mg\/(?:day|kg|m2|dose)|mg|mcg|micrograms?|μg|g\b|kg|ml|mL|l\b|mmol\/[lL]|mmol|mol\/[lL]|umol\/[lL]|µmol\/[lL]|ng\/ml|units?\/?\w*|iu\b|×10\^?\d*\/?l?|x10\^?\d*\/?l?|×10⁹\/l|hours?|hrs?|h\b|days?|weeks?|wk\b|months?|minutes?|mins?|years?|°c|mmhg|bpm)\b/giu; + /\b\d+(?:[.,]\d+)?(?:\s*[-–—]\s*\d+(?:[.,]\d+)?)?\s*(?:×10\^?\d*\/?l?|x10\^?\d*\/?l?|mg\/(?:day|kg|m2|dose)|mg|mcg|micrograms?|μg|g\b|kg|ml|mL|l\b|mmol\/[lL]|mmol|mol\/[lL]|umol\/[lL]|µmol\/[lL]|ng\/ml|units?\/?\w*|iu\b|hours?|hrs?|h\b|days?|weeks?|wk\b|months?|minutes?|mins?|years?|°c|mmhg|bpm)\b|\b\d+(?:[.,]\d+)?\s*%/giu; // Decimal numbers and ranges that, while not unit-bearing, are very likely // clinical thresholds in context (e.g. "ANC 2.0", "INR 2-3"). We only treat a @@ -27,45 +32,48 @@ const NUMERIC_TOKEN_PATTERN = // flagging incidental integers. const SIGNIFICANT_BARE_NUMBER_PATTERN = /\b\d+(?:[.,]\d+)?\s*[-–—]\s*\d+(?:[.,]\d+)?\b|\b\d+\.\d+\b/gu; +// Unicode superscript digits ⁰¹²³⁴⁵⁶⁷⁸⁹ → ASCII. Sources and answers mix +// "×10⁹/L" with "x10^9/L"; folding superscripts to ASCII before extraction and +// matching keeps ANC/WBC thresholds (central to the clozapine use-case) from +// being mangled or silently dropped. +const SUPERSCRIPT_DIGITS: Record = { + "⁰": "0", + "¹": "1", + "²": "2", + "³": "3", + "⁴": "4", + "⁵": "5", + "⁶": "6", + "⁷": "7", + "⁸": "8", + "⁹": "9", +}; + +function foldSuperscripts(text: string): string { + return text.replace(/[⁰¹²³⁴⁵⁶⁷⁸⁹]/g, (ch) => SUPERSCRIPT_DIGITS[ch] ?? ch); +} + function normalizeNumericToken(raw: string): string { - return raw + return foldSuperscripts(raw) .toLowerCase() .replace(/\s+/g, "") .replace(/[–—]/g, "-") .replace(/,/g, ".") .replace(/µ/g, "μ") + .replace(/×10\^?(\d+)/g, "x10^$1") + .replace(/x10(\d)/g, "x10^$1") .trim(); } -// Build a normalized haystack from a single text blob so token lookups are -// whitespace/case/dash insensitive (matches how normalizeNumericToken folds). -function normalizeHaystack(text: string): string { - return text - .toLowerCase() - .replace(/[–—]/g, "-") - .replace(/,/g, ".") - .replace(/µ/g, "μ") - .replace(/\s+/g, " "); -} - -// A token "appears" in the source if either the spaced or the de-spaced form is -// present, so "2.0 mg" in the answer matches "2.0mg" or "2.0 mg" in the source. -function haystackContainsToken(haystack: string, despacedHaystack: string, token: string): boolean { - const normalized = token; // already normalized/de-spaced by caller - if (despacedHaystack.includes(normalized)) return true; - // Also try a spaced variant: re-insert a space between number and unit. - const spaced = normalized.replace(/^(\d+(?:\.\d+)?(?:-\d+(?:\.\d+)?)?)/, "$1 "); - return haystack.includes(spaced); -} - export function extractNumericTokens(text: string): string[] { if (!text) return []; + const folded = foldSuperscripts(text); const tokens = new Set(); - for (const match of text.matchAll(NUMERIC_TOKEN_PATTERN)) { + for (const match of folded.matchAll(NUMERIC_TOKEN_PATTERN)) { const normalized = normalizeNumericToken(match[0]); if (normalized) tokens.add(normalized); } - for (const match of text.matchAll(SIGNIFICANT_BARE_NUMBER_PATTERN)) { + for (const match of folded.matchAll(SIGNIFICANT_BARE_NUMBER_PATTERN)) { const normalized = normalizeNumericToken(match[0]); if (normalized) tokens.add(normalized); } @@ -97,9 +105,15 @@ export type NumericVerification = { hasUnverifiedNumbers: boolean; }; -// Verify every numeric token in `answerText` against the combined text of the +// Verify every numeric token in `answerText` against the numeric tokens of the // chunks the answer actually cites. Only cited chunks count — an answer that // cites nothing has no support, so all of its numbers are unverified. +// +// B1: matching is by SET MEMBERSHIP of normalized numeric tokens, never by +// substring. Substring matching let "2.5 mg" match inside "12.5 mg" (a 5x dose +// error) and "500 mg" inside "1500 mg". We extract the source's tokens with the +// SAME extractor used on the answer, so a number is verified only when an exact +// normalized token (e.g. "2.5mg") is present in the source's token set. export function verifyAnswerNumbers( answerText: string, citations: Array>, @@ -112,17 +126,16 @@ export function verifyAnswerNumbers( const citedIds = new Set(citations.map((citation) => citation.chunk_id)); const citedResults = results.filter((result) => citedIds.has(result.id)); - // If no citation maps to a known chunk, fall back to all results so we don't - // over-flag during transitional states; cited-only is preferred when present. - const sourceResults = citedResults.length > 0 ? citedResults : results; - - const combined = sourceResults.map(sourceTextForResult).join(" \n "); - const haystack = normalizeHaystack(combined); - const despacedHaystack = haystack.replace(/\s+/g, ""); + // N1: fail closed. If nothing the answer cites maps to a known chunk, we have + // no scoped evidence — treat every numeric token as unverified rather than + // matching against the full unfiltered result set (which could "verify" a + // number that lives in a chunk the answer never cited). + if (citedResults.length === 0) { + return { answerTokens, unverifiedTokens: answerTokens, hasUnverifiedNumbers: answerTokens.length > 0 }; + } - const unverifiedTokens = answerTokens.filter( - (token) => !haystackContainsToken(haystack, despacedHaystack, token), - ); + const sourceTokens = sourceNumericTokenSet(citedResults); + const unverifiedTokens = answerTokens.filter((token) => !sourceTokens.has(token)); return { answerTokens, @@ -131,5 +144,18 @@ export function verifyAnswerNumbers( }; } +// Build the union of normalized numeric tokens across a set of source chunks +// using the same extractor applied to the answer, so verification is an exact +// token-membership test rather than a substring scan. +function sourceNumericTokenSet(results: SearchResult[]): Set { + const tokens = new Set(); + for (const result of results) { + for (const token of extractNumericTokens(sourceTextForResult(result))) { + tokens.add(token); + } + } + return tokens; +} + export const VERIFY_AGAINST_SOURCE_NOTE = "CRITICAL: Some figures in this answer could not be matched verbatim to the cited sources — verify against the source documents before acting."; diff --git a/src/lib/rag-routing.ts b/src/lib/rag-routing.ts index 5a31fff996..a0f4379d23 100644 --- a/src/lib/rag-routing.ts +++ b/src/lib/rag-routing.ts @@ -335,10 +335,16 @@ export function chooseAnswerRoute(args: { export function shouldRetryWithStrongAfterFast(args: { route: AnswerRoute; - answer: Pick; + answer: Pick; results: SearchResult[]; }) { if (args.route.mode !== "fast") return false; + // A structured-parse failure is a generation/format problem, not a weak- + // retrieval signal. Since B5 made that fallback fail closed (ungrounded, no + // citations), retrying with the strong model would re-run generation (often + // re-truncating) instead of letting extractive recovery use the retrieved + // sources. Skip the strong retry here; extractive recovery handles it. + if (args.answer.routingReason === "structured_parse_fallback") return false; if (args.answer.grounded && args.answer.confidence !== "unsupported" && args.answer.citations.length > 0) { return false; } diff --git a/src/lib/rag.ts b/src/lib/rag.ts index de4897446b..fac30744ee 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -617,13 +617,17 @@ function normalizeSearchResults(results: SearchResult[]) { } function safeFallbackAnswer(raw: string, results: SearchResult[], query?: string): RagAnswer { - const citations = compactCitations(results); - const confidence = deriveConfidence(results, citations.length); - return { + // B5: on model-JSON parse failure we cannot trust any model-asserted citation + // mapping. Do NOT back-fill all retrieved chunks as citations and stamp the + // answer grounded — that re-introduces exactly the back-fill GEN-C3 removed, + // hidden in the error path. Treat a parse failure as ungrounded/unsupported, + // and still run the numeric faithfulness gate over the salvaged prose so any + // dose/threshold it contains is surfaced as unverified rather than trusted. + const answer: RagAnswer = { answer: boldHighYieldClinicalText(sanitizeAnswerText(raw) || machineReadableFallbackAnswer, query), - grounded: citations.length > 0 && confidence !== "unsupported", - confidence, - citations, + grounded: false, + confidence: "unsupported", + citations: [], sources: results, routingReason: "structured_parse_fallback", answerSections: [], @@ -631,6 +635,7 @@ function safeFallbackAnswer(raw: string, results: SearchResult[], query?: string visualEvidence: buildVisualEvidence(results), bestSource: selectBestSourceRecommendation(results), }; + return applyNumericVerification(answer); } function addOpenAIUsage(total: OpenAITokenUsage, usage?: OpenAITokenUsage) { @@ -3395,7 +3400,7 @@ export function parseAnswerJson(raw: string, results: SearchResult[], query?: st answer.routingReason = "ungrounded_no_model_citation"; } // GEN-C2 / GEN-H2: numeric faithfulness gate. - return applyNumericVerification(answer, query); + return applyNumericVerification(answer); } catch { return safeFallbackAnswer(raw, results, query); } @@ -3405,21 +3410,41 @@ export function parseAnswerJson(raw: string, results: SearchResult[], query?: st // answer against the text of its cited chunks. Unsupported figures are recorded // on the answer and an explicit "verify against source" caveat is appended so a // paraphrased/mis-transcribed dose can never read as authoritative. -function applyNumericVerification(answer: RagAnswer, query?: string): RagAnswer { - const verification = verifyAnswerNumbers(answer.answer, answer.citations, answer.sources ?? []); - if (!verification.hasUnverifiedNumbers) return answer; +function applyNumericVerification(answer: RagAnswer): RagAnswer { + const sources = answer.sources ?? []; + const unverified = new Set(); + + // B4: the model is instructed to put dose details in structured + // answerSections (kind medication_dose), so a top-level-only scan never sees + // section-body doses. Verify the top-level answer AND every section body. + // Each section is scoped to its own citation_chunk_ids when present, so a + // dose is only credited against the chunks that section actually cites; + // sections with no citations fall back to the answer-level citations. + const answerVerification = verifyAnswerNumbers(answer.answer, answer.citations, sources); + for (const token of answerVerification.unverifiedTokens) unverified.add(token); + + for (const section of answer.answerSections ?? []) { + const sectionCitations = + section.citation_chunk_ids.length > 0 + ? section.citation_chunk_ids.map((chunk_id) => ({ chunk_id })) + : answer.citations; + const sectionVerification = verifyAnswerNumbers(section.body, sectionCitations, sources); + for (const token of sectionVerification.unverifiedTokens) unverified.add(token); + } + + if (unverified.size === 0) return answer; - answer.unverifiedNumericTokens = verification.unverifiedTokens; + const unverifiedTokens = [...unverified]; + answer.unverifiedNumericTokens = unverifiedTokens; answer.faithfulnessWarning = VERIFY_AGAINST_SOURCE_NOTE; // Surface as a source gap so the UI's existing gap rendering shows it, and // never let an answer with unverified clinical numbers claim high confidence. const caveat: ConflictOrGap = { type: "gap", - message: `${VERIFY_AGAINST_SOURCE_NOTE} Unverified figures: ${verification.unverifiedTokens.join(", ")}.`, + message: `${VERIFY_AGAINST_SOURCE_NOTE} Unverified figures: ${unverifiedTokens.join(", ")}.`, }; answer.conflictsOrGaps = [...(answer.conflictsOrGaps ?? []), caveat]; if (answer.confidence === "high") answer.confidence = "medium"; - void query; return answer; } @@ -4015,7 +4040,13 @@ ${buildRagSourceBlock(contextResults, { query: answerFocusQuery, queryClass })}` total_latency_ms: Date.now() - startedAt, }; - if (answer.citations.length > 0 && isUnusableGeneratedAnswer(answer)) { + // B5: a structured_parse_fallback answer now fails closed with zero + // citations, so we can no longer gate extractive recovery on the parsed + // answer's citations. buildExtractiveAnswer derives its own source-backed + // citations from the retrieved results, so trigger recovery whenever the + // generated answer is unusable and we have retrieved results to extract from. + const canRecoverExtractively = answer.citations.length > 0 || answerInputResults.length > 0; + if (canRecoverExtractively && isUnusableGeneratedAnswer(answer)) { answer = buildExtractiveAnswer({ query: args.query, queryClass, diff --git a/tests/answer-verification.test.ts b/tests/answer-verification.test.ts index 05ba6fd442..f0a7983605 100644 --- a/tests/answer-verification.test.ts +++ b/tests/answer-verification.test.ts @@ -62,6 +62,75 @@ describe("answer-verification (GEN-C2 / GEN-H2)", () => { expect(verification.unverifiedTokens).toContain("12.5mg"); }); + // B1: substring matching previously let a wrong dose verify against a longer + // number ("2.5 mg" inside "12.5 mg" = a 5x dose error). Matching is now by + // exact normalized token membership, so these must be flagged UNVERIFIED. + it("flags a dose that is only a substring of a source number (B1: 2.5 vs 12.5)", () => { + const result = source({ content: "Start clozapine 12.5 mg on day 1." }); + const verification = verifyAnswerNumbers("Give 2.5 mg.", [{ chunk_id: "chunk-1" }], [result]); + expect(verification.hasUnverifiedNumbers).toBe(true); + expect(verification.unverifiedTokens).toContain("2.5mg"); + }); + + it("flags 500 mg against a source that only contains 1500 mg (B1)", () => { + const result = source({ content: "Maximum dose is 1500 mg per day." }); + const verification = verifyAnswerNumbers("Use 500 mg.", [{ chunk_id: "chunk-1" }], [result]); + expect(verification.hasUnverifiedNumbers).toBe(true); + expect(verification.unverifiedTokens).toContain("500mg"); + }); + + it("flags 2.0 against a source that only contains 12.0 (B1)", () => { + const result = source({ content: "Threshold is 12.0 units." }); + const verification = verifyAnswerNumbers("Use 2.0 units.", [{ chunk_id: "chunk-1" }], [result]); + expect(verification.hasUnverifiedNumbers).toBe(true); + expect(verification.unverifiedTokens).toContain("2.0units"); + }); + + it("still verifies an exact dose match (B1)", () => { + const result = source({ content: "Start clozapine 12.5 mg on day 1." }); + const verification = verifyAnswerNumbers("Give 12.5 mg.", [{ chunk_id: "chunk-1" }], [result]); + expect(verification.hasUnverifiedNumbers).toBe(false); + expect(verification.unverifiedTokens).toEqual([]); + }); + + // B2: unicode superscript ANC/WBC thresholds must be extracted whole and + // match a source written in either superscript (×10⁹/L) or ASCII (x10^9/L). + it("extracts a unicode superscript threshold token whole (B2)", () => { + const tokens = extractNumericTokens("ANC below 2.0 ×10⁹/L."); + expect(tokens.some((t) => t.includes("x10^9"))).toBe(true); + expect(tokens.some((t) => t === "2.0x10")).toBe(false); + }); + + it("matches a superscript answer threshold against an ASCII source (B2)", () => { + const result = source({ content: "Withhold if ANC below 2.0 x10^9/L." }); + const verification = verifyAnswerNumbers("Withhold below 2.0 ×10⁹/L.", [{ chunk_id: "chunk-1" }], [result]); + expect(verification.hasUnverifiedNumbers).toBe(false); + }); + + // B3: the percentage branch never matched because of a trailing \b. Percentages + // must now extract, and a percentage mismatch must be flagged. + it("extracts percentage tokens (B3)", () => { + const tokens = extractNumericTokens("Seen in 80% and 50% of cases."); + expect(tokens).toContain("80%"); + expect(tokens).toContain("50%"); + }); + + it("flags a percentage absent from the cited source (B3)", () => { + const result = source({ content: "Occurs in 80% of patients." }); + const verification = verifyAnswerNumbers("Occurs in 50% of patients.", [{ chunk_id: "chunk-1" }], [result]); + expect(verification.hasUnverifiedNumbers).toBe(true); + expect(verification.unverifiedTokens).toContain("50%"); + }); + + // N1: with no cited chunk mapping to a known result, fail closed — numbers are + // unverified rather than checked against the full unfiltered result set. + it("fails closed when no citation maps to a known chunk (N1)", () => { + const uncited = source({ id: "chunk-2", content: "Dose is 12.5 mg." }); + const verification = verifyAnswerNumbers("Give 12.5 mg.", [{ chunk_id: "missing" }], [uncited]); + expect(verification.hasUnverifiedNumbers).toBe(true); + expect(verification.unverifiedTokens).toContain("12.5mg"); + }); + it("matches numbers in table facts of a cited chunk", () => { const result = source({ content: "See monitoring table.", diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index 1199f13a73..54a536f070 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -702,11 +702,17 @@ describe("private document API access", () => { expect(client.from).not.toHaveBeenCalled(); }); - it("refuses to retry a job a live worker still holds (IDX-C3)", async () => { - const freshLock = new Date(Date.now() - 60_000).toISOString(); + it("refuses to retry a job a live worker still holds (IDX-C3, B6)", async () => { + // B6: the reset is a single conditional UPDATE guarded on status/locked_at. + // A fresh 'processing' lock means the WHERE clause matches 0 rows, so the + // update resolves with no row → refuse with 409. const client = createSupabaseMock((call) => { if (call.table === "ingestion_jobs" && call.operation === "select") { - return ok({ id: "job-1", document_id: documentId, batch_id: null, status: "processing", locked_at: freshLock }); + return ok({ id: "job-1", document_id: documentId, batch_id: null, status: "processing", locked_at: null }); + } + if (call.table === "ingestion_jobs" && call.operation === "update") { + // Guard rejected the reset: no row affected. + return ok(null); } return ok([]); }); @@ -720,19 +726,22 @@ describe("private document API access", () => { expect(response.status).toBe(409); expect(String((await payload(response)).error)).toContain("still being processed"); - // Must not reset the document or re-queue the job while the lock is fresh. + // The guarded update must carry the status/stale-lock filter atomically. + const jobUpdate = client.calls.find((call) => call.table === "ingestion_jobs" && call.operation === "update"); + expect(jobUpdate?.filters.some((f) => f.column === "id")).toBe(true); + expect(jobUpdate?.orFilters.some((f) => f.includes("status.neq.processing") && f.includes("locked_at"))).toBe(true); + // Must NOT reset the document when the guard refused the job reset. expect(client.calls.some((call) => call.table === "documents" && call.operation === "update")).toBe(false); - expect(client.calls.some((call) => call.table === "ingestion_jobs" && call.operation === "update")).toBe(false); }); - it("re-queues a stale-locked job without resetting the live index (IDX-C3, IDX-H1)", async () => { - const staleLock = new Date(Date.now() - 60 * 60_000).toISOString(); + it("re-queues a stale/non-processing job without resetting the live index (IDX-C3, IDX-H1, B6)", async () => { const client = createSupabaseMock((call) => { if (call.table === "ingestion_jobs" && call.operation === "select") { - return ok({ id: "job-1", document_id: documentId, batch_id: null, status: "processing", locked_at: staleLock }); + return ok({ id: "job-1", document_id: documentId, batch_id: null, status: "failed", locked_at: null }); } if (call.table === "documents" && call.operation === "update") return ok([]); if (call.table === "ingestion_jobs" && call.operation === "update") { + // Guard allowed the reset: one row affected. return ok({ id: "job-1", document_id: documentId, status: "pending" }); } return ok([]); diff --git a/tests/rag-trust.test.ts b/tests/rag-trust.test.ts index 6428807aa2..cf0450e034 100644 --- a/tests/rag-trust.test.ts +++ b/tests/rag-trust.test.ts @@ -34,12 +34,27 @@ function source(overrides: Partial = {}): SearchResult { } describe("RAG trust validation", () => { - it("falls back safely when model JSON is invalid", () => { + // B5: on model-JSON parse failure the fallback must fail closed — it must NOT + // back-fill retrieved chunks as citations or stamp the answer grounded (that + // is exactly the back-fill GEN-C3 removed). It must drop to ungrounded / + // unsupported with no citations. + it("falls back safely (ungrounded, no citation back-fill) when model JSON is invalid (B5)", () => { const answer = parseAnswerJson("not json", [source()]); expect(answer.answer).toBe("not json"); - expect(answer.citations).toHaveLength(1); - expect(answer.grounded).toBe(true); + expect(answer.citations).toHaveLength(0); + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + }); + + // B5: a salvaged dose in the parse-failure path must still run the numeric + // gate and surface the verify-against-source caveat rather than read as trusted. + it("runs the numeric gate on parse-failure prose and flags unverified doses (B5)", () => { + const answer = parseAnswerJson("Give 500 mg now.", [source()]); + + expect(answer.grounded).toBe(false); + expect(answer.unverifiedNumericTokens).toContain("500mg"); + expect(answer.faithfulnessWarning).toBeTruthy(); }); it("rejects hallucinated citations that are not retrieved chunks", () => { @@ -414,6 +429,58 @@ describe("RAG trust validation", () => { expect(answer.faithfulnessWarning).toBeUndefined(); }); + // B4: a dose that lives only in an answerSections[].body (kind medication_dose) + // and is absent from the cited chunks must be flagged — the gate previously + // scanned only the top-level answer string. + it("flags a dose present only in a medication_dose section body (B4)", () => { + const answer = parseAnswerJson( + JSON.stringify({ + answer: "Titrate clozapine cautiously.", + grounded: true, + confidence: "high", + citations: [{ chunk_id: "chunk-1" }], + answerSections: [ + { + heading: "Medication/dose details", + kind: "medication_dose", + supportLevel: "direct", + body: "Start at 200 mg on day one.", + citation_chunk_ids: ["chunk-1"], + }, + ], + }), + [source({ content: "Start clozapine 12.5 mg on day one, then titrate slowly." })], + ); + + expect(answer.unverifiedNumericTokens).toContain("200mg"); + expect(answer.faithfulnessWarning).toBeTruthy(); + expect(answer.confidence).not.toBe("high"); + }); + + it("does not flag a section dose that is present in the cited source (B4)", () => { + const answer = parseAnswerJson( + JSON.stringify({ + answer: "Titrate clozapine cautiously.", + grounded: true, + confidence: "medium", + citations: [{ chunk_id: "chunk-1" }], + answerSections: [ + { + heading: "Medication/dose details", + kind: "medication_dose", + supportLevel: "direct", + body: "Start at 12.5 mg on day one.", + citation_chunk_ids: ["chunk-1"], + }, + ], + }), + [source({ content: "Start clozapine 12.5 mg on day one, then titrate slowly." })], + ); + + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + expect(answer.faithfulnessWarning).toBeUndefined(); + }); + // GEN-H1: prompt-injection neutralization + fences. it("neutralizes instruction-like phrases in source content and fences the source block (GEN-H1)", () => { const block = buildRagSourceBlock([