From 4f03aeffe599327c0ebf39a87da8293210bca87d Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:00:12 +0800 Subject: [PATCH 1/2] test(eval): add force-embedding flag + 10 vector-exercising golden cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The golden retrieval set was 100% lexical fast-path (embedding_skipped_rate=1.0), so it could not measure whether a re-index changes vector/embedding retrieval quality. - forceEmbedding option on searchChunksWithTelemetry (SearchChunksArgs): bypasses every lexical text-fast-path so retrieval always exercises the embedding/vector stage. Diagnostic/eval-only; folded into the search cache key; never set on production paths. - eval-retrieval.ts: per-case `forceEmbedding` field + a global `--force-embedding` flag. - 10 `vector-*` cases (psychiatric monographs: PTSD, OCD, panic, anorexia, GAD, Tourette, postnatal, bipolar, ADHD, opioid) with forceEmbedding=true. Each is a clinical query that must be answered by vector retrieval of the right monograph — verified live at document_recall@5=1.0, content_recall@5=1.0, all via strategy=hybrid (embedding used). Rationale: forcing embedding is the correct instrument for re-index measurement — you want to measure the vector index directly, not have a lexical shortcut mask a regression. Wording alone can't reliably force the vector path (the fast-path is driven by emergent lexical-match strength), so the flag makes these probes deterministic. Live golden eval: 34/34 pass (24 existing + 10 new), no regression. verify:cheap green (980). Co-Authored-By: Claude Fable 5 --- scripts/eval-retrieval.ts | 11 ++ scripts/fixtures/rag-retrieval-golden.json | 194 +++++++++++++++++++++ src/lib/rag.ts | 11 +- 3 files changed, 213 insertions(+), 3 deletions(-) diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 0a4332f2f0..7218ae7319 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -19,6 +19,10 @@ const goldenCaseSchema = z.object({ expectedContentTerms: z.array(contentExpectationSchema).default([]), topK: z.number().int().positive().default(8), expectTableEvidence: z.boolean().default(false), + // Bypass the lexical text-fast-path so this case always exercises the embedding/vector + // index. Use for "vector-*" probes that would otherwise be answered by a lexical shortcut, + // so a re-index's effect on vector retrieval is actually measured. + forceEmbedding: z.boolean().optional(), }); const goldenCasesSchema = z.array(goldenCaseSchema); @@ -37,6 +41,7 @@ type EvalArgs = { caseTimeoutMs: number; p90BudgetMs: number; p50BudgetMs: number; + forceEmbedding: boolean; }; export type GoldenRetrievalResult = { @@ -116,6 +121,7 @@ function parseArgs(argv: string[]): EvalArgs { caseTimeoutMs: inferredMode === "latency" ? 25_000 : 0, p90BudgetMs: 20_000, p50BudgetMs: 8_000, + forceEmbedding: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -138,6 +144,10 @@ function parseArgs(argv: string[]): EvalArgs { if (args.caseTimeoutMs <= 0) args.caseTimeoutMs = 25_000; continue; } + if (token === "--force-embedding") { + args.forceEmbedding = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -743,6 +753,7 @@ async function main() { topK: testCase.topK, minSimilarity: 0.12, skipCache: args.mode !== "latency", + forceEmbedding: testCase.forceEmbedding || args.forceEmbedding, }), ); const searchOutcome = await withCaseTimeout(searchPromise, args.caseTimeoutMs); diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index b31176b768..5f3cdabb48 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -224,5 +224,199 @@ ], "topK": 12, "expectTableEvidence": true + }, + { + "id": "vector-ptsd", + "query": "How is post-traumatic stress disorder managed in a person with intrusive flashbacks, nightmares, hyperarousal and avoidance after a traumatic event?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Traumatic Stress" + ], + "expectedContentTerms": [ + [ + "trauma", + "traumatic", + "ptsd", + "flashback" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-ocd", + "query": "How is obsessive-compulsive disorder managed in a person with distressing intrusive obsessions and repetitive compulsive rituals?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Obsessive" + ], + "expectedContentTerms": [ + [ + "obsess", + "compuls", + "intrusive", + "ocd" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-panic", + "query": "How is panic disorder managed in a person with recurrent unexpected panic attacks, palpitations and anticipatory anxiety?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Panic" + ], + "expectedContentTerms": [ + [ + "panic", + "attack", + "anxiety" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-anorexia", + "query": "How is anorexia nervosa managed in a person with severe dietary restriction, intense fear of weight gain and body-image disturbance?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Anorexia" + ], + "expectedContentTerms": [ + [ + "anorexia", + "eating", + "weight" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-gad-worry", + "query": "How is a patient managed who has persistent, excessive worry about many everyday things that they find very hard to control, along with restlessness and muscle tension?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Generalised Anxiety" + ], + "expectedContentTerms": [ + [ + "worry", + "anxiety", + "generalised" + ], + [ + "cbt", + "ssri", + "snri", + "antidepressant", + "psychotherapy" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-tourette", + "query": "How is Tourette syndrome managed in a child with chronic motor tics and vocal tics?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Tourette" + ], + "expectedContentTerms": [ + [ + "tic", + "tics", + "tourette" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-postnatal", + "query": "How is postnatal (postpartum) depression managed in a new mother who develops low mood and poor bonding with her infant in the first weeks postpartum?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Postnatal" + ], + "expectedContentTerms": [ + [ + "postnatal", + "postpartum", + "perinatal", + "depression" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-bipolar", + "query": "How is bipolar disorder managed in an adult with recurrent episodes of mania and depression?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Bipolar" + ], + "expectedContentTerms": [ + [ + "bipolar", + "mania", + "manic", + "mood" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-adhd", + "query": "How is attention deficit hyperactivity disorder managed in an adult with inattention, distractibility and impulsivity?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Attention Deficit Hyperactivity" + ], + "expectedContentTerms": [ + [ + "attention", + "adhd", + "hyperactiv", + "impuls" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-opioid", + "query": "How is opioid use disorder managed in a person dependent on heroin?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Opioid Use Disorder" + ], + "expectedContentTerms": [ + [ + "opioid", + "heroin", + "opiate", + "methadone", + "buprenorphine" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true } ] diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 6764accd2a..9f8e6f059b 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -307,6 +307,10 @@ export type SearchChunksArgs = { // Internal: set when this call is a re-run on a trigram-corrected query, to prevent the // unsupported-short-circuit typo-correction path from recursing more than once. typoCorrected?: boolean; + // Diagnostic/eval-only: bypass every lexical text-fast-path so retrieval always exercises + // the embedding/vector stage. Lets the golden eval measure the vector index directly for a + // re-index, instead of being masked by lexical shortcuts. Never set on production paths. + forceEmbedding?: boolean; }; export type AnswerProgressEvent = { @@ -1455,6 +1459,7 @@ function scopedSearchCacheKey(args: SearchChunksArgs, queryClass?: RagQueryClass args.ownerId ?? "anonymous", scopeKey(args), retrievalPlanCacheQuery(args, queryClass, queryVariants), + args.forceEmbedding ? "force-embedding" : "", ].join("|"); } @@ -5516,7 +5521,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const baseTextFastPath = decideTextFastPath(args.query, baseTextResults, queryClassification.queryClass); - if (shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { + if (!args.forceEmbedding && shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { textFastResults = await attachPageVisualEvidence(supabase, baseTextResults); textFastResults = applySecondStageRerankIfNeeded({ queryClass: queryClassification.queryClass, @@ -5567,7 +5572,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry.rerank_latency_ms += Date.now() - rerankStartedAt; const boostedTextFastPath = decideTextFastPath(args.query, textFastResults, queryClassification.queryClass); - if (boostedTextFastPath.returnFastPath) { + if (!args.forceEmbedding && boostedTextFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath(telemetry, boostedTextFastPath.reason); telemetry.retrieval_strategy = "text_fast_path"; recordSearchScoreTelemetry(telemetry, textFastResults); @@ -5673,7 +5678,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentLookupResults, queryClassification.queryClass, ); - if (documentLookupFastPath.returnFastPath) { + if (!args.forceEmbedding && documentLookupFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath( telemetry, documentLookupFastPath.reason ? `document_lookup_fast_path:${documentLookupFastPath.reason}` : null, From b62f5163a43767c0f70476923113e6fd7ea758e3 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 23:02:29 +0800 Subject: [PATCH 2/2] =?UTF-8?q?test(eval):=20address=20Codex=20review=20?= =?UTF-8?q?=E2=80=94=20isolate=20vector,=20key=20cache,=20propagate=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shared rag_response_cache: key retrievalPlanCacheQuery by forceEmbedding so a cached non-forced text-fast-path result can't satisfy a forced vector run (comment 3520042053). - Bypass the evidence-coverage-gate early return under forceEmbedding, so forced cases can't return before the embedding stage (comment 3520042055). - Isolate the vector index for forced cases: drop the pre-embedding lexical/memory/table candidates and run match_document_chunks_hybrid vector-only (empty query_text), so a broken vector index can't be masked by the lexical text-candidate path (comments 3520042061). - eval-quality.ts (eval:quality:release) now forwards testCase.forceEmbedding so the release gate honours the vector-* annotations instead of taking lexical shortcuts (comment 3520042059). - Prettier-format the fixture. Re-validated live under isolation: 34/34 golden pass, document_recall@5=1.0, content_recall@5=1.0, all 10 vector-* cases strategy=hybrid embedding:1. verify:cheap green (980). Co-Authored-By: Claude Fable 5 --- scripts/eval-quality.ts | 1 + scripts/fixtures/rag-retrieval-golden.json | 133 ++++----------------- src/lib/rag.ts | 15 ++- 3 files changed, 33 insertions(+), 116 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 52cbc50f2d..96475014ce 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -753,6 +753,7 @@ async function runRetrievalQualityCases(args: { topK: testCase.topK, minSimilarity: 0.12, skipCache: true, + forceEmbedding: testCase.forceEmbedding, }), ); const latencyMs = diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index 5f3cdabb48..34b5d4f368 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -229,17 +229,8 @@ "id": "vector-ptsd", "query": "How is post-traumatic stress disorder managed in a person with intrusive flashbacks, nightmares, hyperarousal and avoidance after a traumatic event?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Traumatic Stress" - ], - "expectedContentTerms": [ - [ - "trauma", - "traumatic", - "ptsd", - "flashback" - ] - ], + "expectedDocumentSubstrings": ["Traumatic Stress"], + "expectedContentTerms": [["trauma", "traumatic", "ptsd", "flashback"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -248,17 +239,8 @@ "id": "vector-ocd", "query": "How is obsessive-compulsive disorder managed in a person with distressing intrusive obsessions and repetitive compulsive rituals?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Obsessive" - ], - "expectedContentTerms": [ - [ - "obsess", - "compuls", - "intrusive", - "ocd" - ] - ], + "expectedDocumentSubstrings": ["Obsessive"], + "expectedContentTerms": [["obsess", "compuls", "intrusive", "ocd"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -267,16 +249,8 @@ "id": "vector-panic", "query": "How is panic disorder managed in a person with recurrent unexpected panic attacks, palpitations and anticipatory anxiety?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Panic" - ], - "expectedContentTerms": [ - [ - "panic", - "attack", - "anxiety" - ] - ], + "expectedDocumentSubstrings": ["Panic"], + "expectedContentTerms": [["panic", "attack", "anxiety"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -285,16 +259,8 @@ "id": "vector-anorexia", "query": "How is anorexia nervosa managed in a person with severe dietary restriction, intense fear of weight gain and body-image disturbance?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Anorexia" - ], - "expectedContentTerms": [ - [ - "anorexia", - "eating", - "weight" - ] - ], + "expectedDocumentSubstrings": ["Anorexia"], + "expectedContentTerms": [["anorexia", "eating", "weight"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -303,22 +269,10 @@ "id": "vector-gad-worry", "query": "How is a patient managed who has persistent, excessive worry about many everyday things that they find very hard to control, along with restlessness and muscle tension?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Generalised Anxiety" - ], + "expectedDocumentSubstrings": ["Generalised Anxiety"], "expectedContentTerms": [ - [ - "worry", - "anxiety", - "generalised" - ], - [ - "cbt", - "ssri", - "snri", - "antidepressant", - "psychotherapy" - ] + ["worry", "anxiety", "generalised"], + ["cbt", "ssri", "snri", "antidepressant", "psychotherapy"] ], "topK": 8, "expectTableEvidence": false, @@ -328,16 +282,8 @@ "id": "vector-tourette", "query": "How is Tourette syndrome managed in a child with chronic motor tics and vocal tics?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Tourette" - ], - "expectedContentTerms": [ - [ - "tic", - "tics", - "tourette" - ] - ], + "expectedDocumentSubstrings": ["Tourette"], + "expectedContentTerms": [["tic", "tics", "tourette"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -346,17 +292,8 @@ "id": "vector-postnatal", "query": "How is postnatal (postpartum) depression managed in a new mother who develops low mood and poor bonding with her infant in the first weeks postpartum?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Postnatal" - ], - "expectedContentTerms": [ - [ - "postnatal", - "postpartum", - "perinatal", - "depression" - ] - ], + "expectedDocumentSubstrings": ["Postnatal"], + "expectedContentTerms": [["postnatal", "postpartum", "perinatal", "depression"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -365,17 +302,8 @@ "id": "vector-bipolar", "query": "How is bipolar disorder managed in an adult with recurrent episodes of mania and depression?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Bipolar" - ], - "expectedContentTerms": [ - [ - "bipolar", - "mania", - "manic", - "mood" - ] - ], + "expectedDocumentSubstrings": ["Bipolar"], + "expectedContentTerms": [["bipolar", "mania", "manic", "mood"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -384,17 +312,8 @@ "id": "vector-adhd", "query": "How is attention deficit hyperactivity disorder managed in an adult with inattention, distractibility and impulsivity?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Attention Deficit Hyperactivity" - ], - "expectedContentTerms": [ - [ - "attention", - "adhd", - "hyperactiv", - "impuls" - ] - ], + "expectedDocumentSubstrings": ["Attention Deficit Hyperactivity"], + "expectedContentTerms": [["attention", "adhd", "hyperactiv", "impuls"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true @@ -403,18 +322,8 @@ "id": "vector-opioid", "query": "How is opioid use disorder managed in a person dependent on heroin?", "expectedQueryClass": "broad_summary", - "expectedDocumentSubstrings": [ - "Opioid Use Disorder" - ], - "expectedContentTerms": [ - [ - "opioid", - "heroin", - "opiate", - "methadone", - "buprenorphine" - ] - ], + "expectedDocumentSubstrings": ["Opioid Use Disorder"], + "expectedContentTerms": [["opioid", "heroin", "opiate", "methadone", "buprenorphine"]], "topK": 8, "expectTableEvidence": false, "forceEmbedding": true diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 9f8e6f059b..1620405c6e 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -1433,7 +1433,7 @@ function stableHash(value: string) { export function retrievalPlanCacheQuery( args: Pick< SearchChunksArgs, - "query" | "documentId" | "documentIds" | "ownerId" | "queryMode" | "topK" | "minSimilarity" + "query" | "documentId" | "documentIds" | "ownerId" | "queryMode" | "topK" | "minSimilarity" | "forceEmbedding" >, queryClass?: RagQueryClass, queryVariants: string[] = [], @@ -1449,6 +1449,7 @@ export function retrievalPlanCacheQuery( `topK:${args.topK ?? 8}`, `min:${args.minSimilarity ?? 0.15}`, `rag:${ragDeepMemoryVersion}`, + `force:${args.forceEmbedding ? 1 : 0}`, ].join("|"); return queryCacheKeyForStorage(cacheKey); } @@ -1459,7 +1460,6 @@ function scopedSearchCacheKey(args: SearchChunksArgs, queryClass?: RagQueryClass args.ownerId ?? "anonymous", scopeKey(args), retrievalPlanCacheQuery(args, queryClass, queryVariants), - args.forceEmbedding ? "force-embedding" : "", ].join("|"); } @@ -5705,7 +5705,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const coverageGate = evaluateEvidenceCoverageGate(args.query, coverageGateResults, queryClassification.queryClass); applyCoverageGateTelemetry(telemetry, coverageGate, coverageGate.accepted); - if (coverageGate.accepted) { + if (!args.forceEmbedding && coverageGate.accepted) { telemetry.retrieval_strategy = coverageGate.strategy; recordSearchScoreTelemetry(telemetry, coverageGateResults); setCachedSearch(args, coverageGateResults, telemetry, queryVariants); @@ -5749,6 +5749,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { latencyMs: telemetry.embedding_latency_ms, }); + if (args.forceEmbedding) { + // Force-embedding eval isolation: drop the lexical / memory-card / table candidates gathered + // before embedding so the returned results reflect the embedding-driven retrieval layers only + // (otherwise a broken vector index could still be masked by the lexical text candidate path). + textFastResults = []; + } + // A1: the embedding-field, index-unit, and chunk-hybrid RPCs each depend only on the // already-computed query embedding and have no data dependency on one another, so run // them concurrently instead of as three sequential Supabase round-trips. The two helper @@ -5785,7 +5792,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { const startedAt = Date.now(); const { data, error } = await supabase.rpc("match_document_chunks_hybrid", { query_embedding: embedding as unknown as string, - query_text: textSearchQuery, + query_text: args.forceEmbedding ? "" : textSearchQuery, match_count: candidateCount, min_similarity: minSimilarity, document_filters: documentFilterList ?? undefined,