diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 52cbc50f2d..96475014ce 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -753,6 +753,7 @@ async function runRetrievalQualityCases(args: { topK: testCase.topK, minSimilarity: 0.12, skipCache: true, + forceEmbedding: testCase.forceEmbedding, }), ); const latencyMs = diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 0a4332f2f0..7218ae7319 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -19,6 +19,10 @@ const goldenCaseSchema = z.object({ expectedContentTerms: z.array(contentExpectationSchema).default([]), topK: z.number().int().positive().default(8), expectTableEvidence: z.boolean().default(false), + // Bypass the lexical text-fast-path so this case always exercises the embedding/vector + // index. Use for "vector-*" probes that would otherwise be answered by a lexical shortcut, + // so a re-index's effect on vector retrieval is actually measured. + forceEmbedding: z.boolean().optional(), }); const goldenCasesSchema = z.array(goldenCaseSchema); @@ -37,6 +41,7 @@ type EvalArgs = { caseTimeoutMs: number; p90BudgetMs: number; p50BudgetMs: number; + forceEmbedding: boolean; }; export type GoldenRetrievalResult = { @@ -116,6 +121,7 @@ function parseArgs(argv: string[]): EvalArgs { caseTimeoutMs: inferredMode === "latency" ? 25_000 : 0, p90BudgetMs: 20_000, p50BudgetMs: 8_000, + forceEmbedding: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -138,6 +144,10 @@ function parseArgs(argv: string[]): EvalArgs { if (args.caseTimeoutMs <= 0) args.caseTimeoutMs = 25_000; continue; } + if (token === "--force-embedding") { + args.forceEmbedding = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -743,6 +753,7 @@ async function main() { topK: testCase.topK, minSimilarity: 0.12, skipCache: args.mode !== "latency", + forceEmbedding: testCase.forceEmbedding || args.forceEmbedding, }), ); const searchOutcome = await withCaseTimeout(searchPromise, args.caseTimeoutMs); diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index b31176b768..34b5d4f368 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -224,5 +224,108 @@ ], "topK": 12, "expectTableEvidence": true + }, + { + "id": "vector-ptsd", + "query": "How is post-traumatic stress disorder managed in a person with intrusive flashbacks, nightmares, hyperarousal and avoidance after a traumatic event?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Traumatic Stress"], + "expectedContentTerms": [["trauma", "traumatic", "ptsd", "flashback"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-ocd", + "query": "How is obsessive-compulsive disorder managed in a person with distressing intrusive obsessions and repetitive compulsive rituals?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Obsessive"], + "expectedContentTerms": [["obsess", "compuls", "intrusive", "ocd"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-panic", + "query": "How is panic disorder managed in a person with recurrent unexpected panic attacks, palpitations and anticipatory anxiety?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Panic"], + "expectedContentTerms": [["panic", "attack", "anxiety"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-anorexia", + "query": "How is anorexia nervosa managed in a person with severe dietary restriction, intense fear of weight gain and body-image disturbance?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Anorexia"], + "expectedContentTerms": [["anorexia", "eating", "weight"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-gad-worry", + "query": "How is a patient managed who has persistent, excessive worry about many everyday things that they find very hard to control, along with restlessness and muscle tension?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Generalised Anxiety"], + "expectedContentTerms": [ + ["worry", "anxiety", "generalised"], + ["cbt", "ssri", "snri", "antidepressant", "psychotherapy"] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-tourette", + "query": "How is Tourette syndrome managed in a child with chronic motor tics and vocal tics?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Tourette"], + "expectedContentTerms": [["tic", "tics", "tourette"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-postnatal", + "query": "How is postnatal (postpartum) depression managed in a new mother who develops low mood and poor bonding with her infant in the first weeks postpartum?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Postnatal"], + "expectedContentTerms": [["postnatal", "postpartum", "perinatal", "depression"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-bipolar", + "query": "How is bipolar disorder managed in an adult with recurrent episodes of mania and depression?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Bipolar"], + "expectedContentTerms": [["bipolar", "mania", "manic", "mood"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-adhd", + "query": "How is attention deficit hyperactivity disorder managed in an adult with inattention, distractibility and impulsivity?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Attention Deficit Hyperactivity"], + "expectedContentTerms": [["attention", "adhd", "hyperactiv", "impuls"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-opioid", + "query": "How is opioid use disorder managed in a person dependent on heroin?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": ["Opioid Use Disorder"], + "expectedContentTerms": [["opioid", "heroin", "opiate", "methadone", "buprenorphine"]], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true } ] diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 6764accd2a..1620405c6e 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -307,6 +307,10 @@ export type SearchChunksArgs = { // Internal: set when this call is a re-run on a trigram-corrected query, to prevent the // unsupported-short-circuit typo-correction path from recursing more than once. typoCorrected?: boolean; + // Diagnostic/eval-only: bypass every lexical text-fast-path so retrieval always exercises + // the embedding/vector stage. Lets the golden eval measure the vector index directly for a + // re-index, instead of being masked by lexical shortcuts. Never set on production paths. + forceEmbedding?: boolean; }; export type AnswerProgressEvent = { @@ -1429,7 +1433,7 @@ function stableHash(value: string) { export function retrievalPlanCacheQuery( args: Pick< SearchChunksArgs, - "query" | "documentId" | "documentIds" | "ownerId" | "queryMode" | "topK" | "minSimilarity" + "query" | "documentId" | "documentIds" | "ownerId" | "queryMode" | "topK" | "minSimilarity" | "forceEmbedding" >, queryClass?: RagQueryClass, queryVariants: string[] = [], @@ -1445,6 +1449,7 @@ export function retrievalPlanCacheQuery( `topK:${args.topK ?? 8}`, `min:${args.minSimilarity ?? 0.15}`, `rag:${ragDeepMemoryVersion}`, + `force:${args.forceEmbedding ? 1 : 0}`, ].join("|"); return queryCacheKeyForStorage(cacheKey); } @@ -5516,7 +5521,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const baseTextFastPath = decideTextFastPath(args.query, baseTextResults, queryClassification.queryClass); - if (shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { + if (!args.forceEmbedding && shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { textFastResults = await attachPageVisualEvidence(supabase, baseTextResults); textFastResults = applySecondStageRerankIfNeeded({ queryClass: queryClassification.queryClass, @@ -5567,7 +5572,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry.rerank_latency_ms += Date.now() - rerankStartedAt; const boostedTextFastPath = decideTextFastPath(args.query, textFastResults, queryClassification.queryClass); - if (boostedTextFastPath.returnFastPath) { + if (!args.forceEmbedding && boostedTextFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath(telemetry, boostedTextFastPath.reason); telemetry.retrieval_strategy = "text_fast_path"; recordSearchScoreTelemetry(telemetry, textFastResults); @@ -5673,7 +5678,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentLookupResults, queryClassification.queryClass, ); - if (documentLookupFastPath.returnFastPath) { + if (!args.forceEmbedding && documentLookupFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath( telemetry, documentLookupFastPath.reason ? `document_lookup_fast_path:${documentLookupFastPath.reason}` : null, @@ -5700,7 +5705,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const coverageGate = evaluateEvidenceCoverageGate(args.query, coverageGateResults, queryClassification.queryClass); applyCoverageGateTelemetry(telemetry, coverageGate, coverageGate.accepted); - if (coverageGate.accepted) { + if (!args.forceEmbedding && coverageGate.accepted) { telemetry.retrieval_strategy = coverageGate.strategy; recordSearchScoreTelemetry(telemetry, coverageGateResults); setCachedSearch(args, coverageGateResults, telemetry, queryVariants); @@ -5744,6 +5749,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { latencyMs: telemetry.embedding_latency_ms, }); + if (args.forceEmbedding) { + // Force-embedding eval isolation: drop the lexical / memory-card / table candidates gathered + // before embedding so the returned results reflect the embedding-driven retrieval layers only + // (otherwise a broken vector index could still be masked by the lexical text candidate path). + textFastResults = []; + } + // A1: the embedding-field, index-unit, and chunk-hybrid RPCs each depend only on the // already-computed query embedding and have no data dependency on one another, so run // them concurrently instead of as three sequential Supabase round-trips. The two helper @@ -5780,7 +5792,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { const startedAt = Date.now(); const { data, error } = await supabase.rpc("match_document_chunks_hybrid", { query_embedding: embedding as unknown as string, - query_text: textSearchQuery, + query_text: args.forceEmbedding ? "" : textSearchQuery, match_count: candidateCount, min_similarity: minSimilarity, document_filters: documentFilterList ?? undefined,