From 5ffa042d686a542de3333ebccbd903b6422124a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 13:03:56 +0000 Subject: [PATCH 1/8] fix(rag): stop caching soft-tail unsupported-short-circuit zero results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The soft-tail bucket of the unsupported short-circuit (low confidence, few expanded terms, no deterministic exclusion match) can be a false negative for genuinely in-corpus bare topics — corpus grounding only credits a document *title* match as a topic anchor, so a term like "catatonia" that is well represented in chunk content but never appears in a document title returns "inconclusive" rather than "in_corpus_topic", and falls through to a nondeterministic LLM classifier call. Caching that zero made a single unlucky classification sticky for every later caller within the cache TTL. searchChunksWithTelemetry now skips the cache write only for that specific soft-tail bucket (isUnsupportedSoftTailAnalysis); the three deterministic exclusion patterns ahead of it in shouldShortCircuitUnsupportedSearch are stable true negatives and stay cached as before. Also exposes telemetry.corpus_grounding on the /api/search response, which was already computed internally (rag.ts) but dropped by the route's hand-picked telemetry subset — needed to diagnose this class of false negative without direct Supabase access. No retrieval/ranking decision logic changed — this is a caching-write decision plus an additive observability field. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni5rjjPcnbYUg8m8xaFhgc --- src/app/api/search/route.ts | 1 + src/lib/rag/rag.ts | 12 +- ...ag-unsupported-short-circuit-cache.test.ts | 139 ++++++++++++++++++ 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 tests/rag-unsupported-short-circuit-cache.test.ts diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 9569f80066..cc465f5c04 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -897,6 +897,7 @@ async function buildScopedSearchPayload( weak_source_count: relevance.weakSourceCount, retrieval_strategy: search.telemetry.retrieval_strategy, retrieval_plan: search.telemetry.retrieval_plan, + corpus_grounding: search.telemetry.corpus_grounding, smart_api_intent: smartApiPlan.intent, smart_api_response_mode: smartApiPlan.responseMode, smart_api_display_mode: smartApiPlan.displayMode, diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index c8fda5a1be..da2d2154b2 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -1740,7 +1740,17 @@ export async function searchChunksWithTelemetry( telemetry.embedding_skip_reason = "unsupported_short_circuit"; telemetry.retrieval_strategy = "unsupported_short_circuit"; recordSearchScoreTelemetry(telemetry, []); - await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); + // Finding #11 follow-up: the soft-tail bucket (low confidence, few expanded terms, no + // deterministic exclusion match) can be resolved by a nondeterministic LLM classifier call + // or by corpus grounding returning "inconclusive" rather than "in_corpus_topic" for content + // that lacks a matching document title (e.g. "catatonia" is well represented in chunk text + // but no document is titled "Catatonia"). Caching that zero would make a single unlucky + // classification sticky for every later caller within the cache TTL, so this specific bucket + // is deliberately never cached. The three deterministic exclusion patterns above it in + // shouldShortCircuitUnsupportedSearch are stable true negatives and stay cached as before. + if (!isUnsupportedSoftTailAnalysis(retrievalQuery, queryAnalysis)) { + await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); + } return finishSearch(searchTiming, { results: [] as SearchResult[], telemetry }); } diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts new file mode 100644 index 0000000000..5b08c0403c --- /dev/null +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -0,0 +1,139 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// Finding #11 follow-up (handover 2026-08-06): the soft-tail unsupported short-circuit can be a +// false negative for genuinely in-corpus bare topics (e.g. "catatonia" is well represented in +// chunk content but no document is *titled* "Catatonia", so corpus grounding returns +// "inconclusive" rather than rescuing it, and the query falls through to the nondeterministic +// LLM classifier). Caching that zero made the false negative sticky for every later caller within +// the cache TTL. searchChunksWithTelemetry (src/lib/rag/rag.ts) must skip the cache write only for +// that specific soft-tail bucket, while still caching the three deterministic exclusion patterns +// (unavailable-document noise, clearly-outside-corpus terms, clearly-non-clinical consumer terms) +// that are stable true negatives. + +class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { + select() { + return this; + } + in() { + return this; + } + eq() { + return this; + } + is() { + return this; + } + neq() { + return this; + } + or() { + return this; + } + order() { + return this; + } + limit() { + return Promise.resolve({ data: [], error: null }); + } + then( + onfulfilled?: ((value: { data: unknown[]; error: null }) => TResult1 | PromiseLike) | null, + onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null, + ): PromiseLike { + return Promise.resolve({ data: [], error: null }).then(onfulfilled, onrejected); + } +} + +async function loadSearch(corpusGroundingVerdict: "inconclusive" | "out_of_corpus" | "in_corpus_topic") { + const setCachedSearch = vi.fn(async () => undefined); + + vi.doMock("@/lib/rag/rag-cache", async () => { + const actual = await vi.importActual("@/lib/rag/rag-cache"); + return { + ...actual, + cacheIndexingVersion: vi.fn(async () => "test-indexing-version"), + getCachedSearch: vi.fn(async () => null), + getSharedCachedSearch: vi.fn(async () => null), + setCachedSearch, + }; + }); + vi.doMock("@/lib/rag/rag-retrieval-variants", async () => { + const actual = await vi.importActual( + "@/lib/rag/rag-retrieval-variants", + ); + return { + ...actual, + fetchEnabledRagAliases: vi.fn(async () => []), + }; + }); + vi.doMock("@/lib/corpus-grounding", () => ({ + classifyCorpusGrounding: vi.fn(async () => ({ verdict: corpusGroundingVerdict, anchorTerms: [], absentTerms: [] })), + })); + vi.doMock("@/lib/rag/rag-provider", () => ({ + isSourceOnlyMode: () => true, + allowsAutoDegrade: () => true, + sourceOnlyReason: () => "source_only", + classifyProviderFailure: () => "provider_failure", + SOURCE_ONLY_EMBEDDING_SKIP_REASON: "source_only", + })); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc: vi.fn(async () => ({ data: [], error: null })), + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.stubEnv("OPENAI_API_KEY", ""); + + const { searchChunksWithTelemetry } = await import("../src/lib/rag/rag"); + return { searchChunksWithTelemetry, setCachedSearch }; +} + +const ownerId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + +describe("unsupported short-circuit cache write", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.unstubAllEnvs(); + }); + + it("does not cache a zero result for the soft-tail bucket (corpus grounding inconclusive)", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).not.toHaveBeenCalled(); + }, 60_000); + + it("still caches the deterministic unavailable-document-noise short circuit", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); + + const result = await searchChunksWithTelemetry({ + query: "Show me the airport travel policy", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + + it("rescues an in-corpus bare topic before ever reaching the short circuit", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("in_corpus_topic"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.telemetry.retrieval_strategy).not.toBe("unsupported_short_circuit"); + expect(setCachedSearch).not.toHaveBeenCalled(); + }, 60_000); +}); From 9512d3efbeddffb9f025a6f23da8c0ec34db4f56 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 13:08:19 +0000 Subject: [PATCH 2/8] docs(ledger): record review for PR #1646 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni5rjjPcnbYUg8m8xaFhgc --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index aac566c3fd..87d64f86a2 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -661,3 +661,4 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-08-05 | claude/design-system-1616-colors-aw538h | 98b65ae9f222e621da8b5bca75d0b0f25d05ca09 | prlanded | merged, squash 98b65ae verified content-identical to branch tip e66ecaa (empty diff) | vitest ckb-v2-token-contract (26 passed), vitest pwa-manifest (11 passed), live Playwright render check, CI green (pr-required) | | 2026-08-05 | cursor/privacy-page-mockups-2ff6 | 7c82f92a447986178ba12d6f8b7a447bb63e91ef | Run PR sweep | resolved privacy/page conflict with #1621 standalone shell; fixed Devin double scroll-pad + scrollIntoView yank | merge resolved; prettier | | 2026-08-05 | cursor/privacy-page-mockups-2ff6 | 32c4406cb728176933b669779d4f16fd245534bb | Run PR sweep | supersede: desktop index sticky top tracks measured StickySignalChrome height; prior row checks lacked decisive prettier output | prettier --check mockup+ledger: All matched files use Prettier code style!; ResizeObserver sticky chrome height for desktop index | +| 2026-08-06 | claude/implement-97vpz7 | 5ffa042d686a542de3333ebccbd903b6422124a7 | src/lib/rag/rag.ts, src/app/api/search/route.ts, tests/rag-unsupported-short-circuit-cache.test.ts (RAG soft-tail unsupported-short-circuit cache fix + corpus_grounding telemetry exposure) | PR #1646 opened (draft); no retrieval/ranking behaviour change; verified: lint, typecheck, full unit suite (513 files/5413 tests), eval:rag:offline, build, check:bundle-budget | lint,typecheck,test,eval:rag:offline,build,check:bundle-budget | From d8a730c6682428daca0f8cbd1fe7c77f9cf44c70 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 13:57:33 +0000 Subject: [PATCH 3/8] fix(rag): forward corpus_grounding telemetry on shared search-cache hits getSharedCachedSearch reconstructs its returned telemetry from a hand-picked field allowlist that dropped corpus_grounding, so the diagnostic field added in the previous commit went missing on cross-process shared-cache hits even though it was stored in the cached payload. Forward it like every other optional telemetry field on that path. Addresses a P2 finding from automated PR review on #1646. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni5rjjPcnbYUg8m8xaFhgc --- src/lib/rag/rag-cache.ts | 1 + tests/rag-shared-cache.test.ts | 63 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/src/lib/rag/rag-cache.ts b/src/lib/rag/rag-cache.ts index 276e0f19de..ea42480664 100644 --- a/src/lib/rag/rag-cache.ts +++ b/src/lib/rag/rag-cache.ts @@ -542,6 +542,7 @@ export async function getSharedCachedSearch( shared_cache_status: "hit", shared_cache_miss_reason: null, query_class: payload.telemetry?.query_class, + corpus_grounding: payload.telemetry?.corpus_grounding, vector_candidate_count: payload.telemetry?.vector_candidate_count, text_candidate_count: payload.telemetry?.text_candidate_count, embedding_field_count: payload.telemetry?.embedding_field_count, diff --git a/tests/rag-shared-cache.test.ts b/tests/rag-shared-cache.test.ts index 9daaae8965..a74f1eacf3 100644 --- a/tests/rag-shared-cache.test.ts +++ b/tests/rag-shared-cache.test.ts @@ -62,4 +62,67 @@ describe("shared RAG search cache", () => { expect(result).toEqual({ kind: "miss", reason: "unknown_filter_miss" }); expect(sharedCacheReads).toBe(1); }); + + it("forwards corpus_grounding from the stored payload on a shared-cache hit", async () => { + vi.resetModules(); + + vi.doMock("@/lib/env", () => ({ + env: { + RAG_SEARCH_CACHE_TTL_MS: 60_000, + RAG_SEARCH_CACHE_SIZE: 200, + RAG_PERSIST_RAW_QUERY_TEXT: false, + RAG_QUERY_HASH_SECRET: "test-query-hash-secret", + }, + isDemoMode: () => false, + isLocalNoAuthMode: () => false, + })); + vi.doMock("@/lib/deep-memory", () => ({ ragDeepMemoryVersion: "test-rag-version" })); + vi.doMock("@/lib/clinical-search", () => ({ + buildClinicalTextSearchQuery: (query: string) => query.trim(), + })); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + from: (table: string) => { + const builder = { + select: () => builder, + eq: () => builder, + is: () => builder, + in: () => builder, + or: () => builder, + gt: () => builder, + order: () => builder, + limit: () => builder, + maybeSingle: async () => { + if (table !== "rag_response_cache") return { data: null, error: null }; + return { + data: { + payload: { + results: [], + telemetry: { query_class: "unsupported_or_general", corpus_grounding: "in_corpus_topic" }, + }, + }, + error: null, + }; + }, + then: (resolve: (value: { data: unknown[]; error: null }) => unknown) => + Promise.resolve({ + data: + table === "documents" ? [{ id: "doc-1", updated_at: "2026-07-14T00:00:00.000Z", metadata: {} }] : [], + error: null, + }).then(resolve), + }; + return builder; + }, + }), + })); + + const { getSharedCachedSearch } = await import("../src/lib/rag/rag-cache"); + const result = await getSharedCachedSearch({ + query: "catatonia", + ownerId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + }); + + expect(result?.kind).toBe("hit"); + expect(result?.kind === "hit" && result.telemetry.corpus_grounding).toBe("in_corpus_topic"); + }); }); From e903009bd79ef88a9730afd665242927fc60dc0d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 14:55:19 +0000 Subject: [PATCH 4/8] fix(rag): do not memoize a rejected soft-tail classifier verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analyzeQueryWithClassifierFallback memoizes every classifier verdict — accepted or rejected — for 15 minutes so the same query gets a consistent result within a session (Finding #11 interim fix). For the soft-tail bucket (the same fragile, low-confidence case the unsupported short-circuit treats specially), that made a rejected verdict sticky for 15 minutes, 15x longer than the 60s search-cache TTL the previous commit stopped writing to — so that fix alone did not meaningfully unstick a repeat "catatonia"-style query. A rejected verdict for that specific bucket is no longer memoized, so a repeat query gets a fresh classifier attempt instead of reproducing the same rejection for the rest of the TTL window. Accepted verdicts, and rejected verdicts outside the soft-tail bucket, keep the existing determinism guarantee unchanged. Addresses a P1 finding from automated PR review on #1646. RAG impact: this changes how often the *same* query can get a *different* classification within a session — for the soft-tail bucket only, and only in the direction of more classifier calls (never fewer), so it can only recover additional in-corpus topics, not lose previously-supported ones. A live eval-canary confirmation is still owed per this repo's RAG-ranking-protection rules before this is fully trusted in production. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni5rjjPcnbYUg8m8xaFhgc --- src/lib/rag/rag.ts | 16 ++++++++++- tests/rag-classifier-memo.test.ts | 46 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index da2d2154b2..e5e2b37525 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -1152,7 +1152,21 @@ export async function analyzeQueryWithClassifierFallback( try { const verdict = await awaitWithCallerSignal(pending, opts?.signal); - storeClassifierVerdictMemo(memoKey, verdict); + // Finding #11 follow-up: a rejected verdict (still unsupported_or_general / confidence < + // 0.58) is normally memoized for the full 15-minute TTL alongside accepted ones, so the + // same nondeterministic LLM call never gets a second chance within a session — Codex review + // on PR #1646 identified this as the dominant stickiness behind the "catatonia" false + // negative, longer-lived than the 60s search-cache TTL. For the soft-tail bucket + // specifically (the same fragile, low-confidence, no-deterministic-exclusion-match case the + // unsupported short-circuit treats specially — see isUnsupportedSoftTailAnalysis), a + // rejected verdict is not memoized, so a repeat query gets a fresh classifier attempt + // instead of reproducing the same rejection for the rest of the TTL window. Accepted + // verdicts, and rejected verdicts for every other query shape, keep the original + // determinism guarantee. + const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; + if (!(rejected && isUnsupportedSoftTailAnalysis(query, analysis))) { + storeClassifierVerdictMemo(memoKey, verdict); + } return applyClassifierVerdict(analysis, verdict); } catch (error) { if ( diff --git a/tests/rag-classifier-memo.test.ts b/tests/rag-classifier-memo.test.ts index 60958362bb..288ffe94bd 100644 --- a/tests/rag-classifier-memo.test.ts +++ b/tests/rag-classifier-memo.test.ts @@ -85,6 +85,52 @@ describe("classifier verdict memoization", () => { expect(second).toBe(analysis); }); + it("does not memoize a rejected verdict for the soft-tail bucket, so a repeat query can recover on the second call", async () => { + // "catatonia" is a bare single-word query: low confidence, no medications/thresholds/title + // terms, few expanded terms — exactly the soft-tail bucket the unsupported short-circuit + // treats specially (isUnsupportedSoftTailAnalysis true), unlike fallbackQueryAnalysis's + // multi-word fixture above (confidence 0.45, just above the 0.42 soft-tail ceiling). The + // two-call proof Codex review asked for on PR #1646: reject on call 1, recover on call 2 — + // that recovery is only reachable if the rejection from call 1 was not memoized. + const query = "catatonia"; + const mock = vi + .fn() + .mockResolvedValueOnce(classifierResponse({ confidence: 0.3 })) + .mockResolvedValueOnce(classifierResponse()); + const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); + // Mirror production: searchChunksWithTelemetry always passes opts.corpusGrounding, and a + // corpus-grounding verdict of "inconclusive" (no matching document title, so the query can't + // be deterministically rescued) is what routes a bare in-corpus topic like "catatonia" to the + // classifier at all — otherwise the Finding #2 deterministic short-query fallback (rag.ts + // ~1122-1134) rescues it before ever calling the classifier, and this test would prove + // nothing about the memo. Setting corpusGrounding directly reproduces that gate without + // depending on the corpus-grounding module's live database call. + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + expect(analysis.needsClassifierFallback).toBe(true); + + const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); + const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); + + expect(mock).toHaveBeenCalledTimes(2); + expect(first).toBe(analysis); + expect(first.queryClass).toBe("unsupported_or_general"); + expect(second.queryClass).toBe("broad_summary"); + expect(second.needsClassifierFallback).toBe(false); + }); + + it("still memoizes a rejected verdict outside the soft-tail bucket (existing determinism guarantee)", async () => { + const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); + const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + + const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); + const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); + + expect(mock).toHaveBeenCalledTimes(1); + expect(first).toBe(analysis); + expect(second).toBe(analysis); + }); + it("sends only supported structural constraints to Structured Outputs", async () => { const mock = vi.fn(async () => classifierResponse()); const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); From 9fb19515d46450ab67867a54e5a14280fc55d20d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 15:10:23 +0000 Subject: [PATCH 5/8] fix(rag): bound the classifier-memo skip and stop skipping deterministic out_of_corpus caching Two follow-up findings from Devin review on the previous commit: - The rejected-soft-tail-verdict memo skip was unbounded: every repeat of a never-recovering query re-invoked the paid OpenAI classifier forever, and a borderline query could flip between zero and non-zero results on every single request instead of within a stable window. Rejected soft-tail verdicts now use a short 60s memo TTL (matching the default search-cache TTL) instead of skipping the memo entirely, bounding both the retry rate and the classifier-call cost while still recovering much faster than the original 15-minute TTL. - isUnsupportedSoftTailAnalysis only inspects the query text and deterministic analysis, not queryAnalysis.corpusGrounding, so a query with a genuine "out_of_corpus" verdict (a deterministic, corpus-derived true negative reached without any LLM call) was also excluded from the search-cache write, forcing every repeat to redo the corpus-grounding and trigram-correction RPCs for an answer that will never change. Excluded "out_of_corpus" from the cache-write skip so it caches like the other deterministic exclusion patterns. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni5rjjPcnbYUg8m8xaFhgc --- src/lib/rag/rag.ts | 40 ++++++--------- tests/rag-classifier-memo.test.ts | 49 ++++++++++++++----- ...ag-unsupported-short-circuit-cache.test.ts | 19 +++++++ 3 files changed, 70 insertions(+), 38 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index e5e2b37525..9fbcf0f6d0 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -951,6 +951,9 @@ type ClassifierVerdict = z.infer; // a query's classification for the whole TTL. The full corpus-grounded relevance fix remains // scoped to RAG optimisation Phase 2. const classifierVerdictMemoTtlMs = 15 * 60 * 1000; +// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (see +// isUnsupportedSoftTailAnalysis) instead of the full 15-minute TTL or unbounded re-classification. +const rejectedSoftTailMemoTtlMs = 60 * 1000; const classifierVerdictMemoMaxEntries = 500; const classifierVerdictMemo = new Map(); const classifierVerdictInflight = new Map>(); @@ -970,12 +973,12 @@ function classifierVerdictMemoKey(query: string, analysis: ClinicalQueryAnalysis } /** Store classifier verdict memo. */ -function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict) { +function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict, ttlMs = classifierVerdictMemoTtlMs) { if (classifierVerdictMemo.size >= classifierVerdictMemoMaxEntries) { const oldestKey = classifierVerdictMemo.keys().next().value; if (oldestKey !== undefined) classifierVerdictMemo.delete(oldestKey); } - classifierVerdictMemo.set(key, { expiresAt: Date.now() + classifierVerdictMemoTtlMs, verdict }); + classifierVerdictMemo.set(key, { expiresAt: Date.now() + ttlMs, verdict }); } /** Reset classifier verdict memo for tests. */ @@ -1152,21 +1155,10 @@ export async function analyzeQueryWithClassifierFallback( try { const verdict = await awaitWithCallerSignal(pending, opts?.signal); - // Finding #11 follow-up: a rejected verdict (still unsupported_or_general / confidence < - // 0.58) is normally memoized for the full 15-minute TTL alongside accepted ones, so the - // same nondeterministic LLM call never gets a second chance within a session — Codex review - // on PR #1646 identified this as the dominant stickiness behind the "catatonia" false - // negative, longer-lived than the 60s search-cache TTL. For the soft-tail bucket - // specifically (the same fragile, low-confidence, no-deterministic-exclusion-match case the - // unsupported short-circuit treats specially — see isUnsupportedSoftTailAnalysis), a - // rejected verdict is not memoized, so a repeat query gets a fresh classifier attempt - // instead of reproducing the same rejection for the rest of the TTL window. Accepted - // verdicts, and rejected verdicts for every other query shape, keep the original - // determinism guarantee. + // Finding #11 follow-up: bounded TTL for a rejected soft-tail verdict — see the constant above. const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; - if (!(rejected && isUnsupportedSoftTailAnalysis(query, analysis))) { - storeClassifierVerdictMemo(memoKey, verdict); - } + const softTail = rejected && isUnsupportedSoftTailAnalysis(query, analysis); + storeClassifierVerdictMemo(memoKey, verdict, softTail ? rejectedSoftTailMemoTtlMs : undefined); return applyClassifierVerdict(analysis, verdict); } catch (error) { if ( @@ -1754,15 +1746,13 @@ export async function searchChunksWithTelemetry( telemetry.embedding_skip_reason = "unsupported_short_circuit"; telemetry.retrieval_strategy = "unsupported_short_circuit"; recordSearchScoreTelemetry(telemetry, []); - // Finding #11 follow-up: the soft-tail bucket (low confidence, few expanded terms, no - // deterministic exclusion match) can be resolved by a nondeterministic LLM classifier call - // or by corpus grounding returning "inconclusive" rather than "in_corpus_topic" for content - // that lacks a matching document title (e.g. "catatonia" is well represented in chunk text - // but no document is titled "Catatonia"). Caching that zero would make a single unlucky - // classification sticky for every later caller within the cache TTL, so this specific bucket - // is deliberately never cached. The three deterministic exclusion patterns above it in - // shouldShortCircuitUnsupportedSearch are stable true negatives and stay cached as before. - if (!isUnsupportedSoftTailAnalysis(retrievalQuery, queryAnalysis)) { + // Finding #11 follow-up: a soft-tail zero can come from a nondeterministic LLM call, so + // don't cache it — except "out_of_corpus", a deterministic corpus-derived true negative + // (classifyCorpusGrounding) reached without any LLM call, which stays cached like the + // deterministic exclusion patterns above it. + const skipCacheWrite = + isUnsupportedSoftTailAnalysis(retrievalQuery, queryAnalysis) && queryAnalysis.corpusGrounding !== "out_of_corpus"; + if (!skipCacheWrite) { await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); } return finishSearch(searchTiming, { results: [] as SearchResult[], telemetry }); diff --git a/tests/rag-classifier-memo.test.ts b/tests/rag-classifier-memo.test.ts index 288ffe94bd..605ab1d293 100644 --- a/tests/rag-classifier-memo.test.ts +++ b/tests/rag-classifier-memo.test.ts @@ -85,30 +85,53 @@ describe("classifier verdict memoization", () => { expect(second).toBe(analysis); }); - it("does not memoize a rejected verdict for the soft-tail bucket, so a repeat query can recover on the second call", async () => { - // "catatonia" is a bare single-word query: low confidence, no medications/thresholds/title - // terms, few expanded terms — exactly the soft-tail bucket the unsupported short-circuit - // treats specially (isUnsupportedSoftTailAnalysis true), unlike fallbackQueryAnalysis's - // multi-word fixture above (confidence 0.45, just above the 0.42 soft-tail ceiling). The - // two-call proof Codex review asked for on PR #1646: reject on call 1, recover on call 2 — - // that recovery is only reachable if the rejection from call 1 was not memoized. + // "catatonia" is a bare single-word query: low confidence, no medications/thresholds/title + // terms, few expanded terms — exactly the soft-tail bucket the unsupported short-circuit + // treats specially (isUnsupportedSoftTailAnalysis true), unlike fallbackQueryAnalysis's + // multi-word fixture above (confidence 0.45, just above the 0.42 soft-tail ceiling). + function softTailQueryAnalysis( + analyzeClinicalQuery: (typeof import("../src/lib/clinical-search"))["analyzeClinicalQuery"], + ) { const query = "catatonia"; - const mock = vi - .fn() - .mockResolvedValueOnce(classifierResponse({ confidence: 0.3 })) - .mockResolvedValueOnce(classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); // Mirror production: searchChunksWithTelemetry always passes opts.corpusGrounding, and a // corpus-grounding verdict of "inconclusive" (no matching document title, so the query can't // be deterministically rescued) is what routes a bare in-corpus topic like "catatonia" to the // classifier at all — otherwise the Finding #2 deterministic short-query fallback (rag.ts - // ~1122-1134) rescues it before ever calling the classifier, and this test would prove + // ~1122-1134) rescues it before ever calling the classifier, and these tests would prove // nothing about the memo. Setting corpusGrounding directly reproduces that gate without // depending on the corpus-grounding module's live database call. const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; expect(analysis.needsClassifierFallback).toBe(true); + return { query, analysis }; + } + + it("still memoizes a rejected soft-tail verdict within its short TTL (bounded, not unlimited retries)", async () => { + const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); + const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); + const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery); + + const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); + const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); + + expect(mock).toHaveBeenCalledTimes(1); + expect(first).toBe(analysis); + expect(second).toBe(analysis); + }); + + it("re-calls the classifier for a rejected soft-tail verdict after its short TTL expires, so it can recover", async () => { + // The two-call proof Codex review asked for on PR #1646: reject on call 1, recover on call 2 + // — reachable only once the short soft-tail TTL (60s, well under the full 15-minute TTL) + // expires, never on an immediate repeat (see the previous test). + vi.useFakeTimers({ now: new Date("2026-07-06T00:00:00Z") }); + const mock = vi + .fn() + .mockResolvedValueOnce(classifierResponse({ confidence: 0.3 })) + .mockResolvedValueOnce(classifierResponse()); + const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); + const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); + vi.setSystemTime(new Date("2026-07-06T00:01:01Z")); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); expect(mock).toHaveBeenCalledTimes(2); diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts index 5b08c0403c..bcf4d19f1e 100644 --- a/tests/rag-unsupported-short-circuit-cache.test.ts +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -124,6 +124,25 @@ describe("unsupported short-circuit cache write", () => { expect(setCachedSearch).toHaveBeenCalledTimes(1); }, 60_000); + it("still caches a deterministic out-of-corpus zero even though it looks soft-tail-shaped", async () => { + // Devin review on PR #1646: isUnsupportedSoftTailAnalysis only looks at the query text and + // deterministic analysis, not queryAnalysis.corpusGrounding, so a real "out_of_corpus" verdict + // (a deterministic, corpus-derived true negative reached without any LLM call) must be + // excluded from the soft-tail cache-write skip, or every repeat re-runs classifyCorpusGrounding + // and the trigram-correction RPC for a query that will always come back empty. + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("out_of_corpus"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + it("rescues an in-corpus bare topic before ever reaching the short circuit", async () => { const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("in_corpus_topic"); From 068ea2b7344252d04ef352ea447efa80b597a9de Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 15:23:26 +0000 Subject: [PATCH 6/8] fix(rag): only skip the soft-tail cache write when a classifier was reachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analyzeQueryWithClassifierFallback returns before any classifier call when OPENAI_API_KEY is absent (rag.ts:1139), so in source-only/offline deployments the soft-tail short-circuit outcome is fully deterministic: same query, same corpus-grounding verdict, same empty result every time. The cache-write skip from the previous two commits didn't account for this, so every repeat of the same query in that deployment mode redid classifyCorpusGrounding (and, when not source-only, the trigram-correction RPC) for an answer that could never change. The skip is now additionally gated on OPENAI_API_KEY being present, since that's the only condition under which a nondeterministic classifier call could actually have produced the verdict. Updated the existing test that had been (correctly, at the time) pinning the no-key case to the skip behavior, and added a new test proving the with-key case — the genuinely nondeterministic one — still skips the cache write. Addresses a further Devin review finding on PR #1646. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ni5rjjPcnbYUg8m8xaFhgc --- src/lib/rag/rag.ts | 14 +++--- ...ag-unsupported-short-circuit-cache.test.ts | 43 +++++++++++++++++-- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 9fbcf0f6d0..77441082d2 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -951,8 +951,7 @@ type ClassifierVerdict = z.infer; // a query's classification for the whole TTL. The full corpus-grounded relevance fix remains // scoped to RAG optimisation Phase 2. const classifierVerdictMemoTtlMs = 15 * 60 * 1000; -// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (see -// isUnsupportedSoftTailAnalysis) instead of the full 15-minute TTL or unbounded re-classification. +// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (isUnsupportedSoftTailAnalysis). const rejectedSoftTailMemoTtlMs = 60 * 1000; const classifierVerdictMemoMaxEntries = 500; const classifierVerdictMemo = new Map(); @@ -1746,12 +1745,13 @@ export async function searchChunksWithTelemetry( telemetry.embedding_skip_reason = "unsupported_short_circuit"; telemetry.retrieval_strategy = "unsupported_short_circuit"; recordSearchScoreTelemetry(telemetry, []); - // Finding #11 follow-up: a soft-tail zero can come from a nondeterministic LLM call, so - // don't cache it — except "out_of_corpus", a deterministic corpus-derived true negative - // (classifyCorpusGrounding) reached without any LLM call, which stays cached like the - // deterministic exclusion patterns above it. + // Finding #11 follow-up: skip caching a soft-tail zero only when a nondeterministic LLM + // classifier could actually have produced it (OPENAI_API_KEY present, rag.ts:1139) and + // corpus grounding didn't already deterministically decide it ("out_of_corpus"). const skipCacheWrite = - isUnsupportedSoftTailAnalysis(retrievalQuery, queryAnalysis) && queryAnalysis.corpusGrounding !== "out_of_corpus"; + Boolean(env.OPENAI_API_KEY) && + isUnsupportedSoftTailAnalysis(retrievalQuery, queryAnalysis) && + queryAnalysis.corpusGrounding !== "out_of_corpus"; if (!skipCacheWrite) { await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); } diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts index bcf4d19f1e..f7c1ca385c 100644 --- a/tests/rag-unsupported-short-circuit-cache.test.ts +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -43,7 +43,10 @@ class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { } } -async function loadSearch(corpusGroundingVerdict: "inconclusive" | "out_of_corpus" | "in_corpus_topic") { +async function loadSearch( + corpusGroundingVerdict: "inconclusive" | "out_of_corpus" | "in_corpus_topic", + options: { openAiApiKey?: string } = {}, +) { const setCachedSearch = vi.fn(async () => undefined); vi.doMock("@/lib/rag/rag-cache", async () => { @@ -68,6 +71,18 @@ async function loadSearch(corpusGroundingVerdict: "inconclusive" | "out_of_corpu vi.doMock("@/lib/corpus-grounding", () => ({ classifyCorpusGrounding: vi.fn(async () => ({ verdict: corpusGroundingVerdict, anchorTerms: [], absentTerms: [] })), })); + // A rejected classifier verdict — only reachable, and only mock-invoked, when the test opts + // into a non-empty OPENAI_API_KEY below (Devin review: the classifier is unreachable at all + // without a key, so that case must stay deterministic and cacheable). + vi.doMock("@/lib/openai", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + generateParsedTextResult: vi.fn(async () => ({ + parsed: { queryClass: "unsupported_or_general", confidence: 0.3, reasons: ["test"], expandedTerms: [] }, + })), + }; + }); vi.doMock("@/lib/rag/rag-provider", () => ({ isSourceOnlyMode: () => true, allowsAutoDegrade: () => true, @@ -81,7 +96,7 @@ async function loadSearch(corpusGroundingVerdict: "inconclusive" | "out_of_corpu from: vi.fn(() => new EmptyQuery()), }), })); - vi.stubEnv("OPENAI_API_KEY", ""); + vi.stubEnv("OPENAI_API_KEY", options.openAiApiKey ?? ""); const { searchChunksWithTelemetry } = await import("../src/lib/rag/rag"); return { searchChunksWithTelemetry, setCachedSearch }; @@ -96,8 +111,10 @@ describe("unsupported short-circuit cache write", () => { vi.unstubAllEnvs(); }); - it("does not cache a zero result for the soft-tail bucket (corpus grounding inconclusive)", async () => { - const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); + it("does not cache a zero result for the soft-tail bucket when a classifier could have decided it (OPENAI_API_KEY set)", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive", { + openAiApiKey: "test-key", + }); const result = await searchChunksWithTelemetry({ query: "catatonia", @@ -110,6 +127,24 @@ describe("unsupported short-circuit cache write", () => { expect(setCachedSearch).not.toHaveBeenCalled(); }, 60_000); + it("caches a soft-tail zero when no classifier was ever reachable (no OPENAI_API_KEY — deterministic)", async () => { + // Devin review on PR #1646: without a key, analyzeQueryWithClassifierFallback returns before + // any classifier call (rag.ts:1139), so the "inconclusive" corpus-grounding verdict is the + // whole story — same query, same DB state, same empty result every time. Skipping the cache + // write here bought nothing but repeat classifyCorpusGrounding + trigram-RPC cost. + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + it("still caches the deterministic unavailable-document-noise short circuit", async () => { const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); From 00ab7bfd34684bc854d15a3f28987674098a7130 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 6 Aug 2026 15:30:26 +0000 Subject: [PATCH 7/8] fix(rag): skip soft-tail unsupported answer cache and harden soft-tail tests The search-layer soft-tail skip left /api/answer still caching the same unsupported refusal for RAG_ANSWER_CACHE_TTL_MS (5 minutes). Extract the shared skip predicate into rag-query-guard and apply it on the unsupported answer path when the empty result came from the soft-tail short circuit and a classifier was reachable. Also pin soft-tail vs non-soft-tail fixtures with isUnsupportedSoftTailAnalysis, drop the duplicate non-soft-tail memo test, and stop asserting setCachedSearch on the in-corpus rescue path (it could pass for the wrong reason). Ratchets the rag.ts maintainability budget to 4362 for the answer-path call site; the decision logic lives in rag-query-guard.ts. Co-Authored-By: Cursor Grok 4.5 Co-authored-by: BigSimmo --- scripts/check-maintainability-budgets.mjs | 3 +- src/lib/rag/rag-query-guard.ts | 37 ++++++ src/lib/rag/rag.ts | 29 +++-- tests/rag-classifier-memo.test.ts | 68 ++++++----- tests/rag-query-guard-soft-tail-cache.test.ts | 106 ++++++++++++++++++ ...ag-unsupported-short-circuit-cache.test.ts | 6 +- 6 files changed, 201 insertions(+), 48 deletions(-) create mode 100644 tests/rag-query-guard-soft-tail-cache.test.ts diff --git a/scripts/check-maintainability-budgets.mjs b/scripts/check-maintainability-budgets.mjs index baa30fd526..65de282d21 100644 --- a/scripts/check-maintainability-budgets.mjs +++ b/scripts/check-maintainability-budgets.mjs @@ -7,7 +7,8 @@ const budgets = new Map([ ["src/components/ClinicalDashboard.tsx", 4140], // Evidence coverage, per-request hydration, and second-stage ranking live in // focused rag modules; keep the reclaimed budget so it cannot silently drift back. - ["src/lib/rag/rag.ts", 4351], + // 4362: soft-tail answer-cache skip call site (logic in rag-query-guard.ts). + ["src/lib/rag/rag.ts", 4362], ["src/components/DocumentViewer.tsx", 1734], ["supabase/functions/indexing-v3-agent/index.ts", 2191], ]); diff --git a/src/lib/rag/rag-query-guard.ts b/src/lib/rag/rag-query-guard.ts index eb96d97a69..dd994ed160 100644 --- a/src/lib/rag/rag-query-guard.ts +++ b/src/lib/rag/rag-query-guard.ts @@ -32,3 +32,40 @@ export function isUnsupportedSoftTailAnalysis(query: string, analysis: ClinicalQ if (clearlyNonClinicalConsumerPattern.test(query)) return false; return analysis.confidence <= 0.42 && analysis.expandedTerms.length <= 5; } + +/** + * Soft-tail zeros should skip search/answer cache writes only when a nondeterministic + * classifier call could have produced them. Without an API key the classifier path is + * unreachable (`analyzeQueryWithClassifierFallback` returns early), and an + * `"out_of_corpus"` grounding verdict is a deterministic corpus-derived true negative — + * both stay cacheable. + */ +export function shouldSkipUnsupportedSoftTailCacheWrite( + query: string, + analysis: ClinicalQueryAnalysis, + options: { + openAiApiKeyPresent: boolean; + corpusGrounding?: ClinicalQueryAnalysis["corpusGrounding"]; + }, +): boolean { + if (!options.openAiApiKeyPresent) return false; + const grounding = options.corpusGrounding ?? analysis.corpusGrounding; + if (grounding === "out_of_corpus") return false; + return isUnsupportedSoftTailAnalysis(query, analysis); +} + +/** Answer-path counterpart: only skip when the empty unsupported refusal came from the soft-tail short circuit. */ +export function shouldSkipUnsupportedSoftTailAnswerCacheWrite(args: { + resultCount: number; + retrievalStrategy: string | undefined; + query: string; + analysis: ClinicalQueryAnalysis; + openAiApiKeyPresent: boolean; + corpusGrounding?: ClinicalQueryAnalysis["corpusGrounding"]; +}): boolean { + if (args.resultCount > 0 || args.retrievalStrategy !== "unsupported_short_circuit") return false; + return shouldSkipUnsupportedSoftTailCacheWrite(args.query, args.analysis, { + openAiApiKeyPresent: args.openAiApiKeyPresent, + corpusGrounding: args.corpusGrounding, + }); +} diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 77441082d2..b32bc1a404 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -204,6 +204,8 @@ export { retrievalPlanForQueryClass, type SearchChunksArgs, type SearchTelemetry import { clearlyOutsideCorpusMedicalPattern, isUnsupportedSoftTailAnalysis, + shouldSkipUnsupportedSoftTailAnswerCacheWrite, + shouldSkipUnsupportedSoftTailCacheWrite, unavailableDocumentNoisePattern, } from "@/lib/rag/rag-query-guard"; export { shouldShortCircuitUnsupportedSearch } from "@/lib/rag/rag-query-guard"; @@ -1745,14 +1747,12 @@ export async function searchChunksWithTelemetry( telemetry.embedding_skip_reason = "unsupported_short_circuit"; telemetry.retrieval_strategy = "unsupported_short_circuit"; recordSearchScoreTelemetry(telemetry, []); - // Finding #11 follow-up: skip caching a soft-tail zero only when a nondeterministic LLM - // classifier could actually have produced it (OPENAI_API_KEY present, rag.ts:1139) and - // corpus grounding didn't already deterministically decide it ("out_of_corpus"). - const skipCacheWrite = - Boolean(env.OPENAI_API_KEY) && - isUnsupportedSoftTailAnalysis(retrievalQuery, queryAnalysis) && - queryAnalysis.corpusGrounding !== "out_of_corpus"; - if (!skipCacheWrite) { + // Skip only when a reachable classifier could have produced a nondeterministic soft-tail zero. + if ( + !shouldSkipUnsupportedSoftTailCacheWrite(retrievalQuery, queryAnalysis, { + openAiApiKeyPresent: Boolean(env.OPENAI_API_KEY), + }) + ) { await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); } return finishSearch(searchTiming, { results: [] as SearchResult[], telemetry }); @@ -2961,7 +2961,18 @@ async function answerQuestionWithScopeUncoalesced( }, }); - if (answerRouteResultCanBeCached(routeDeadline)) + // Soft-tail unsupported refusals must not stick in the 5-minute answer cache. + if ( + answerRouteResultCanBeCached(routeDeadline) && + !shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: results.length, + retrievalStrategy: search.telemetry.retrieval_strategy, + query: answerFocusQuery, + analysis: queryAnalysis, + openAiApiKeyPresent: Boolean(env.OPENAI_API_KEY), + corpusGrounding: search.telemetry.corpus_grounding, + }) + ) await setCachedAnswer(args, finalizedAnswer, { indexingVersionAtRetrievalStart }); routeDeadline.dispose(); return finalizedAnswer; diff --git a/tests/rag-classifier-memo.test.ts b/tests/rag-classifier-memo.test.ts index 605ab1d293..814f45116e 100644 --- a/tests/rag-classifier-memo.test.ts +++ b/tests/rag-classifier-memo.test.ts @@ -21,8 +21,9 @@ async function loadWithClassifierMock(mock: ReturnType) { }); const rag = await import("../src/lib/rag/rag"); const { analyzeClinicalQuery } = await import("../src/lib/clinical-search"); + const { isUnsupportedSoftTailAnalysis } = await import("../src/lib/rag/rag-query-guard"); rag.resetClassifierVerdictMemoForTests(); - return { rag, analyzeClinicalQuery }; + return { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis }; } function classifierResponse(overrides: Record = {}) { @@ -39,20 +40,24 @@ function classifierResponse(overrides: Record = {}) { function fallbackQueryAnalysis( analyzeClinicalQuery: (typeof import("../src/lib/clinical-search"))["analyzeClinicalQuery"], + isUnsupportedSoftTailAnalysis: (typeof import("../src/lib/rag/rag-query-guard"))["isUnsupportedSoftTailAnalysis"], ) { // A bare condition query above the short-query deterministic fallback threshold still needs // the LLM fallback (confidence below 0.58 with class unsupported_or_general). const query = "bipolar disorder long term care"; const analysis = analyzeClinicalQuery(query); expect(analysis.needsClassifierFallback).toBe(true); + // Pin the non-soft-tail shape explicitly so a scoring tweak cannot silently move this + // fixture into the soft-tail bucket (and vice versa) without failing loudly. + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(false); return { query, analysis }; } describe("classifier verdict memoization", () => { it("does not re-call the model for a repeated query and returns an identical verdict", async () => { const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -73,8 +78,8 @@ describe("classifier verdict memoization", () => { it("memoizes rejected verdicts so a rejection is also deterministic", async () => { const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -91,6 +96,7 @@ describe("classifier verdict memoization", () => { // multi-word fixture above (confidence 0.45, just above the 0.42 soft-tail ceiling). function softTailQueryAnalysis( analyzeClinicalQuery: (typeof import("../src/lib/clinical-search"))["analyzeClinicalQuery"], + isUnsupportedSoftTailAnalysis: (typeof import("../src/lib/rag/rag-query-guard"))["isUnsupportedSoftTailAnalysis"], ) { const query = "catatonia"; // Mirror production: searchChunksWithTelemetry always passes opts.corpusGrounding, and a @@ -102,13 +108,16 @@ describe("classifier verdict memoization", () => { // depending on the corpus-grounding module's live database call. const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; expect(analysis.needsClassifierFallback).toBe(true); + // Pin soft-tail eligibility explicitly so a scoring tweak cannot silently move this + // fixture out of the soft-tail bucket without failing loudly. + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(true); return { query, analysis }; } it("still memoizes a rejected soft-tail verdict within its short TTL (bounded, not unlimited retries)", async () => { const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -127,8 +136,8 @@ describe("classifier verdict memoization", () => { .fn() .mockResolvedValueOnce(classifierResponse({ confidence: 0.3 })) .mockResolvedValueOnce(classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); vi.setSystemTime(new Date("2026-07-06T00:01:01Z")); @@ -141,23 +150,10 @@ describe("classifier verdict memoization", () => { expect(second.needsClassifierFallback).toBe(false); }); - it("still memoizes a rejected verdict outside the soft-tail bucket (existing determinism guarantee)", async () => { - const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); - - const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); - const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); - - expect(mock).toHaveBeenCalledTimes(1); - expect(first).toBe(analysis); - expect(second).toBe(analysis); - }); - it("sends only supported structural constraints to Structured Outputs", async () => { const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -175,8 +171,8 @@ describe("classifier verdict memoization", () => { expandedTerms: Array.from({ length: 11 }, (_, index) => `term-${index}`), }), ); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -189,8 +185,8 @@ describe("classifier verdict memoization", () => { it("threads a pseudonymous safety identifier for an authenticated classifier request", async () => { vi.stubEnv("OPENAI_SAFETY_IDENTIFIER_SECRET", "test-secret-that-is-at-least-thirty-two-characters"); const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); await rag.analyzeQueryWithClassifierFallback(query, analysis, { ownerId: "owner-a" }); @@ -202,8 +198,8 @@ describe("classifier verdict memoization", () => { it("does not memoize transport errors — the next request retries the classifier", async () => { const mock = vi.fn().mockRejectedValueOnce(new Error("timeout")).mockResolvedValueOnce(classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -221,8 +217,8 @@ describe("classifier verdict memoization", () => { resolveCall = resolve; }), ); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const firstPromise = rag.analyzeQueryWithClassifierFallback(query, analysis); const secondPromise = rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -236,8 +232,8 @@ describe("classifier verdict memoization", () => { it("skips the classifier when fallback is not required", async () => { const mock = vi.fn(); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const fallback = { ...analysis, needsClassifierFallback: false, queryClass: "unsupported_or_general" as const }; const result = await rag.analyzeQueryWithClassifierFallback(query, fallback); @@ -249,8 +245,8 @@ describe("classifier verdict memoization", () => { it("re-calls the model after the memo TTL expires", async () => { vi.useFakeTimers({ now: new Date("2026-07-06T00:00:00Z") }); const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); await rag.analyzeQueryWithClassifierFallback(query, analysis); vi.setSystemTime(new Date("2026-07-06T00:16:00Z")); diff --git a/tests/rag-query-guard-soft-tail-cache.test.ts b/tests/rag-query-guard-soft-tail-cache.test.ts new file mode 100644 index 0000000000..dcab1f0fb3 --- /dev/null +++ b/tests/rag-query-guard-soft-tail-cache.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeClinicalQuery } from "@/lib/clinical-search"; +import { + isUnsupportedSoftTailAnalysis, + shouldSkipUnsupportedSoftTailAnswerCacheWrite, + shouldSkipUnsupportedSoftTailCacheWrite, +} from "@/lib/rag/rag-query-guard"; + +describe("shouldSkipUnsupportedSoftTailCacheWrite", () => { + it("skips only when a classifier could have decided a soft-tail inconclusive zero", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(true); + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + }), + ).toBe(true); + }); + + it("does not skip when no classifier is reachable", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: false, + }), + ).toBe(false); + }); + + it("does not skip a deterministic out_of_corpus zero even with a key present", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "out_of_corpus" as const }; + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(true); + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + }), + ).toBe(false); + }); + + it("does not skip deterministic exclusion patterns outside the soft-tail bucket", () => { + const query = "Show me the airport travel policy"; + const analysis = analyzeClinicalQuery(query); + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(false); + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(false); + }); + + it("honours an explicit corpusGrounding override over analysis.corpusGrounding", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + corpusGrounding: "out_of_corpus", + }), + ).toBe(false); + }); + + it("skips the answer-cache write only for empty unsupported_short_circuit soft-tail zeros", () => { + const query = "catatonia"; + const analysis = analyzeClinicalQuery(query); + + expect( + shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: 0, + retrievalStrategy: "unsupported_short_circuit", + query, + analysis, + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(true); + expect( + shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: 1, + retrievalStrategy: "unsupported_short_circuit", + query, + analysis, + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(false); + expect( + shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: 0, + retrievalStrategy: "text_fast_path", + query, + analysis, + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(false); + }); +}); diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts index f7c1ca385c..01e7d10405 100644 --- a/tests/rag-unsupported-short-circuit-cache.test.ts +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -179,7 +179,7 @@ describe("unsupported short-circuit cache write", () => { }, 60_000); it("rescues an in-corpus bare topic before ever reaching the short circuit", async () => { - const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("in_corpus_topic"); + const { searchChunksWithTelemetry } = await loadSearch("in_corpus_topic"); const result = await searchChunksWithTelemetry({ query: "catatonia", @@ -187,7 +187,9 @@ describe("unsupported short-circuit cache write", () => { lexicalOnly: true, }); + // Pin the rescue itself. Do not assert on setCachedSearch here: with an empty mocked + // Supabase client the retrieval path may write nothing for unrelated reasons, so a + // "not called" expectation would pass for the wrong reason if candidates later appear. expect(result.telemetry.retrieval_strategy).not.toBe("unsupported_short_circuit"); - expect(setCachedSearch).not.toHaveBeenCalled(); }, 60_000); }); From 482a7497aa69c30e7e7e00ebf7e0965e306ecc0f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 6 Aug 2026 15:31:16 +0000 Subject: [PATCH 8/8] docs(ledger): record soft-tail answer-cache fix review for PR #1646 Co-Authored-By: Cursor Grok 4.5 Co-authored-by: BigSimmo --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 1d9caa3ae5..03497796eb 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -664,3 +664,4 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-08-06 | claude/implement-97vpz7 | 5ffa042d686a542de3333ebccbd903b6422124a7 | src/lib/rag/rag.ts, src/app/api/search/route.ts, tests/rag-unsupported-short-circuit-cache.test.ts (RAG soft-tail unsupported-short-circuit cache fix + corpus_grounding telemetry exposure) | PR #1646 opened (draft); no retrieval/ranking behaviour change; verified: lint, typecheck, full unit suite (513 files/5413 tests), eval:rag:offline, build, check:bundle-budget | lint,typecheck,test,eval:rag:offline,build,check:bundle-budget | | 2026-08-06 | a24f74fdf0134487a03dce37dd9f1e9bd18502f5 | a24f74fdf0134487a03dce37dd9f1e9bd18502f5 | PR #1614 post-merge RAG index restoration audit | Pass - guard-only migration, no DDL, no ranking/RPC change; pr-policy ragRanking=false so no eval-canary required; 1 P3 doc nit (#248 renumber note says 237->246, row is #248) | check:migration-role; npx vitest run tests/supabase-schema.test.ts (74 passed); check:outstanding-issues | | 2026-08-06 | PR #1614 / codex/restore-rag-indexes-20260804 | a24f74fdf0134487a03dce37dd9f1e9bd18502f5 | PR #1614 post-merge RAG index restoration audit | Pass - guard-only migration, no DDL, no ranking/RPC change; pr-policy ragRanking=false so no eval-canary required; 1 P3 doc nit (#248 renumber note says 237->246, row is #248); supersedes 2026-08-06 row (ref column mistakenly held commit SHA instead of PR ref, breaking ledger:lookup per Devin/Sentry review on PR #1636) | check:migration-role; npx vitest run tests/supabase-schema.test.ts (74 passed); check:outstanding-issues | +| 2026-08-06 | claude/implement-97vpz7 | 00ab7bfd34684bc854d15a3f28987674098a7130 | PR #1646 soft-tail answer-cache skip + soft-tail test hardening | fixed — answer-path soft-tail skip via rag-query-guard helpers; soft-tail fixture pins; duplicate memo test removed; in-corpus assert narrowed; budget 4362 | test:rag-query-guard+unsupported-cache+classifier-memo 22/22,check:maintainability-budgets 4362/4362 |