From ab87c91caed04633e1fd38ba7d74d1553d0837ee Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:25:31 +0800 Subject: [PATCH 1/2] fix(rag): harden alias cache on transient failure; document finding #11 deferral MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchEnabledRagAliases no longer caches an empty result on a transient rag_aliases read failure. Caching [] suppressed alias-based query expansion (and could let an alias-rescuable query short-circuit) for the whole TTL; it now returns empty for the failing call only and retries next call. Also documents in docs/process-hardening.md why finding #11 (nondeterministic "unsupported" retrieval — bipolar/anorexia intermittently returning 0 results) has no safe Phase-1 fix: the deterministic classifier cannot distinguish in-corpus from out-of-corpus topics, and removing the LLM-gated soft-tail short-circuit regresses eval unsupported_correct 1.0->0.79 in this broad corpus. Deferred to Phase 2 corpus-grounded relevance (IDF/semantic + RC6). Co-Authored-By: Claude Opus 4.8 --- docs/process-hardening.md | 7 +++++++ src/lib/rag.ts | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/process-hardening.md b/docs/process-hardening.md index b43cbbd37c..7908482ace 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -99,3 +99,10 @@ This document turns the current process review into phased, durable repo practic - `GlobalMockupSearchShell` (aka `GlobalSearchShell`, used by the `forms`/`services`/`favourites`/`medications` layouts) wrapped `GlobalMockupSearchShellClient` in a `` whose **fallback also rendered `props.children` inside `#main-content`** — the same subtree the client body renders. Because `useSearchParams()` forces that boundary to the fallback on the server, the page subtree was emitted twice and both copies could persist, producing duplicate `id="main-content"` and duplicate `data-testid` on every shell page. It surfaced as `ui-smoke.spec.ts:1103` failing with a strict-mode violation (two `data-testid="acamprosate-medication-page"` `
` elements on `/medications/acamprosate`). - Fix: the Suspense fallback renders a **neutral placeholder only** — never `props.children`. Rule: do not render the resolved content inside its own Suspense fallback; the fallback is a loading state, not a second copy of the page. + +## Nondeterministic "unsupported" retrieval — finding #11 needs Phase 2 (2026-07-03) + +- **Symptom (rag-hybrid-findings finding #11):** valid clinical topics phrased as bare low-confidence queries ("bipolar disorder", "anorexia management") intermittently return 0 results (`unsupported_short_circuit`) instead of an answer. +- **Root cause (confirmed live):** the soft-tail branch of `shouldShortCircuitUnsupportedSearch` (`src/lib/rag.ts`) fires on `analysis.queryClass === "unsupported_or_general"`, and that class is set by `analyzeQueryWithClassifierFallback`, which calls a **generative LLM classifier** (6s timeout, `reasoningEffort:"low"`, uncached) for low-confidence queries. That call is nondeterministic: it reclassifies "bipolar disorder" to a supported class on some runs (→ answered) and declines/times-out on others (→ short-circuited to 0). Reproduced with a same-process ×N probe. +- **Why a clean Phase-1 fix does NOT exist:** the deterministic analyzer carries **no signal** distinguishing in-corpus topics from out-of-corpus (bipolar, anorexia, gout, DKA all produce identical `unsupported_or_general`, confidence ≈ 0.40, `canonicalTerms` = query tokens). Only the corpus can tell them apart. Removing the soft-tail short-circuit fixes the valid topics but **regresses `unsupported_correct_rate` 1.0 → 0.79** on `eval:quality --rag-only`: a lexical distinctive-term relevance gate in `chooseAnswerRoute` either over-refuses legitimate semantic/vector matches (whose exact term is not in the retrieved text — e.g. the `rag-routing.test.ts` "admission" fixtures) or under-refuses invented terms ("florbizone syndrome") that score strongly on generic scaffolding words ("syndrome"/"management"). The corpus is broad (gout 13 / crohn 49 / appendicitis 30 / angioplasty 32 / bipolar 719 chunks), so almost no common medical term is truly absent — grounded low-confidence answers, not fabrication, are the realistic worst case for real terms; only genuinely invented terms should refuse. +- **Decision (2026-07-03):** the risky change (soft-tail removal + relevance gate + invented-term eval controls) was **reverted**; only a safe, independent hardening was kept — `fetchEnabledRagAliases` no longer caches `[]` on a transient `rag_aliases` read error (which would suppress alias expansion for the whole TTL). Finding #11 is **re-scoped into RAG optimisation Phase 2** ("fit retrieval to content"), where it should be fixed with corpus-grounded relevance — IDF/corpus-frequency weighting of query terms and/or the semantic relevance model, plus the data-driven query-understanding vocabulary (RC6) so the deterministic classifier recognises in-corpus topics up front. Any gate must keep `eval:quality --rag-only` `unsupported_correct_rate` at 1.0 (add invented-term controls like "florbizone syndrome management" / "quxbyria disorder treatment" once it can pass them) while letting valid bare topics answer. A cheaper interim option worth measuring first: **classifier-verdict memoization** to at least make the current behaviour deterministic per query. diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 55dcf66dd5..924de0fd4d 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -2002,7 +2002,9 @@ async function fetchEnabledRagAliases( ragAliasCache.set(cacheKey, { aliases: merged, expiresAt: Date.now() + ragAliasCacheTtlMs }); return merged; } catch { - ragAliasCache.set(cacheKey, { aliases: [], expiresAt: Date.now() + ragAliasCacheTtlMs }); + // Do not cache an empty result on a transient rag_aliases read failure: caching [] would suppress + // alias-based query expansion (and could let an alias-rescuable query short-circuit) for the whole + // TTL. Return empty for this call only and retry on the next call. return []; } } From 0a3fbe69e6c0d400a9fc818149b59fe06e20d347 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:25:32 +0800 Subject: [PATCH 2/2] fix(rag): deterministic ranking tiebreak below the engineered signals finalScore is clamped to [0,1], so heavily-boosted results saturate at 1.0 and tie. rankClinicalResults now falls through to the pre-clamp boost magnitude (recovering the boost engineering the clamp discards) and finally a stable id comparison, so fully-tied results no longer order by arbitrary retrieval order (a residual run-to-run nondeterminism). Both levels fire only after score/tieBreakScore have tied, so ranking quality is unchanged: golden retrieval stays 23/23 with mrr@10=0.7283 identical to baseline. Co-Authored-By: Claude Opus 4.8 --- src/lib/clinical-search.ts | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 8878204b5d..daa908e686 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -1679,6 +1679,19 @@ export function rankClinicalResults(query: string, results: SearchResult[]) { const hasImageEvidence = (result.images ?? []).some((image) => isClinicalImageEvidence(image)); const imageEvidencePenalty = wantsImageEvidence && !hasImageEvidence ? -0.04 : 0; const score = explanation.finalScore + imageEvidencePenalty; + // finalScore is clamped to [0,1], so heavily-boosted results saturate at 1.0 and tie on `score`. + // The pre-clamp magnitude (the inner boost sum before the outer clamp) still carries the boost + // engineering that the clamp discards, so we use it as a deep tiebreak below the engineered + // rankingTieBreakScore — it can only discriminate results that are otherwise tied, never reorder + // above tieBreakScore. + const preClampFinalScore = + explanation.weightedHybridScore + + explanation.titleBoost + + explanation.metadataBoost + + explanation.clinicalSignalBoost + + explanation.rrfBoost + + explanation.penalty + + imageEvidencePenalty; return { result, explanation: { @@ -1688,9 +1701,20 @@ export function rankClinicalResults(query: string, results: SearchResult[]) { }, score, tieBreakScore: rankingTieBreakScore(query, result, explanation), + preClampFinalScore, }; }) - .sort((a, b) => b.score - a.score || b.tieBreakScore - a.tieBreakScore || b.result.similarity - a.result.similarity) + // Final `id` comparison is a stable, deterministic tiebreak so fully-tied results no longer order + // by arbitrary retrieval order (a residual run-to-run nondeterminism); it fires only after every + // meaningful signal has tied, so it never wastes the boost/relevance engineering above it. + .sort( + (a, b) => + b.score - a.score || + b.tieBreakScore - a.tieBreakScore || + b.preClampFinalScore - a.preClampFinalScore || + b.result.similarity - a.result.similarity || + a.result.id.localeCompare(b.result.id), + ) .map((entry, index) => ({ ...entry.result, score_explanation: {