diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 0998eed456..adc7b8eeb4 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -120,3 +120,10 @@ For each: trace which module-scope helpers/icons/types it uses; move solely-cons - **Migration `20260703020000_clinical_registry_records.sql` (applied live 2026-07-03 with explicit user approval)** created `public.clinical_registry_records` (owner-scoped structured Services/Forms records — 30 cols, JSONB render payloads, conservative `source_status`/`validation_status` governance columns) and the `public.clinical_registry_record_sources` join table (record ↔ verifying corpus document, FK cascade). Both are service-role-only RLS (enabled + revoked from anon/authenticated + a single `for all to service_role` policy); ownership is enforced in the API layer, matching the documents model. Verified post-apply: both tables present with correct columns, RLS enabled, 0 rows; `get_advisors(security)` returns no lints. `supabase/schema.sql` mirrors the migration and `tests/supabase-schema.test.ts` asserts the shape. - **Version-recording nuance:** the MCP `apply_migration` timestamps the history row in UTC, so live recorded the migration as version `20260702183308` (name `clinical_registry_records`) while the repo file is `20260703020000_...` (Australia/Perth date). Because the migration is fully idempotent (`create table/index if not exists`, `drop trigger/policy if exists` + create), a later `supabase db push` re-applying the repo file is a harmless no-op that only adds a second history row — the same known duplicate-version churn already present in live history. Do not rewrite history to reconcile; treat as a caution. - **Remaining step (user):** seed the registry per owner with `npm run registry:seed -- --owner-id --write --confirm`. Until seeded, authenticated users see the honest empty-registry state; demo/env-less deployments are unaffected. See [[PR #209]]. + +## Nondeterministic "unsupported" retrieval — finding #11 needs Phase 2 (2026-07-03) + +- **Symptom (rag-hybrid-findings finding #11):** valid clinical topics phrased as bare low-confidence queries ("bipolar disorder", "anorexia management") intermittently return 0 results (`unsupported_short_circuit`) instead of an answer. +- **Root cause (confirmed live):** the soft-tail branch of `shouldShortCircuitUnsupportedSearch` (`src/lib/rag.ts`) fires on `analysis.queryClass === "unsupported_or_general"`, and that class is set by `analyzeQueryWithClassifierFallback`, which calls a **generative LLM classifier** (6s timeout, `reasoningEffort:"low"`, uncached) for low-confidence queries. That call is nondeterministic: it reclassifies "bipolar disorder" to a supported class on some runs (→ answered) and declines/times-out on others (→ short-circuited to 0). Reproduced with a same-process ×N probe. +- **Why a clean Phase-1 fix does NOT exist:** the deterministic analyzer carries **no signal** distinguishing in-corpus topics from out-of-corpus (bipolar, anorexia, gout, DKA all produce identical `unsupported_or_general`, confidence ≈ 0.40, `canonicalTerms` = query tokens). Only the corpus can tell them apart. Removing the soft-tail short-circuit fixes the valid topics but **regresses `unsupported_correct_rate` 1.0 → 0.79** on `eval:quality --rag-only`: a lexical distinctive-term relevance gate in `chooseAnswerRoute` either over-refuses legitimate semantic/vector matches (whose exact term is not in the retrieved text — e.g. the `rag-routing.test.ts` "admission" fixtures) or under-refuses invented terms ("florbizone syndrome") that score strongly on generic scaffolding words ("syndrome"/"management"). The corpus is broad (gout 13 / crohn 49 / appendicitis 30 / angioplasty 32 / bipolar 719 chunks), so almost no common medical term is truly absent — grounded low-confidence answers, not fabrication, are the realistic worst case for real terms; only genuinely invented terms should refuse. +- **Decision (2026-07-03):** the risky change (soft-tail removal + relevance gate + invented-term eval controls) was **reverted**; only a safe, independent hardening was kept — `fetchEnabledRagAliases` no longer caches `[]` on a transient `rag_aliases` read error (which would suppress alias expansion for the whole TTL). Finding #11 is **re-scoped into RAG optimisation Phase 2** ("fit retrieval to content"), where it should be fixed with corpus-grounded relevance — IDF/corpus-frequency weighting of query terms and/or the semantic relevance model, plus the data-driven query-understanding vocabulary (RC6) so the deterministic classifier recognises in-corpus topics up front. Any gate must keep `eval:quality --rag-only` `unsupported_correct_rate` at 1.0 (add invented-term controls like "florbizone syndrome management" / "quxbyria disorder treatment" once it can pass them) while letting valid bare topics answer. A cheaper interim option worth measuring first: **classifier-verdict memoization** to at least make the current behaviour deterministic per query. diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 8878204b5d..daa908e686 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -1679,6 +1679,19 @@ export function rankClinicalResults(query: string, results: SearchResult[]) { const hasImageEvidence = (result.images ?? []).some((image) => isClinicalImageEvidence(image)); const imageEvidencePenalty = wantsImageEvidence && !hasImageEvidence ? -0.04 : 0; const score = explanation.finalScore + imageEvidencePenalty; + // finalScore is clamped to [0,1], so heavily-boosted results saturate at 1.0 and tie on `score`. + // The pre-clamp magnitude (the inner boost sum before the outer clamp) still carries the boost + // engineering that the clamp discards, so we use it as a deep tiebreak below the engineered + // rankingTieBreakScore — it can only discriminate results that are otherwise tied, never reorder + // above tieBreakScore. + const preClampFinalScore = + explanation.weightedHybridScore + + explanation.titleBoost + + explanation.metadataBoost + + explanation.clinicalSignalBoost + + explanation.rrfBoost + + explanation.penalty + + imageEvidencePenalty; return { result, explanation: { @@ -1688,9 +1701,20 @@ export function rankClinicalResults(query: string, results: SearchResult[]) { }, score, tieBreakScore: rankingTieBreakScore(query, result, explanation), + preClampFinalScore, }; }) - .sort((a, b) => b.score - a.score || b.tieBreakScore - a.tieBreakScore || b.result.similarity - a.result.similarity) + // Final `id` comparison is a stable, deterministic tiebreak so fully-tied results no longer order + // by arbitrary retrieval order (a residual run-to-run nondeterminism); it fires only after every + // meaningful signal has tied, so it never wastes the boost/relevance engineering above it. + .sort( + (a, b) => + b.score - a.score || + b.tieBreakScore - a.tieBreakScore || + b.preClampFinalScore - a.preClampFinalScore || + b.result.similarity - a.result.similarity || + a.result.id.localeCompare(b.result.id), + ) .map((entry, index) => ({ ...entry.result, score_explanation: { diff --git a/src/lib/rag.ts b/src/lib/rag.ts index b86f09a98e..7c44e480f6 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -2002,7 +2002,9 @@ async function fetchEnabledRagAliases( ragAliasCache.set(cacheKey, { aliases: merged, expiresAt: Date.now() + ragAliasCacheTtlMs }); return merged; } catch { - ragAliasCache.set(cacheKey, { aliases: [], expiresAt: Date.now() + ragAliasCacheTtlMs }); + // Do not cache an empty result on a transient rag_aliases read failure: caching [] would suppress + // alias-based query expansion (and could let an alias-rescuable query short-circuit) for the whole + // TTL. Return empty for this call only and retry on the next call. return []; } }