From 7185298ea716e37934687f6ba8dcf9ac0d82aaef Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 13:05:34 +0000 Subject: [PATCH 1/2] Fix answer search: port PR #315 Supabase-key fallback to the stream route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UI's answer search calls POST /api/answer/stream, but PR #315's Supabase API-key configuration fallback only landed on /api/answer and /api/search — so document search recovered while answer search kept failing with a generic stream error. Port the non-production demo fallback to the stream route (extracting its inline demo payload builder for reuse) and tag key-configuration errors with a stable `supabase_api_key_configuration` code in the production SSE error payload so operators can diagnose it from the client. Also apply the documented finding #11 interim hardening: memoize the generative query classifier's definitive verdicts per normalized query (10 min TTL) so bare low-confidence queries like "bipolar disorder" stop nondeterministically short-circuiting to "unsupported" between runs. Transient classifier failures are not cached, mirroring the rag_aliases lesson. No retrieval selection/scoring behavior changes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0179yo6uA4zAxx2hMKS88MXG --- src/app/api/answer/stream/route.ts | 73 +++++++++++------ src/lib/rag.ts | 78 ++++++++++++------ tests/private-access-routes.test.ts | 68 ++++++++++++++++ tests/rag-classifier-memo.test.ts | 119 ++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 46 deletions(-) create mode 100644 tests/rag-classifier-memo.test.ts diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index b8509d1025..fe84ff7e4c 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -20,6 +20,7 @@ import { sourceGovernanceWarnings, } from "@/lib/source-governance"; import { createAdminClient } from "@/lib/supabase/admin"; +import { isSupabaseApiKeyConfigurationError, nonProductionSupabaseDemoFallbackReason } from "@/lib/supabase/errors"; import { AuthenticationError, unauthorizedResponse } from "@/lib/supabase/auth"; import { logger } from "@/lib/logger"; import { parseJsonBody } from "@/lib/validation/body"; @@ -69,6 +70,34 @@ function rateLimitStream(rateLimit: ApiRateLimitResult) { ); } +function buildDemoAnswerPayload(body: AnswerBody, fallbackReason?: string) { + const demo = demoAnswer(body.query, body.documentId, body.documentIds); + const answerFocusQuery = queryForClinicalMode(body.query, body.queryMode); + const sources = annotateSearchResults(answerFocusQuery, demo.sources); + const relevance = buildEvidenceRelevance(answerFocusQuery, sources); + return { + ...demo, + sources, + relevance, + smartPanel: demo.smartPanel ? { ...demo.smartPanel, relevance } : demo.smartPanel, + smartApiPlan: buildSmartRagApiPlan({ + query: answerFocusQuery, + queryClass: queryClassForClinicalMode(body.queryMode) ?? classifyRagQuery(answerFocusQuery).queryClass, + results: sources, + routeMode: demo.routingMode, + retrievalStrategy: "hybrid", + }), + demoMode: true, + ...(fallbackReason + ? { + degradedMode: { active: true, reason: fallbackReason }, + fallbackMode: "non_production_demo", + fallbackReason, + } + : {}), + }; +} + function streamErrorPayload(error: unknown) { if (error instanceof PublicApiError) { return { @@ -78,6 +107,16 @@ function streamErrorPayload(error: unknown) { }; } + // Production has no demo fallback for a misconfigured Supabase key, so tag the + // SSE error with a stable code operators can spot in the client/network tab. + if (isSupabaseApiKeyConfigurationError(error)) { + return { + message: "Answer generation failed. Retry with a narrower question.", + status: 500, + details: { code: "supabase_api_key_configuration" }, + }; + } + if (error instanceof Error) { return { message: "Answer generation failed. Retry with a narrower question.", @@ -142,27 +181,7 @@ function streamAnswer(body: AnswerBody, ownerId?: string, signal?: AbortSignal, body.documentId && !body.documentIds?.length && scope?.activeFilterCount === 0, ); const answer = isDemoMode() - ? (() => { - const demo = demoAnswer(body.query, body.documentId, body.documentIds); - const answerFocusQuery = queryForClinicalMode(body.query, body.queryMode); - const sources = annotateSearchResults(answerFocusQuery, demo.sources); - const relevance = buildEvidenceRelevance(answerFocusQuery, sources); - return { - ...demo, - sources, - relevance, - smartPanel: demo.smartPanel ? { ...demo.smartPanel, relevance } : demo.smartPanel, - smartApiPlan: buildSmartRagApiPlan({ - query: answerFocusQuery, - queryClass: - queryClassForClinicalMode(body.queryMode) ?? classifyRagQuery(answerFocusQuery).queryClass, - results: sources, - routeMode: demo.routingMode, - retrievalStrategy: "hybrid", - }), - demoMode: true, - }; - })() + ? buildDemoAnswerPayload(body) : await answerQuestionWithScope({ query: body.query, documentId: singleDocumentScope ? body.documentId : undefined, @@ -206,8 +225,16 @@ function streamAnswer(body: AnswerBody, ownerId?: string, signal?: AbortSignal, }); } catch (error) { logStreamError(error); - const streamError = streamErrorPayload(error); - send("error", { error: streamError.message, status: streamError.status, details: streamError.details }); + // Parity with /api/answer (PR #315): outside production, a misconfigured + // Supabase API key degrades to a visible demo answer instead of a stream + // error — the UI's answer search uses this route, not /api/answer. + const fallbackReason = nonProductionSupabaseDemoFallbackReason(error); + if (fallbackReason) { + send("final", buildDemoAnswerPayload(body, fallbackReason)); + } else { + const streamError = streamErrorPayload(error); + send("error", { error: streamError.message, status: streamError.status, details: streamError.details }); + } } finally { controller.close(); } diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 68d07c8c31..ff9dc9cf9d 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -1215,6 +1215,52 @@ function uniqueTextValues(values: Array, limit = 32) return output; } +type QueryClassifierVerdict = z.infer; + +// Finding #11 interim hardening (docs/process-hardening.md): the generative query +// classifier is nondeterministic, so the same bare low-confidence query ("bipolar +// disorder") could answer on one run and short-circuit to "unsupported" on the next. +// Memoize the classifier's verdict per normalized query so behaviour is deterministic +// within the TTL. Definitive verdicts — accepted reclassifications AND declines — are +// cached; transient failures (timeout, parse error) are NOT cached, mirroring the +// rag_aliases lesson, so a flaky call retries on the next request. +const queryClassifierVerdictCacheTtlMs = 10 * 60_000; +const queryClassifierVerdictCacheMaxSize = 500; +const queryClassifierVerdictCache = new Map(); + +function rememberQueryClassifierVerdict(cacheKey: string, verdict: QueryClassifierVerdict | null) { + if (queryClassifierVerdictCache.size >= queryClassifierVerdictCacheMaxSize) { + const oldestKey = queryClassifierVerdictCache.keys().next().value; + if (oldestKey !== undefined) queryClassifierVerdictCache.delete(oldestKey); + } + queryClassifierVerdictCache.set(cacheKey, { verdict, expiresAt: Date.now() + queryClassifierVerdictCacheTtlMs }); +} + +function applyQueryClassifierVerdict(analysis: ClinicalQueryAnalysis, parsed: QueryClassifierVerdict) { + return { + ...analysis, + queryClass: parsed.queryClass, + confidence: Math.max(analysis.confidence, parsed.confidence), + needsClassifierFallback: false, + needsSynthesis: + analysis.needsSynthesis || + parsed.queryClass === "comparison" || + parsed.queryClass === "broad_summary" || + parsed.queryClass === "medication_dose_risk", + expandedTerms: uniqueTextValues([...analysis.expandedTerms, ...parsed.expandedTerms], 36), + queryRewrite: { + ...analysis.queryRewrite, + expansions: uniqueTextValues([...analysis.queryRewrite.expansions, ...parsed.expandedTerms], 48), + searchQuery: uniqueTextValues( + [analysis.queryRewrite.searchQuery, ...analysis.queryRewrite.expansions, ...parsed.expandedTerms], + 60, + ).join(" "), + reasons: uniqueTextValues([...analysis.queryRewrite.reasons, ...parsed.reasons, "classifier_fallback"], 16), + }, + reasons: uniqueTextValues([...analysis.reasons, ...parsed.reasons, "classifier_fallback"], 12), + } satisfies ClinicalQueryAnalysis; +} + async function analyzeQueryWithClassifierFallback(query: string, analysis: ClinicalQueryAnalysis) { if ( // Fail closed before any generative model call: an adversarial-manipulation @@ -1229,6 +1275,12 @@ async function analyzeQueryWithClassifierFallback(query: string, analysis: Clini } if (!analysis.needsClassifierFallback || !env.OPENAI_API_KEY) return analysis; + const cacheKey = normalizedCacheQuery(query); + const cached = queryClassifierVerdictCache.get(cacheKey); + if (cached && cached.expiresAt > Date.now()) { + return cached.verdict ? applyQueryClassifierVerdict(analysis, cached.verdict) : analysis; + } + try { const result = await generateStructuredTextResult( [ @@ -1262,29 +1314,9 @@ async function analyzeQueryWithClassifierFallback(query: string, analysis: Clini }, ); const parsed = queryClassifierParseSchema.parse(JSON.parse(result.text)); - if (parsed.confidence < 0.58 || parsed.queryClass === "unsupported_or_general") return analysis; - return { - ...analysis, - queryClass: parsed.queryClass, - confidence: Math.max(analysis.confidence, parsed.confidence), - needsClassifierFallback: false, - needsSynthesis: - analysis.needsSynthesis || - parsed.queryClass === "comparison" || - parsed.queryClass === "broad_summary" || - parsed.queryClass === "medication_dose_risk", - expandedTerms: uniqueTextValues([...analysis.expandedTerms, ...parsed.expandedTerms], 36), - queryRewrite: { - ...analysis.queryRewrite, - expansions: uniqueTextValues([...analysis.queryRewrite.expansions, ...parsed.expandedTerms], 48), - searchQuery: uniqueTextValues( - [analysis.queryRewrite.searchQuery, ...analysis.queryRewrite.expansions, ...parsed.expandedTerms], - 60, - ).join(" "), - reasons: uniqueTextValues([...analysis.queryRewrite.reasons, ...parsed.reasons, "classifier_fallback"], 16), - }, - reasons: uniqueTextValues([...analysis.reasons, ...parsed.reasons, "classifier_fallback"], 12), - } satisfies ClinicalQueryAnalysis; + const verdict = parsed.confidence < 0.58 || parsed.queryClass === "unsupported_or_general" ? null : parsed; + rememberQueryClassifierVerdict(cacheKey, verdict); + return verdict ? applyQueryClassifierVerdict(analysis, verdict) : analysis; } catch { return analysis; } diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index b4c6b3d101..a096b75862 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -3367,6 +3367,74 @@ describe("private document API access", () => { expect(answerQuestionWithScope).not.toHaveBeenCalled(); }); + it("falls back to visible demo answers on the answer stream only outside production when Supabase rejects the API key", async () => { + const answerQuestionWithScope = vi.fn(async () => ({ + answer: "Live answer", + grounded: true, + confidence: "medium", + citations: [], + sources: [], + })); + const client = createSupabaseMock((call) => + call.table === "documents" && call.operation === "select" ? fail("Unregistered API key") : ok([]), + ); + mockRuntime(client, { answerQuestionWithScope }); + const { POST } = await import("../src/app/api/answer/stream/route"); + + const response = await POST( + request("/api/answer/stream", { + method: "POST", + body: JSON.stringify({ query: "clozapine monitoring" }), + }), + ); + const body = await response.text(); + const final = ssePayload(body, "final"); + + expect(response.status).toBe(200); + expect(body).not.toContain("event: error"); + expect(final).toMatchObject({ + demoMode: true, + fallbackMode: "non_production_demo", + fallbackReason: "supabase_api_key_configuration_unavailable", + degradedMode: { active: true, reason: "supabase_api_key_configuration_unavailable" }, + }); + expect(String(final.answer)).toContain("Synthetic"); + expect(answerQuestionWithScope).not.toHaveBeenCalled(); + }); + + it("does not fall back to demo answers on the answer stream in production when Supabase rejects the API key", async () => { + vi.stubEnv("NODE_ENV", "production"); + const answerQuestionWithScope = vi.fn(async () => ({ + answer: "Live answer", + grounded: true, + confidence: "medium", + citations: [], + sources: [], + })); + const client = createSupabaseMock((call) => + call.table === "documents" && call.operation === "select" ? fail("Unregistered API key") : ok([]), + ); + mockRuntime(client, { answerQuestionWithScope }); + const { POST } = await import("../src/app/api/answer/stream/route"); + + const response = await POST( + request("/api/answer/stream", { + method: "POST", + body: JSON.stringify({ query: "clozapine monitoring" }), + }), + ); + const body = await response.text(); + const streamError = ssePayload(body, "error"); + + expect(body).not.toContain("event: final"); + expect(streamError).toMatchObject({ + error: "Answer generation failed. Retry with a narrower question.", + status: 500, + details: { code: "supabase_api_key_configuration" }, + }); + expect(answerQuestionWithScope).not.toHaveBeenCalled(); + }); + it("uses an anonymous in-memory limiter for managed local no-auth search", async () => { const searchChunksWithTelemetry = vi.fn(async () => ({ results: [], diff --git a/tests/rag-classifier-memo.test.ts b/tests/rag-classifier-memo.test.ts new file mode 100644 index 0000000000..12799df1b1 --- /dev/null +++ b/tests/rag-classifier-memo.test.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// Finding #11 interim hardening: the generative query classifier's verdict is +// memoized per normalized query, so a bare low-confidence query behaves the same +// on repeat runs instead of intermittently short-circuiting to "unsupported". + +class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { + select() { + return this; + } + + in() { + return this; + } + + eq() { + return this; + } + + neq() { + return this; + } + + is() { + return this; + } + + order() { + return this; + } + + limit() { + return Promise.resolve({ data: [], error: null }); + } + + then( + onfulfilled?: ((value: { data: unknown[]; error: null }) => TResult1 | PromiseLike) | null, + onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null, + ): PromiseLike { + return Promise.resolve({ data: [], error: null }).then(onfulfilled, onrejected); + } +} + +async function searchTwice(classifierResult: (() => Promise<{ text: string }>) | Error) { + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + const rpc = vi.fn(async (name: string) => { + if (name === "correct_clinical_query_terms") return { data: null, error: null }; + return { data: [], error: null }; + }); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + + const generateStructuredTextResult = vi.fn(async () => { + if (classifierResult instanceof Error) throw classifierResult; + return { + ...(await classifierResult()), + model: "gpt-4.1-mini", + operation: "text_generation", + latencyMs: 5, + requestId: "req_classifier_memo", + usage: { input_tokens: 40, output_tokens: 20, total_tokens: 60 }, + }; + }); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry: vi.fn(async () => { + throw new Error("embeddings must not run for short-circuited queries"); + }), + generateStructuredTextResult, + })); + + const { searchChunksWithTelemetry } = await import("../src/lib/rag"); + const search = () => + searchChunksWithTelemetry({ + query: "bipolar disorder", + ownerId: undefined, + allowGlobalSearch: true, + }); + const first = await search(); + const second = await search(); + return { first, second, generateStructuredTextResult }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.unstubAllEnvs(); +}); + +describe("query classifier verdict memoization", () => { + it("memoizes a definitive classifier decline so repeat queries stay deterministic", async () => { + const { first, second, generateStructuredTextResult } = await searchTwice(async () => ({ + text: JSON.stringify({ + queryClass: "unsupported_or_general", + confidence: 0.9, + reasons: ["not_retrieval"], + expandedTerms: [], + }), + })); + + expect(first.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(second.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(generateStructuredTextResult).toHaveBeenCalledTimes(1); + }); + + it("does not memoize transient classifier failures so the next request can retry", async () => { + const { first, second, generateStructuredTextResult } = await searchTwice(new Error("classifier timed out")); + + expect(first.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(second.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(generateStructuredTextResult).toHaveBeenCalledTimes(2); + }); +}); From f9db52e6c95e0ad333de4e831b59faebb03a7fd1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 13:15:20 +0000 Subject: [PATCH 2/2] Detect disabled legacy Supabase keys as key-configuration errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live: the project's legacy anon/service_role JWTs were disabled on 2026-07-05, and PostgREST rejects them with "Legacy API keys are disabled" — which the existing unregistered/invalid matcher missed, so neither the non-production demo fallback nor the tagged stream error code fired for the exact failure breaking answer search. Match that message (and "Secret API key required") in isSupabaseApiKeyConfigurationError, with unit coverage. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0179yo6uA4zAxx2hMKS88MXG --- src/lib/supabase/errors.ts | 11 +++++++++- tests/supabase-errors.test.ts | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 tests/supabase-errors.test.ts diff --git a/src/lib/supabase/errors.ts b/src/lib/supabase/errors.ts index b69557a159..24ea4756d3 100644 --- a/src/lib/supabase/errors.ts +++ b/src/lib/supabase/errors.ts @@ -8,7 +8,16 @@ function errorMessage(error: unknown) { } export function isSupabaseApiKeyConfigurationError(error: unknown) { - return /\b(?:unregistered|invalid)\s+api\s+key\b/i.test(errorMessage(error)); + const message = errorMessage(error); + return ( + /\b(?:unregistered|invalid)\s+api\s+key\b/i.test(message) || + // Post-rotation messages confirmed live (2026-07-06): PostgREST returns + // "Legacy API keys are disabled" once the anon/service_role JWTs are turned + // off, and "Secret API key required" when a publishable key is sent to a + // secret-only endpoint. Both mean the configured key is wrong, not the query. + /\blegacy\s+api\s+keys?\s+(?:are\s+)?disabled\b/i.test(message) || + /\bsecret\s+api\s+key\s+required\b/i.test(message) + ); } export function nonProductionSupabaseDemoFallbackReason(error: unknown) { diff --git a/tests/supabase-errors.test.ts b/tests/supabase-errors.test.ts new file mode 100644 index 0000000000..9a2a091f9d --- /dev/null +++ b/tests/supabase-errors.test.ts @@ -0,0 +1,41 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + isSupabaseApiKeyConfigurationError, + nonProductionSupabaseDemoFallbackReason, +} from "../src/lib/supabase/errors"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("Supabase API key configuration error detection", () => { + it.each([ + "Unregistered API key", + "Invalid API key", + // Confirmed live 2026-07-06 after key rotation: legacy anon/service_role + // JWTs disabled on 2026-07-05 produce this exact message. + "Legacy API keys are disabled", + "Secret API key required", + ])("matches %j", (message) => { + expect(isSupabaseApiKeyConfigurationError(new Error(message))).toBe(true); + expect(isSupabaseApiKeyConfigurationError({ message })).toBe(true); + }); + + it.each(["JWT expired", "row-level security violation", "network timeout"])( + "does not match unrelated error %j", + (message) => { + expect(isSupabaseApiKeyConfigurationError(new Error(message))).toBe(false); + }, + ); + + it("maps disabled legacy keys to the demo fallback reason outside production", () => { + expect(nonProductionSupabaseDemoFallbackReason(new Error("Legacy API keys are disabled"))).toBe( + "supabase_api_key_configuration_unavailable", + ); + }); + + it("never returns a fallback reason in production", () => { + vi.stubEnv("NODE_ENV", "production"); + expect(nonProductionSupabaseDemoFallbackReason(new Error("Legacy API keys are disabled"))).toBeNull(); + }); +});