diff --git a/eslint.config.mjs b/eslint.config.mjs index e5dee54988..0f68a1f11f 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -12,8 +12,10 @@ const eslintConfig = defineConfig([ "out/**", "build/**", "coverage/**", + ".claude/**", "playwright-report/**", "test-results/**", + ".tmp-visual/**", "sample-documents/**", "scratch/**", "next-env.d.ts", diff --git a/next.config.ts b/next.config.ts index 0ce9f06ba6..a6ba686e78 100644 --- a/next.config.ts +++ b/next.config.ts @@ -2,6 +2,7 @@ import type { NextConfig } from "next"; const isDevelopment = process.env.NODE_ENV === "development"; const scriptSrc = `script-src 'self' 'unsafe-inline'${isDevelopment ? " 'unsafe-eval'" : ""}; `; +const upgradeInsecureRequests = isDevelopment ? "" : "upgrade-insecure-requests; "; const securityHeaders = [ { key: "X-Content-Type-Options", value: "nosniff" }, @@ -20,7 +21,7 @@ const securityHeaders = [ "object-src 'none'; " + "frame-ancestors 'none'; " + "form-action 'self'; " + - "upgrade-insecure-requests; " + + upgradeInsecureRequests + "img-src 'self' data: blob: https:; " + "media-src 'self' https:; " + "connect-src 'self' https://sjrfecxgysukkwxsowpy.supabase.co https://*.supabase.co https://api.openai.com; " + diff --git a/package.json b/package.json index 04ca223b07..b0e9c29cc1 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "import:docs": "tsx scripts/import-documents.ts", "import:docs:20": "tsx scripts/import-documents.ts --queue-batch-size 20", "enrich:documents": "tsx scripts/enrich-documents.ts", + "enrich:backfill": "tsx scripts/backfill-enrichment.ts", "tags:backfill": "tsx scripts/backfill-document-tags.ts", "index:backfill": "tsx scripts/backfill-smart-index.ts", "check:supabase-project": "tsx scripts/check-supabase-project.ts", @@ -54,7 +55,9 @@ "workflow:status": "node ../.local-dev/workflow-status.mjs", "workflow:verify": "node ../.local-dev/workflow-verify.mjs", "workflow:deps": "node ../.local-dev/workflow-deps.mjs", - "workflow:clean-state": "node ../.local-dev/workflow-clean-state.mjs" + "workflow:clean-state": "node ../.local-dev/workflow-clean-state.mjs", + "workflow:export": "node ../.local-dev/workflow-export.mjs", + "workflow:handoff": "node ../.local-dev/workflow-handoff.mjs" }, "dependencies": { "@next/env": "^16.2.9", diff --git a/scripts/backfill-enrichment.ts b/scripts/backfill-enrichment.ts new file mode 100644 index 0000000000..1ee382db69 --- /dev/null +++ b/scripts/backfill-enrichment.ts @@ -0,0 +1,523 @@ +import { createHash } from "node:crypto"; +import { loadEnvConfig } from "@next/env"; +import type { SupabaseClient } from "@supabase/supabase-js"; + +loadEnvConfig(process.cwd()); + +type SupabaseAdmin = SupabaseClient; + +let createAdminClient: () => SupabaseAdmin; +let assertSupabaseHealthy: (health: unknown, label?: string) => void; +let probeSupabaseHealth: (supabase: SupabaseAdmin) => Promise; +let embedTexts: (texts: string[]) => Promise; +let upsertDocumentDeepMemory: typeof import("@/lib/deep-memory").upsertDocumentDeepMemory; +let upsertDocumentEnrichment: typeof import("@/lib/document-enrichment").upsertDocumentEnrichment; +let ragDeepMemoryVersion: string; +let ragEnrichmentVersion: string; +let documentIntelligenceVersion: string; + +type BackfillArgs = { + limit: number; + documentId?: string; + ownerId?: string; + includeCurrent: boolean; + dryRun: boolean; + retryAttempts: number; +}; + +type BackfillDocument = { + id: string; + owner_id: string | null; + title: string; + file_name: string; + source_path: string | null; + metadata: Record | null; +}; + +type BackfillChunk = { + id: string; + document_id: string; + page_number: number | null; + chunk_index: number; + section_heading: string | null; + section_path?: string[] | null; + anchor_id?: string | null; + content: string; + image_ids?: string[] | null; + metadata?: Record | null; +}; + +type BackfillImage = { + id: string; + page_number: number | null; + caption: string | null; + image_type: string | null; + labels?: string[] | null; + source_kind?: string | null; + clinical_relevance_score?: number | null; + metadata?: Record | null; +}; + +const pageSize = 1000; +const stageDelayMs = 1000; + +function parseArgs(argv: string[]): BackfillArgs { + const args: BackfillArgs = { + limit: 25, + ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID, + includeCurrent: false, + dryRun: false, + retryAttempts: 6, + }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith("--")) continue; + if (token === "--include-current") { + args.includeCurrent = true; + continue; + } + if (token === "--dry-run") { + args.dryRun = true; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + index += 1; + if (token === "--limit") args.limit = Number.parseInt(value, 10); + if (token === "--document-id") args.documentId = value; + if (token === "--owner-id") args.ownerId = value; + if (token === "--retry-attempts") args.retryAttempts = Number.parseInt(value, 10); + } + + if (!Number.isInteger(args.limit) || args.limit <= 0) throw new Error("--limit must be a positive integer."); + if (!Number.isInteger(args.retryAttempts) || args.retryAttempts <= 0) { + throw new Error("--retry-attempts must be a positive integer."); + } + return args; +} + +function metadataRecord(metadata: unknown): Record { + return metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? { ...(metadata as Record) } + : {}; +} + +function hashContent(content: string) { + return createHash("md5").update(content).digest("hex"); +} + +function compactSearchText(value: unknown, limit = 1200) { + const compact = String(value ?? "") + .replace(/\s+/g, " ") + .trim(); + if (!compact) return ""; + return compact.length > limit ? compact.slice(0, limit).trim() : compact; +} + +function isRateLimitError(error: unknown) { + const message = formatError(error); + return /\b(?:429|rate limit|rate_limit|too many requests)\b/i.test(message); +} + +function formatError(error: unknown) { + if (error instanceof Error && error.message) return error.message; + if (error instanceof Error && error.stack) return error.stack; + if (error && typeof error === "object") { + const record = error as Record; + const parts = [record.message, record.details, record.hint, record.code] + .map((value) => (typeof value === "string" ? value.trim() : "")) + .filter(Boolean); + if (parts.length) return parts.join(" | "); + const serialized = JSON.stringify(record); + if (serialized && serialized !== "{}") return serialized; + } + const fallback = String(error ?? ""); + return fallback.trim() || "Unknown error"; +} + +function supabaseErrorMessage(error: unknown) { + return formatError(error); +} + +async function sleep(ms: number) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function updateDocumentStage( + supabase: SupabaseAdmin, + document: BackfillDocument, + stage: string, + patch: Record = {}, +) { + const metadata = { + ...metadataRecord(document.metadata), + ...patch, + enrichment_status: patch.enrichment_status ?? (stage === "completed" ? "completed" : "processing"), + enrichment_stage: stage, + document_intelligence_version: documentIntelligenceVersion, + document_intelligence_updated_at: new Date().toISOString(), + }; + document.metadata = metadata; + const { error } = await supabase.from("documents").update({ metadata }).eq("id", document.id); + if (error) throw new Error(supabaseErrorMessage(error)); +} + +async function loadDocuments(supabase: SupabaseAdmin, args: BackfillArgs) { + let query = supabase + .from("documents") + .select("id,owner_id,title,file_name,source_path,metadata") + .eq("status", "indexed") + .order("created_at", { ascending: true }) + .limit(args.documentId ? 1 : Math.max(args.limit * 8, args.limit)); + if (args.ownerId) query = query.eq("owner_id", args.ownerId); + if (args.documentId) query = query.eq("id", args.documentId); + if (!args.includeCurrent) + query = query.or("metadata->>enrichment_status.eq.pending,metadata->>enrichment_status.is.null"); + const { data, error } = await query; + if (error) throw new Error(supabaseErrorMessage(error)); + return ((data ?? []) as BackfillDocument[]).slice(0, args.limit); +} + +async function selectRowsInPages( + supabase: SupabaseAdmin, + table: "document_chunks" | "document_images", + select: string, + documentId: string, + searchableOnly = false, +) { + const rows: T[] = []; + for (let offset = 0; ; offset += pageSize) { + let query = supabase.from(table).select(select).eq("document_id", documentId); + if (searchableOnly) query = query.eq("searchable", true); + const { data, error } = await query.range(offset, offset + pageSize - 1); + if (error) throw new Error(supabaseErrorMessage(error)); + rows.push(...((data ?? []) as T[])); + if (!data || data.length < pageSize) break; + } + return rows; +} + +async function loadEvidence(supabase: SupabaseAdmin, documentId: string) { + const [chunks, images] = await Promise.all([ + selectRowsInPages( + supabase, + "document_chunks", + "id,document_id,page_number,chunk_index,section_heading,section_path,anchor_id,content,image_ids,metadata", + documentId, + ), + selectRowsInPages( + supabase, + "document_images", + "id,page_number,caption,image_type,labels,source_kind,clinical_relevance_score,metadata", + documentId, + true, + ), + ]); + chunks.sort((a, b) => Number(a.chunk_index ?? 0) - Number(b.chunk_index ?? 0)); + images.sort((a, b) => Number(b.clinical_relevance_score ?? 0) - Number(a.clinical_relevance_score ?? 0)); + return { chunks, images }; +} + +async function insertDocumentLevelEmbeddingFields(args: { + supabase: SupabaseAdmin; + document: BackfillDocument; + chunks: BackfillChunk[]; + summary: string | null; +}) { + const sourceChunkId = args.chunks[0]?.id; + if (!sourceChunkId) return []; + const inputs = [ + { + field_type: "document_title", + content: compactSearchText(`${args.document.title} ${args.document.file_name}`, 600), + }, + { + field_type: "document_summary", + content: compactSearchText(`${args.document.title} ${args.summary ?? ""}`, 1200), + }, + ].filter((input) => input.content); + const existing = await args.supabase + .from("document_embedding_fields") + .select("field_type,content_hash") + .eq("document_id", args.document.id) + .in( + "field_type", + inputs.map((input) => input.field_type), + ); + if (existing.error) throw new Error(supabaseErrorMessage(existing.error)); + const existingKeys = new Set((existing.data ?? []).map((row) => `${row.field_type}:${row.content_hash}`)); + const missing = inputs.filter((input) => !existingKeys.has(`${input.field_type}:${hashContent(input.content)}`)); + if (missing.length === 0) return inputs.map((input) => input.field_type); + + const embeddings = await embedTexts(missing.map((input) => input.content)); + const rows = missing.map((input, index) => ({ + owner_id: args.document.owner_id, + document_id: args.document.id, + source_chunk_id: sourceChunkId, + field_type: input.field_type, + content: input.content, + content_hash: hashContent(input.content), + embedding: embeddings[index], + metadata: { + source: "document_level_backfill", + rag_indexing_version: ragDeepMemoryVersion, + document_intelligence_version: documentIntelligenceVersion, + }, + })); + const { error } = await args.supabase.from("document_embedding_fields").insert(rows); + if (error) throw new Error(supabaseErrorMessage(error)); + return inputs.map((input) => input.field_type); +} + +async function countRows(supabase: SupabaseAdmin, table: string, documentId: string) { + const query = supabase.from(table).select("document_id", { count: "exact", head: true }).eq("document_id", documentId); + const result = await query; + if (result.error) throw new Error(supabaseErrorMessage(result.error)); + return result.count ?? 0; +} + +type BackfillVerificationCounts = { + summaries: number; + labels: number; + sections: number; + memoryCards: number; + indexUnits: number; + embeddingFields: number; + qualityRows: number; +}; + +async function readBackfillCounts(supabase: SupabaseAdmin, documentId: string): Promise { + const [summaries, labels, sections, memoryCards, indexUnits, embeddingFields, qualityRows] = await Promise.all([ + countRows(supabase, "document_summaries", documentId), + countRows(supabase, "document_labels", documentId), + countRows(supabase, "document_sections", documentId), + countRows(supabase, "document_memory_cards", documentId), + countRows(supabase, "document_index_units", documentId), + countRows(supabase, "document_embedding_fields", documentId), + countRows(supabase, "document_index_quality", documentId), + ]); + return { summaries, labels, sections, memoryCards, indexUnits, embeddingFields, qualityRows }; +} + +function missingBackfillCounts(counts: BackfillVerificationCounts) { + return Object.entries(counts) + .filter(([, count]) => count <= 0) + .map(([key]) => key); +} + +async function verifyBackfill(supabase: SupabaseAdmin, documentId: string) { + const counts = await readBackfillCounts(supabase, documentId); + const missing = missingBackfillCounts(counts); + if (missing.length) throw new Error(`Backfill verification failed; missing ${missing.join(", ")}.`); + return counts; +} + +async function countIndexUnitsByType(supabase: SupabaseAdmin, documentId: string) { + const rows: { unit_type: string }[] = []; + for (let offset = 0; ; offset += pageSize) { + const { data, error } = await supabase + .from("document_index_units") + .select("unit_type") + .eq("document_id", documentId) + .range(offset, offset + pageSize - 1); + if (error) throw new Error(supabaseErrorMessage(error)); + rows.push(...((data ?? []) as { unit_type: string }[])); + if (!data || data.length < pageSize) break; + } + return rows.reduce>((counts, row) => { + counts[row.unit_type] = (counts[row.unit_type] ?? 0) + 1; + return counts; + }, {}); +} + +async function markBackfillCompleted(args: { + supabase: SupabaseAdmin; + document: BackfillDocument; + counts: BackfillVerificationCounts; + generatedLabelCount?: number; + sectionCount?: number; + memoryCardCount?: number; + indexUnitCount?: number; + documentEmbeddingFieldTypes?: string[]; +}) { + const indexUnitCountsByType = await countIndexUnitsByType(args.supabase, args.document.id); + const patch: Record = { + enrichment_status: "completed", + enrichment_error: null, + rag_enrichment_version: ragEnrichmentVersion, + rag_indexing_version: ragDeepMemoryVersion, + rag_memory_version: ragDeepMemoryVersion, + rag_enrichment_updated_at: new Date().toISOString(), + rag_memory_updated_at: new Date().toISOString(), + generated_label_count: args.generatedLabelCount ?? args.counts.labels, + section_count: args.sectionCount ?? args.counts.sections, + memory_card_count: args.memoryCardCount ?? args.counts.memoryCards, + index_unit_count: args.indexUnitCount ?? args.counts.indexUnits, + index_unit_counts_by_type: indexUnitCountsByType, + backfill_verification_counts: args.counts, + }; + if (args.documentEmbeddingFieldTypes) patch.document_embedding_field_types = args.documentEmbeddingFieldTypes; + await updateDocumentStage(args.supabase, args.document, "completed", patch); +} + +async function tryFinalizeExistingBackfill(supabase: SupabaseAdmin, document: BackfillDocument) { + const counts = await readBackfillCounts(supabase, document.id); + const missing = Object.entries(counts) + .filter(([, count]) => count <= 0) + .map(([key]) => key); + if (missing.length) return null; + await markBackfillCompleted({ supabase, document, counts }); + return counts; +} + +async function processDocument(supabase: SupabaseAdmin, document: BackfillDocument) { + await updateDocumentStage(supabase, document, "loading_evidence"); + const evidence = await loadEvidence(supabase, document.id); + if (evidence.chunks.length === 0) throw new Error("Document has no indexed chunks to enrich."); + + const finalizedCounts = await tryFinalizeExistingBackfill(supabase, document); + if (finalizedCounts) { + return { + reusedExisting: true, + labels: finalizedCounts.labels, + sections: finalizedCounts.sections, + memoryCards: finalizedCounts.memoryCards, + indexUnits: finalizedCounts.indexUnits, + counts: finalizedCounts, + }; + } + + await updateDocumentStage(supabase, document, "generating_enrichment", { + enrichment_error: null, + enrichment_chunk_count: evidence.chunks.length, + enrichment_image_count: evidence.images.length, + }); + const enrichment = await upsertDocumentEnrichment({ + supabase, + document, + chunks: evidence.chunks, + images: evidence.images, + }); + + await updateDocumentStage(supabase, document, "building_deep_memory"); + const memory = await upsertDocumentDeepMemory({ + supabase, + document, + chunks: evidence.chunks, + images: evidence.images, + summary: enrichment.summary.summary, + }); + + await updateDocumentStage(supabase, document, "embedding_document_profile"); + const documentEmbeddingFieldTypes = await insertDocumentLevelEmbeddingFields({ + supabase, + document, + chunks: evidence.chunks, + summary: enrichment.summary.summary, + }); + + await updateDocumentStage(supabase, document, "verifying_backfill"); + const counts = await verifyBackfill(supabase, document.id); + await markBackfillCompleted({ + supabase, + document, + counts, + generatedLabelCount: enrichment.labels.length, + sectionCount: memory.sections.length, + memoryCardCount: memory.memoryCards.length, + indexUnitCount: memory.indexUnits.length, + documentEmbeddingFieldTypes: documentEmbeddingFieldTypes, + }); + + return { + labels: enrichment.labels.length, + sections: memory.sections.length, + memoryCards: memory.memoryCards.length, + indexUnits: memory.indexUnits.length, + counts, + }; +} + +async function main() { + const [deepMemoryModule, enrichmentModule, indexUnitModule, envModule, openAiModule, adminModule, healthModule] = + await Promise.all([ + import("@/lib/deep-memory"), + import("@/lib/document-enrichment"), + import("@/lib/document-index-units"), + import("@/lib/env"), + import("@/lib/openai"), + import("@/lib/supabase/admin"), + import("@/lib/supabase/health"), + ]); + upsertDocumentDeepMemory = deepMemoryModule.upsertDocumentDeepMemory; + ragDeepMemoryVersion = deepMemoryModule.ragDeepMemoryVersion; + upsertDocumentEnrichment = enrichmentModule.upsertDocumentEnrichment; + ragEnrichmentVersion = enrichmentModule.ragEnrichmentVersion; + documentIntelligenceVersion = indexUnitModule.documentIntelligenceVersion; + embedTexts = openAiModule.embedTexts; + createAdminClient = adminModule.createAdminClient; + assertSupabaseHealthy = healthModule.assertSupabaseHealthy as typeof assertSupabaseHealthy; + probeSupabaseHealth = healthModule.probeSupabaseHealth as typeof probeSupabaseHealth; + + const args = parseArgs(process.argv.slice(2)); + envModule.requireServerEnv(); + envModule.requireOpenAIEnv(); + + const supabase = createAdminClient(); + assertSupabaseHealthy(await probeSupabaseHealth(supabase), "Enrichment backfill"); + const documents = await loadDocuments(supabase, args); + console.log( + JSON.stringify({ + event: "backfill_start", + limit: args.limit, + documentCount: documents.length, + ownerId: args.ownerId ?? null, + documentId: args.documentId ?? null, + dryRun: args.dryRun, + version: documentIntelligenceVersion, + }), + ); + if (args.dryRun) return; + + let completed = 0; + let failed = 0; + for (const document of documents) { + let attempt = 0; + for (;;) { + attempt += 1; + try { + const result = await processDocument(supabase, document); + completed += 1; + console.log(JSON.stringify({ event: "backfill_completed", documentId: document.id, attempt, ...result })); + await sleep(stageDelayMs); + break; + } catch (error) { + const message = formatError(error); + if (isRateLimitError(error) && attempt < args.retryAttempts) { + const delayMs = Math.min(120_000, 8000 * 2 ** (attempt - 1)); + console.warn( + JSON.stringify({ event: "backfill_rate_limited", documentId: document.id, attempt, retryInMs: delayMs }), + ); + await sleep(delayMs); + continue; + } + failed += 1; + await updateDocumentStage(supabase, document, "failed", { + enrichment_status: "failed", + enrichment_error: message, + }).catch(() => undefined); + console.error(JSON.stringify({ event: "backfill_failed", documentId: document.id, attempt, error: message })); + break; + } + } + } + + console.log(JSON.stringify({ event: "backfill_done", completed, failed, total: documents.length })); + if (failed > 0) process.exitCode = 1; +} + +main().catch((error) => { + console.error(formatError(error)); + process.exitCode = 1; +}); diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index 127be9a104..b2f6c2b8fe 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -133,6 +133,12 @@ function strictEnrichmentVersionRequired() { ); } +function maxPendingEnrichmentAllowed() { + const raw = process.env.RAG_MAX_PENDING_ENRICHMENT ?? "0"; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0; +} + function missingSchemaMessage(error: { message: string } | Error | null | undefined) { const message = error instanceof Error ? error.message : error?.message; if (!message) return ""; @@ -444,6 +450,7 @@ async function main() { ); }); const requireCurrentEnrichmentVersion = strictEnrichmentVersionRequired(); + const pendingEnrichmentLimit = maxPendingEnrichmentAllowed(); const missingEmbeddingResult = await supabase .from("document_chunks") @@ -536,6 +543,9 @@ async function main() { if (requireCurrentEnrichmentVersion && documentsMissingCurrentDeepMemoryVersion > 0) { issues.push(`indexed documents missing current deep-memory version: ${documentsMissingCurrentDeepMemoryVersion}`); } + if (pendingIndexedDocuments.length > pendingEnrichmentLimit) { + issues.push(`pending enrichment queue exceeds limit: ${pendingIndexedDocuments.length}/${pendingEnrichmentLimit}`); + } if (actionableFailedJobs.length > 0) issues.push(`actionable failed ingestion jobs: ${actionableFailedJobs.length}`); if ((cleanupIssuesResult.data ?? []).length > 0) { issues.push(`pending or failed storage cleanup jobs: ${(cleanupIssuesResult.data ?? []).length}`); @@ -561,7 +571,7 @@ async function main() { `Indexed documents: completed=${indexedDocuments.length}; pending=${pendingIndexedDocuments.length}; empty=${emptyIndexedDocuments.length}`, ); if (pendingIndexedDocuments.length > 0) { - console.log(`Pending enrichment queue: ${pendingIndexedDocuments.length}`); + console.log(`Pending enrichment queue: ${pendingIndexedDocuments.length}; limit=${pendingEnrichmentLimit}`); } console.log(`Chunk-count mismatches: ${documentsWithChunkCountMismatch.length}`); console.log( diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts index d53101ceeb..c4c07fc65b 100644 --- a/scripts/enrich-documents.ts +++ b/scripts/enrich-documents.ts @@ -176,7 +176,9 @@ async function loadEvidence(supabase: SupabaseAdmin, documentId: string) { for (let start = 0; ; start += 1000) { const { data, error } = await supabase .from("document_chunks") - .select("id,document_id,page_number,chunk_index,section_heading,content,image_ids,metadata") + .select( + "id,document_id,page_number,chunk_index,section_heading,section_path,anchor_id,content,image_ids,metadata", + ) .eq("document_id", documentId) .order("chunk_index", { ascending: true }) .range(start, start + 999); diff --git a/scripts/ensure-local-server.mjs b/scripts/ensure-local-server.mjs index f2b210d82c..8258c40e7a 100644 --- a/scripts/ensure-local-server.mjs +++ b/scripts/ensure-local-server.mjs @@ -12,6 +12,11 @@ const maxPort = 65535; const identityPath = "/api/local-project-id"; const logPath = path.join(projectRoot, "dev-server.log"); const printUrlOnly = process.argv.slice(2).includes("--print-url"); +const debugEnabled = process.env.ENSURE_DEBUG === "1"; + +function debug(message) { + if (debugEnabled) console.error(`[ensure-local-server] ${message}`); +} function localUrl(port) { return `http://localhost:${port}`; @@ -46,7 +51,22 @@ async function isPortBusy(port) { function requestJson(url, timeoutMs = 3500) { return new Promise((resolve) => { - const request = http.get(url, { timeout: timeoutMs }, (response) => { + let settled = false; + let request; + + const settle = (value) => { + if (settled) return; + settled = true; + clearTimeout(fallback); + resolve(value); + }; + + const fallback = setTimeout(() => { + request?.destroy(); + settle(null); + }, timeoutMs + 500); + + request = http.get(url, { timeout: timeoutMs }, (response) => { let body = ""; response.setEncoding("utf8"); response.on("data", (chunk) => { @@ -54,93 +74,109 @@ function requestJson(url, timeoutMs = 3500) { }); response.on("end", () => { try { - resolve(JSON.parse(body)); + settle(JSON.parse(body)); } catch { - resolve(null); + settle(null); } }); }); + request.on("timeout", () => { request.destroy(); - resolve(null); + settle(null); }); - request.on("error", () => resolve(null)); + request.on("error", () => settle(null)); }); } -async function isThisProject(port) { - for (let attempt = 0; attempt < 3; attempt += 1) { +async function isThisProject(port, attempts = 3) { + for (let attempt = 0; attempt < attempts; attempt += 1) { const payload = await requestJson(`http://localhost:${port}${identityPath}`); + debug(`identity attempt ${attempt + 1} on ${port}: ${JSON.stringify(payload)}`); if (payload?.appName === appName && payload?.projectId === localProjectId(projectRoot)) return true; - if (attempt < 2) await sleep(250); + if (attempt < attempts - 1) await sleep(250); } return false; } async function findExistingProjectServer(startPort) { for (let port = startPort; port <= projectPortEnd; port += 1) { - if (await isThisProject(port)) return port; + if (await isThisProject(port, 1)) return port; } return null; } async function findStartPort(startPort) { for (let port = startPort; port <= maxPort; port += 1) { - if (await isThisProject(port)) return { port, alreadyRunning: true }; + if (await isThisProject(port, 1)) return { port, alreadyRunning: true }; if (!(await isPortBusy(port))) return { port, alreadyRunning: false }; } throw new Error(`No free local port found from ${startPort} to ${maxPort}.`); } function startDevServer(port) { + debug(`starting dev server on ${port}`); const out = fs.openSync(logPath, "a"); const err = fs.openSync(logPath, "a"); - const child = spawn(process.execPath, [path.join("scripts", "dev-free-port.mjs"), "--port", String(port)], { - cwd: projectRoot, - detached: true, - env: { ...process.env, PORT: String(port) }, - stdio: ["ignore", out, err], - windowsHide: true, - }); - child.unref(); + try { + const child = spawn(process.execPath, [path.join("scripts", "dev-free-port.mjs"), "--port", String(port)], { + cwd: projectRoot, + detached: true, + env: { ...process.env, PORT: String(port) }, + stdio: ["ignore", out, err], + windowsHide: true, + }); + child.unref(); + } finally { + fs.closeSync(out); + fs.closeSync(err); + } } async function waitForProject(port) { for (let attempt = 0; attempt < 90; attempt += 1) { - if (await isThisProject(port)) return true; + if (await isThisProject(port, 1)) return true; + debug(`waiting for project on ${port}: attempt ${attempt + 1}`); await sleep(500); } return false; } -const stablePort = stableProjectPort(projectRoot); -const existingPort = await findExistingProjectServer(stablePort); +async function main() { + const stablePort = stableProjectPort(projectRoot); + debug(`stable port ${stablePort}`); + const existingPort = await findExistingProjectServer(stablePort); + debug(`existing port ${existingPort ?? "none"}`); -if (existingPort) { - console.log(printUrlOnly ? localUrl(existingPort) : `Clinical KB is already running at ${localUrl(existingPort)}`); - process.exit(0); -} + if (existingPort) { + console.log(printUrlOnly ? localUrl(existingPort) : `Clinical KB is already running at ${localUrl(existingPort)}`); + return 0; + } -const target = await findStartPort(stablePort); + const target = await findStartPort(stablePort); + debug(`target ${target.port}, alreadyRunning=${target.alreadyRunning}`); -if (target.alreadyRunning) { - console.log(printUrlOnly ? localUrl(target.port) : `Clinical KB is already running at ${localUrl(target.port)}`); - process.exit(0); -} + if (target.alreadyRunning) { + console.log(printUrlOnly ? localUrl(target.port) : `Clinical KB is already running at ${localUrl(target.port)}`); + return 0; + } -if (target.port !== stablePort && !printUrlOnly) { - console.log( - `Stable project port ${stablePort} is serving another local project; starting Clinical KB at ${localUrl(target.port)}`, - ); -} + if (target.port !== stablePort && !printUrlOnly) { + console.log( + `Stable project port ${stablePort} is serving another local project; starting Clinical KB at ${localUrl(target.port)}`, + ); + } + + startDevServer(target.port); -startDevServer(target.port); + if (await waitForProject(target.port)) { + console.log(printUrlOnly ? localUrl(target.port) : `Clinical KB is running at ${localUrl(target.port)}`); + if (!printUrlOnly) console.log(`Server log: ${logPath}`); + return 0; + } -if (await waitForProject(target.port)) { - console.log(printUrlOnly ? localUrl(target.port) : `Clinical KB is running at ${localUrl(target.port)}`); - if (!printUrlOnly) console.log(`Server log: ${logPath}`); - process.exit(0); + console.error(`Clinical KB did not become ready at ${localUrl(target.port)}. Check ${logPath}`); + return 1; } -console.error(`Clinical KB did not become ready at ${localUrl(target.port)}. Check ${logPath}`); -process.exit(1); +process.exitCode = await main(); diff --git a/scripts/playwright-base-url.ts b/scripts/playwright-base-url.ts index 53410c1115..314d8b7abc 100644 --- a/scripts/playwright-base-url.ts +++ b/scripts/playwright-base-url.ts @@ -1,6 +1,6 @@ -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import path from "node:path"; -import { appName, localProjectId } from "./local-server-utils.mjs"; +import { appName, localProjectId, stableProjectPort } from "./local-server-utils.mjs"; const projectRoot = path.resolve(__dirname, ".."); const ensureScript = path.join(projectRoot, "scripts", "ensure-local-server.mjs"); @@ -42,14 +42,48 @@ function verifyLocalProjectIdentity(baseUrl: string) { } } +function tryVerifiedLocalProjectUrl(baseUrl: string) { + try { + verifyLocalProjectIdentity(baseUrl); + return baseUrl; + } catch { + return null; + } +} + +function findExistingLocalProjectUrl() { + const stablePort = stableProjectPort(projectRoot); + return tryVerifiedLocalProjectUrl(`http://localhost:${stablePort}`); +} + export function getPlaywrightBaseUrl() { - const output = execFileSync(process.execPath, [ensureScript, "--print-url"], { + const configuredBaseUrl = process.env.PLAYWRIGHT_BASE_URL; + if (configuredBaseUrl) { + if (!localUrlPattern.test(configuredBaseUrl)) { + throw new Error(`PLAYWRIGHT_BASE_URL must be a localhost URL, received: ${configuredBaseUrl}`); + } + verifyLocalProjectIdentity(configuredBaseUrl); + return configuredBaseUrl; + } + + const existingUrl = findExistingLocalProjectUrl(); + if (existingUrl) return existingUrl; + + const result = spawnSync(process.execPath, [ensureScript, "--print-url"], { cwd: projectRoot, encoding: "utf8", - stdio: ["ignore", "pipe", "inherit"], - }).trim(); + stdio: ["ignore", "pipe", "pipe"], + }); + const output = (result.stdout ?? "").trim(); if (!localUrlPattern.test(output)) { + if (result.error) throw result.error; + if (result.status !== 0) { + const diagnostic = (result.stderr ?? "").trim(); + throw new Error( + `ensure-local-server failed before printing a localhost URL${diagnostic ? `: ${diagnostic}` : "."}`, + ); + } throw new Error(`Expected ensure-local-server to print a localhost URL, received: ${output || ""}`); } diff --git a/src/app/api/answer/route.ts b/src/app/api/answer/route.ts index 36899f0b6b..fdd5b37654 100644 --- a/src/app/api/answer/route.ts +++ b/src/app/api/answer/route.ts @@ -1,15 +1,19 @@ import { NextResponse } from "next/server"; import { z } from "zod"; import { demoAnswer } from "@/lib/demo-data"; -import { isDemoMode } from "@/lib/env"; +import { isDemoMode, isLocalNoAuthMode } from "@/lib/env"; import { answerQuestionWithScope } from "@/lib/rag"; import { jsonError, PublicApiError } from "@/lib/http"; -import { consumePublicAnswerRateLimit } from "@/lib/public-rate-limit"; +import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { classifyRagQuery } from "@/lib/clinical-search"; import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { clinicalQueryModeSchema, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode"; import { resolveSearchScope, searchScopeFiltersSchema } from "@/lib/search-scope"; -import { sourceGovernanceWarnings } from "@/lib/source-governance"; +import { + hasDangerSourceGovernanceWarning, + sourceGovernanceRefusalAnswer, + sourceGovernanceWarnings, +} from "@/lib/source-governance"; import { createAdminClient } from "@/lib/supabase/admin"; import * as serverAuth from "@/lib/supabase/auth"; @@ -48,12 +52,14 @@ export async function POST(request: Request) { const supabase = createAdminClient(); const user = await serverAuth.requireAuthenticatedUser(request, supabase); - const rateLimit = consumePublicAnswerRateLimit(request.headers); + const rateLimit = await consumeApiRateLimit({ + supabase, + ownerId: user.id, + bucket: "answer", + allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(), + }); if (rateLimit.limited) { - return NextResponse.json( - { error: "Too many public answer requests. Retry shortly." }, - { status: 429, headers: { "Retry-After": String(rateLimit.retryAfterSeconds) } }, - ); + return rateLimitJsonResponse("Too many answer requests. Retry shortly.", rateLimit); } const scope = await resolveSearchScope({ @@ -86,13 +92,26 @@ export async function POST(request: Request) { queryMode: body.queryMode, skipCache: body.skipCache, }); + const warnings = sourceGovernanceWarnings({ + results: answer.sources ?? [], + relevance: answer.relevance ?? answer.smartPanel?.relevance ?? null, + }); + if (hasDangerSourceGovernanceWarning(warnings)) { + return NextResponse.json({ + ...answer, + answer: sourceGovernanceRefusalAnswer, + grounded: false, + confidence: "unsupported", + citations: [], + scope: { ...scope, queryMode: body.queryMode }, + sourceGovernanceWarnings: warnings, + }); + } + return NextResponse.json({ ...answer, scope: { ...scope, queryMode: body.queryMode }, - sourceGovernanceWarnings: sourceGovernanceWarnings({ - results: answer.sources ?? [], - relevance: answer.relevance ?? answer.smartPanel?.relevance ?? null, - }), + sourceGovernanceWarnings: warnings, }); } catch (error) { if (error instanceof serverAuth.AuthenticationError) { diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index 9952977519..b877ee33e1 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -1,15 +1,19 @@ import { z } from "zod"; import { demoAnswer } from "@/lib/demo-data"; -import { isDemoMode } from "@/lib/env"; +import { isDemoMode, isLocalNoAuthMode } from "@/lib/env"; import { PublicApiError, jsonError } from "@/lib/http"; -import { consumePublicAnswerRateLimit, type PublicRateLimitResult } from "@/lib/public-rate-limit"; +import { consumeApiRateLimit, type ApiRateLimitResult } from "@/lib/api-rate-limit"; import { answerQuestionWithScope, type AnswerProgressEvent } from "@/lib/rag"; import { classifyRagQuery } from "@/lib/clinical-search"; import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { clinicalQueryModeSchema, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode"; import { resolveSearchScope, searchScopeFiltersSchema } from "@/lib/search-scope"; -import { sourceGovernanceWarnings } from "@/lib/source-governance"; +import { + hasDangerSourceGovernanceWarning, + sourceGovernanceRefusalAnswer, + sourceGovernanceWarnings, +} from "@/lib/source-governance"; import { createAdminClient } from "@/lib/supabase/admin"; import { requireAuthenticatedUser } from "@/lib/supabase/auth"; @@ -30,10 +34,10 @@ function encodeSse(event: string, data: unknown) { return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; } -function rateLimitStream(rateLimit: PublicRateLimitResult) { +function rateLimitStream(rateLimit: ApiRateLimitResult) { return new Response( encodeSse("error", { - error: "Too many public answer requests. Retry shortly.", + error: "Too many answer requests. Retry shortly.", status: 429, details: { retryAfterSeconds: rateLimit.retryAfterSeconds, resetAt: rateLimit.resetAt }, }), @@ -50,7 +54,11 @@ function rateLimitStream(rateLimit: PublicRateLimitResult) { function streamErrorPayload(error: unknown) { if (error instanceof PublicApiError) { - return { message: error.message, status: error.status, details: error.details }; + return { + message: error.message, + status: error.status, + details: error.details?.code ? { code: error.details.code } : undefined, + }; } if (error instanceof Error) { @@ -146,13 +154,27 @@ function streamAnswer(body: AnswerBody, ownerId?: string) { skipCache: body.skipCache, onProgress, }); + const warnings = sourceGovernanceWarnings({ + results: answer.sources ?? [], + relevance: answer.relevance ?? answer.smartPanel?.relevance ?? null, + }); + if (hasDangerSourceGovernanceWarning(warnings)) { + send("final", { + ...answer, + answer: sourceGovernanceRefusalAnswer, + grounded: false, + confidence: "unsupported", + citations: [], + scope: scope ? { ...scope, queryMode: body.queryMode } : undefined, + sourceGovernanceWarnings: warnings, + }); + return; + } + send("final", { ...answer, scope: scope ? { ...scope, queryMode: body.queryMode } : undefined, - sourceGovernanceWarnings: sourceGovernanceWarnings({ - results: answer.sources ?? [], - relevance: answer.relevance ?? answer.smartPanel?.relevance ?? null, - }), + sourceGovernanceWarnings: warnings, }); } catch (error) { logStreamError(error); @@ -181,7 +203,12 @@ export async function POST(request: Request) { const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); - const rateLimit = consumePublicAnswerRateLimit(request.headers); + const rateLimit = await consumeApiRateLimit({ + supabase, + ownerId: user.id, + bucket: "answer", + allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(), + }); if (rateLimit.limited) return rateLimitStream(rateLimit); return streamAnswer(body, user.id); diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 7785efb0b4..54e015757d 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -3,6 +3,7 @@ import { env, isDemoMode } from "@/lib/env"; import { upsertDocumentEnrichment } from "@/lib/document-enrichment"; import { upsertDocumentDeepMemory } from "@/lib/deep-memory"; import { jsonError } from "@/lib/http"; +import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; import { probeSupabaseHealth } from "@/lib/supabase/health"; @@ -71,6 +72,9 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); const mode = await readMode(request); + const rateLimit = await consumeApiRateLimit({ supabase, ownerId: user.id, bucket: "document_reindex" }); + if (rateLimit.limited) + return rateLimitJsonResponse("Too many document reindex requests. Retry shortly.", rateLimit); const { data: document, error: documentError } = await supabase .from("documents") diff --git a/src/app/api/documents/[id]/summarize/route.ts b/src/app/api/documents/[id]/summarize/route.ts index 53e9521922..f779aa8e68 100644 --- a/src/app/api/documents/[id]/summarize/route.ts +++ b/src/app/api/documents/[id]/summarize/route.ts @@ -3,6 +3,7 @@ import { demoSummary, getDemoDocument } from "@/lib/demo-data"; import { isDemoMode } from "@/lib/env"; import { summarizeDocument } from "@/lib/rag"; import { jsonError } from "@/lib/http"; +import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; @@ -20,6 +21,9 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); + const rateLimit = await consumeApiRateLimit({ supabase, ownerId: user.id, bucket: "document_summarize" }); + if (rateLimit.limited) + return rateLimitJsonResponse("Too many document summary requests. Retry shortly.", rateLimit); return NextResponse.json(await summarizeDocument(id, user.id)); } catch (error) { if (error instanceof AuthenticationError) { diff --git a/src/app/api/documents/bulk/reindex/route.ts b/src/app/api/documents/bulk/reindex/route.ts index 15e88db900..35360121d2 100644 --- a/src/app/api/documents/bulk/reindex/route.ts +++ b/src/app/api/documents/bulk/reindex/route.ts @@ -4,6 +4,7 @@ import { upsertDocumentDeepMemory } from "@/lib/deep-memory"; import { upsertDocumentEnrichment } from "@/lib/document-enrichment"; import { env, isDemoMode } from "@/lib/env"; import { jsonError, PublicApiError } from "@/lib/http"; +import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { invalidateRagCachesForOwner } from "@/lib/rag"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; @@ -69,6 +70,9 @@ export async function POST(request: Request) { const supabase = createAdminClient(); const user = await requireAuthenticatedUser(request, supabase); + const rateLimit = await consumeApiRateLimit({ supabase, ownerId: user.id, bucket: "bulk_reindex" }); + if (rateLimit.limited) return rateLimitJsonResponse("Too many bulk reindex requests. Retry shortly.", rateLimit); + const documentIds = Array.from(new Set(parsed.data.documentIds)); const { data: documents, error: documentError } = await supabase .from("documents") diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 7109327707..3c50ea09b5 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -1,7 +1,7 @@ import { NextResponse } from "next/server"; import { z } from "zod"; import { demoSearch } from "@/lib/demo-data"; -import { isDemoMode } from "@/lib/env"; +import { isDemoMode, isLocalNoAuthMode } from "@/lib/env"; import { buildSmartPanel, buildVisualEvidence, diversifySearchResults } from "@/lib/evidence"; import { annotateDocumentMatches, annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; import { fetchRelatedDocuments, toDocumentMatch } from "@/lib/document-enrichment"; @@ -12,7 +12,7 @@ import { classifyRagQuery, normalizedClinicalSearchTokens } from "@/lib/clinical import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { createAdminClient } from "@/lib/supabase/admin"; import * as serverAuth from "@/lib/supabase/auth"; -import { consumePublicSearchRateLimit } from "@/lib/public-rate-limit"; +import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { clinicalQueryModeSchema, queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode"; import { resolveSearchScope, searchScopeFiltersSchema } from "@/lib/search-scope"; import { sourceGovernanceWarnings } from "@/lib/source-governance"; @@ -753,19 +753,16 @@ export async function POST(request: Request) { const user = await serverAuth.requireAuthenticatedUser(request, supabase); ownerId = user.id; - const rateLimit = consumePublicSearchRateLimit(request.headers); + const rateLimit = await consumeApiRateLimit({ + supabase, + ownerId, + bucket: "search", + allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(), + }); if (rateLimit.limited) { - return NextResponse.json( - { - error: "Search is temporarily rate limited because too many requests were received. Retry shortly.", - retryAfterSeconds: rateLimit.retryAfterSeconds, - }, - { - status: 429, - headers: { - "Retry-After": String(rateLimit.retryAfterSeconds), - }, - }, + return rateLimitJsonResponse( + "Search is temporarily rate limited because too many requests were received. Retry shortly.", + rateLimit, ); } @@ -787,6 +784,9 @@ export async function POST(request: Request) { if (error instanceof z.ZodError) { return jsonError(error, 400); } + if (error instanceof PublicApiError) { + return jsonError(error, error.status); + } if (error instanceof Error && error.message.trim()) { const code = classifySearchFailure(error); const failurePayload = { diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 9da31cfd23..9b6271d780 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -30,7 +30,7 @@ import { WifiOff, X, } from "lucide-react"; -import { FormEvent, ReactNode, useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from "react"; +import { FormEvent, useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from "react"; import { AccessibleTable } from "@/components/AccessibleTable"; import { DocumentTagCloud } from "@/components/DocumentTagCloud"; import { DocumentManagementActions, type DocumentDeleteResult } from "@/components/DocumentManagementActions"; @@ -54,7 +54,6 @@ import { iconTilePremium, fieldLabel, metadataPill, - navPill, panel, panelSubtle, primaryControl, @@ -71,12 +70,12 @@ import { toneSuccess, toneWarning, } from "@/components/ui-primitives"; -import { Sheet } from "@/components/ui/sheet"; import { AUTH_EMAIL_STORAGE_KEY, useAuthSession } from "@/lib/supabase/client"; import { SafeBoldText } from "@/components/SafeBoldText"; import { AnswerEmptyState, AnswerSkeleton, CopyButton } from "@/components/clinical-dashboard/answer-status"; import { useTheme } from "@/components/clinical-dashboard/use-theme"; import { StatusBadge, StrengthBadge } from "@/components/clinical-dashboard/badges"; +import { GuideDialog, GuideTrigger, SectionHeading, UtilityDrawer } from "@/components/clinical-dashboard/dashboard-shell"; import { MasterSearchHeader } from "@/components/clinical-dashboard/master-search-header"; import { DocumentSearchResultsPanel, @@ -169,7 +168,6 @@ const documentPageSize = 150; const activeIndexingPollFallbackMs = 5_000; const setupRecheckPollMs = 60_000; const indexingWorkDetailsPollMs = 15_000; -const mobileSheetMediaQuery = "(max-width: 639px)"; const stagedDashboardExtraction = { answerSurface: true, } as const; @@ -490,54 +488,6 @@ function SourceImage({ ); } -function SectionHeading({ - icon: Icon, - title, - description, - action, - testId, - hideDescriptionOnMobile = false, - compactMobile = false, -}: { - icon: typeof Search; - title: string; - description?: string; - action?: ReactNode; - testId?: string; - hideDescriptionOnMobile?: boolean; - compactMobile?: boolean; -}) { - const alignWhenCompact = compactMobile && hideDescriptionOnMobile ? "items-center sm:items-start" : "items-start"; - - return ( -
-
- - - -
-

{title}

- {description && ( -

- {description} -

- )} -
-
- {action} -
- ); -} - function ScopeAndGovernanceNotice({ scope, warnings, @@ -3794,115 +3744,6 @@ function LibraryHealthStrip({ ); } -function UtilityDrawer({ - id, - title, - icon: Icon, - summary, - mobileSummary, - children, - defaultOpen = false, - open: controlledOpen, - onOpenChange, - className, - mobileInline = false, -}: { - id?: string; - title: string; - icon: typeof FileText; - summary?: string; - mobileSummary?: string; - children: ReactNode; - defaultOpen?: boolean; - open?: boolean; - onOpenChange?: (open: boolean) => void; - className?: string; - mobileInline?: boolean; -}) { - const [uncontrolledOpen, setUncontrolledOpen] = useState(defaultOpen); - const [usesSheet, setUsesSheet] = useState(false); - const open = controlledOpen ?? uncontrolledOpen; - const setOpen = useCallback( - (nextOpen: boolean) => { - if (controlledOpen === undefined) setUncontrolledOpen(nextOpen); - onOpenChange?.(nextOpen); - }, - [controlledOpen, onOpenChange], - ); - - useEffect(() => { - const mediaQuery = window.matchMedia(mobileSheetMediaQuery); - const sync = () => setUsesSheet(mediaQuery.matches); - sync(); - mediaQuery.addEventListener("change", sync); - return () => mediaQuery.removeEventListener("change", sync); - }, []); - - return ( - <> - - -
{ - const nextOpen = event.currentTarget.open; - if (nextOpen !== open) setOpen(nextOpen); - }} - className={cn("group", mobileInline ? "block" : "hidden sm:block", panelSubtle, className)} - > - - - - - - - {title} - {summary && {summary}} - - - - - {open && (!usesSheet || mobileInline) &&
{children}
} -
- - setOpen(false)} - title={title} - description={mobileSummary ?? summary} - closeLabel={`Close ${title}`} - > -
{children}
-
- - ); -} - function DrawerGroupLabel({ title }: { title: string }) { return (

{title}

@@ -4253,72 +4094,6 @@ function MobileSectionFab({ ); } -const guideSections = [ - { - title: "Ask and verify", - body: "Ask a focused guideline question, then verify linked citations and source passages before use.", - }, - { - title: "Top source and citations", - body: "Use Top source, citation chips, and source cards to open the relevant document page and check the retrieved evidence.", - }, - { - title: "Scope", - body: "Use document scope controls when a question should search only selected guidelines rather than every indexed source.", - }, - { - title: "Quotes, images, sources", - body: "Bottom nav jumps to quotes, diagrams, and source passages. Empty sections had no citations.", - }, - { - title: "Upload and indexing", - body: "Real uploads require Supabase, OpenAI setup, the database schema, and the worker. Demo mode is synthetic only.", - }, - { - title: "Copying text", - body: "Copied drafts are not final clinical notes. Keep the provenance footer and verify source material before using copied text.", - }, -] as const; - -function GuideDialog({ open, onClose }: { open: boolean; onClose: () => void }) { - return ( - -
- {guideSections.map((section) => ( -
-

{section.title}

-

{section.body}

-
- ))} -
-
- ); -} - -function GuideTrigger({ onOpen }: { onOpen: () => void }) { - return ( -
- -
- ); -} - function answerReferencesDocument(answer: RagAnswer | null, documentId: string) { if (!answer) return false; return ( diff --git a/src/components/clinical-dashboard/dashboard-shell.tsx b/src/components/clinical-dashboard/dashboard-shell.tsx new file mode 100644 index 0000000000..b97feeaab0 --- /dev/null +++ b/src/components/clinical-dashboard/dashboard-shell.tsx @@ -0,0 +1,240 @@ +"use client"; + +import { BookOpen, ChevronDown, type LucideIcon } from "lucide-react"; +import { ReactNode, useCallback, useEffect, useState } from "react"; + +import { Sheet } from "@/components/ui/sheet"; +import { + clinicalDivider, + cn, + iconTilePremium, + navPill, + panelSubtle, + sourceCard, + textMuted, +} from "@/components/ui-primitives"; + +const mobileSheetMediaQuery = "(max-width: 639px)"; + +export function SectionHeading({ + icon: Icon, + title, + description, + action, + testId, + hideDescriptionOnMobile = false, + compactMobile = false, +}: { + icon: LucideIcon; + title: string; + description?: string; + action?: ReactNode; + testId?: string; + hideDescriptionOnMobile?: boolean; + compactMobile?: boolean; +}) { + const alignWhenCompact = compactMobile && hideDescriptionOnMobile ? "items-center sm:items-start" : "items-start"; + + return ( +
+
+ + + +
+

{title}

+ {description && ( +

+ {description} +

+ )} +
+
+ {action} +
+ ); +} + +export function UtilityDrawer({ + id, + title, + icon: Icon, + summary, + mobileSummary, + children, + defaultOpen = false, + open: controlledOpen, + onOpenChange, + className, + mobileInline = false, +}: { + id?: string; + title: string; + icon: LucideIcon; + summary?: string; + mobileSummary?: string; + children: ReactNode; + defaultOpen?: boolean; + open?: boolean; + onOpenChange?: (open: boolean) => void; + className?: string; + mobileInline?: boolean; +}) { + const [uncontrolledOpen, setUncontrolledOpen] = useState(defaultOpen); + const [usesSheet, setUsesSheet] = useState(false); + const open = controlledOpen ?? uncontrolledOpen; + const setOpen = useCallback( + (nextOpen: boolean) => { + if (controlledOpen === undefined) setUncontrolledOpen(nextOpen); + onOpenChange?.(nextOpen); + }, + [controlledOpen, onOpenChange], + ); + + useEffect(() => { + const mediaQuery = window.matchMedia(mobileSheetMediaQuery); + const sync = () => setUsesSheet(mediaQuery.matches); + sync(); + mediaQuery.addEventListener("change", sync); + return () => mediaQuery.removeEventListener("change", sync); + }, []); + + return ( + <> + + +
{ + const nextOpen = event.currentTarget.open; + if (nextOpen !== open) setOpen(nextOpen); + }} + className={cn("group", mobileInline ? "block" : "hidden sm:block", panelSubtle, className)} + > + + + + + + + {title} + {summary && {summary}} + + + + + {open && (!usesSheet || mobileInline) &&
{children}
} +
+ + setOpen(false)} + title={title} + description={mobileSummary ?? summary} + closeLabel={`Close ${title}`} + > +
{children}
+
+ + ); +} + +const guideSections = [ + { + title: "Ask and verify", + body: "Ask a focused guideline question, then verify linked citations and source passages before use.", + }, + { + title: "Top source and citations", + body: "Use Top source, citation chips, and source cards to open the relevant document page and check the retrieved evidence.", + }, + { + title: "Scope", + body: "Use document scope controls when a question should search only selected guidelines rather than every indexed source.", + }, + { + title: "Quotes, images, sources", + body: "Bottom nav jumps to quotes, diagrams, and source passages. Empty sections had no citations.", + }, + { + title: "Upload and indexing", + body: "Real uploads require Supabase, OpenAI setup, the database schema, and the worker. Demo mode is synthetic only.", + }, + { + title: "Copying text", + body: "Copied drafts are not final clinical notes. Keep the provenance footer and verify source material before using copied text.", + }, +] as const; + +export function GuideDialog({ open, onClose }: { open: boolean; onClose: () => void }) { + return ( + +
+ {guideSections.map((section) => ( +
+

{section.title}

+

{section.body}

+
+ ))} +
+
+ ); +} + +export function GuideTrigger({ onOpen }: { onOpen: () => void }) { + return ( +
+ +
+ ); +} diff --git a/src/components/clinical-dashboard/index.ts b/src/components/clinical-dashboard/index.ts index 2a0f18b17a..df78e0976f 100644 --- a/src/components/clinical-dashboard/index.ts +++ b/src/components/clinical-dashboard/index.ts @@ -2,6 +2,7 @@ export { ClinicalDashboard } from "./ClinicalDashboard"; export { AnswerEmptyState, AnswerSkeleton, CopyButton } from "./answer-status"; export { useTheme } from "./use-theme"; export { StatusBadge, StrengthBadge } from "./badges"; +export { GuideDialog, GuideTrigger, SectionHeading, UtilityDrawer } from "./dashboard-shell"; export { MasterSearchHeader } from "./master-search-header"; export { DocumentSearchResultsPanel, MatchExplanationChips } from "./document-search-results"; export type { SearchFacets } from "./document-search-results"; diff --git a/src/components/clinical-dashboard/master-search-header.tsx b/src/components/clinical-dashboard/master-search-header.tsx index 8502570a00..5e7db21518 100644 --- a/src/components/clinical-dashboard/master-search-header.tsx +++ b/src/components/clinical-dashboard/master-search-header.tsx @@ -172,6 +172,11 @@ export function MasterSearchHeader({ if (restoreFocus) scopeSummaryRef.current?.focus(); }, []); + const closeScopeSheet = useCallback(() => { + setScopeSheetOpen(false); + window.requestAnimationFrame(() => scopeSummaryRef.current?.focus()); + }, []); + useEffect(() => { const mediaQuery = window.matchMedia(mobileSheetMediaQuery); const sync = () => setUsesScopeSheet(mediaQuery.matches); @@ -660,13 +665,16 @@ export function MasterSearchHeader({ {loading ? : } {submitLabel} - <> + {usesScopeSheet ? ( - + ) : (
{ @@ -685,13 +693,15 @@ export function MasterSearchHeader({ setScopeOpen(open); if (open) window.setTimeout(() => scopeFilterInputRef.current?.focus(), 0); }} - className="group relative hidden sm:block" + className="group relative" > { + scopeSummaryRef.current = element; + }} data-testid="scope-trigger" className="flex min-h-[44px] cursor-pointer list-none items-center justify-center gap-1.5 whitespace-nowrap rounded-[var(--radius-lg)] border border-white/15 bg-white/7 px-0 text-sm font-semibold text-slate-100 shadow-[var(--shadow-tight)] transition motion-safe:duration-150 hover:border-white/25 hover:bg-white/12 sm:gap-2 sm:px-3 sm:text-xs" - aria-label={usesScopeSheet ? "Open desktop document scope" : "Open document scope"} + aria-label="Open document scope" aria-expanded={scopeOpen} > @@ -716,31 +726,30 @@ export function MasterSearchHeader({ {renderScopeRows()}
+ )} - setScopeSheetOpen(false)} - title="Document scope" - description="Choose documents and filters for the next search." - closeLabel="Close document scope" - initialFocusRef={scopeFilterInputRef} - contentClassName="sm:hidden" + +
-
-
- Document scope - {scopeSummary} -
- {scopePreview ? ( -

{scopePreview}

- ) : null} - {renderScopeRows()} +
+ Document scope + {scopeSummary}
- - + {scopePreview ? ( +

{scopePreview}

+ ) : null} + {renderScopeRows()} +
+
diff --git a/src/components/ui/sheet.tsx b/src/components/ui/sheet.tsx index edd803c738..f2dd3ba3fc 100644 --- a/src/components/ui/sheet.tsx +++ b/src/components/ui/sheet.tsx @@ -64,7 +64,10 @@ export function Sheet({ const first = focusable[0]; const last = focusable[focusable.length - 1]; - if (event.shiftKey && document.activeElement === first) { + if (panelRef.current && !panelRef.current.contains(document.activeElement)) { + event.preventDefault(); + (event.shiftKey ? last : first).focus(); + } else if (event.shiftKey && document.activeElement === first) { event.preventDefault(); last.focus(); } else if (!event.shiftKey && document.activeElement === last) { diff --git a/src/lib/api-rate-limit.ts b/src/lib/api-rate-limit.ts new file mode 100644 index 0000000000..6f8463fdf3 --- /dev/null +++ b/src/lib/api-rate-limit.ts @@ -0,0 +1,149 @@ +import { NextResponse } from "next/server"; +import { PublicApiError } from "@/lib/http"; +import type { createAdminClient } from "@/lib/supabase/admin"; + +export type ApiRateLimitBucket = "answer" | "search" | "document_summarize" | "document_reindex" | "bulk_reindex"; + +export type ApiRateLimitResult = { + limited: boolean; + limit: number; + remaining: number; + retryAfterSeconds: number; + resetAt: string; +}; + +const apiRateLimitDefaults = { + answer: { limit: 30, windowSeconds: 60 }, + search: { limit: 240, windowSeconds: 60 }, + document_summarize: { limit: 12, windowSeconds: 60 }, + document_reindex: { limit: 6, windowSeconds: 60 }, + bulk_reindex: { limit: 2, windowSeconds: 60 }, +} as const satisfies Record; + +type SupabaseAdmin = ReturnType; + +type RateLimitRpcRow = { + limited?: boolean; + limit_value?: number; + remaining?: number; + retry_after_seconds?: number; + reset_at?: string; +}; + +type InMemoryRateLimitWindow = { + windowStartMs: number; + requestCount: number; +}; + +type GlobalWithRateLimitFallback = typeof globalThis & { + __clinicalKbInMemoryApiRateLimits?: Map; +}; + +const inMemoryApiRateLimits = ((globalThis as GlobalWithRateLimitFallback).__clinicalKbInMemoryApiRateLimits ??= + new Map()); + +export class ApiRateLimitUnavailableError extends PublicApiError { + constructor() { + super("Rate limit check is temporarily unavailable.", 503, { code: "rate_limit_unavailable" }); + this.name = "ApiRateLimitUnavailableError"; + } +} + +function parseRateLimitRow(data: unknown): RateLimitRpcRow | null { + if (Array.isArray(data)) return (data[0] as RateLimitRpcRow | undefined) ?? null; + return data && typeof data === "object" ? (data as RateLimitRpcRow) : null; +} + +export async function consumeApiRateLimit(args: { + supabase: SupabaseAdmin; + ownerId: string; + bucket: ApiRateLimitBucket; + limit?: number; + windowSeconds?: number; + allowInMemoryFallbackOnUnavailable?: boolean; +}): Promise { + const defaults = apiRateLimitDefaults[args.bucket]; + const limit = args.limit ?? defaults.limit; + const windowSeconds = args.windowSeconds ?? defaults.windowSeconds; + const { data, error } = await args.supabase.rpc("consume_api_rate_limit", { + p_owner_id: args.ownerId, + p_bucket: args.bucket, + p_limit: limit, + p_window_seconds: windowSeconds, + }); + + if (error) { + if (args.allowInMemoryFallbackOnUnavailable) { + console.warn("Durable API rate limit check unavailable; using local in-memory fallback.", { + bucket: args.bucket, + code: error.code, + message: error.message, + }); + return consumeInMemoryApiRateLimit({ ownerId: args.ownerId, bucket: args.bucket, limit, windowSeconds }); + } + throw new ApiRateLimitUnavailableError(); + } + const row = parseRateLimitRow(data); + if (!row || typeof row.limited !== "boolean") { + if (args.allowInMemoryFallbackOnUnavailable) { + console.warn("Durable API rate limit check returned an invalid payload; using local in-memory fallback.", { + bucket: args.bucket, + }); + return consumeInMemoryApiRateLimit({ ownerId: args.ownerId, bucket: args.bucket, limit, windowSeconds }); + } + throw new ApiRateLimitUnavailableError(); + } + + return { + limited: row.limited, + limit: Number(row.limit_value ?? limit), + remaining: Number(row.remaining ?? 0), + retryAfterSeconds: Math.max(1, Number(row.retry_after_seconds ?? windowSeconds)), + resetAt: String(row.reset_at ?? new Date(Date.now() + windowSeconds * 1000).toISOString()), + }; +} + +function consumeInMemoryApiRateLimit({ + ownerId, + bucket, + limit, + windowSeconds, +}: { + ownerId: string; + bucket: ApiRateLimitBucket; + limit: number; + windowSeconds: number; +}): ApiRateLimitResult { + const now = Date.now(); + const windowMs = windowSeconds * 1000; + const key = `${ownerId}:${bucket}`; + const current = inMemoryApiRateLimits.get(key); + const windowStartMs = current && now - current.windowStartMs < windowMs ? current.windowStartMs : now; + const requestCount = (current && current.windowStartMs === windowStartMs ? current.requestCount : 0) + 1; + const resetAtMs = windowStartMs + windowMs; + + inMemoryApiRateLimits.set(key, { windowStartMs, requestCount }); + + return { + limited: requestCount > limit, + limit, + remaining: Math.max(limit - requestCount, 0), + retryAfterSeconds: Math.max(1, Math.ceil((resetAtMs - now) / 1000)), + resetAt: new Date(resetAtMs).toISOString(), + }; +} + +export function rateLimitJsonResponse(message: string, rateLimit: ApiRateLimitResult) { + return NextResponse.json( + { + error: message, + retryAfterSeconds: rateLimit.retryAfterSeconds, + }, + { + status: 429, + headers: { + "Retry-After": String(rateLimit.retryAfterSeconds), + }, + }, + ); +} diff --git a/src/lib/deep-memory.ts b/src/lib/deep-memory.ts index 3bce46d189..a562a772c7 100644 --- a/src/lib/deep-memory.ts +++ b/src/lib/deep-memory.ts @@ -1,6 +1,11 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { buildClinicalTextSearchQuery, classifyRagQuery, normalizedClinicalSearchTokens } from "@/lib/clinical-search"; -import { buildDocumentIndexUnitInputs, embeddingTextForDocumentIndexUnit } from "@/lib/document-index-units"; +import { + buildDocumentIndexUnitInputs, + countDocumentIndexUnitsByType, + documentIntelligenceVersion, + embeddingTextForDocumentIndexUnit, +} from "@/lib/document-index-units"; import { isClinicalImageEvidence } from "@/lib/image-filtering"; import { fallbackModelIndexProfile, @@ -27,6 +32,8 @@ type MemoryChunk = { page_number: number | null; chunk_index: number; section_heading: string | null; + section_path?: string[] | null; + anchor_id?: string | null; content: string; image_ids?: string[] | null; metadata?: Record | null; @@ -468,7 +475,66 @@ function embeddingText(card: BuiltMemoryCard) { return `${card.title}\n${card.card_type}\n${card.content}\nTerms: ${card.normalized_terms.join(", ")}`; } -async function stampDeepMemoryVersion(args: { supabase: SupabaseClient; documentId: string }) { +function slugForAnchor(value: string) { + return ( + value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/(^-|-$)/g, "") + .slice(0, 48) || "section" + ); +} + +function derivedChunkAnchor(chunk: MemoryChunk) { + const page = Number.isFinite(chunk.page_number) ? `p${chunk.page_number}` : "pna"; + const heading = slugForAnchor(chunk.section_heading || chunk.section_path?.join(" ") || "chunk"); + return `${page}-c${chunk.chunk_index}-${heading}`.slice(0, 80); +} + +async function repairMissingChunkAnchors(supabase: SupabaseClient, chunks: MemoryChunk[]) { + const missing = chunks.filter((chunk) => chunk.anchor_id === null); + let repaired = 0; + + for (let start = 0; start < missing.length; start += 10) { + const batch = missing.slice(start, start + 10); + await Promise.all( + batch.map(async (chunk) => { + const anchor = derivedChunkAnchor(chunk); + const { error } = await supabase + .from("document_chunks") + .update({ + anchor_id: anchor, + metadata: { + ...metadataRecord(chunk.metadata), + anchor_id: anchor, + anchor_repaired_by: documentIntelligenceVersion, + }, + }) + .eq("id", chunk.id) + .is("anchor_id", null); + if (error) throw new Error(error.message); + chunk.anchor_id = anchor; + chunk.metadata = { + ...metadataRecord(chunk.metadata), + anchor_id: anchor, + anchor_repaired_by: documentIntelligenceVersion, + }; + repaired += 1; + }), + ); + } + + return repaired; +} + +async function stampDeepMemoryVersion(args: { + supabase: SupabaseClient; + documentId: string; + sectionCount: number; + memoryCardCount: number; + indexUnitCountsByType: Record; + repairedAnchorCount: number; +}) { const stampedAt = new Date().toISOString(); const { data: doc, error: fetchError } = await args.supabase @@ -487,6 +553,13 @@ async function stampDeepMemoryVersion(args: { supabase: SupabaseClient; document rag_indexing_version: ragDeepMemoryVersion, rag_memory_version: ragDeepMemoryVersion, rag_memory_updated_at: stampedAt, + document_intelligence_version: documentIntelligenceVersion, + document_intelligence_updated_at: stampedAt, + section_count: args.sectionCount, + memory_card_count: args.memoryCardCount, + index_unit_count: Object.values(args.indexUnitCountsByType).reduce((sum, count) => sum + count, 0), + index_unit_counts_by_type: args.indexUnitCountsByType, + repaired_anchor_count: args.repairedAnchorCount, }, }) .eq("id", args.documentId); @@ -517,6 +590,7 @@ async function stampDeepMemoryVersion(args: { supabase: SupabaseClient; document rag_indexing_version: ragDeepMemoryVersion, rag_memory_version: ragDeepMemoryVersion, rag_memory_updated_at: stampedAt, + document_intelligence_version: documentIntelligenceVersion, }, }) .eq("id", chunk.id); @@ -629,7 +703,15 @@ export async function upsertDocumentDeepMemory(args: { } } - await stampDeepMemoryVersion({ supabase: args.supabase, documentId: args.document.id }); + const repairedAnchorCount = await repairMissingChunkAnchors(args.supabase, args.chunks); + await stampDeepMemoryVersion({ + supabase: args.supabase, + documentId: args.document.id, + sectionCount: sections.length, + memoryCardCount: cards.length, + indexUnitCountsByType: countDocumentIndexUnitsByType(indexUnits), + repairedAnchorCount, + }); return { sections, memoryCards: cards, indexUnits, modelProfile }; } diff --git a/src/lib/document-index-units.ts b/src/lib/document-index-units.ts index 880b7f87b6..9cc0b597e1 100644 --- a/src/lib/document-index-units.ts +++ b/src/lib/document-index-units.ts @@ -4,6 +4,7 @@ import type { ModelIndexProfile, ModelIndexProfileItem, ModelIndexTableFact } fr import type { ClinicalDocument, DocumentSectionMemory } from "@/lib/types"; export const documentIndexUnitVersion = "document-index-units-v1" as const; +export const documentIntelligenceVersion = "document-intelligence-v2" as const; export type DocumentIndexUnitType = | "document_profile" @@ -13,6 +14,10 @@ export type DocumentIndexUnitType = | "table_fact" | "askable_question" | "clinical_fact" + | "threshold" + | "workflow_step" + | "medication_monitoring" + | "alias" | "vocabulary_term"; export type DocumentIndexUnitInput = { @@ -52,6 +57,14 @@ function compact(value: unknown, limit = 900) { return compacted.length <= limit ? compacted : `${compacted.slice(0, limit - 3).trim()}...`; } +const sentenceBoundary = /(?<=[.!?])\s+|\n+/; +const thresholdPattern = + /\b(?:threshold|cut[\s-]?off|level|range|score|scale|criteria|criterion|maximum|minimum|baseline|anc|fbc|wbc|neutrophil|withhold|cease|stop|urgent|review|<|>|<=|>=|\d+(?:\.\d+)?\s*(?:mg|mcg|mmol|x\s*10\^?9\/l|%))\b/i; +const medicationMonitoringPattern = + /\b(?:clozapine|lithium|antipsychotic|benzodiazepine|olanzapine|lorazepam|diazepam|haloperidol|depot|lai|neuroleptic|dose|mg|mcg|route|oral|im\b|po\b|titrate|monitor|fbc|anc|level|toxicity)\b/i; +const workflowPattern = + /\b(?:workflow|pathway|process|procedure|step|refer|review|document|record|complete|form|responsib\w*|follow[- ]?up|appointment|escalat\w*|urgent|required|must|should)\b/i; + function termsFor(...values: unknown[]) { return Array.from( new Set( @@ -68,6 +81,47 @@ function termsFor(...values: unknown[]) { ).slice(0, 48); } +function splitSourceSentences(content: string) { + return content + .replace(/\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]/g, " ") + .split(sentenceBoundary) + .map((sentence) => compact(sentence, 520)) + .filter((sentence) => sentence.length >= 20); +} + +function deterministicTypedCandidates(chunk: IndexUnitChunk) { + const candidates: Array<{ unit_type: DocumentIndexUnitType; title: string; content: string; score: number }> = []; + const sentences = splitSourceSentences(chunk.content); + const add = (unit_type: DocumentIndexUnitType, title: string, content: string, score: number) => { + if (!content) return; + if (candidates.some((candidate) => candidate.unit_type === unit_type && candidate.content === content)) return; + candidates.push({ unit_type, title, content, score }); + }; + + for (const sentence of sentences) { + if (thresholdPattern.test(sentence)) { + add("threshold", chunk.section_heading || "Threshold", sentence, 0.7); + } + if (medicationMonitoringPattern.test(sentence)) { + add("medication_monitoring", chunk.section_heading || "Medication monitoring", sentence, 0.68); + } + if (workflowPattern.test(sentence)) { + add("workflow_step", chunk.section_heading || "Workflow step", sentence, 0.64); + } + if (candidates.length >= 6) break; + } + + return candidates.slice(0, 4); +} + +function unitTypeForClinicalItem(item: ModelIndexProfileItem): DocumentIndexUnitType { + const text = `${item.title} ${item.content}`; + if (thresholdPattern.test(text)) return "threshold"; + if (medicationMonitoringPattern.test(text)) return "medication_monitoring"; + if (workflowPattern.test(text)) return "workflow_step"; + return "clinical_fact"; +} + function firstChunk(chunks: IndexUnitChunk[], chunkIds: string[] = []) { if (chunkIds.length) { const direct = chunks.find((chunk) => chunkIds.includes(chunk.id)); @@ -119,6 +173,7 @@ function buildUnit(args: { extraction_mode: args.extraction_mode, metadata: { document_index_unit_version: documentIndexUnitVersion, + document_intelligence_version: documentIntelligenceVersion, chunk_index: args.sourceChunk?.chunk_index ?? null, section_heading: args.sourceChunk?.section_heading ?? null, ...args.metadata, @@ -247,6 +302,24 @@ export function buildDocumentIndexUnitInputs(args: { metadata: { source: "document_chunks" }, }), ); + + for (const candidate of deterministicTypedCandidates(chunk)) { + add( + buildUnit({ + document: args.document, + unit_type: candidate.unit_type, + sourceChunk: chunk, + title: candidate.title, + content: candidate.content, + quality_score: candidate.score, + extraction_mode: "deterministic", + metadata: { + source: "deterministic_chunk_signal", + typed_signal: candidate.unit_type, + }, + }), + ); + } } for (const item of args.modelProfile?.sections ?? []) { @@ -277,8 +350,8 @@ export function buildDocumentIndexUnitInputs(args: { document: args.document, chunks: args.chunks, item, - unit_type: "clinical_fact", - metadata: { source: "model_clinical_facts" }, + unit_type: unitTypeForClinicalItem(item), + metadata: { source: "model_clinical_facts", original_unit_type: "clinical_fact" }, }), ); } @@ -290,7 +363,7 @@ export function buildDocumentIndexUnitInputs(args: { add( buildUnit({ document: args.document, - unit_type: "vocabulary_term", + unit_type: "alias", sourceChunk, title: alias.canonical, content: `${alias.alias} means ${alias.canonical}`, @@ -316,6 +389,13 @@ export function buildDocumentIndexUnitInputs(args: { }); } +export function countDocumentIndexUnitsByType(units: Array>) { + return units.reduce>((counts, unit) => { + counts[unit.unit_type] = (counts[unit.unit_type] ?? 0) + 1; + return counts; + }, {}); +} + export function embeddingTextForDocumentIndexUnit(unit: DocumentIndexUnitInput) { return [ `Type: ${unit.unit_type}`, diff --git a/src/lib/extractors/document.ts b/src/lib/extractors/document.ts index d0e9da9877..b92110242c 100644 --- a/src/lib/extractors/document.ts +++ b/src/lib/extractors/document.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import ExcelJS from "exceljs"; @@ -57,72 +57,79 @@ async function extractPdf(buffer: Buffer) { await writeFile(pdfPath, buffer); try { - return await runPythonPdfExtractor(pdfPath, imageDir); + const extracted = await runPythonPdfExtractor(pdfPath, imageDir); + return { ...extracted, temporaryPaths: [tempRoot] }; } catch { // Fallback for developer machines without PyMuPDF/pytesseract. It still // indexes text PDFs, but scanned PDFs and image extraction need the Python // helper dependencies listed in worker/python/requirements.txt. const parser = new PDFParse({ data: buffer }); - const parsed = await parser.getText(); - const imageResult = await parser.getImage({ - imageBuffer: true, - imageDataUrl: true, - imageThreshold: 20, - }); - const images: ExtractedDocument["images"] = []; - for (const page of imageResult.pages) { - for (const [index, image] of page.images.entries()) { - const dataUrlMatch = image.dataUrl?.match(/^data:(.*?);base64,(.*)$/); - const mimeType = dataUrlMatch?.[1] ?? "image/png"; - const extension = mimeType.includes("jpeg") ? "jpg" : "png"; - const outputPath = path.join(imageDir, `fallback-page-${page.pageNumber}-image-${index + 1}.${extension}`); - const bytes = dataUrlMatch ? Buffer.from(dataUrlMatch[2], "base64") : Buffer.from(image.data); - await writeFile(outputPath, bytes); - images.push({ - pageNumber: page.pageNumber, - path: outputPath, - mimeType, - bbox: null, - width: null, - height: null, - sourceKind: "fallback", - metadata: { source_kind: "fallback" }, - }); + try { + const parsed = await parser.getText(); + const imageResult = await parser.getImage({ + imageBuffer: true, + imageDataUrl: true, + imageThreshold: 20, + }); + const images: ExtractedDocument["images"] = []; + for (const page of imageResult.pages) { + for (const [index, image] of page.images.entries()) { + const dataUrlMatch = image.dataUrl?.match(/^data:(.*?);base64,(.*)$/); + const mimeType = dataUrlMatch?.[1] ?? "image/png"; + const extension = mimeType.includes("jpeg") ? "jpg" : "png"; + const outputPath = path.join(imageDir, `fallback-page-${page.pageNumber}-image-${index + 1}.${extension}`); + const bytes = dataUrlMatch ? Buffer.from(dataUrlMatch[2], "base64") : Buffer.from(image.data); + await writeFile(outputPath, bytes); + images.push({ + pageNumber: page.pageNumber, + path: outputPath, + mimeType, + bbox: null, + width: null, + height: null, + sourceKind: "fallback", + metadata: { source_kind: "fallback" }, + }); + } + } + await parser.destroy(); + + // IDX-H3: the JS fallback cannot OCR. A scanned / image-only page yields little or no + // embedded text, so without flagging it the document would index as near-empty yet still + // be marked "indexed" — invisible to retrieval. Mark any page that has image content but + // below-threshold text as needsOcr so index_quality surfaces it (and the worker refuses + // to mark an image-only PDF as fully indexed). + const JS_FALLBACK_MIN_TEXT_CHARS = 40; + const imageCountByPage = new Map(); + for (const image of images) { + if (image.pageNumber === null) continue; + imageCountByPage.set(image.pageNumber, (imageCountByPage.get(image.pageNumber) ?? 0) + 1); } - } - await parser.destroy(); - - // IDX-H3: the JS fallback cannot OCR. A scanned / image-only page yields little or no - // embedded text, so without flagging it the document would index as near-empty yet still - // be marked "indexed" — invisible to retrieval. Mark any page that has image content but - // below-threshold text as needsOcr so index_quality surfaces it (and the worker refuses - // to mark an image-only PDF as fully indexed). - const JS_FALLBACK_MIN_TEXT_CHARS = 40; - const imageCountByPage = new Map(); - for (const image of images) { - if (image.pageNumber === null) continue; - imageCountByPage.set(image.pageNumber, (imageCountByPage.get(image.pageNumber) ?? 0) + 1); - } - - const rawPages = - parsed.pages.length > 0 - ? parsed.pages.map((page) => ({ pageNumber: page.num, text: page.text || "" })) - : [{ pageNumber: 1, text: parsed.text || "" }]; - const pages = rawPages.map((page) => { - const textLength = page.text.trim().length; - const hasImages = (imageCountByPage.get(page.pageNumber) ?? 0) > 0; - const needsOcr = textLength < JS_FALLBACK_MIN_TEXT_CHARS && hasImages; - return { pageNumber: page.pageNumber, text: page.text, ocrUsed: false, needsOcr }; - }); + const rawPages = + parsed.pages.length > 0 + ? parsed.pages.map((page) => ({ pageNumber: page.num, text: page.text || "" })) + : [{ pageNumber: 1, text: parsed.text || "" }]; + + const pages = rawPages.map((page) => { + const textLength = page.text.trim().length; + const hasImages = (imageCountByPage.get(page.pageNumber) ?? 0) > 0; + const needsOcr = textLength < JS_FALLBACK_MIN_TEXT_CHARS && hasImages; + return { pageNumber: page.pageNumber, text: page.text, ocrUsed: false, needsOcr }; + }); + + const warnings = ["Used JavaScript PDF fallback; install Python PDF/OCR prerequisites for scanned PDFs."]; + const ocrNeededPages = pages.filter((page) => page.needsOcr).length; + if (ocrNeededPages > 0) { + warnings.push(`needs_ocr: ${ocrNeededPages} page(s) appear image-only and were not OCR'd by the JS fallback.`); + } - const warnings = ["Used JavaScript PDF fallback; install Python PDF/OCR prerequisites for scanned PDFs."]; - const ocrNeededPages = pages.filter((page) => page.needsOcr).length; - if (ocrNeededPages > 0) { - warnings.push(`needs_ocr: ${ocrNeededPages} page(s) appear image-only and were not OCR'd by the JS fallback.`); + return { pages, images, warnings, temporaryPaths: [tempRoot] }; + } catch (fallbackError) { + await parser.destroy().catch(() => undefined); + await rm(tempRoot, { recursive: true, force: true }).catch(() => undefined); + throw fallbackError; } - - return { pages, images, warnings }; } } @@ -132,31 +139,37 @@ async function extractDocx(buffer: Buffer) { const tempRoot = await mkdtemp(path.join(tmpdir(), "clinical-kb-docx-")); const images: ExtractedDocument["images"] = []; - const media = Object.keys(zip.files).filter((name) => name.startsWith("word/media/")); - for (const [index, name] of media.entries()) { - const file = zip.files[name]; - if (file.dir) continue; - const bytes = await file.async("nodebuffer"); - const ext = path.extname(name).toLowerCase() || ".png"; - const mimeType = ext === ".jpg" || ext === ".jpeg" ? "image/jpeg" : ext === ".webp" ? "image/webp" : "image/png"; - const outputPath = path.join(tempRoot, `docx-image-${index}${ext}`); - await writeFile(outputPath, bytes); - images.push({ - pageNumber: null, - path: outputPath, - mimeType, - bbox: null, - width: null, - height: null, - sourceKind: "embedded", - metadata: { source_kind: "docx_media", file_name: name }, - }); - } + try { + const media = Object.keys(zip.files).filter((name) => name.startsWith("word/media/")); + for (const [index, name] of media.entries()) { + const file = zip.files[name]; + if (file.dir) continue; + const bytes = await file.async("nodebuffer"); + const ext = path.extname(name).toLowerCase() || ".png"; + const mimeType = ext === ".jpg" || ext === ".jpeg" ? "image/jpeg" : ext === ".webp" ? "image/webp" : "image/png"; + const outputPath = path.join(tempRoot, `docx-image-${index}${ext}`); + await writeFile(outputPath, bytes); + images.push({ + pageNumber: null, + path: outputPath, + mimeType, + bbox: null, + width: null, + height: null, + sourceKind: "embedded", + metadata: { source_kind: "docx_media", file_name: name }, + }); + } - return { - pages: [{ pageNumber: 1, text: raw.value || "", ocrUsed: false }], - images, - } satisfies ExtractedDocument; + return { + pages: [{ pageNumber: 1, text: raw.value || "", ocrUsed: false }], + images, + temporaryPaths: [tempRoot], + } satisfies ExtractedDocument; + } catch (error) { + await rm(tempRoot, { recursive: true, force: true }).catch(() => undefined); + throw error; + } } async function extractXlsx(buffer: Buffer) { diff --git a/src/lib/model-index-extraction.ts b/src/lib/model-index-extraction.ts index d11cd9385d..a9c1c228bd 100644 --- a/src/lib/model-index-extraction.ts +++ b/src/lib/model-index-extraction.ts @@ -1,5 +1,10 @@ import { env } from "@/lib/env"; import { expandClinicalVocabularyText } from "@/lib/clinical-vocabulary"; +import { + buildCoveragePromptNote, + buildIndexingCoverageProfile, + selectCoverageAwarePromptChunks, +} from "@/lib/indexing-coverage"; import { generateStructuredTextResponse } from "@/lib/openai"; import { cleanClinicalSummaryText, sourceTextForModel } from "@/lib/source-text-sanitizer"; @@ -279,9 +284,21 @@ function buildPrompt(args: { chunks: ModelIndexChunk[]; images: ModelIndexImage[]; }) { - const chunks = args.chunks.slice(0, 90); + const selectedChunks = selectCoverageAwarePromptChunks(args.chunks, 90); + const chunks = selectedChunks.chunks; + const coverage = buildIndexingCoverageProfile({ chunks: args.chunks, images: args.images }); const imageBlock = args.images - .slice(0, 40) + .map((image) => ({ + image, + score: + (image.source_kind === "table_crop" ? 4 : 0) + + (image.source_kind === "diagram_crop" ? 3 : 0) + + (image.caption ? 1 : 0) + + (image.labels?.length ? 1 : 0), + })) + .sort((a, b) => b.score - a.score || Number(a.image.page_number ?? 0) - Number(b.image.page_number ?? 0)) + .slice(0, 60) + .map((item) => item.image) .map((image) => { const metadata = image.metadata ?? {}; return [ @@ -332,6 +349,9 @@ Document: ${args.document.title} File: ${args.document.file_name} Source path: ${args.document.source_path ?? "unknown"} Vocabulary hints already known locally: ${vocabularyHints.join(", ") || "none"} +Coverage strategy: ${selectedChunks.strategy} + +${buildCoveragePromptNote({ profile: coverage, selectedChunkIds: chunks.map((chunk) => chunk.id) })} Text chunks: ${sourceBlock || "No source text."} diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 5a5da812d8..62440f9ec2 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -386,6 +386,7 @@ export type SearchTelemetry = { memory_top_score?: number; index_unit_count?: number; index_unit_top_score?: number; + retrieval_layer_counts?: Record; weighted_top_score?: number; rrf_top_score?: number; top_score?: number; @@ -448,6 +449,15 @@ function recordSearchScoreTelemetry(telemetry: SearchTelemetry, results: SearchR telemetry.score_spread = Number(Math.max(0, telemetry.top_score - telemetry.second_top_score).toFixed(4)); telemetry.score_distinct_documents = new Set(results.map((result) => result.document_id)).size; telemetry.retrieval_candidate_count = results.length; + telemetry.retrieval_layer_counts = results.reduce>((counts, result) => { + const layers = new Set(["chunk"]); + if (result.memory_cards?.length) layers.add("memory_card"); + if (result.index_unit?.unit_type) layers.add(`index_unit:${result.index_unit.unit_type}`); + if (result.match_explanation?.tableHit || result.index_unit?.unit_type === "table_fact") layers.add("table_fact"); + if (result.match_explanation?.fieldType) layers.add(`field:${result.match_explanation.fieldType}`); + for (const layer of layers) counts[layer] = (counts[layer] ?? 0) + 1; + return counts; + }, {}); } const citationSchema = z.object({ @@ -3053,16 +3063,6 @@ function isEssentialSimpleQuestionSection(section: Pick isEssentialSimpleQuestionSection(section)); - answer.answerSections = essentialSection ? [essentialSection] : []; - return answer; -} - export async function searchChunksWithTelemetry(args: SearchChunksArgs) { assertGlobalSearchAllowed(args); const cached = getCachedSearch(args); diff --git a/src/lib/source-governance.ts b/src/lib/source-governance.ts index ba2d28b256..67649a8968 100644 --- a/src/lib/source-governance.ts +++ b/src/lib/source-governance.ts @@ -27,6 +27,9 @@ export type GroupedSourceGovernanceWarning = { titles: string[]; }; +export const sourceGovernanceRefusalAnswer = + "I cannot provide a source-backed clinical answer because one or more supporting sources are marked outdated or have poor extraction quality. Review or reindex the source material before using it for clinical guidance."; + function isLocalMetadataText(value: string) { return /\b(?:wa|western australia|perth|north metropolitan|east metropolitan|south metropolitan|health service)\b/i.test( value, @@ -193,3 +196,7 @@ export function groupSourceGovernanceWarnings(warnings: SourceGovernanceWarning[ const severityRank = { danger: 0, warning: 1, info: 2 } satisfies Record; return Array.from(grouped.values()).sort((a, b) => severityRank[a.severity] - severityRank[b.severity]); } + +export function hasDangerSourceGovernanceWarning(warnings: SourceGovernanceWarning[]) { + return warnings.some((warning) => warning.severity === "danger"); +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 00de8e0815..cceb3acbf2 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -813,6 +813,7 @@ export type ExtractedDocument = { pages: ExtractedPage[]; images: ExtractedImage[]; warnings?: string[]; + temporaryPaths?: string[]; }; export type ChunkInput = { diff --git a/supabase/migrations/20260623030000_api_rate_limits.sql b/supabase/migrations/20260623030000_api_rate_limits.sql new file mode 100644 index 0000000000..6c3ef6f210 --- /dev/null +++ b/supabase/migrations/20260623030000_api_rate_limits.sql @@ -0,0 +1,104 @@ +set search_path = public, extensions, pg_temp; + +create table if not exists public.api_rate_limits ( + owner_id uuid not null references auth.users(id) on delete cascade, + bucket text not null, + window_start timestamptz not null default now(), + request_count integer not null default 0 check (request_count >= 0), + updated_at timestamptz not null default now(), + primary key (owner_id, bucket), + constraint api_rate_limits_bucket_nonempty check (btrim(bucket) <> '') +); + +create index if not exists api_rate_limits_bucket_updated_idx + on public.api_rate_limits(bucket, updated_at desc); + +create or replace function public.consume_api_rate_limit( + p_owner_id uuid, + p_bucket text, + p_limit integer, + p_window_seconds integer +) +returns table ( + limited boolean, + limit_value integer, + remaining integer, + retry_after_seconds integer, + reset_at timestamptz +) +language plpgsql +security definer +set search_path = public, extensions, pg_temp +as $$ +declare + v_now timestamptz := now(); + v_window_start timestamptz := v_now; + v_count integer; + v_reset_at timestamptz; +begin + if p_owner_id is null then + raise exception 'owner_id is required'; + end if; + if p_bucket is null or btrim(p_bucket) = '' then + raise exception 'bucket is required'; + end if; + if p_limit < 1 then + raise exception 'limit must be positive'; + end if; + if p_window_seconds < 1 then + raise exception 'window must be positive'; + end if; + + loop + update public.api_rate_limits + set + window_start = case + when window_start + make_interval(secs => p_window_seconds) <= v_now then v_window_start + else window_start + end, + request_count = case + when window_start + make_interval(secs => p_window_seconds) <= v_now then 1 + else request_count + 1 + end, + updated_at = v_now + where owner_id = p_owner_id + and bucket = p_bucket + returning request_count, window_start + make_interval(secs => p_window_seconds) + into v_count, v_reset_at; + + exit when found; + + begin + insert into public.api_rate_limits(owner_id, bucket, window_start, request_count, updated_at) + values (p_owner_id, p_bucket, v_window_start, 1, v_now) + returning request_count, window_start + make_interval(secs => p_window_seconds) + into v_count, v_reset_at; + exit; + exception when unique_violation then + -- Another request created the bucket first; retry and update it atomically. + end; + end loop; + + return query + select + v_count > p_limit as limited, + p_limit as limit_value, + greatest(p_limit - v_count, 0) as remaining, + greatest(1, ceiling(extract(epoch from (v_reset_at - v_now)))::integer) as retry_after_seconds, + v_reset_at as reset_at; +end; +$$; + +revoke all privileges on table public.api_rate_limits from public, anon, authenticated; +grant select, insert, update, delete on table public.api_rate_limits to service_role; + +revoke execute on function public.consume_api_rate_limit(uuid, text, integer, integer) from public, anon, authenticated; +grant execute on function public.consume_api_rate_limit(uuid, text, integer, integer) to service_role; + +alter table public.api_rate_limits enable row level security; + +drop policy if exists "api rate limits service role all" on public.api_rate_limits; +create policy "api rate limits service role all" on public.api_rate_limits + for all to service_role + using (true) + with check (true); diff --git a/supabase/migrations/20260623043000_document_intelligence_v2_units.sql b/supabase/migrations/20260623043000_document_intelligence_v2_units.sql new file mode 100644 index 0000000000..f21c17a972 --- /dev/null +++ b/supabase/migrations/20260623043000_document_intelligence_v2_units.sql @@ -0,0 +1,96 @@ +alter table public.document_index_units + drop constraint if exists document_index_units_unit_type_check; + +alter table public.document_index_units + add constraint document_index_units_unit_type_check + check ( + unit_type in ( + 'document_profile', + 'section_summary', + 'page_text', + 'chunk_evidence', + 'table_fact', + 'askable_question', + 'clinical_fact', + 'threshold', + 'workflow_step', + 'medication_monitoring', + 'alias', + 'vocabulary_term' + ) + ); + +create or replace function public.match_document_index_units_hybrid( + query_embedding extensions.vector(1536), + query_text text, + match_count integer default 24, + min_similarity double precision default 0.1, + document_filters uuid[] default null, + owner_filter uuid default null +) +returns table ( + id uuid, + document_id uuid, + source_chunk_id uuid, + source_image_id uuid, + unit_type text, + title text, + content text, + page_start integer, + page_end integer, + heading_path text[], + normalized_terms text[], + source_span jsonb, + quality_score real, + extraction_mode text, + similarity double precision, + text_rank double precision, + hybrid_score double precision, + metadata jsonb +) +language sql +stable +set search_path = public, extensions, pg_temp +as $$ + with query as ( + select websearch_to_tsquery('english', coalesce(query_text, '')) as tsq, + regexp_split_to_array(lower(coalesce(query_text, '')), '\s+') as terms + ), + ranked as ( + select u.id, u.document_id, u.source_chunk_id, u.source_image_id, u.unit_type, u.title, u.content, u.page_start, + u.page_end, u.heading_path, u.normalized_terms, u.source_span, u.quality_score, u.extraction_mode, + (1 - (u.embedding <=> query_embedding))::double precision as similarity, + (ts_rank_cd(u.search_tsv, query.tsq) + + case when u.normalized_terms && query.terms then 0.25 else 0 end + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact', 'threshold', 'workflow_step', 'medication_monitoring', 'alias') then 0.06 + when u.unit_type = 'section_summary' then 0.03 + else 0 end + )::double precision as text_rank, + u.metadata + from public.document_index_units u + join public.documents d on d.id = u.document_id + cross join query + where d.status = 'indexed' + and (document_filters is null or u.document_id = any(document_filters)) + and (owner_filter is null or d.owner_id = owner_filter) + and u.source_chunk_id is not null + and (u.search_tsv @@ query.tsq or u.normalized_terms && query.terms) + order by text_rank desc, similarity desc + limit greatest(match_count * 3, 48) + ) + select id, document_id, source_chunk_id, source_image_id, unit_type, title, content, page_start, page_end, heading_path, + normalized_terms, source_span, quality_score, extraction_mode, similarity, text_rank, + ( + (similarity * 0.52) + + (least(text_rank, 1) * 0.28) + + (quality_score * 0.12) + + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) + + (case when unit_type in ('askable_question', 'threshold', 'table_fact') then 0.04 + when unit_type in ('workflow_step', 'medication_monitoring') then 0.03 + else 0 end) + )::double precision as hybrid_score, + metadata + from ranked + order by hybrid_score desc, similarity desc, text_rank desc + limit match_count; +$$; diff --git a/supabase/schema.sql b/supabase/schema.sql index b426b3ea3d..6f52cc1e0f 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -463,6 +463,16 @@ create table if not exists public.rag_retrieval_logs ( created_at timestamptz not null default now() ); +create table if not exists public.api_rate_limits ( + owner_id uuid not null references auth.users(id) on delete cascade, + bucket text not null, + window_start timestamptz not null default now(), + request_count integer not null default 0 check (request_count >= 0), + updated_at timestamptz not null default now(), + primary key (owner_id, bucket), + constraint api_rate_limits_bucket_nonempty check (btrim(bucket) <> '') +); + create table if not exists public.storage_cleanup_jobs ( id uuid primary key default gen_random_uuid(), owner_id uuid references auth.users(id) on delete set null, @@ -653,6 +663,8 @@ create index if not exists rag_retrieval_logs_miss_idx where is_miss = true; create index if not exists rag_retrieval_logs_strategy_idx on public.rag_retrieval_logs(retrieval_strategy, created_at desc); +create index if not exists api_rate_limits_bucket_updated_idx + on public.api_rate_limits(bucket, updated_at desc); -- Redundant single-column FK indexes removed (covered by composite indexes -- with the same leading column, e.g. document_chunks_document_idx). @@ -668,6 +680,81 @@ begin end; $$; +create or replace function public.consume_api_rate_limit( + p_owner_id uuid, + p_bucket text, + p_limit integer, + p_window_seconds integer +) +returns table ( + limited boolean, + limit_value integer, + remaining integer, + retry_after_seconds integer, + reset_at timestamptz +) +language plpgsql +security definer +set search_path = public, extensions, pg_temp +as $$ +declare + v_now timestamptz := now(); + v_window_start timestamptz := v_now; + v_count integer; + v_reset_at timestamptz; +begin + if p_owner_id is null then + raise exception 'owner_id is required'; + end if; + if p_bucket is null or btrim(p_bucket) = '' then + raise exception 'bucket is required'; + end if; + if p_limit < 1 then + raise exception 'limit must be positive'; + end if; + if p_window_seconds < 1 then + raise exception 'window must be positive'; + end if; + + loop + update public.api_rate_limits + set + window_start = case + when window_start + make_interval(secs => p_window_seconds) <= v_now then v_window_start + else window_start + end, + request_count = case + when window_start + make_interval(secs => p_window_seconds) <= v_now then 1 + else request_count + 1 + end, + updated_at = v_now + where owner_id = p_owner_id + and bucket = p_bucket + returning request_count, window_start + make_interval(secs => p_window_seconds) + into v_count, v_reset_at; + + exit when found; + + begin + insert into public.api_rate_limits(owner_id, bucket, window_start, request_count, updated_at) + values (p_owner_id, p_bucket, v_window_start, 1, v_now) + returning request_count, window_start + make_interval(secs => p_window_seconds) + into v_count, v_reset_at; + exit; + exception when unique_violation then + end; + end loop; + + return query + select + v_count > p_limit as limited, + p_limit as limit_value, + greatest(p_limit - v_count, 0) as remaining, + greatest(1, ceiling(extract(epoch from (v_reset_at - v_now)))::integer) as retry_after_seconds, + v_reset_at as reset_at; +end; +$$; + drop trigger if exists import_batches_updated_at on public.import_batches; create trigger import_batches_updated_at before update on public.import_batches @@ -2018,6 +2105,7 @@ grant select, insert, update, delete on table public.rag_query_misses, public.rag_aliases, public.rag_response_cache, + public.api_rate_limits, public.storage_cleanup_jobs, public.rag_retrieval_logs to service_role; @@ -2064,6 +2152,7 @@ alter table public.rag_queries enable row level security; alter table public.rag_query_misses enable row level security; alter table public.rag_aliases enable row level security; alter table public.rag_response_cache enable row level security; +alter table public.api_rate_limits enable row level security; alter table public.storage_cleanup_jobs enable row level security; alter table public.rag_retrieval_logs enable row level security; @@ -2152,6 +2241,11 @@ create policy "rag response cache service role all" on public.rag_response_cache using (true) with check (true); +create policy "api rate limits service role all" on public.api_rate_limits + for all to service_role + using (true) + with check (true); + create policy "storage cleanup owner read" on public.storage_cleanup_jobs for select to authenticated using (owner_id = (select auth.uid())); @@ -2167,7 +2261,7 @@ create table if not exists public.document_index_units ( id uuid primary key default gen_random_uuid(), owner_id uuid references auth.users(id) on delete set null, document_id uuid not null references public.documents(id) on delete cascade, - unit_type text not null check (unit_type in ('document_profile', 'section_summary', 'page_text', 'chunk_evidence', 'table_fact', 'askable_question', 'clinical_fact', 'vocabulary_term')), + unit_type text not null check (unit_type in ('document_profile', 'section_summary', 'page_text', 'chunk_evidence', 'table_fact', 'askable_question', 'clinical_fact', 'threshold', 'workflow_step', 'medication_monitoring', 'alias', 'vocabulary_term')), source_chunk_id uuid references public.document_chunks(id) on delete cascade, source_image_id uuid references public.document_images(id) on delete set null, page_start integer, @@ -2242,7 +2336,7 @@ as $$ (1 - (u.embedding <=> query_embedding))::double precision as similarity, (ts_rank_cd(u.search_tsv, query.tsq) + case when u.normalized_terms && query.terms then 0.25 else 0 end - + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact') then 0.06 + + case when u.unit_type in ('askable_question', 'table_fact', 'clinical_fact', 'threshold', 'workflow_step', 'medication_monitoring', 'alias') then 0.06 when u.unit_type = 'section_summary' then 0.03 else 0 end )::double precision as text_rank, @@ -2265,7 +2359,9 @@ as $$ + (least(text_rank, 1) * 0.28) + (quality_score * 0.12) + (case when extraction_mode in ('model_heavy', 'hybrid') then 0.04 else 0 end) - + (case when unit_type = 'askable_question' then 0.04 else 0 end) + + (case when unit_type in ('askable_question', 'threshold', 'table_fact') then 0.04 + when unit_type in ('workflow_step', 'medication_monitoring') then 0.03 + else 0 end) )::double precision as hybrid_score, metadata from ranked diff --git a/tests/deep-memory.test.ts b/tests/deep-memory.test.ts index 9325560b78..b86b98eeb3 100644 --- a/tests/deep-memory.test.ts +++ b/tests/deep-memory.test.ts @@ -259,8 +259,8 @@ describe("deep RAG memory indexing", () => { insertedMemoryRows.push(...payload); return Promise.resolve({ data: null, error: null }); }, - select: (columns: string) => ({ - eq: (column: string, value: unknown) => { + select: () => ({ + eq: () => { if (table === "documents") { return { single: async () => ({ data: { metadata: {} }, error: null }), @@ -278,7 +278,7 @@ describe("deep RAG memory indexing", () => { }, }), update: (payload: Record) => ({ - eq: (column: string, value: unknown) => { + eq: () => { updatedRows.set(table, [...(updatedRows.get(table) ?? []), payload]); return Promise.resolve({ data: null, error: null }); }, diff --git a/tests/document-index-units.test.ts b/tests/document-index-units.test.ts new file mode 100644 index 0000000000..2fff4b3261 --- /dev/null +++ b/tests/document-index-units.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { + buildDocumentIndexUnitInputs, + countDocumentIndexUnitsByType, + documentIntelligenceVersion, +} from "../src/lib/document-index-units"; + +const document = { + id: "doc-1", + owner_id: "owner-1", + title: "Clozapine Monitoring", + file_name: "clozapine.pdf", +}; + +describe("document index units", () => { + it("creates typed deterministic and model-backed clinical index units", () => { + const units = buildDocumentIndexUnitInputs({ + document, + chunks: [ + { + id: "chunk-1", + document_id: "doc-1", + page_number: 4, + chunk_index: 0, + section_heading: "Monitoring", + section_path: ["Monitoring"], + content: + "If ANC is < 1.5, stop clozapine and seek urgent review. Monitor FBC weekly and document the workflow step.", + metadata: {}, + }, + ], + modelProfile: { + sections: [], + askable_questions: [ + { + title: "What ANC threshold stops clozapine?", + content: "What ANC threshold requires clozapine to stop?", + source_chunk_ids: ["chunk-1"], + source_image_ids: [], + confidence: 0.9, + }, + ], + clinical_facts: [ + { + title: "ANC stop threshold", + content: "ANC below 1.5 requires stopping clozapine and urgent review.", + source_chunk_ids: ["chunk-1"], + source_image_ids: [], + confidence: 0.92, + }, + ], + table_facts: [], + aliases: [ + { + alias: "ANC", + canonical: "absolute neutrophil count", + alias_type: "clinical_term", + source_chunk_ids: ["chunk-1"], + confidence: 0.9, + }, + ], + quality_issues: [], + model: "test-model", + version: "model-heavy-index-v1", + }, + }); + + expect(units.map((unit) => unit.unit_type)).toEqual( + expect.arrayContaining(["threshold", "medication_monitoring", "workflow_step", "askable_question", "alias"]), + ); + expect(units.every((unit) => unit.metadata.document_intelligence_version === documentIntelligenceVersion)).toBe( + true, + ); + expect(countDocumentIndexUnitsByType(units)).toMatchObject({ + threshold: expect.any(Number), + askable_question: 1, + alias: 1, + }); + }); +}); diff --git a/tests/model-index-extraction.test.ts b/tests/model-index-extraction.test.ts new file mode 100644 index 0000000000..f546e3be1f --- /dev/null +++ b/tests/model-index-extraction.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + generateStructuredTextResponse: vi.fn(), +})); + +vi.mock("@/lib/env", () => ({ + env: { + OPENAI_ANSWER_MODEL: "gpt-test", + OPENAI_STRONG_ANSWER_MODEL: "gpt-strong-test", + }, +})); + +vi.mock("@/lib/openai", () => ({ + generateStructuredTextResponse: mocks.generateStructuredTextResponse, +})); + +import { generateModelIndexProfile } from "@/lib/model-index-extraction"; + +describe("model index extraction", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("uses coverage-aware chunk selection so late high-yield content reaches the model prompt", async () => { + mocks.generateStructuredTextResponse.mockResolvedValueOnce( + JSON.stringify({ + sections: [], + askable_questions: [], + clinical_facts: [], + table_facts: [], + aliases: [], + quality_issues: [], + }), + ); + + await generateModelIndexProfile({ + document: { title: "Large Clozapine Protocol", file_name: "large-clozapine.pdf" }, + chunks: Array.from({ length: 140 }, (_, index) => ({ + id: `chunk-${index}`, + page_number: index + 1, + chunk_index: index, + section_heading: index % 20 === 0 ? `Section ${index}` : null, + content: + index === 132 + ? "If ANC is < 1.5, stop clozapine and seek urgent specialist review." + : `Routine source content ${index}.`, + })), + images: [], + }); + + const prompt = String(mocks.generateStructuredTextResponse.mock.calls[0]?.[0] ?? ""); + expect(prompt).toContain("Coverage strategy: coverage_spread_high_yield_headings"); + expect(prompt).toContain("chunk_id: chunk-132"); + expect(prompt).toContain("remain indexed and retrievable"); + }); +}); diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index 7e014c5886..12124af654 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -36,6 +36,17 @@ function fail(message: string): QueryResult { return { data: null, error: { message } }; } +function rateLimitRow(overrides: Partial> = {}) { + return { + limited: false, + limit_value: 100, + remaining: 99, + retry_after_seconds: 60, + reset_at: new Date(Date.now() + 60_000).toISOString(), + ...overrides, + }; +} + class QueryBuilder implements PromiseLike { constructor( private readonly call: QueryCall, @@ -159,7 +170,14 @@ function createSupabaseMock(resolve: QueryResolver = () => ok([])) { ? { data: { user: { id: userId } }, error: null } : { data: { user: null }, error: { message: "Invalid token" } }, ); - const rpc = vi.fn(async () => ok([])); + const rpc = vi.fn(async (name: string) => + name === "consume_api_rate_limit" + ? { + data: [rateLimitRow()], + error: null, + } + : ok([]), + ); const client = { auth: { getUser, admin: { listUsers } }, calls, @@ -611,25 +629,29 @@ describe("private document API access", () => { } return ok([]); }); - client.rpc.mockResolvedValue({ - data: [ - { - document_id: documentId, - labels: [ - { - id: "label-1", - document_id: documentId, - label: "agitation", - label_type: "topic", - source: "generated", - confidence: 0.9, - }, - ], - summary: "High-yield agitation management guidance.", - }, - ], - error: null, - }); + client.rpc.mockImplementation(async (name: string) => + name === "consume_api_rate_limit" + ? { data: [rateLimitRow()], error: null } + : { + data: [ + { + document_id: documentId, + labels: [ + { + id: "label-1", + document_id: documentId, + label: "agitation", + label_type: "topic", + source: "generated", + confidence: 0.9, + }, + ], + summary: "High-yield agitation management guidance.", + }, + ], + error: null, + }, + ); mockRuntime(client, { searchChunksWithTelemetry: vi.fn(async () => ({ results: [ @@ -1500,6 +1522,13 @@ describe("private document API access", () => { sources: [], })); const client = createSupabaseMock(); + client.rpc.mockImplementation(async (name: string, args?: Record) => + name === "consume_api_rate_limit" && args?.p_bucket === "answer" + ? { data: [rateLimitRow({ limited: true, limit_value: 30, remaining: 0 })], error: null } + : name === "consume_api_rate_limit" + ? { data: [rateLimitRow()], error: null } + : ok([]), + ); mockRuntime(client, { searchChunksWithTelemetry, answerQuestionWithScope }); const answerRoute = await import("../src/app/api/answer/route"); @@ -1511,16 +1540,14 @@ describe("private document API access", () => { body: JSON.stringify({ query: "monitoring" }), }); - for (let index = 0; index < 30; index += 1) { - const response = await answerRoute.POST(answerRequest()); - expect(response.status).toBe(200); - } - const limited = await answerRoute.POST(answerRequest()); expect(limited.status).toBe(429); expect(limited.headers.get("Retry-After")).toBe("60"); - expect(await payload(limited)).toEqual({ error: "Too many public answer requests. Retry shortly." }); - expect(answerQuestionWithScope).toHaveBeenCalledTimes(30); + expect(await payload(limited)).toEqual({ + error: "Too many answer requests. Retry shortly.", + retryAfterSeconds: 60, + }); + expect(answerQuestionWithScope).not.toHaveBeenCalled(); const searchResponse = await searchRoute.POST( authenticatedRequest("/api/search", { @@ -1532,6 +1559,14 @@ describe("private document API access", () => { expect(searchResponse.status).toBe(200); expect(searchChunksWithTelemetry).toHaveBeenCalledWith(expect.objectContaining({ ownerId: userId })); + expect(client.rpc).toHaveBeenCalledWith( + "consume_api_rate_limit", + expect.objectContaining({ p_owner_id: userId, p_bucket: "answer" }), + ); + expect(client.rpc).toHaveBeenCalledWith( + "consume_api_rate_limit", + expect.objectContaining({ p_owner_id: userId, p_bucket: "search" }), + ); }); it("rate limits abnormal authenticated search bursts with retry metadata", async () => { @@ -1551,15 +1586,14 @@ describe("private document API access", () => { }, })); const client = createSupabaseMock(); + client.rpc.mockImplementation(async (name: string, args?: Record) => + name === "consume_api_rate_limit" && args?.p_bucket === "search" + ? { data: [rateLimitRow({ limited: true, limit_value: 2, remaining: 0 })], error: null } + : name === "consume_api_rate_limit" + ? { data: [rateLimitRow()], error: null } + : ok([]), + ); mockRuntime(client, { searchChunksWithTelemetry }); - vi.doMock("@/lib/public-rate-limit", async () => { - const actual = await vi.importActual("@/lib/public-rate-limit"); - return { - ...actual, - consumePublicSearchRateLimit: (headers: Headers) => - actual.consumePublicSearchRateLimit(headers, Date.now(), { limit: 2, windowMs: 60_000 }), - }; - }); const searchRoute = await import("../src/app/api/search/route"); const searchRequest = () => authenticatedRequest("/api/search", { @@ -1568,9 +1602,6 @@ describe("private document API access", () => { body: JSON.stringify({ query: "monitoring", includeRelatedDocuments: false }), }); - expect((await searchRoute.POST(searchRequest())).status).toBe(200); - expect((await searchRoute.POST(searchRequest())).status).toBe(200); - const limited = await searchRoute.POST(searchRequest()); expect(limited.status).toBe(429); expect(limited.headers.get("Retry-After")).toBe("60"); @@ -1578,7 +1609,80 @@ describe("private document API access", () => { error: "Search is temporarily rate limited because too many requests were received. Retry shortly.", retryAfterSeconds: 60, }); - expect(searchChunksWithTelemetry).toHaveBeenCalledTimes(2); + expect(searchChunksWithTelemetry).not.toHaveBeenCalled(); + }); + + it("fails closed when the durable rate limit check is unavailable", async () => { + const searchChunksWithTelemetry = vi.fn(async () => ({ + results: [], + telemetry: { + search_cache_hit: false, + text_fast_path_latency_ms: 0, + embedding_skipped: true, + embedding_latency_ms: 0, + embedding_cache_hit: false, + supabase_rpc_latency_ms: 0, + rerank_latency_ms: 0, + retrieval_strategy: "text_fast_path", + }, + })); + const client = createSupabaseMock(); + client.rpc.mockImplementation(async (name: string) => + name === "consume_api_rate_limit" ? fail("limiter table unavailable") : ok([]), + ); + mockRuntime(client, { searchChunksWithTelemetry }); + const { POST } = await import("../src/app/api/search/route"); + + const response = await POST( + authenticatedRequest("/api/search", { + method: "POST", + body: JSON.stringify({ query: "monitoring", includeRelatedDocuments: false }), + }), + ); + + expect(response.status).toBe(503); + expect(await payload(response)).toEqual({ error: "Rate limit check is temporarily unavailable." }); + expect(searchChunksWithTelemetry).not.toHaveBeenCalled(); + }); + + it("uses an in-memory limiter fallback for managed local no-auth search when the durable check is unavailable", async () => { + const searchChunksWithTelemetry = vi.fn(async () => ({ + results: [], + telemetry: { + search_cache_hit: false, + text_fast_path_latency_ms: 0, + embedding_skipped: true, + embedding_latency_ms: 0, + embedding_cache_hit: false, + supabase_rpc_latency_ms: 0, + rerank_latency_ms: 0, + retrieval_strategy: "text_fast_path", + }, + })); + const client = createSupabaseMock(); + client.auth.admin.listUsers.mockResolvedValueOnce({ + data: { users: [{ id: userId, email: "clinician@example.test" }], nextPage: 0 }, + error: null, + }); + client.rpc.mockImplementation(async (name: string) => + name === "consume_api_rate_limit" ? fail("limiter table unavailable") : ok([]), + ); + mockRuntime(client, { searchChunksWithTelemetry }, { localNoAuth: true, localOwnerEmail: "clinician@example.test" }); + const { POST } = await import("../src/app/api/search/route"); + + const response = await POST( + localPortRequest(4298, "/api/search", { + method: "POST", + body: JSON.stringify({ query: "monitoring", includeRelatedDocuments: false }), + }), + ); + + expect(response.status).toBe(200); + expect(searchChunksWithTelemetry).toHaveBeenCalledWith(expect.objectContaining({ ownerId: userId })); + expect(client.rpc).toHaveBeenCalledWith( + "consume_api_rate_limit", + expect.objectContaining({ p_owner_id: userId, p_bucket: "search" }), + ); }); it("coalesces identical in-flight authenticated search requests", async () => { @@ -1674,21 +1778,16 @@ describe("private document API access", () => { sources: [], })); const client = createSupabaseMock(); + client.rpc.mockImplementation(async (name: string, args?: Record) => + name === "consume_api_rate_limit" && args?.p_bucket === "answer" + ? { data: [rateLimitRow({ limited: true, limit_value: 30, remaining: 0 })], error: null } + : name === "consume_api_rate_limit" + ? { data: [rateLimitRow()], error: null } + : ok([]), + ); mockRuntime(client, { answerQuestionWithScope }); - const answerRoute = await import("../src/app/api/answer/route"); const streamRoute = await import("../src/app/api/answer/stream/route"); - const answerRequest = () => - authenticatedRequest("/api/answer", { - method: "POST", - headers: { "x-real-ip": "203.0.113.11" }, - body: JSON.stringify({ query: "monitoring" }), - }); - - for (let index = 0; index < 30; index += 1) { - const response = await answerRoute.POST(answerRequest()); - expect(response.status).toBe(200); - } const response = await streamRoute.POST( authenticatedRequest("/api/answer/stream", { @@ -1703,8 +1802,195 @@ describe("private document API access", () => { expect(response.headers.get("Retry-After")).toBe("60"); expect(body).toContain("event: error"); expect(body).toContain('"status":429'); - expect(body).toContain("Too many public answer requests. Retry shortly."); - expect(answerQuestionWithScope).toHaveBeenCalledTimes(30); + expect(body).toContain("Too many answer requests. Retry shortly."); + expect(answerQuestionWithScope).not.toHaveBeenCalled(); + }); + + it("uses an in-memory limiter fallback for managed local no-auth streaming answers when the durable check is unavailable", async () => { + const answerQuestionWithScope = vi.fn(async () => ({ + answer: "Owned evidence.", + grounded: true, + confidence: "medium", + citations: [], + sources: [], + })); + const client = createSupabaseMock(); + client.auth.admin.listUsers.mockResolvedValueOnce({ + data: { users: [{ id: userId, email: "clinician@example.test" }], nextPage: 0 }, + error: null, + }); + client.rpc.mockImplementation(async (name: string) => + name === "consume_api_rate_limit" ? fail("limiter table unavailable") : ok([]), + ); + mockRuntime(client, { answerQuestionWithScope }, { localNoAuth: true, localOwnerEmail: "clinician@example.test" }); + const { POST } = await import("../src/app/api/answer/stream/route"); + + const response = await POST( + localPortRequest(4298, "/api/answer/stream", { + method: "POST", + body: JSON.stringify({ query: "monitoring", documentId: otherDocumentId }), + }), + ); + const body = await response.text(); + + expect(response.status).toBe(200); + expect(body).toContain("event: final"); + expect(answerQuestionWithScope).toHaveBeenCalledWith( + expect.objectContaining({ ownerId: userId, documentId: otherDocumentId, onProgress: expect.any(Function) }), + ); + expect(client.rpc).toHaveBeenCalledWith( + "consume_api_rate_limit", + expect.objectContaining({ p_owner_id: userId, p_bucket: "answer" }), + ); + }); + + it("does not stream internal PublicApiError details to clients", async () => { + const answerQuestionWithScope = vi.fn(async () => { + const { PublicApiError } = await import("../src/lib/http"); + throw new PublicApiError("Stream failed safely.", 503, { + code: "stream_failed", + causeMessage: "secret table public.private_data does not exist", + sqlState: "42P01", + }); + }); + const client = createSupabaseMock(); + mockRuntime(client, { answerQuestionWithScope }); + const { POST } = await import("../src/app/api/answer/stream/route"); + + const response = await POST( + authenticatedRequest("/api/answer/stream", { + method: "POST", + body: JSON.stringify({ query: "monitoring", documentId: otherDocumentId }), + }), + ); + const body = await response.text(); + + expect(response.status).toBe(200); + expect(body).toContain("Stream failed safely."); + expect(body).toContain("stream_failed"); + expect(body).not.toContain("private_data"); + expect(body).not.toContain("42P01"); + }); + + it("refuses answer responses backed by danger-class source governance warnings", async () => { + const answerQuestionWithScope = vi.fn(async () => ({ + answer: "Use the old protocol.", + grounded: true, + confidence: "high", + citations: [{ chunk_id: "chunk-1", page_number: 1, quote: "old protocol", document_id: documentId }], + sources: [ + { + id: "chunk-1", + document_id: documentId, + title: "Outdated guideline", + file_name: "old.pdf", + page_number: 1, + chunk_index: 0, + section_heading: null, + content: "old protocol", + image_ids: [], + similarity: 0.9, + source_metadata: { + source_title: "Outdated guideline", + publisher: "Local WA service", + jurisdiction: "WA", + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "outdated", + clinical_validation_status: "approved", + extraction_quality: "good", + }, + images: [], + }, + ], + })); + const client = createSupabaseMock(); + mockRuntime(client, { answerQuestionWithScope }); + const { POST } = await import("../src/app/api/answer/route"); + + const response = await POST( + authenticatedRequest("/api/answer", { + method: "POST", + body: JSON.stringify({ query: "monitoring", documentId }), + }), + ); + const body = await payload(response); + + expect(response.status).toBe(200); + expect(body.grounded).toBe(false); + expect(body.confidence).toBe("unsupported"); + expect(body.citations).toEqual([]); + expect(String(body.answer)).toContain("cannot provide a source-backed clinical answer"); + expect(body.sourceGovernanceWarnings).toEqual([ + expect.objectContaining({ code: "outdated_source", severity: "danger" }), + ]); + }); + + it("rate limits document summarization before OpenAI work", async () => { + const summarizeDocument = vi.fn(async () => ({ summary: "Expensive summary" })); + const client = createSupabaseMock(); + client.rpc.mockImplementation(async (name: string, args?: Record) => + name === "consume_api_rate_limit" && args?.p_bucket === "document_summarize" + ? { data: [rateLimitRow({ limited: true, limit_value: 12, remaining: 0 })], error: null } + : name === "consume_api_rate_limit" + ? { data: [rateLimitRow()], error: null } + : ok([]), + ); + mockRuntime(client, { summarizeDocument }); + const { POST } = await import("../src/app/api/documents/[id]/summarize/route"); + + const response = await POST(authenticatedRequest(`/api/documents/${documentId}/summarize`, { method: "POST" }), { + params: Promise.resolve({ id: documentId }), + }); + + expect(response.status).toBe(429); + expect(await payload(response)).toMatchObject({ + error: "Too many document summary requests. Retry shortly.", + retryAfterSeconds: 60, + }); + expect(summarizeDocument).not.toHaveBeenCalled(); + }); + + it("rate limits single and bulk reindex before enrichment or queue work", async () => { + const client = createSupabaseMock(); + client.rpc.mockImplementation(async (name: string, args?: Record) => + name === "consume_api_rate_limit" && (args?.p_bucket === "document_reindex" || args?.p_bucket === "bulk_reindex") + ? { data: [rateLimitRow({ limited: true, limit_value: 1, remaining: 0 })], error: null } + : name === "consume_api_rate_limit" + ? { data: [rateLimitRow()], error: null } + : ok([]), + ); + const upsertDocumentEnrichment = vi.fn(); + const upsertDocumentDeepMemory = vi.fn(); + mockRuntime(client, { invalidateRagCachesForOwner: vi.fn() }); + vi.doMock("@/lib/document-enrichment", () => ({ upsertDocumentEnrichment })); + vi.doMock("@/lib/deep-memory", () => ({ upsertDocumentDeepMemory })); + const singleRoute = await import("../src/app/api/documents/[id]/reindex/route"); + const bulkRoute = await import("../src/app/api/documents/bulk/reindex/route"); + + const singleResponse = await singleRoute.POST( + authenticatedRequest(`/api/documents/${documentId}/reindex`, { + method: "POST", + body: JSON.stringify({ mode: "enrichment" }), + }), + { params: Promise.resolve({ id: documentId }) }, + ); + const bulkResponse = await bulkRoute.POST( + authenticatedRequest("/api/documents/bulk/reindex", { + method: "POST", + body: JSON.stringify({ documentIds: [documentId], mode: "enrichment" }), + }), + ); + + expect(singleResponse.status).toBe(429); + expect(bulkResponse.status).toBe(429); + expect(upsertDocumentEnrichment).not.toHaveBeenCalled(); + expect(upsertDocumentDeepMemory).not.toHaveBeenCalled(); + expect(client.calls.some((call) => call.table === "documents")).toBe(false); }); it("returns a generic not found response when summarizing an unowned document", async () => { diff --git a/tests/private-rag-access.test.ts b/tests/private-rag-access.test.ts index c103a50174..e627eae7a0 100644 --- a/tests/private-rag-access.test.ts +++ b/tests/private-rag-access.test.ts @@ -9,7 +9,7 @@ const allowedRateLimit = { limit: 100, remaining: 99, retryAfterSeconds: 0, - resetAt: Date.now() + 60_000, + resetAt: new Date(Date.now() + 60_000).toISOString(), }; function isRecord(value: unknown): value is Record { @@ -123,10 +123,13 @@ function mockRuntime(options: { demoMode?: boolean } = {}) { requireAuthenticatedUser, unauthorizedResponse, })); - vi.doMock("@/lib/public-rate-limit", () => ({ - consumePublicAnswerRateLimit: vi.fn(() => allowedRateLimit), - consumePublicSearchRateLimit: vi.fn(() => allowedRateLimit), - })); + vi.doMock("@/lib/api-rate-limit", async () => { + const actual = await vi.importActual("@/lib/api-rate-limit"); + return { + ...actual, + consumeApiRateLimit: vi.fn(async () => allowedRateLimit), + }; + }); vi.doMock("@/lib/demo-data", () => ({ demoAnswer, demoSearch })); vi.doMock("@/lib/rag", () => ({ answerQuestionWithScope, searchChunksWithTelemetry })); vi.doMock("@/lib/document-enrichment", () => ({ diff --git a/tests/source-governance.test.ts b/tests/source-governance.test.ts index 88d513edc2..45ca1624e7 100644 --- a/tests/source-governance.test.ts +++ b/tests/source-governance.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { groupSourceGovernanceWarnings, sourceGovernanceWarnings } from "../src/lib/source-governance"; +import { + groupSourceGovernanceWarnings, + hasDangerSourceGovernanceWarning, + sourceGovernanceWarnings, +} from "../src/lib/source-governance"; import type { SearchResult } from "../src/lib/types"; function result(overrides: Partial = {}): SearchResult { @@ -115,4 +119,11 @@ describe("source governance warnings", () => { titles: ["A", "B"], }); }); + + it("identifies danger-class warnings for server-side answer refusal", () => { + const warnings = sourceGovernanceWarnings({ results: [result()] }); + + expect(hasDangerSourceGovernanceWarning(warnings)).toBe(true); + expect(hasDangerSourceGovernanceWarning(warnings.filter((warning) => warning.severity !== "danger"))).toBe(false); + }); }); diff --git a/tests/supabase-schema.test.ts b/tests/supabase-schema.test.ts index d53490886e..7e266b4c23 100644 --- a/tests/supabase-schema.test.ts +++ b/tests/supabase-schema.test.ts @@ -82,6 +82,18 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("delete from public.document_sections where document_id = p_document_id;"); }); + it("supports service-role-only durable API rate limiting", () => { + expect(schema).toContain("create table if not exists public.api_rate_limits"); + expect(schema).toContain("primary key (owner_id, bucket)"); + expect(schema).toContain("create or replace function public.consume_api_rate_limit"); + expect(schema).toContain("returns table ( limited boolean, limit_value integer, remaining integer"); + expect(schema).toContain("grant select, insert, update, delete on table"); + expect(schema).toContain("public.api_rate_limits,"); + expect(schema).toContain("alter table public.api_rate_limits enable row level security"); + expect(schema).toContain('create policy "api rate limits service role all"'); + expect(schema).not.toMatch(/grant [^;]*public\.api_rate_limits[^;]* to authenticated;/); + }); + it("stores deep structured memory privately for source-backed answers", () => { expect(schema).toContain("create table if not exists public.document_sections"); expect(schema).toContain("create table if not exists public.document_memory_cards"); @@ -204,6 +216,10 @@ describe("Supabase schema Data API grants", () => { expect(schema).toContain("create table if not exists public.document_index_units"); expect(schema).toContain("'document_profile'"); expect(schema).toContain("'askable_question'"); + expect(schema).toContain("'threshold'"); + expect(schema).toContain("'workflow_step'"); + expect(schema).toContain("'medication_monitoring'"); + expect(schema).toContain("'alias'"); expect(schema).toContain("'vocabulary_term'"); expect(schema).toContain("source_span jsonb"); expect(schema).toContain("create index if not exists document_index_units_embedding_hnsw_idx"); diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index 08b26a058c..a19db5a6c4 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -53,90 +53,6 @@ async function mockPrivateUnauthenticatedApi(page: Page) { }); } -async function seedAuthenticatedSession(page: Page) { - await page.addInitScript(() => { - const expiresAt = Math.floor(Date.now() / 1000) + 60 * 60; - window.localStorage.setItem( - "sb-sjrfecxgysukkwxsowpy-auth-token", - JSON.stringify({ - access_token: "test-access-token", - refresh_token: "test-refresh-token", - token_type: "bearer", - expires_in: 3600, - expires_at: expiresAt, - user: { - id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", - aud: "authenticated", - role: "authenticated", - email: "test@example.com", - app_metadata: { provider: "email", providers: ["email"] }, - user_metadata: { email: "test@example.com" }, - created_at: new Date().toISOString(), - }, - }), - ); - }); -} - -async function mockPrivateAuthenticatedApi(page: Page) { - await page.route(/\/api\/setup-status$/, async (route) => { - await route.fulfill({ - json: { demoMode: false, checks: readySetupChecks }, - }); - }); - await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => { - await route.fulfill({ - json: { - documents: [], - pagination: { limit: 150, offset: 0, total: 0, nextOffset: 0, hasMore: false }, - }, - }); - }); - await page.route(/\/api\/ingestion\/jobs(?:\?.*)?$/, async (route) => { - await route.fulfill({ json: { jobs: [] } }); - }); - await page.route(/\/api\/ingestion\/batches(?:\?.*)?$/, async (route) => { - await route.fulfill({ - json: { - batches: [ - { - id: "batch-1", - owner_id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", - name: "Duplicate dry run", - source_root: "D:\\Clinical PDFs", - include_glob: "**/*.pdf", - status: "completed", - total_files: 3, - queued_files: 1, - skipped_files: 2, - failed_files: 0, - total_bytes: 1024, - metadata: {}, - completed_at: "2026-05-27T00:00:00.000Z", - created_at: "2026-05-27T00:00:00.000Z", - updated_at: "2026-05-27T00:00:00.000Z", - }, - ], - }, - }); - }); - await page.route(/\/api\/upload$/, async (route) => { - await route.fulfill({ - json: { - duplicate: true, - duplicateReason: "exact_content_hash", - document: { - id: "11111111-1111-4111-8111-111111111111", - title: "Existing guideline", - file_name: "guideline.pdf", - status: "indexed", - }, - message: 'Exact copy already exists as "Existing guideline"; no duplicate job was queued.', - }, - }); - }); -} - function answerStreamBody(payload: unknown) { return [ `event: progress\ndata: ${JSON.stringify({ stage: "retrieving", message: "Searching indexed documents." })}`, @@ -739,11 +655,11 @@ test.describe("Clinical KB UI smoke coverage", () => { await waitForDemoDashboardReady(page); await page - .getByRole("button", { - name: "Use sample question: What clozapine monitoring items are shown in the table image?", - }) - .click(); - await page.getByRole("button", { name: "Generate source-backed answer" }).click(); + .getByLabel("Search indexed guidelines by question or keyword") + .fill("What clozapine monitoring items are shown in the table image?"); + const submitAnswer = page.getByRole("button", { name: "Generate source-backed answer" }); + await expect(submitAnswer).toBeEnabled(); + await submitAnswer.click(); const clinicalTable = page.getByTestId("clinical-action-view").getByTestId("clinical-detail-table").first(); await expect(clinicalTable).toBeVisible(); @@ -975,15 +891,29 @@ test.describe("Clinical KB UI smoke coverage", () => { test("document viewer private missing source state is coherent", async ({ page }) => { await page.setViewportSize({ width: 390, height: 820 }); await mockPrivateUnauthenticatedApi(page); + await page.route(/\/api\/documents\/[^/]+(?:\?.*)?$/, async (route) => { + await route.fulfill({ + status: 404, + json: { error: "Document not found." }, + }); + }); + await page.route(/\/api\/documents\/[^/]+\/signed-url(?:\?.*)?$/, async (route) => { + await route.fulfill({ + status: 404, + json: { error: "Document not found." }, + }); + }); await gotoApp( page, "/documents/11111111-1111-4111-8111-111111111111?page=1&chunk=44444444-4444-4444-8444-444444444442", ); - await expect( - page.getByTestId("pdf-preview").getByText(/Sign in to open private source documents\.|Document not found\./), - ).toBeVisible({ timeout: 30000 }); - await expect(page.getByRole("heading", { level: 1, name: /Sign in required|Source unavailable/ })).toBeVisible(); + await expect(page.getByRole("heading", { level: 1, name: /Sign in required|Source unavailable/ })).toBeVisible({ + timeout: 30000, + }); + await expect(page.locator("body")).toContainText( + /Sign in to open private source documents\.|Document not found\.|Supabase browser authentication is not configured for private source documents\./, + ); await expect(page.getByRole("button", { name: "Summarise document" })).toBeDisabled(); await expect(page.locator("body")).not.toContainText("loading source"); await expect(page.locator("body")).not.toContainText("Loading source metadata"); @@ -1036,10 +966,9 @@ test.describe("Clinical KB UI smoke coverage", () => { await expectNoPageHorizontalOverflow(page); }); - test("duplicate upload warning and exact-copy batch count are visible", async ({ page }) => { + test("upload drawer disables uploads in demo mode", async ({ page }) => { await page.setViewportSize({ width: 414, height: 820 }); - await seedAuthenticatedSession(page); - await mockPrivateAuthenticatedApi(page); + await mockDemoApi(page); await gotoApp(page, "/"); await scrollDashboardToBottom(page); @@ -1050,19 +979,14 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(uploadDrawer).toBeVisible(); await uploadDrawer.getByRole("tab", { name: /Jobs/ }).click(); - await expect(uploadDrawer.getByText("2 exact copies skipped")).toBeVisible(); + await expect(uploadDrawer.getByText("Indexing progress")).toBeVisible(); await uploadDrawer.getByRole("tab", { name: /Upload/ }).click(); - await expect(uploadDrawer.getByRole("button", { name: "Queue document" })).toBeEnabled({ timeout: 30000 }); - await uploadDrawer.locator('input[name="file"]').setInputFiles({ - name: "guideline.pdf", - mimeType: "application/pdf", - buffer: Buffer.from("%PDF-1.7"), - }); - await uploadDrawer.getByRole("button", { name: "Queue document" }).click(); - await expect( - uploadDrawer.getByText('Exact copy already exists as "Existing guideline"; no duplicate job was queued.'), + uploadDrawer.getByText( + "Demo mode is read-only. Configure Supabase, OpenAI, and the local worker before uploading private guideline files.", + ), ).toBeVisible(); + await expect(uploadDrawer.locator('input[name="file"]')).toBeDisabled(); await expectNoPageHorizontalOverflow(page); }); diff --git a/worker/main.ts b/worker/main.ts index e59d349c9d..bb9e679db9 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { env } from "../src/lib/env"; @@ -62,6 +62,12 @@ const maxSupabaseBackoffMs = env.WORKER_HEALTH_BACKOFF_MS; const analyzeRagTablesThrottleMs = 45_000; let lastAnalyzeRagTablesAt = 0; +type OptionalIndexWriteIssue = { + stage: string; + message: string; + code?: string | null; +}; + function supabaseStageError( stage: string, error: { message?: string; code?: string; details?: string; hint?: string }, @@ -228,8 +234,25 @@ function workerBackoffMs(failures: number) { return Math.min(maxSupabaseBackoffMs, env.WORKER_POLL_MS * 2 ** Math.max(0, failures - 1)); } -function optionalIndexWriteWarning(stage: string, error: unknown) { - console.warn(`Optional ${stage} write failed`, safeErrorLogDetails(error)); +function optionalIndexWriteWarning(stage: string, error: unknown): OptionalIndexWriteIssue { + const details = safeErrorLogDetails(error); + console.warn(`Optional ${stage} write failed`, details); + return { + stage, + message: String(details.message ?? "Optional index write failed."), + code: typeof details.code === "string" ? details.code : null, + }; +} + +async function cleanupExtractedTemporaryPaths(extracted: ExtractedDocument | null) { + const temporaryPaths = Array.from(new Set(extracted?.temporaryPaths ?? [])); + for (const temporaryPath of temporaryPaths) { + try { + await rm(temporaryPath, { recursive: true, force: true }); + } catch (error) { + console.warn("Temporary extraction cleanup failed", safeErrorLogDetails(error)); + } + } } async function refreshRagTableStats() { @@ -802,6 +825,7 @@ function buildIndexQualityPayload(args: { sectionCount: number; memoryCardCount: number; documentEmbeddingFieldTypes?: string[]; + optionalIndexWriteIssues?: OptionalIndexWriteIssue[]; }) { const assessment = assessDocumentIndexQuality({ metrics: args.metrics, @@ -811,16 +835,21 @@ function buildIndexQualityPayload(args: { memoryCardCount: args.memoryCardCount, documentEmbeddingFieldTypes: args.documentEmbeddingFieldTypes, }); + const optionalIssues = args.optionalIndexWriteIssues ?? []; + const optionalIssueMessages = optionalIssues.map((issue) => `Optional ${issue.stage} write failed.`); + const extractionQuality = + optionalIssues.length > 0 && assessment.extractionQuality === "good" ? "partial" : assessment.extractionQuality; return { document_id: args.job.document_id, owner_id: args.job.documents.owner_id, quality_score: assessment.qualityScore, - extraction_quality: assessment.extractionQuality, - issues: assessment.issues, + extraction_quality: extractionQuality, + issues: [...assessment.issues, ...optionalIssueMessages], metrics: { ...args.metrics, ...assessment.metrics, + optional_index_write_issues: optionalIssues, search_eval_hit_rate: null, }, updated_at: new Date().toISOString(), @@ -879,6 +908,7 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { const imageResult = await uploadAndCaptionImages(job, extracted, pagesByNumber); const { insertedImages } = imageResult; const indexGenerationId = randomUUID(); + const optionalIndexWriteIssues: OptionalIndexWriteIssue[] = []; await updateJob(job.id, { stage: "chunking", progress: 72 }); const chunkMetadata = { @@ -953,7 +983,7 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { if (fieldsError) throw supabaseStageError("insert section-context embedding fields", fieldsError); } } catch (error) { - optionalIndexWriteWarning("section-context embedding field", error); + optionalIndexWriteIssues.push(optionalIndexWriteWarning("section-context embedding field", error)); } } } @@ -969,7 +999,10 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { })); if (tableFacts.length > 0) { const { error: factsError } = await supabase.from("document_table_facts").insert(tableFacts); - if (factsError) optionalIndexWriteWarning("table fact", supabaseStageError("insert table facts", factsError)); + if (factsError) + optionalIndexWriteIssues.push( + optionalIndexWriteWarning("table fact", supabaseStageError("insert table facts", factsError)), + ); } const additionalFieldInputs = buildAdditionalEmbeddingFieldInputs({ @@ -999,7 +1032,7 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { throw supabaseStageError("insert supplemental embedding fields", additionalFieldsError); } } catch (error) { - optionalIndexWriteWarning("supplemental embedding field", error); + optionalIndexWriteIssues.push(optionalIndexWriteWarning("supplemental embedding field", error)); } } @@ -1012,6 +1045,7 @@ async function insertEmbeddedChunks(job: JobRow, extracted: ExtractedDocument) { skippedImages: imageResult.skippedImages, imageSkipReasons: imageResult.skipReasons, imageTypeCounts: imageResult.imageTypeCounts, + optionalIndexWriteIssues, }; } @@ -1080,12 +1114,13 @@ async function processJob(job: JobRow) { }); await updateDocument(job.document_id, { status: "processing", error_message: null }); await updateBatch(job.batch_id); + let extracted: ExtractedDocument | null = null; try { await resetDocumentIndex(job.document_id); const buffer = await downloadDocument(job.documents.storage_path); await updateJobProgress(job.id, { stage: "extracting text/images", progress: 20 }); - const extracted = await extractDocument({ + extracted = await extractDocument({ buffer, fileName: job.documents.file_name, mimeType: job.documents.file_type, @@ -1102,6 +1137,7 @@ async function processJob(job: JobRow) { skippedImages, imageSkipReasons, imageTypeCounts, + optionalIndexWriteIssues, } = await insertEmbeddedChunks(job, extracted); const metrics = extractionMetrics(extracted, skippedImages, imageSkipReasons, imageTypeCounts); @@ -1112,6 +1148,7 @@ async function processJob(job: JobRow) { insertedImages, sectionCount: 0, memoryCardCount: 0, + optionalIndexWriteIssues, }); const { error: initialQualityError } = await supabase .from("document_index_quality") @@ -1143,6 +1180,7 @@ async function processJob(job: JobRow) { index_quality_score: initialQuality.quality_score, index_quality_issues: initialQuality.issues, index_quality_metrics: initialQuality.metrics, + optional_index_write_issues: optionalIndexWriteIssues, embedding_model: env.OPENAI_EMBEDDING_MODEL, ...metrics, }, @@ -1189,6 +1227,7 @@ async function processJob(job: JobRow) { sectionCount, memoryCardCount, documentEmbeddingFieldTypes, + optionalIndexWriteIssues, }); const { error: qualityError } = await supabase .from("document_index_quality") @@ -1224,6 +1263,7 @@ async function processJob(job: JobRow) { index_quality_score: finalQuality.quality_score, index_quality_issues: finalQuality.issues, index_quality_metrics: finalQuality.metrics, + optional_index_write_issues: optionalIndexWriteIssues, embedding_model: env.OPENAI_EMBEDDING_MODEL, ...metrics, }, @@ -1263,6 +1303,8 @@ async function processJob(job: JobRow) { errorMessage: message, }); } + } finally { + await cleanupExtractedTemporaryPaths(extracted); } }