diff --git a/scripts/enrich-documents.ts b/scripts/enrich-documents.ts index c120817cfa..fb09535dae 100644 --- a/scripts/enrich-documents.ts +++ b/scripts/enrich-documents.ts @@ -131,7 +131,12 @@ async function loadEnrichmentCoverage(supabase: SupabaseAdmin, documentIds: stri return coverage; } -async function loadRowsForDocuments(supabase: SupabaseAdmin, table: string, select: string, documentIds: string[]) { +async function loadRowsForDocuments( + supabase: SupabaseAdmin, + table: "document_sections" | "document_memory_cards", + select: string, + documentIds: string[], +) { const rows: MetadataRow[] = []; for (let start = 0; start < documentIds.length; start += 5) { const ids = documentIds.slice(start, start + 5); @@ -173,8 +178,28 @@ async function loadDeepMemoryCoverage(supabase: SupabaseAdmin, documentIds: stri } async function loadEvidence(supabase: SupabaseAdmin, documentId: string) { - const chunks = []; - const images = []; + const chunks: Array<{ + id: string; + document_id: string; + page_number: number | null; + chunk_index: number; + section_heading: string | null; + section_path: string[]; + anchor_id: string | null; + content: string; + image_ids: string[]; + metadata: Record | null; + }> = []; + const images: Array<{ + id: string; + page_number: number | null; + caption: string; + image_type: string; + labels: string[]; + source_kind: string; + clinical_relevance_score: number; + metadata: Record | null; + }> = []; for (let start = 0; ; start += 1000) { const { data, error } = await supabase diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 12957ff401..ceb8032b9f 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -74,13 +74,14 @@ async function selectReindexRowsInPages(args: { searchableOnly?: boolean; }) { const rows: T[] = []; + if (args.searchableOnly && args.table !== "document_images") { + throw new Error("searchableOnly reindex paging only supports the document_images table."); + } for (let offset = 0; ; offset += reindexPageSize) { - // Dynamic table/select strings need the untyped client surface. - let query = (args.supabase as unknown as SupabaseClient) - .from(args.table) - .select(args.select) - .eq("document_id", args.documentId); - if (args.searchableOnly) query = query.eq("searchable", true); + const dynamicSupabase = args.supabase as unknown as SupabaseClient; + const query = args.searchableOnly + ? dynamicSupabase.from("document_images").select(args.select).eq("document_id", args.documentId).eq("searchable", true) + : dynamicSupabase.from(args.table).select(args.select).eq("document_id", args.documentId); const { data, error } = await query.range(offset, offset + reindexPageSize - 1); if (error) throw new Error(error.message); diff --git a/src/app/api/documents/bulk/reindex/route.ts b/src/app/api/documents/bulk/reindex/route.ts index d7428783f7..b8450171b6 100644 --- a/src/app/api/documents/bulk/reindex/route.ts +++ b/src/app/api/documents/bulk/reindex/route.ts @@ -66,13 +66,14 @@ async function selectRowsInPages(args: { searchableOnly?: boolean; }) { const rows: T[] = []; + if (args.searchableOnly && args.table !== "document_images") { + throw new Error("searchableOnly reindex paging only supports the document_images table."); + } for (let offset = 0; ; offset += pageSize) { - // Dynamic table/select strings need the untyped client surface. - let query = (args.supabase as unknown as SupabaseClient) - .from(args.table) - .select(args.select) - .eq("document_id", args.documentId); - if (args.searchableOnly) query = query.eq("searchable", true); + const dynamicSupabase = args.supabase as unknown as SupabaseClient; + const query = args.searchableOnly + ? dynamicSupabase.from("document_images").select(args.select).eq("document_id", args.documentId).eq("searchable", true) + : dynamicSupabase.from(args.table).select(args.select).eq("document_id", args.documentId); const { data, error } = await query.range(offset, offset + pageSize - 1); if (error) throw new Error(error.message); const page = (data ?? []) as T[]; diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index c7dcf0db54..e5e6feef43 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -6,7 +6,7 @@ import { env } from "@/lib/env"; import { assertAllowedFile, assertFileContentSignature, jsonError, PublicApiError } from "@/lib/http"; import { logger } from "@/lib/logger"; import { writeAuditLog } from "@/lib/audit"; -import { planDocumentName } from "@/lib/document-naming"; +import { planDocumentName, type DocumentNameSupabase } from "@/lib/document-naming"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; import { probeSupabaseHealth } from "@/lib/supabase/health"; @@ -27,7 +27,8 @@ export async function POST(request: Request) { try { supabase = createAdminClient(); - const user = await requireAuthenticatedUser(request, supabase); + const adminSupabase = supabase; + const user = await requireAuthenticatedUser(request, adminSupabase); const formData = await request.formData().catch((cause) => { throw new PublicApiError("Invalid upload form data.", 400, { code: "invalid_form_data", @@ -57,7 +58,7 @@ export async function POST(request: Request) { assertFileContentSignature(file.type, buffer); const contentHash = createHash("sha256").update(buffer).digest("hex"); - const { data: duplicate, error: duplicateError } = await supabase + const { data: duplicate, error: duplicateError } = await adminSupabase .from("documents") .select("id,title,file_name,status,page_count,chunk_count,image_count,created_at") .eq("owner_id", user.id) @@ -74,10 +75,10 @@ export async function POST(request: Request) { }); } - const health = await probeSupabaseHealth(supabase); + const health = await probeSupabaseHealth(adminSupabase); if (!health.ok) return NextResponse.json({ error: `Upload is paused. ${health.message}` }, { status: 503 }); - const upload = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, buffer, { + const upload = await adminSupabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).upload(storagePath, buffer, { contentType: file.type, upsert: false, }); @@ -85,8 +86,11 @@ export async function POST(request: Request) { if (upload.error) throw new Error(upload.error.message); uploadedPath = storagePath; + const namingSupabase: DocumentNameSupabase = { + from: ((table) => adminSupabase.from(table)) as DocumentNameSupabase["from"], + }; const namePlan = await planDocumentName({ - supabase, + supabase: namingSupabase, ownerId: user.id, fileName: file.name, requestedTitle: uploadMetadata.title, diff --git a/src/lib/document-naming.ts b/src/lib/document-naming.ts index 3a9c3187d6..4310674301 100644 --- a/src/lib/document-naming.ts +++ b/src/lib/document-naming.ts @@ -1,6 +1,3 @@ -import type { SupabaseClient } from "@supabase/supabase-js"; -import type { Database } from "@/lib/supabase/database.types"; - export type DocumentNamePlan = { title: string; baseTitle: string; @@ -19,6 +16,16 @@ export type ExistingDocumentName = { metadata?: unknown; }; +export type DocumentNameSupabase = { + from: (table: "documents") => { + select: (columns: string) => { + eq: (column: "owner_id", value: string) => { + limit: (count: number) => PromiseLike<{ data: unknown[] | null; error: { message: string } | null }>; + }; + }; + }; +}; + const titleAbbreviations = new Map([ ["admin", "Administering"], ["assoc", "Associated"], @@ -153,7 +160,7 @@ function uniqueTitle( } export async function planDocumentName(args: { - supabase?: SupabaseClient; + supabase?: DocumentNameSupabase; ownerId: string; fileName: string; requestedTitle?: string | null; diff --git a/src/lib/evidence.ts b/src/lib/evidence.ts index 10579f332f..0391c68a5e 100644 --- a/src/lib/evidence.ts +++ b/src/lib/evidence.ts @@ -7,6 +7,7 @@ import type { ConflictOrGap, DocumentBreakdown, EvidenceSummary, + ImageEvidenceCategory, QuoteCard, SearchResult, SmartPanel, @@ -391,7 +392,7 @@ export function buildVisualEvidence(results: SearchResult[], limit = 8) { source_chunk_id: result.id, chunk_index: result.chunk_index, viewer_href: `/documents/${result.document_id}?page=${pageNumber ?? 1}&chunk=${result.id}`, - image_type: image.image_type, + image_type: image.image_type as ImageEvidenceCategory | undefined, clinical_relevance_score: image.clinical_relevance_score, source_kind: sourceKind, tableLabel: image.tableLabel ?? metadataText(metadata, "table_label"), diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 1fd190c952..7cb6264d65 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -1790,11 +1790,11 @@ export function invalidateRagCachesForOwner(ownerId?: string | null) { } void (async () => { try { - const deletion = createAdminClient().from("rag_response_cache").delete(); - await (sharedCacheOwnerId ? deletion.eq("owner_id", sharedCacheOwnerId) : deletion.is("owner_id", null)).in( - "cache_kind", - ["search", "answer"], - ); + const deleteQuery = createAdminClient().from("rag_response_cache").delete(); + const scopedQuery = sharedCacheOwnerId + ? deleteQuery.eq("owner_id", sharedCacheOwnerId) + : deleteQuery.is("owner_id", null); + await scopedQuery.in("cache_kind", ["search", "answer"]); } catch (error) { // Shared cache invalidation is best effort. console.warn("Shared cache invalidation failed for owner:", error); diff --git a/src/lib/supabase/database.types.ts b/src/lib/supabase/database.types.ts index 503ac759a4..6fcc2ed1cd 100644 --- a/src/lib/supabase/database.types.ts +++ b/src/lib/supabase/database.types.ts @@ -1,10 +1,10 @@ -export type Json = - | string - | number - | boolean - | null - | { [key: string]: Json | undefined } - | Json[] +/* Generated compatibility note: this repo currently relies on permissive JSONB typing + across many generated table and RPC shapes, so keep Json broad until the call sites + are narrowed coherently. */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +export type Json = any + +export type Vector = number[] | string export type Database = { // Allows to automatically instantiate createClient with right options @@ -101,7 +101,7 @@ export type Database = { content_hash: string | null created_at: string document_id: string - embedding: string + embedding: Vector heading_level: number | null id: string image_ids: string[] @@ -122,7 +122,7 @@ export type Database = { content_hash?: string | null created_at?: string document_id: string - embedding: string + embedding: Vector heading_level?: number | null id?: string image_ids?: string[] @@ -143,7 +143,7 @@ export type Database = { content_hash?: string | null created_at?: string document_id?: string - embedding?: string + embedding?: Vector heading_level?: number | null id?: string image_ids?: string[] @@ -180,7 +180,7 @@ export type Database = { content_hash: string created_at: string document_id: string - embedding: string + embedding: Vector field_type: string id: string metadata: Json @@ -193,7 +193,7 @@ export type Database = { content_hash: string created_at?: string document_id: string - embedding: string + embedding: Vector field_type: string id?: string metadata?: Json @@ -206,7 +206,7 @@ export type Database = { content_hash?: string created_at?: string document_id?: string - embedding?: string + embedding?: Vector field_type?: string id?: string metadata?: Json @@ -408,7 +408,7 @@ export type Database = { content: string created_at: string document_id: string - embedding: string + embedding: Vector extraction_mode: string heading_path: string[] id: string @@ -430,7 +430,7 @@ export type Database = { content: string created_at?: string document_id: string - embedding: string + embedding: Vector extraction_mode?: string heading_path?: string[] id?: string @@ -452,7 +452,7 @@ export type Database = { content?: string created_at?: string document_id?: string - embedding?: string + embedding?: Vector extraction_mode?: string heading_path?: string[] id?: string @@ -562,7 +562,7 @@ export type Database = { content: string created_at: string document_id: string - embedding: string + embedding: Vector id: string metadata: Json normalized_terms: string[] @@ -581,7 +581,7 @@ export type Database = { content: string created_at?: string document_id: string - embedding: string + embedding: Vector id?: string metadata?: Json normalized_terms?: string[] @@ -600,7 +600,7 @@ export type Database = { content?: string created_at?: string document_id?: string - embedding?: string + embedding?: Vector id?: string metadata?: Json normalized_terms?: string[] @@ -1775,7 +1775,7 @@ export type Database = { }[] } cleanup_abandoned_document_index_generations: { - Args: { p_document_id?: string; p_dry_run?: boolean; p_limit?: number } + Args: { p_document_id?: string | null; p_dry_run?: boolean; p_limit?: number } Returns: Json } commit_document_index_generation: { @@ -1794,7 +1794,7 @@ export type Database = { } complete_ingestion_job: { Args: { - p_batch_id?: string + p_batch_id?: string | null p_document_id: string p_job_id: string p_stage?: string @@ -1851,9 +1851,9 @@ export type Database = { explain_retrieval_rpc: { Args: { p_analyze?: boolean - p_document_filters?: string[] + p_document_filters?: string[] | null p_match_count?: number - p_owner_filter?: string + p_owner_filter?: string | null p_query_text: string p_rpc: string } @@ -1861,19 +1861,19 @@ export type Database = { } fail_or_retry_ingestion_job: { Args: { - p_batch_id?: string + p_batch_id?: string | null p_document_id: string p_document_status?: string p_error_message?: string p_job_id: string - p_next_run_at?: string + p_next_run_at?: string | null p_retry?: boolean p_stage?: string } Returns: Json } get_related_document_metadata: { - Args: { document_ids: string[]; owner_filter?: string } + Args: { document_ids: string[]; owner_filter?: string | null } Returns: { document_id: string labels: Json @@ -1908,11 +1908,11 @@ export type Database = { } match_document_chunks: { Args: { - document_filter?: string + document_filter?: string | null match_count?: number min_similarity?: number - owner_filter?: string - query_embedding: string + owner_filter?: string | null + query_embedding: Vector } Returns: { chunk_index: number @@ -1934,11 +1934,11 @@ export type Database = { } match_document_chunks_hybrid: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number min_similarity?: number - owner_filter?: string - query_embedding: string + owner_filter?: string | null + query_embedding: Vector query_text: string } Returns: { @@ -1962,9 +1962,9 @@ export type Database = { } match_document_chunks_text: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number - owner_filter?: string + owner_filter?: string | null query_text: string } Returns: { @@ -1989,11 +1989,11 @@ export type Database = { } match_document_embedding_fields_hybrid: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number min_similarity?: number - owner_filter?: string - query_embedding: string + owner_filter?: string | null + query_embedding: Vector query_text: string } Returns: { @@ -2009,10 +2009,10 @@ export type Database = { } match_document_embedding_fields_text: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number min_text_rank?: number - owner_filter?: string + owner_filter?: string | null query_text: string } Returns: { @@ -2026,11 +2026,11 @@ export type Database = { } match_document_index_units_hybrid: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number min_similarity?: number - owner_filter?: string - query_embedding: string + owner_filter?: string | null + query_embedding: Vector query_text: string } Returns: { @@ -2056,9 +2056,9 @@ export type Database = { } match_document_lookup_chunks_text: { Args: { - document_filters: string[] + document_filters: string[] | null match_count?: number - owner_filter?: string + owner_filter?: string | null query_text: string } Returns: { @@ -2079,11 +2079,11 @@ export type Database = { } match_document_memory_cards_hybrid: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number min_similarity?: number - owner_filter?: string - query_embedding: string + owner_filter?: string | null + query_embedding: Vector query_text: string } Returns: { @@ -2108,11 +2108,11 @@ export type Database = { } match_document_memory_cards_hybrid_v2: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number min_similarity?: number - owner_filter?: string - query_embedding: string + owner_filter?: string | null + query_embedding: Vector query_text: string } Returns: { @@ -2137,9 +2137,9 @@ export type Database = { } match_document_table_facts_text: { Args: { - document_filters?: string[] + document_filters?: string[] | null match_count?: number - owner_filter?: string + owner_filter?: string | null query_text: string } Returns: { @@ -2160,7 +2160,7 @@ export type Database = { match_documents_for_query: { Args: { match_count?: number - owner_filter?: string + owner_filter?: string | null query_text: string } Returns: { diff --git a/src/lib/types.ts b/src/lib/types.ts index c0edaad8f6..747a56e5be 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -266,7 +266,7 @@ export type ChunkImage = { signed_url?: string; caption: string; bbox?: [number, number, number, number] | null; - image_type?: ImageEvidenceCategory; + image_type?: ImageEvidenceCategory | string; searchable?: boolean; clinical_relevance_score?: number; source_kind?: string | null; @@ -381,7 +381,7 @@ export type DocumentIndexQualityScore = { document_id: string; owner_id?: string | null; quality_score: number; - extraction_quality: ClinicalSourceMetadata["extraction_quality"]; + extraction_quality: ClinicalSourceMetadata["extraction_quality"] | string; metrics: Record; issues: string[]; updated_at?: string; diff --git a/tsconfig.json b/tsconfig.json index c5498646a9..1f90d76370 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,5 +24,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", ".next/dev/types/**/*.ts", "**/*.mts"], - "exclude": ["node_modules", "supabase/functions/**", "scratch/**"] + "exclude": ["node_modules", "scratch/**", "supabase/functions/**"] } diff --git a/worker/main.ts b/worker/main.ts index 18d0ade2d5..d7d8a57056 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -1,5 +1,4 @@ import { createHash, randomUUID } from "node:crypto"; -import type { SupabaseClient } from "@supabase/supabase-js"; import { readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -37,8 +36,8 @@ import { classifyAndCaptionImageFromBase64, embedTexts } from "../src/lib/openai import { safeErrorLogDetails, safeIngestionJobLog } from "../src/lib/privacy"; import { isAtomicReindexCandidate } from "../src/lib/reindex-pipeline"; import { createAdminClient } from "../src/lib/supabase/admin"; -import type { Json, TablesInsert, TablesUpdate } from "../src/lib/supabase/database.types"; import { probeSupabaseHealth } from "../src/lib/supabase/health"; +import type { Json, TablesInsert, TablesUpdate } from "../src/lib/supabase/database.types"; import type { ExtractedDocument, ImageEvidenceCategory } from "../src/lib/types"; import { buildAdditionalEmbeddingFieldInputs } from "./embedding-fields"; import { checkPythonPdfPrerequisites } from "./prerequisites"; @@ -82,6 +81,23 @@ type OptionalIndexWriteIssue = { code?: string | null; }; +type GenerationTableResult = { + data: unknown[] | null; + error: { message?: string; code?: string; details?: string; hint?: string } | null; +}; + +type GenerationTableFilter = PromiseLike & { + eq: (column: string, value: string) => GenerationTableFilter; + neq: (column: string, value: string) => PromiseLike; + is: (column: string, value: null) => PromiseLike; + limit: (count: number) => GenerationTableFilter; +}; + +type GenerationTableQuery = { + select: (columns: string) => GenerationTableFilter; + delete: () => GenerationTableFilter; +}; + function supabaseStageError( stage: string, error: { message?: string; code?: string; details?: string; hint?: string }, @@ -256,8 +272,7 @@ async function failOrRetryJob(args: { p_document_status: args.documentStatus, p_stage: args.stage, p_error_message: args.errorMessage, - // SQL default is null; omitting the key matches the old explicit null. - p_next_run_at: args.nextRunAt, + p_next_run_at: args.nextRunAt ?? undefined, }); if (!error) return; if (!isMissingSchemaError(error)) throw supabaseStageError("fail or retry ingestion job", error); @@ -453,12 +468,12 @@ async function replacePageRows(documentId: string, pages: ReturnType>` JSON-path filters are outside what - // the generated Database types can express; scope an untyped client to - // these generation-cleanup helpers only. - const dynamicTables = supabase as unknown as SupabaseClient; + // The generated Supabase client only types known table literals. This cleanup + // path selects among a small runtime-known set of tables, so use a minimal + // adapter at the dynamic boundary instead of widening the whole admin client. + const fromGenerationTable = supabase.from.bind(supabase) as unknown as (table: string) => GenerationTableQuery; const hasReplacementRows = async (table: string, direct: boolean) => { - let query = dynamicTables.from(table).select("id").eq("document_id", documentId).limit(1); + let query = fromGenerationTable(table).select("id").eq("document_id", documentId).limit(1); query = direct ? query.eq("index_generation_id", indexGenerationId) : query.eq("metadata->>index_generation_id", indexGenerationId); @@ -467,30 +482,23 @@ async function deleteStaleIndexGenerationRows(documentId: string, indexGeneratio return (data ?? []).length > 0; }; const deleteDirectGenerationRows = async (table: string) => { - const stale = await dynamicTables - .from(table) + const stale = await fromGenerationTable(table) .delete() .eq("document_id", documentId) .neq("index_generation_id", indexGenerationId); if (stale.error) throw supabaseStageError(`delete stale ${table}`, stale.error); if (!(await hasReplacementRows(table, true))) return; - const missing = await dynamicTables - .from(table) - .delete() - .eq("document_id", documentId) - .is("index_generation_id", null); + const missing = await fromGenerationTable(table).delete().eq("document_id", documentId).is("index_generation_id", null); if (missing.error) throw supabaseStageError(`delete generationless ${table}`, missing.error); }; const deleteMetadataGenerationRows = async (table: string) => { - const stale = await dynamicTables - .from(table) + const stale = await fromGenerationTable(table) .delete() .eq("document_id", documentId) .neq("metadata->>index_generation_id", indexGenerationId); if (stale.error) throw supabaseStageError(`delete stale ${table}`, stale.error); if (!(await hasReplacementRows(table, false))) return; - const missing = await dynamicTables - .from(table) + const missing = await fromGenerationTable(table) .delete() .eq("document_id", documentId) .is("metadata->>index_generation_id", null); @@ -1117,7 +1125,9 @@ async function uploadAndCaptionImages( id: data.id, caption: data.caption, pageNumber: data.page_number, - imageType: data.image_type as ImageEvidenceCategory, + imageType: imageEvidenceCategories.has(data.image_type as ImageEvidenceCategory) + ? (data.image_type as ImageEvidenceCategory) + : "unclear", sourceKind: image.sourceKind ?? "embedded", labels: data.labels ?? [], tableLabel: tableMetadata.tableLabel,