diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 60e084fee0..bd0a57a08a 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -740,8 +740,13 @@ export function classifyQueryIntent(query: string): IntentSignals { }; } +const clinicalQueryAnalysisCache = new Map(); +const clinicalQueryAnalysisCacheLimit = 32; + export function analyzeClinicalQuery(query: string): ClinicalQueryAnalysis { const originalQuery = query.trim(); + const cached = clinicalQueryAnalysisCache.get(originalQuery); + if (cached) return structuredClone(cached); const normalizedQuery = normalizeAnalysisText(originalQuery); const corrected = correctedTokens(originalQuery); const corrections = tokens(originalQuery) @@ -810,7 +815,7 @@ export function analyzeClinicalQuery(query: string): ClinicalQueryAnalysis { vocabularyTerms, }); - const analysis = { + const analysis: ClinicalQueryAnalysis = { originalQuery, normalizedQuery, queryClass, @@ -836,7 +841,12 @@ export function analyzeClinicalQuery(query: string): ClinicalQueryAnalysis { needsClassifierFallback: confidence < 0.58 && queryClass === "unsupported_or_general", }; - return { ...analysis }; + clinicalQueryAnalysisCache.set(originalQuery, analysis); + if (clinicalQueryAnalysisCache.size > clinicalQueryAnalysisCacheLimit) { + const oldestKey = clinicalQueryAnalysisCache.keys().next().value; + if (oldestKey !== undefined) clinicalQueryAnalysisCache.delete(oldestKey); + } + return structuredClone(analysis); } export function classifyRagQuery(query: string): RagQueryClassification { diff --git a/src/lib/reindex-pipeline.ts b/src/lib/reindex-pipeline.ts index 0468c0eedd..f533434530 100644 --- a/src/lib/reindex-pipeline.ts +++ b/src/lib/reindex-pipeline.ts @@ -37,7 +37,10 @@ export function isAtomicReindexCandidate(document: { status?: string | null; met return document.status === "indexed"; } -export function isCommittedGenerationMetadata(args: { rowMetadata?: unknown; committedGeneration?: string | null }) { +export function isCommittedGenerationMetadata(args: { + rowMetadata?: unknown; + committedGeneration?: string | null; +}) { const rowGeneration = committedIndexGeneration(args.rowMetadata); if (!rowGeneration) return true; return Boolean(args.committedGeneration) && rowGeneration === args.committedGeneration; diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 7e0283187b..baf7528257 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -179,6 +179,27 @@ describe("image-aware chunks", () => { }); }); +describe("buildChunks dedupe", () => { + it("dedupes same-page chunks despite punctuation and table-label noise", () => { + const chunks = buildChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: "Table: Lithium monitoring", + metadata: { content_hash: "abc", embedding_model: "text-embedding-3-small" }, + }, + { + documentId: "doc-1", + pageNumber: 1, + pageText: "Lithium-monitoring", + metadata: { content_hash: "abc", embedding_model: "text-embedding-3-small" }, + }, + ]); + + expect(chunks.map((chunk) => chunk.content)).toEqual(["Table: Lithium monitoring"]); + }); +}); + describe("section-aware chunking groundwork", () => { it("carries the previous section path onto a following page without a new heading", () => { const chunks = buildChunks([ diff --git a/tests/worker-visual-capture.test.ts b/tests/worker-visual-capture.test.ts index 3b88e56c79..b59d696281 100644 --- a/tests/worker-visual-capture.test.ts +++ b/tests/worker-visual-capture.test.ts @@ -41,7 +41,7 @@ describe("worker visual capture hardening", () => { expect(workerSource).toContain("await deleteStaleIndexGenerationRows(args.documentId, args.indexGenerationId)"); expect(workerSource).toContain("async function deleteStaleIndexGenerationRows"); expect(workerSource).toContain("`${imagePrefix}/${indexGenerationId}/image-${index + 1}${ext}`"); - expect(workerSource).toContain('indexing_v3_agent_repair_reason: "core_index_committed"'); + expect(workerSource).toContain("indexing_v3_agent_repair_reason: null"); }); it("uses the strict completion RPC when inline enrichment succeeds", () => { diff --git a/worker/main.ts b/worker/main.ts index 20c494efba..c696ec95fc 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -816,17 +816,27 @@ async function uploadAndCaptionImages( env.WORKER_MAX_CAPTIONED_IMAGES_PER_PAGE, ); + // Keep selection, de-dupe, and budget checks sequential so the chosen images + // are deterministic; only the expensive cache/model calls run concurrently. + type CaptionTask = { + candidate: (typeof scoredCandidates)[number]; + index: number; + image: ExtractedDocument["images"][number]; + preparedImage: (typeof preparedImages)[number]; + perceptualHash: string; + imageHash: string; + nearbyText: string | undefined; + tableMetadata: ReturnType; + contextHash: string; + presetClassification: ImageClassification | null; + }; + + const captionTasks: CaptionTask[] = []; for (const candidate of scoredCandidates) { const index = candidate.originalIndex; const image = extracted.images[index]; - await updateJobProgress(job.id, { - stage: `captioning image ${index + 1}/${extracted.images.length}`, - progress: Math.min(70, 35 + Math.round((index / Math.max(extracted.images.length, 1)) * 25)), - }); - const preparedImage = preparedImages[index]; const imageHash = preparedImage.imageHash; - const perceptualHash = preparedImage.perceptualHash; const skipReason = cheapImageSkipReason({ bytesLength: preparedImage.bytesLength, imageHash, @@ -857,41 +867,77 @@ async function uploadAndCaptionImages( noteSkippedImage(skipReasons, lowSignalSkipReason); continue; } - let classification: ImageClassification | null = + const presetClassification: ImageClassification | null = image.sourceKind === "table_crop" ? nonClinicalTableClassification({ tableMetadata, sourceKind: image.sourceKind }) : null; - let classificationCacheHit = false; - const usesModelCaptionBudget = !classification; - if (usesModelCaptionBudget && !selectedCaptionCandidateIndexes.has(index)) { + if (!presetClassification && !selectedCaptionCandidateIndexes.has(index)) { skippedImages += 1; noteSkippedImage(skipReasons, "visual intelligence candidate below caption budget"); continue; } - if (!classification) { - classification = await getCachedImageClassification(job.documents.owner_id, imageHash, contextHash); - classificationCacheHit = Boolean(classification); - } - if (!classification) { - classification = await classifyAndCaptionImageFromBase64({ - base64: preparedImage.bytes.toString("base64"), - mimeType: image.mimeType, - nearbyText, - sourceKind: image.sourceKind ?? null, - candidateType: tableMetadata.candidateType, - tableLabel: tableMetadata.tableLabel, - tableTitle: tableMetadata.tableTitle, - tableRole: tableMetadata.tableRole, - tableText: tableMetadata.tableText, - }); - await setCachedImageClassification({ - ownerId: job.documents.owner_id, - imageHash, - contextHash, - mimeType: image.mimeType, - classification, - }); - } + captionTasks.push({ + candidate, + index, + image, + preparedImage, + perceptualHash: preparedImage.perceptualHash, + imageHash, + nearbyText, + tableMetadata, + contextHash, + presetClassification, + }); + } + + const captionConcurrency = 4; + const resolvedTasks: Array<{ task: CaptionTask; classification: ImageClassification; classificationCacheHit: boolean }> = + []; + for (let start = 0; start < captionTasks.length; start += captionConcurrency) { + const batch = captionTasks.slice(start, start + captionConcurrency); + await updateJobProgress(job.id, { + stage: `captioning images ${start + 1}-${start + batch.length}/${captionTasks.length}`, + progress: Math.min(70, 35 + Math.round(((start + batch.length) / Math.max(captionTasks.length, 1)) * 25)), + }); + const batchResults = await Promise.all( + batch.map(async (task) => { + let classification: ImageClassification | null = task.presetClassification; + let classificationCacheHit = false; + if (!classification) { + classification = await getCachedImageClassification(job.documents.owner_id, task.imageHash, task.contextHash); + classificationCacheHit = Boolean(classification); + } + if (!classification) { + classification = await classifyAndCaptionImageFromBase64({ + base64: task.preparedImage.bytes.toString("base64"), + mimeType: task.image.mimeType, + nearbyText: task.nearbyText, + sourceKind: task.image.sourceKind ?? null, + candidateType: task.tableMetadata.candidateType, + tableLabel: task.tableMetadata.tableLabel, + tableTitle: task.tableMetadata.tableTitle, + tableRole: task.tableMetadata.tableRole, + tableText: task.tableMetadata.tableText, + }); + await setCachedImageClassification({ + ownerId: job.documents.owner_id, + imageHash: task.imageHash, + contextHash: task.contextHash, + mimeType: task.image.mimeType, + classification, + }); + } + return { task, classification, classificationCacheHit }; + }), + ); + resolvedTasks.push(...batchResults); + } + + for (const resolved of resolvedTasks) { + const { task, classificationCacheHit } = resolved; + const { candidate, index, image, preparedImage, perceptualHash, imageHash, nearbyText, tableMetadata, contextHash } = + task; + let classification = resolved.classification; const policyAssessment = assessClinicalImageUse({ imageType: classification.image_type, searchable: classification.searchable, @@ -1497,10 +1543,6 @@ async function processJob(job: JobRow) { index_quality_metrics: initialQuality.metrics, optional_index_write_issues: optionalIndexWriteIssues, embedding_model: env.OPENAI_EMBEDDING_MODEL, - indexing_v3_agent_status: "pending", - indexing_v3_agent_last_error: coreAgentMessage, - indexing_v3_agent_repair_reason: "core_index_committed", - indexing_v3_agent_updated_at: indexedAt, ...metrics, }; await commitDocumentIndexGeneration({