Extract consolidation fixes

src/lib/clinical-search.ts

@@ -740,8 +740,13 @@ export function classifyQueryIntent(query: string): IntentSignals {
  };
 }
 
+const clinicalQueryAnalysisCache = new Map<string, ClinicalQueryAnalysis>();
+const clinicalQueryAnalysisCacheLimit = 32;
+
 export function analyzeClinicalQuery(query: string): ClinicalQueryAnalysis {
  const originalQuery = query.trim();
+ const cached = clinicalQueryAnalysisCache.get(originalQuery);
+ if (cached) return structuredClone(cached);
  const normalizedQuery = normalizeAnalysisText(originalQuery);
  const corrected = correctedTokens(originalQuery);
  const corrections = tokens(originalQuery)
@@ -810,7 +815,7 @@ export function analyzeClinicalQuery(query: string): ClinicalQueryAnalysis {
  vocabularyTerms,
  });
 
- const analysis = {
+ const analysis: ClinicalQueryAnalysis = {
  originalQuery,
  normalizedQuery,
  queryClass,
@@ -836,7 +841,12 @@ export function analyzeClinicalQuery(query: string): ClinicalQueryAnalysis {
  needsClassifierFallback: confidence < 0.58 && queryClass === "unsupported_or_general",
  };
 
- return { ...analysis };
+ clinicalQueryAnalysisCache.set(originalQuery, analysis);
+ if (clinicalQueryAnalysisCache.size > clinicalQueryAnalysisCacheLimit) {
+ const oldestKey = clinicalQueryAnalysisCache.keys().next().value;
+ if (oldestKey !== undefined) clinicalQueryAnalysisCache.delete(oldestKey);
+ }
+ return structuredClone(analysis);
 }
 
 export function classifyRagQuery(query: string): RagQueryClassification {

src/lib/reindex-pipeline.ts

@@ -37,7 +37,10 @@ export function isAtomicReindexCandidate(document: { status?: string | null; met
  return document.status === "indexed";
 }
 
-export function isCommittedGenerationMetadata(args: { rowMetadata?: unknown; committedGeneration?: string | null }) {
+export function isCommittedGenerationMetadata(args: {
+ rowMetadata?: unknown;
+ committedGeneration?: string | null;
+}) {
  const rowGeneration = committedIndexGeneration(args.rowMetadata);
  if (!rowGeneration) return true;
  return Boolean(args.committedGeneration) && rowGeneration === args.committedGeneration;

tests/chunking.test.ts

@@ -179,6 +179,27 @@ describe("image-aware chunks", () => {
  });
 });
 
+describe("buildChunks dedupe", () => {
+ it("dedupes same-page chunks despite punctuation and table-label noise", () => {
+ const chunks = buildChunks([
+ {
+ documentId: "doc-1",
+ pageNumber: 1,
+ pageText: "Table: Lithium monitoring",
+ metadata: { content_hash: "abc", embedding_model: "text-embedding-3-small" },
+ },
+ {
+ documentId: "doc-1",
+ pageNumber: 1,
+ pageText: "Lithium-monitoring",
+ metadata: { content_hash: "abc", embedding_model: "text-embedding-3-small" },
+ },
+ ]);
+
+ expect(chunks.map((chunk) => chunk.content)).toEqual(["Table: Lithium monitoring"]);
+ });
+});
+
 describe("section-aware chunking groundwork", () => {
  it("carries the previous section path onto a following page without a new heading", () => {
  const chunks = buildChunks([

tests/worker-visual-capture.test.ts

@@ -41,7 +41,7 @@ describe("worker visual capture hardening", () => {
  expect(workerSource).toContain("await deleteStaleIndexGenerationRows(args.documentId, args.indexGenerationId)");
  expect(workerSource).toContain("async function deleteStaleIndexGenerationRows");
  expect(workerSource).toContain("`${imagePrefix}/${indexGenerationId}/image-${index + 1}${ext}`");
- expect(workerSource).toContain('indexing_v3_agent_repair_reason: "core_index_committed"');
+ expect(workerSource).toContain("indexing_v3_agent_repair_reason: null");
  });
 
  it("uses the strict completion RPC when inline enrichment succeeds", () => {

worker/main.ts

@@ -816,17 +816,27 @@
  env.WORKER_MAX_CAPTIONED_IMAGES_PER_PAGE,
  );
 
+ // Keep selection, de-dupe, and budget checks sequential so the chosen images
+ // are deterministic; only the expensive cache/model calls run concurrently.
+ type CaptionTask = {
+ candidate: (typeof scoredCandidates)[number];
+ index: number;
+ image: ExtractedDocument["images"][number];
+ preparedImage: (typeof preparedImages)[number];
+ perceptualHash: string;
+ imageHash: string;
+ nearbyText: string | undefined;
+ tableMetadata: ReturnType<typeof imageTableMetadata>;
+ contextHash: string;
+ presetClassification: ImageClassification | null;
+ };
+
+ const captionTasks: CaptionTask[] = [];
  for (const candidate of scoredCandidates) {
  const index = candidate.originalIndex;
  const image = extracted.images[index];
- await updateJobProgress(job.id, {
- stage: `captioning image ${index + 1}/${extracted.images.length}`,
- progress: Math.min(70, 35 + Math.round((index / Math.max(extracted.images.length, 1)) * 25)),
- });
-
  const preparedImage = preparedImages[index];
  const imageHash = preparedImage.imageHash;
- const perceptualHash = preparedImage.perceptualHash;
  const skipReason = cheapImageSkipReason({
  bytesLength: preparedImage.bytesLength,
  imageHash,
@@ -857,41 +867,77 @@
  noteSkippedImage(skipReasons, lowSignalSkipReason);
  continue;
  }
-let classification: ImageClassification | null =
+const presetClassification: ImageClassification | null =
  image.sourceKind === "table_crop"
  ? nonClinicalTableClassification({ tableMetadata, sourceKind: image.sourceKind })
  : null;
- let classificationCacheHit = false;
- const usesModelCaptionBudget = !classification;
- if (usesModelCaptionBudget && !selectedCaptionCandidateIndexes.has(index)) {
+ if (!presetClassification && !selectedCaptionCandidateIndexes.has(index)) {
  skippedImages += 1;
  noteSkippedImage(skipReasons, "visual intelligence candidate below caption budget");
  continue;
  }
- if (!classification) {
- classification = await getCachedImageClassification(job.documents.owner_id, imageHash, contextHash);
- classificationCacheHit = Boolean(classification);
- }
- if (!classification) {
- classification = await classifyAndCaptionImageFromBase64({
- base64: preparedImage.bytes.toString("base64"),
- mimeType: image.mimeType,
- nearbyText,
- sourceKind: image.sourceKind ?? null,
- candidateType: tableMetadata.candidateType,
- tableLabel: tableMetadata.tableLabel,
- tableTitle: tableMetadata.tableTitle,
- tableRole: tableMetadata.tableRole,
- tableText: tableMetadata.tableText,
- });
- await setCachedImageClassification({
- ownerId: job.documents.owner_id,
- imageHash,
- contextHash,
- mimeType: image.mimeType,
- classification,
- });
- }
+ captionTasks.push({
+ candidate,
+ index,
+ image,
+ preparedImage,
+ perceptualHash: preparedImage.perceptualHash,
+ imageHash,
+ nearbyText,
+ tableMetadata,
+ contextHash,
+ presetClassification,
+ });
+ }
+
+ const captionConcurrency = 4;
+ const resolvedTasks: Array<{ task: CaptionTask; classification: ImageClassification; classificationCacheHit: boolean }> =
+ [];
+ for (let start = 0; start < captionTasks.length; start += captionConcurrency) {
+ const batch = captionTasks.slice(start, start + captionConcurrency);
+ await updateJobProgress(job.id, {
+ stage: `captioning images ${start + 1}-${start + batch.length}/${captionTasks.length}`,
+ progress: Math.min(70, 35 + Math.round(((start + batch.length) / Math.max(captionTasks.length, 1)) * 25)),
+ });
+ const batchResults = await Promise.all(
+ batch.map(async (task) => {
+ let classification: ImageClassification | null = task.presetClassification;
+ let classificationCacheHit = false;
+ if (!classification) {
+ classification = await getCachedImageClassification(job.documents.owner_id, task.imageHash, task.contextHash);
+ classificationCacheHit = Boolean(classification);
+ }
+ if (!classification) {
+ classification = await classifyAndCaptionImageFromBase64({
+ base64: task.preparedImage.bytes.toString("base64"),
+ mimeType: task.image.mimeType,
+ nearbyText: task.nearbyText,
+ sourceKind: task.image.sourceKind ?? null,
+ candidateType: task.tableMetadata.candidateType,
+ tableLabel: task.tableMetadata.tableLabel,
+ tableTitle: task.tableMetadata.tableTitle,
+ tableRole: task.tableMetadata.tableRole,
+ tableText: task.tableMetadata.tableText,
+ });
+ await setCachedImageClassification({
+ ownerId: job.documents.owner_id,
+ imageHash: task.imageHash,
+ contextHash: task.contextHash,
+ mimeType: task.image.mimeType,
+ classification,
+ });
+ }
+ return { task, classification, classificationCacheHit };
+ }),
+ );
+ resolvedTasks.push(...batchResults);
+ }
+
+ for (const resolved of resolvedTasks) {
+ const { task, classificationCacheHit } = resolved;
+ const { candidate, index, image, preparedImage, perceptualHash, imageHash, nearbyText, tableMetadata, contextHash } =
+ task;
+ let classification = resolved.classification;
  const policyAssessment = assessClinicalImageUse({
  imageType: classification.image_type,
  searchable: classification.searchable,
@@ -1478,7 +1524,7 @@
  });
 
  const indexedAt = new Date().toISOString();
  const coreAgentMessage = "Core index committed; enrichment pending.";
  const committedCoreMetadata = {
  ...(job.documents.metadata ?? {}),
  indexed_at: indexedAt,
@@ -1497,10 +1543,6 @@
  index_quality_metrics: initialQuality.metrics,
  optional_index_write_issues: optionalIndexWriteIssues,
  embedding_model: env.OPENAI_EMBEDDING_MODEL,
- indexing_v3_agent_status: "pending",
- indexing_v3_agent_last_error: coreAgentMessage,
- indexing_v3_agent_repair_reason: "core_index_committed",
- indexing_v3_agent_updated_at: indexedAt,
  ...metrics,
  };
  await commitDocumentIndexGeneration({