diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 84c6e7af7c..c648d80cbc 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -6821,8 +6821,6 @@ export function ClinicalDashboard() { apiUnavailable={apiUnavailable} setupWarning={setupWarning} facets={searchFacets} - onQueryChange={setQuery} - onSearch={ask} onScopeDocument={scopeOnlyDocument} onAnswerFromDocument={answerFromDocument} onTagSearch={handleTagSearch} diff --git a/src/components/clinical-dashboard/document-search-results.tsx b/src/components/clinical-dashboard/document-search-results.tsx index 22aaa8edf1..46e1cb0558 100644 --- a/src/components/clinical-dashboard/document-search-results.tsx +++ b/src/components/clinical-dashboard/document-search-results.tsx @@ -5,11 +5,8 @@ import { useMemo, useState } from "react"; import { AlertCircle, ChevronDown, - Clock, - ExternalLink, FileText, Filter, - FolderOpen, ListChecks, ShieldAlert, SlidersHorizontal, @@ -257,36 +254,10 @@ function documentOpenHref(document: DocumentMatch) { return `/documents/${document.document_id}?${params.toString()}`; } -const startRows = [ - { - title: "Recent documents", - icon: Clock, - query: "recent documents", - }, - { - title: "Browse library", - icon: FolderOpen, - query: "clinical guideline", - }, - { - title: "Open a source PDF", - icon: ExternalLink, - query: "PDF", - }, -] as const; - -function DocumentSearchHome({ - documentCount, - onSuggestedSearch, -}: { - documentCount: number; - onSuggestedSearch: (query: string) => void; -}) { - const suggestedSearches = ["lithium", "clozapine", "ECT pathway", "monitoring"]; - +function DocumentSearchHome({ documentCount }: { documentCount: number }) { return ( -
-
+
+
@@ -296,61 +267,12 @@ function DocumentSearchHome({

Find guidelines, policies, forms, and source PDFs.

- - {documentCount > 0 ? {documentCount.toLocaleString()} documents indexed : null} -
- -
- {startRows.map((row) => { - const Icon = row.icon; - return ( - - ); - })} -
- -
-

Suggested

-
- {suggestedSearches.map((search) => ( - - ))} +
+ + {documentCount > 0 + ? `${documentCount.toLocaleString()} source${documentCount === 1 ? "" : "s"} indexed` + : "No indexed sources"} +
@@ -436,8 +358,6 @@ export function DocumentSearchResultsPanel({ apiUnavailable, setupWarning, facets: _facets, - onQueryChange, - onSearch, onScopeDocument, onAnswerFromDocument, onTagSearch, @@ -451,8 +371,6 @@ export function DocumentSearchResultsPanel({ apiUnavailable: boolean; setupWarning: string | null; facets?: SearchFacets | null; - onQueryChange: (query: string) => void; - onSearch: () => void; onScopeDocument: (documentId: string) => void; onAnswerFromDocument: (documentId: string) => void; onTagSearch: (tag: SmartDocumentTag | SmartDocumentTagFacet) => void; @@ -503,10 +421,6 @@ export function DocumentSearchResultsPanel({ : trimmedQuery ? "No matching documents" : `${documentCount} document${documentCount === 1 ? "" : "s"}`; - const runSuggestedSearch = (nextQuery: string) => { - onQueryChange(nextQuery); - window.setTimeout(() => onSearch(), 0); - }; return (
@@ -542,7 +456,7 @@ export function DocumentSearchResultsPanel({
) : ( - + ) ) : ( <> diff --git a/src/lib/document-index-units.ts b/src/lib/document-index-units.ts index 13d6447616..cfc065fc85 100644 --- a/src/lib/document-index-units.ts +++ b/src/lib/document-index-units.ts @@ -320,6 +320,7 @@ function visualUnit(args: { extraction_mode: args.profile.confidence >= 0.65 ? "hybrid" : "deterministic", metadata: { source: "visual_intelligence", + generated_by: "local-worker", visual_intelligence_version: visualIntelligenceVersion, image_type: args.image.imageType ?? null, source_kind: args.image.sourceKind ?? null, diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 82d799e2bf..cdd96944e9 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -2472,6 +2472,8 @@ async function searchIndexUnitCandidates(args: { return loadChunksForSignalMatches({ supabase: args.supabase, matches, ownerId: args.ownerId }); } +type MemoryCardCache = Map>; + async function withMemoryBoostedCandidates(args: { supabase: ReturnType; query: string; @@ -2480,15 +2482,26 @@ async function withMemoryBoostedCandidates(args: { ownerId?: string; documentIds?: string[]; matchCount: number; + cardCache?: MemoryCardCache; }) { - const cards = await fetchMemoryCardsForQuery({ - supabase: args.supabase, - query: args.query, - queryEmbedding: args.queryEmbedding, - ownerId: args.ownerId, - documentIds: args.documentIds, - matchCount: Math.max(args.matchCount, 48), - }); + // A3: the memory-card fetch is invoked at several waterfall stages for the same query/owner. + // Memoize per request, keyed by the inputs that actually vary within a request — the query, + // whether an embedding is supplied (embedding vs text-only fetches differ), and the count. + const effectiveMatchCount = Math.max(args.matchCount, 48); + const cacheKey = `${args.query}\0${args.queryEmbedding?.length ? "vec" : "text"}\0${effectiveMatchCount}`; + let cardsPromise = args.cardCache?.get(cacheKey); + if (!cardsPromise) { + cardsPromise = fetchMemoryCardsForQuery({ + supabase: args.supabase, + query: args.query, + queryEmbedding: args.queryEmbedding, + ownerId: args.ownerId, + documentIds: args.documentIds, + matchCount: effectiveMatchCount, + }); + args.cardCache?.set(cacheKey, cardsPromise); + } + const cards = await cardsPromise; if (cards.length === 0) return { results: args.candidates, cards }; const memoryChunkResults = await loadChunksForMemoryCards(args.supabase, cards, args.ownerId); @@ -3646,6 +3659,9 @@ function isEssentialSimpleQuestionSection(section: Pick { + const startedAt = Date.now(); + const candidates = await searchEmbeddingFieldCandidates({ + supabase, + query: args.query, + queryEmbedding: embedding, + ownerId: args.ownerId, + documentIds: documentFilterList, + matchCount: Math.min(candidateCount, 48), + }); + return { candidates, latencyMs: Date.now() - startedAt }; + })(), + (async () => { + const startedAt = Date.now(); + const candidates = await searchIndexUnitCandidates({ + supabase, + query: args.query, + queryEmbedding: embedding, + ownerId: args.ownerId, + documentIds: documentFilterList, + matchCount: Math.min(candidateCount, 64), + }); + return { candidates, latencyMs: Date.now() - startedAt }; + })(), + (async () => { + const startedAt = Date.now(); + const { data, error } = await supabase.rpc("match_document_chunks_hybrid", { + query_embedding: embedding, + query_text: textSearchQuery, + match_count: candidateCount, + min_similarity: minSimilarity, + document_filters: documentFilterList ?? null, + owner_filter: args.ownerId ?? null, + }); + return { data, error, latencyMs: Date.now() - startedAt }; + })(), + ]); + // The three calls overlap, so charge wall-clock once rather than summing per-call latencies. + telemetry.supabase_rpc_latency_ms += Date.now() - parallelRpcStartedAt; + + const embeddingFieldCandidates = embeddingFieldResult.candidates; telemetry.embedding_field_count = embeddingFieldCandidates.length; - const embeddingFieldLatencyMs = Date.now() - embeddingFieldStartedAt; - telemetry.supabase_rpc_latency_ms += embeddingFieldLatencyMs; recordRetrievalLayer(telemetry, "embedding_fields", embeddingFieldCandidates.length, { - latencyMs: embeddingFieldLatencyMs, + latencyMs: embeddingFieldResult.latencyMs, topScore: layerTopScore(embeddingFieldCandidates), }); if (embeddingFieldCandidates.length > 0) { textFastResults = mergeSearchResults(embeddingFieldCandidates, textFastResults); } - const indexUnitStartedAt = Date.now(); - const indexUnitCandidates = await searchIndexUnitCandidates({ - supabase, - query: args.query, - queryEmbedding: embedding, - ownerId: args.ownerId, - documentIds: documentFilterList, - matchCount: Math.min(candidateCount, 64), - }); - const indexUnitLatencyMs = Date.now() - indexUnitStartedAt; - telemetry.supabase_rpc_latency_ms += indexUnitLatencyMs; + const indexUnitCandidates = indexUnitResult.candidates; telemetry.index_unit_count = indexUnitCandidates.length; telemetry.index_unit_top_score = Number( Math.max(0, ...indexUnitCandidates.map((result) => result.hybrid_score ?? result.similarity ?? 0)).toFixed(4), @@ -4005,24 +4049,14 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { textFastResults = mergeSearchResults(indexUnitCandidates, textFastResults); } recordRetrievalLayer(telemetry, "index_units", indexUnitCandidates.length, { - latencyMs: indexUnitLatencyMs, + latencyMs: indexUnitResult.latencyMs, topScore: telemetry.index_unit_top_score, }); - const hybridRpcStartedAt = Date.now(); - const { data: hybridData, error: hybridError } = await supabase.rpc("match_document_chunks_hybrid", { - query_embedding: embedding, - query_text: textSearchQuery, - match_count: candidateCount, - min_similarity: minSimilarity, - document_filters: documentFilterList ?? null, - owner_filter: args.ownerId ?? null, - }); - const hybridLatencyMs = Date.now() - hybridRpcStartedAt; - telemetry.supabase_rpc_latency_ms += hybridLatencyMs; + const { data: hybridData, error: hybridError } = hybridResult; telemetry.vector_candidate_count = hybridData?.length ?? 0; recordRetrievalLayer(telemetry, "hybrid_vector", hybridData?.length ?? 0, { - latencyMs: hybridLatencyMs, + latencyMs: hybridResult.latencyMs, topScore: layerTopScore((hybridData ?? []) as SearchResult[]), }); @@ -4038,6 +4072,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { ownerId: args.ownerId, documentIds: documentFilterList, matchCount: candidateCount, + cardCache: memoryCardCache, }); telemetry.memory_card_count = Math.max(telemetry.memory_card_count ?? 0, memoryBoost.cards.length); telemetry.memory_top_score = Math.max( @@ -4108,6 +4143,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { ownerId: args.ownerId, documentIds: documentFilterList, matchCount: candidateCount, + cardCache: memoryCardCache, }); telemetry.memory_card_count = Math.max(telemetry.memory_card_count ?? 0, memoryBoost.cards.length); telemetry.memory_top_score = Math.max( diff --git a/supabase/functions/indexing-v3-agent/index.ts b/supabase/functions/indexing-v3-agent/index.ts index 1ed0147d7a..2ef58915ce 100644 --- a/supabase/functions/indexing-v3-agent/index.ts +++ b/supabase/functions/indexing-v3-agent/index.ts @@ -1834,7 +1834,11 @@ async function needsVisualArtifacts(job: ClaimedJob): Promise { from public.document_index_units where document_id = ${job.document_id}::uuid and source_image_id is not null - and metadata->>'generated_by' = ${GENERATED_BY} + and ( + metadata->>'generated_by' = ${GENERATED_BY} + or metadata->>'generated_by' = 'local-worker' + or metadata->>'source' = 'visual_intelligence' + ) ) as generated_visual_units `; const row = rows[0] ?? { eligible_images: 0, generated_visual_units: 0 }; diff --git a/supabase/migrations/20260627000000_retrieval_hnsw_ef_search.sql b/supabase/migrations/20260627000000_retrieval_hnsw_ef_search.sql new file mode 100644 index 0000000000..c21a2dce40 --- /dev/null +++ b/supabase/migrations/20260627000000_retrieval_hnsw_ef_search.sql @@ -0,0 +1,31 @@ +-- A2: raise pgvector HNSW ef_search for the vector retrieval functions. +-- +-- The hybrid/vector match functions request up to 128 candidates from their HNSW indexes +-- (limit least(greatest(match_count * 2, 48), 128)), but the pgvector default hnsw.ef_search +-- is 40 — so the index returns at most ~40 quality neighbours, the deeper fetch is wasted, +-- and recall is capped below intent. Pin a higher ef_search per vector function so the index +-- explores enough candidates to fill the requested depth. +-- +-- Body-preserving: ALTER FUNCTION ... SET only attaches a per-function GUC; it does not +-- redefine the function body, so this is low-risk relative to recall gains. The function-level +-- SET reliably applies on every invocation regardless of the connection/role (Supabase +-- PostgREST connects as `authenticator` then SET ROLE service_role, so role-level GUCs would +-- not apply — function-level does). +-- +-- Tunable range ~80-120; comparison-class queries fetch the full 128 and may warrant raising +-- toward 128 at some latency cost. Validate recall vs latency with +-- `npm run eval:retrieval:quality` and `npm run eval:retrieval:latency` on a Supabase branch +-- (and `npm run profile:retrieval` for EXPLAIN plans) before applying to the live project. + +set search_path = public, extensions; + +alter function public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid) + set hnsw.ef_search = 100; diff --git a/supabase/schema.sql b/supabase/schema.sql index 0521f66297..8e19471899 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -3159,3 +3159,19 @@ create policy "document index units owner read" on public.document_index_units for select to authenticated using ( exists (select 1 from public.documents d where d.id = document_id and d.owner_id = (select auth.uid())) ); + +-- A2: raise pgvector HNSW ef_search on the vector retrieval functions so the index explores +-- enough candidates to fill their up-to-128-row fetch (pgvector default ef_search is 40, which +-- caps recall and wastes the deeper fetch). Function-level SET applies on every invocation +-- regardless of connection role. Tunable ~80-120; see migration +-- 20260627000000_retrieval_hnsw_ef_search.sql and validate with `npm run eval:retrieval`. +alter function public.match_document_chunks_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_memory_cards_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_index_units_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_embedding_fields_hybrid(extensions.vector, text, integer, double precision, uuid[], uuid) + set hnsw.ef_search = 100; +alter function public.match_document_chunks(extensions.vector, integer, double precision, uuid, uuid) + set hnsw.ef_search = 100; diff --git a/tests/document-index-units.test.ts b/tests/document-index-units.test.ts index a642b7177a..b39dc78b52 100644 --- a/tests/document-index-units.test.ts +++ b/tests/document-index-units.test.ts @@ -141,6 +141,7 @@ describe("document index units", () => { "visual-intelligence-v1", ); expect(units.find((unit) => unit.unit_type === "risk_matrix_cell")?.metadata).toMatchObject({ + generated_by: "local-worker", source_image_id: "image-1", page_number: 5, }); diff --git a/tests/indexing-v3-agent.test.ts b/tests/indexing-v3-agent.test.ts index 43013717f1..39c8d75bfa 100644 --- a/tests/indexing-v3-agent.test.ts +++ b/tests/indexing-v3-agent.test.ts @@ -154,6 +154,17 @@ describe("indexing-v3-agent behavior", () => { expect(currentQuality.needs_quality_promotion).toBe(false); }); + it("documents that local worker visual units satisfy visual artifact capture", async () => { + const edgeSource = String( + await import("node:fs/promises").then((fs) => + fs.readFile(new URL("../supabase/functions/indexing-v3-agent/index.ts", import.meta.url), "utf8"), + ), + ); + + expect(edgeSource).toContain("metadata->>'generated_by' = 'local-worker'"); + expect(edgeSource).toContain("metadata->>'source' = 'visual_intelligence'"); + }); + it("normalizes metadata counters for repeated idempotent runs", () => { expect(metadataNumber({ indexing_v3_agent_deferral_count: "4" }, "indexing_v3_agent_deferral_count")).toBe(4); expect(metadataNumber({ indexing_v3_agent_deferral_count: "bad" }, "indexing_v3_agent_deferral_count")).toBe(0); diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index bf7930b0d9..002c92a6b1 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -846,11 +846,12 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByRole("main").getByRole("heading", { name: "Documents" })).toBeVisible(); await expect(page.getByTestId("document-search-workspace")).toBeVisible(); await expect(visibleQuestionInput(page)).toBeVisible(); - await expect(page.getByTestId("document-home-overview")).toBeVisible(); - await expect(page.getByRole("button", { name: /Resume Lithium monitoring guideline/i })).toBeVisible(); - await expect(page.getByRole("region", { name: "Document shortcuts" })).toBeVisible(); - await expect(page.getByRole("region", { name: "Suggested searches" })).toBeVisible(); - await expect(page.getByRole("button", { name: "monitoring", exact: true })).toBeVisible(); + await expect(page.getByTestId("document-search-empty-state")).toBeVisible(); + await expect(page.getByText(`${demoDocuments.length} sources indexed`)).toBeVisible(); + await expect(page.getByRole("button", { name: /Resume Lithium monitoring guideline/i })).toHaveCount(0); + await expect(page.getByRole("region", { name: "Document shortcuts" })).toHaveCount(0); + await expect(page.getByRole("region", { name: "Suggested searches" })).toHaveCount(0); + await expect(page.getByRole("button", { name: "monitoring", exact: true })).toHaveCount(0); await expect(page.getByText("Source library workspace")).toHaveCount(0); await expect(page.getByText("Document display")).toHaveCount(0); @@ -863,6 +864,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByText("1 table").first()).toBeVisible(); await expect(page.getByTestId("document-search-workspace")).toContainText("Best match"); await expect(page.getByTestId("document-search-workspace")).toContainText("High relevance"); + await expect(page.getByRole("button", { name: "Lithium", exact: true })).toBeVisible(); await expect(page.getByText("Tag facets")).toHaveCount(0); await expect(page.getByTestId("document-search-workspace")).not.toContainText( /No direct support|Partial support|source support|direct support/i, diff --git a/tests/worker-visual-capture.test.ts b/tests/worker-visual-capture.test.ts index d2040f66bc..471a751118 100644 --- a/tests/worker-visual-capture.test.ts +++ b/tests/worker-visual-capture.test.ts @@ -24,11 +24,20 @@ describe("worker visual capture hardening", () => { it("leaves optional artifact write failures claimable by the Supabase v3 repair agent", () => { expect(workerSource).toContain('const optionalRepairRequired = optionalIndexWriteIssues.length > 0'); + expect(workerSource).toContain('const agentRepairRequired = enrichmentStatus !== "completed" || optionalRepairRequired'); expect(workerSource).toContain('enrichmentStatus = "pending"'); expect(workerSource).toContain('indexing_v3_agent_status: "pending"'); expect(workerSource).toContain('indexing_v3_agent_repair_reason: "optional_index_write_issues"'); }); + it("uses the strict completion RPC when inline enrichment succeeds", () => { + expect(workerSource).toContain('async function completeStrictEnrichmentJob(job: JobRow)'); + expect(workerSource).toContain('complete_strict_enrichment_job'); + expect(workerSource).toContain('p_agent_version: "visual-core-v3"'); + expect(workerSource).toContain('p_visual_indexing_version: "visual-v3"'); + expect(workerSource).toContain('indexing_v3_agent_repair_reason: "strict_completion_gate_blocked"'); + }); + it("invalidates stale image caption cache entries by policy, prompt, and context versions", () => { expect(workerSource).toContain('const imageCaptionCacheVersion = "clinical-image-caption-cache-v2"'); expect(workerSource).toContain('const visionClassificationPromptVersion = "clinical-image-classification-v1"'); diff --git a/worker/main.ts b/worker/main.ts index 22a99a2c7d..aa0ad4aca6 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -6,7 +6,7 @@ import { env } from "../src/lib/env"; import { buildChunks } from "../src/lib/chunking"; import { ragEnrichmentVersion, upsertDocumentEnrichment } from "../src/lib/document-enrichment"; import { ragDeepMemoryVersion, upsertDocumentDeepMemory } from "../src/lib/deep-memory"; -import { extractDocument, fileToBase64 } from "../src/lib/extractors/document"; +import { extractDocument } from "../src/lib/extractors/document"; import { assertEmbeddingDim } from "../src/lib/embedding-dimensions"; import { buildVisualDocumentIndexUnitInputs, @@ -204,6 +204,39 @@ async function completeJob(job: JobRow, stage: string) { await updateBatch(job.batch_id); } +async function completeStrictEnrichmentJob(job: JobRow) { + const { data, error } = await supabase.rpc("complete_strict_enrichment_job", { + p_document_id: job.document_id, + p_job_id: job.id, + p_stage: "indexed; enrichment completed", + p_agent_version: "visual-core-v3", + p_visual_indexing_version: "visual-v3", + }); + + if (error) { + return { + completed: false, + missing: ["strict_completion_rpc_failed"], + message: supabaseStageError("complete strict enrichment job", error).message, + }; + } + + const result = Array.isArray(data) ? (data[0] as Record | undefined) : undefined; + const missing = Array.isArray(result?.missing) ? result.missing.map(String) : []; + if (result?.ok === true && result?.gate_passed === true) { + return { completed: true, missing, message: null }; + } + + return { + completed: false, + missing: missing.length > 0 ? missing : ["strict_completion_gate_blocked"], + message: `Strict enrichment completion blocked: ${JSON.stringify({ + status: typeof result?.status === "string" ? result.status : "missing_result", + missing: missing.length > 0 ? missing : ["strict_completion_gate_blocked"], + })}`, + }; +} + async function failOrRetryJob(args: { job: JobRow; retry: boolean; @@ -659,6 +692,9 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument, const bytes = await readFile(image.path); const imageHash = hashBytes(bytes); return { + // B4: retain the buffer so caption (base64) and storage upload reuse this single + // read instead of re-reading the same file from disk two more times per image. + bytes, imageHash, bytesLength: bytes.length, perceptualHash: lightweightPerceptualHash(bytes, image.width, image.height), @@ -742,7 +778,7 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument, } if (!classification) { classification = await classifyAndCaptionImageFromBase64({ - base64: await fileToBase64(image.path), + base64: preparedImage.bytes.toString("base64"), mimeType: image.mimeType, nearbyText, sourceKind: image.sourceKind ?? null, @@ -820,7 +856,7 @@ async function uploadAndCaptionImages(job: JobRow, extracted: ExtractedDocument, } const ext = path.extname(image.path) || ".png"; - const bytes = await readFile(image.path); + const bytes = preparedImage.bytes; const imagePrefix = job.documents.owner_id ? `${job.documents.owner_id}/images/${job.document_id}` : `local/${job.document_id}`; @@ -1439,39 +1475,68 @@ async function processJob(job: JobRow) { enrichmentErrorMessage = optionalRepairMessage; } + const agentRepairRequired = enrichmentStatus !== "completed" || optionalRepairRequired; + const agentRepairReason = optionalRepairRequired + ? "optional_index_write_issues" + : enrichmentStatus === "failed" + ? "inline_enrichment_failed" + : "enrichment_deferred"; + const finalMetadata = { + ...(job.documents.metadata ?? {}), + indexed_at: indexedAt, + index_generation_id: indexGenerationId, + rag_enrichment_version: ragEnrichmentVersion, + rag_indexing_version: ragDeepMemoryVersion, + rag_memory_version: ragDeepMemoryVersion, + rag_memory_updated_at: enrichmentUpdatedAt, + rag_enrichment_updated_at: enrichmentUpdatedAt, + enrichment_status: enrichmentStatus, + enrichment_error: enrichmentErrorMessage, + section_count: sectionCount, + memory_card_count: memoryCardCount, + extraction_quality: finalQuality.extraction_quality, + index_quality_score: finalQuality.quality_score, + index_quality_issues: finalQuality.issues, + index_quality_metrics: finalQuality.metrics, + optional_index_write_issues: optionalIndexWriteIssues, + ...(agentRepairRequired + ? { + indexing_v3_agent_status: "pending", + indexing_v3_agent_last_error: enrichmentErrorMessage ?? optionalRepairMessage, + indexing_v3_agent_repair_reason: agentRepairReason, + indexing_v3_agent_updated_at: new Date().toISOString(), + } + : {}), + embedding_model: env.OPENAI_EMBEDDING_MODEL, + ...metrics, + }; + await updateDocument(job.document_id, { - metadata: { - ...(job.documents.metadata ?? {}), - indexed_at: indexedAt, - index_generation_id: indexGenerationId, - rag_enrichment_version: ragEnrichmentVersion, - rag_indexing_version: ragDeepMemoryVersion, - rag_memory_version: ragDeepMemoryVersion, - rag_memory_updated_at: enrichmentUpdatedAt, - rag_enrichment_updated_at: enrichmentUpdatedAt, - enrichment_status: enrichmentStatus, - enrichment_error: enrichmentErrorMessage, - section_count: sectionCount, - memory_card_count: memoryCardCount, - extraction_quality: finalQuality.extraction_quality, - index_quality_score: finalQuality.quality_score, - index_quality_issues: finalQuality.issues, - index_quality_metrics: finalQuality.metrics, - optional_index_write_issues: optionalIndexWriteIssues, - ...(optionalRepairRequired - ? { - indexing_v3_agent_status: "pending", - indexing_v3_agent_last_error: enrichmentErrorMessage ?? optionalRepairMessage, - indexing_v3_agent_repair_reason: "optional_index_write_issues", - indexing_v3_agent_updated_at: new Date().toISOString(), - } - : {}), - embedding_model: env.OPENAI_EMBEDDING_MODEL, - ...metrics, - }, + metadata: finalMetadata, }); - await completeJob(job, enrichmentStatus === "completed" ? "indexed" : "indexed; enrichment deferred"); + let completionStage = enrichmentStatus === "completed" ? "indexed" : "indexed; enrichment deferred"; + if (enrichmentStatus === "completed") { + const strictCompletion = await completeStrictEnrichmentJob(job); + if (!strictCompletion.completed) { + completionStage = "indexed; enrichment deferred"; + const strictCompletionMessage = strictCompletion.message ?? "Strict enrichment completion blocked."; + await updateDocument(job.document_id, { + metadata: { + ...finalMetadata, + enrichment_status: "pending", + enrichment_error: strictCompletionMessage, + indexing_v3_agent_status: "pending", + indexing_v3_agent_last_error: strictCompletionMessage, + indexing_v3_agent_repair_reason: "strict_completion_gate_blocked", + indexing_v3_agent_updated_at: new Date().toISOString(), + completion_gate_missing: strictCompletion.missing, + }, + }); + } + } + + await completeJob(job, completionStage); await refreshRagTableStats(); } catch (error) { console.error(`Ingestion job ${job.id} failed:`, error);