From 6b7e98c87502b74c66d1fb895b44439b0a12ff13 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:00:15 +0800 Subject: [PATCH 1/5] fix: close bug-hunter stale-state paths Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 22 ++ src/app/api/documents/[id]/reindex/route.ts | 23 ++- src/app/api/documents/bulk/reindex/route.ts | 23 ++- .../api/ingestion/jobs/[id]/retry/route.ts | 26 ++- src/app/api/upload/route.ts | 30 ++- src/components/forms/form-detail-page.tsx | 17 +- tests/forms-clipboard-fallback.test.ts | 14 ++ tests/private-access-routes.test.ts | 189 +++++++++++++++++- tests/worker-visual-capture.test.ts | 6 + worker/main.ts | 8 +- 10 files changed, 343 insertions(+), 15 deletions(-) create mode 100644 tests/forms-clipboard-fallback.test.ts diff --git a/AGENTS.md b/AGENTS.md index 35b5468330..e9dcda623f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,28 @@ After setup: + + +## Bug-hunter shortcut + +When the user types exactly `bug-hunter` as the entire task message, after trimming surrounding whitespace, treat it as a shortcut for targeted defect discovery. + +Execution rules: + +- Invoke the `bug-hunter` skill first. +- Prioritize reproducible defects over code style, naming, or formatting feedback. +- Trace realistic failure paths: invalid input, empty states, retries, race/concurrency issues, stale state/cache, network/auth failures, permissions, and boundary values. +- For each finding, include trigger, expected behavior, actual risk, and the smallest proof (or targeted test/check) that would catch it. +- If no high-confidence defect is found, explicitly state that and list the most likely residual risk area. + +Scope and safety: + +- Keep the hunt scoped to code touched by the user request unless the defect clearly crosses module boundaries. +- Do not make broad refactors while hunting; propose minimal fixes for confirmed issues. +- Run the smallest focused verification for each confirmed defect, then expand only if needed. + + + # Local server safety diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 2c34d38c93..2e1d749abd 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -101,7 +101,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const { data: document, error: documentError } = await supabase .from("documents") - .select("id,owner_id,title,file_name,source_path,import_batch_id,status,metadata") + .select("id,owner_id,title,file_name,source_path,import_batch_id,status,error_message,page_count,chunk_count,image_count,metadata") .eq("id", id) .eq("owner_id", user.id) .maybeSingle(); @@ -170,6 +170,15 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: } const atomicReindex = isAtomicReindexCandidate(document); + const rollbackDocumentPayload = atomicReindex + ? { error_message: document.error_message ?? null } + : { + status: document.status ?? null, + error_message: document.error_message ?? null, + page_count: document.page_count ?? 0, + chunk_count: document.chunk_count ?? 0, + image_count: document.image_count ?? 0, + }; const { error: updateError } = await supabase .from("documents") .update( @@ -194,7 +203,17 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: .select() .single(); - if (jobError) throw new Error(jobError.message); + if (jobError) { + const { error: rollbackError } = await supabase + .from("documents") + .update(rollbackDocumentPayload) + .eq("id", id) + .eq("owner_id", user.id); + if (rollbackError) { + throw new Error(`Failed to enqueue reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); + } + throw new Error(jobError.message); + } return NextResponse.json({ job }, { status: 201 }); } catch (error) { if (error instanceof AuthenticationError) return unauthorizedResponse(); diff --git a/src/app/api/documents/bulk/reindex/route.ts b/src/app/api/documents/bulk/reindex/route.ts index 864878678e..28286c58bf 100644 --- a/src/app/api/documents/bulk/reindex/route.ts +++ b/src/app/api/documents/bulk/reindex/route.ts @@ -91,7 +91,7 @@ export async function POST(request: Request) { const documentIds = Array.from(new Set(parsed.documentIds)); const { data: documents, error: documentError } = await supabase .from("documents") - .select("id,owner_id,title,file_name,source_path,import_batch_id,status,metadata") + .select("id,owner_id,title,file_name,source_path,import_batch_id,status,error_message,page_count,chunk_count,image_count,metadata") .eq("owner_id", user.id) .in("id", documentIds); if (documentError) throw new Error(documentError.message); @@ -166,6 +166,15 @@ export async function POST(request: Request) { } const atomicReindex = isAtomicReindexCandidate(document); + const rollbackDocumentPayload = atomicReindex + ? { error_message: document.error_message ?? null } + : { + status: document.status ?? null, + error_message: document.error_message ?? null, + page_count: document.page_count ?? 0, + chunk_count: document.chunk_count ?? 0, + image_count: document.image_count ?? 0, + }; const { error: updateError } = await supabase .from("documents") .update( @@ -188,7 +197,17 @@ export async function POST(request: Request) { }) .select("id") .single(); - if (jobError) throw new Error(jobError.message); + if (jobError) { + const { error: rollbackError } = await supabase + .from("documents") + .update(rollbackDocumentPayload) + .eq("id", document.id) + .eq("owner_id", user.id); + if (rollbackError) { + throw new Error(`Failed to enqueue bulk reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); + } + throw new Error(jobError.message); + } results.push({ documentId: document.id, mode: parsed.mode, ok: true, jobId: job.id }); } catch (error) { results.push({ diff --git a/src/app/api/ingestion/jobs/[id]/retry/route.ts b/src/app/api/ingestion/jobs/[id]/retry/route.ts index 4e6b4285f1..e1e52ccdee 100644 --- a/src/app/api/ingestion/jobs/[id]/retry/route.ts +++ b/src/app/api/ingestion/jobs/[id]/retry/route.ts @@ -23,7 +23,9 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const { data: job, error: jobError } = await supabase .from("ingestion_jobs") - .select("id,document_id,batch_id,status,locked_at,documents!inner(owner_id)") + .select( + "id,document_id,batch_id,status,stage,progress,error_message,attempt_count,max_attempts,locked_at,locked_by,next_run_at,completed_at,documents!inner(owner_id)", + ) .eq("id", id) .eq("documents.owner_id", user.id) .maybeSingle(); @@ -81,7 +83,27 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: .update({ status: "queued", error_message: null }) .eq("id", job.document_id) .eq("owner_id", user.id); - if (documentError) throw new Error(documentError.message); + if (documentError) { + const { error: rollbackError } = await supabase + .from("ingestion_jobs") + .update({ + status: job.status, + stage: job.stage, + progress: job.progress, + error_message: job.error_message, + attempt_count: job.attempt_count, + max_attempts: job.max_attempts, + locked_at: job.locked_at, + locked_by: job.locked_by, + next_run_at: job.next_run_at, + completed_at: job.completed_at, + }) + .eq("id", id); + if (rollbackError) { + throw new Error(`${documentError.message}; failed to roll back retried job state: ${rollbackError.message}`); + } + throw new Error(documentError.message); + } return NextResponse.json({ job: data }); } catch (error) { diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index b118535cba..81119e3463 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -24,6 +24,8 @@ const uploadMetadataSchema = z export async function POST(request: Request) { let supabase: ReturnType | null = null; let uploadedPath: string | null = null; + let insertedDocumentId: string | null = null; + let insertedDocumentOwnerId: string | null = null; try { supabase = createAdminClient(); @@ -131,6 +133,8 @@ export async function POST(request: Request) { .single(); if (documentError) throw new Error(documentError.message); + insertedDocumentId = documentId; + insertedDocumentOwnerId = user.id; const { data: job, error: jobError } = await supabase .from("ingestion_jobs") @@ -145,7 +149,19 @@ export async function POST(request: Request) { .select() .single(); - if (jobError) throw new Error(jobError.message); + if (jobError) { + const { error: rollbackDocumentError } = await supabase + .from("documents") + .delete() + .eq("id", documentId) + .eq("owner_id", user.id); + if (rollbackDocumentError) { + throw new Error(`Failed to enqueue ingestion job: ${jobError.message}; rollback failed: ${rollbackDocumentError.message}`); + } + insertedDocumentId = null; + insertedDocumentOwnerId = null; + throw new Error(jobError.message); + } await writeAuditLog(supabase, { ownerId: user.id, @@ -157,6 +173,18 @@ export async function POST(request: Request) { return NextResponse.json({ document, job }, { status: 201 }); } catch (error) { + if (insertedDocumentId && insertedDocumentOwnerId && supabase) { + try { + await supabase.from("documents").delete().eq("id", insertedDocumentId).eq("owner_id", insertedDocumentOwnerId); + } catch (cleanupError) { + logger.error("Upload cleanup failed; document row may be orphaned", { + documentId: insertedDocumentId, + ownerId: insertedDocumentOwnerId, + message: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + }); + } + } + if (uploadedPath && supabase) { try { await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).remove([uploadedPath]); diff --git a/src/components/forms/form-detail-page.tsx b/src/components/forms/form-detail-page.tsx index 7db8180bff..c8c40991f5 100644 --- a/src/components/forms/form-detail-page.tsx +++ b/src/components/forms/form-detail-page.tsx @@ -69,8 +69,12 @@ function readSavedForms() { async function copyText(value: string) { if (navigator.clipboard?.writeText) { - await navigator.clipboard.writeText(value); - return; + try { + await navigator.clipboard.writeText(value); + return; + } catch { + // Fall through to the legacy selection path for restricted browser contexts. + } } const textArea = document.createElement("textarea"); @@ -80,9 +84,12 @@ async function copyText(value: string) { textArea.style.opacity = "0"; document.body.appendChild(textArea); textArea.select(); - const copied = document.execCommand?.("copy"); - document.body.removeChild(textArea); - if (copied === false) throw new Error("copy command rejected"); + try { + const copied = document.execCommand?.("copy"); + if (copied === false) throw new Error("copy command rejected"); + } finally { + document.body.removeChild(textArea); + } } function chipToneClass(tone: ServiceChipTone | null | undefined) { diff --git a/tests/forms-clipboard-fallback.test.ts b/tests/forms-clipboard-fallback.test.ts new file mode 100644 index 0000000000..a41b963293 --- /dev/null +++ b/tests/forms-clipboard-fallback.test.ts @@ -0,0 +1,14 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const source = readFileSync(new URL("../src/components/forms/form-detail-page.tsx", import.meta.url), "utf8"); + +describe("form detail clipboard fallback", () => { + it("falls back to selection-copy when clipboard.writeText rejects", () => { + expect(source).toContain("if (navigator.clipboard?.writeText)"); + expect(source).toContain("await navigator.clipboard.writeText(value)"); + expect(source).toContain("Fall through to the legacy selection path for restricted browser contexts."); + expect(source).toContain("document.execCommand?.(\"copy\")"); + expect(source).toContain("finally {\n document.body.removeChild(textArea);\n }"); + }); +}); diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index 2cff6e9a78..54865f94bd 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -714,7 +714,7 @@ describe("private document API access", () => { id: documentId, title: "Existing guideline", file_name: "guideline.pdf", - status: "indexed", + status: "failed", page_count: 4, chunk_count: 8, image_count: 1, @@ -1065,13 +1065,70 @@ describe("private document API access", () => { expect(client.rpc).not.toHaveBeenCalled(); }); + it("rolls back the job retry when document queue status update fails", async () => { + const previousJob = { + id: "job-1", + document_id: documentId, + batch_id: null, + status: "failed", + stage: "failed", + progress: 42, + error_message: "OCR failed", + attempt_count: 2, + max_attempts: 3, + locked_at: null, + locked_by: null, + next_run_at: null, + completed_at: "2024-01-01T00:00:00.000Z", + }; + + let ingestionUpdateCount = 0; + const client = createSupabaseMock((call) => { + if (call.table === "ingestion_jobs" && call.operation === "select") { + return ok(previousJob); + } + if (call.table === "ingestion_jobs" && call.operation === "update") { + ingestionUpdateCount += 1; + if (ingestionUpdateCount === 1) { + return ok({ id: "job-1", document_id: documentId, status: "pending" }); + } + return ok({ id: "job-1" }); + } + if (call.table === "documents" && call.operation === "update") return fail("documents update failed"); + return ok([]); + }); + mockRuntime(client); + const { POST } = await import("../src/app/api/ingestion/jobs/[id]/retry/route"); + + const response = await POST(authenticatedRequest(`/api/ingestion/jobs/job-1/retry`, { method: "POST" }), { + params: Promise.resolve({ id: "job-1" }), + }); + + expect(response.status).toBe(400); + expect(String((await payload(response)).error)).toBe("Request could not be completed."); + const jobUpdates = client.calls.filter((call) => call.table === "ingestion_jobs" && call.operation === "update"); + expect(jobUpdates).toHaveLength(2); + expect(jobUpdates[1]?.updatePayload).toEqual({ + status: previousJob.status, + stage: previousJob.stage, + progress: previousJob.progress, + error_message: previousJob.error_message, + attempt_count: previousJob.attempt_count, + max_attempts: previousJob.max_attempts, + locked_at: previousJob.locked_at, + locked_by: previousJob.locked_by, + next_run_at: previousJob.next_run_at, + completed_at: previousJob.completed_at, + }); + }); + it("runs enrichment-only reindex for owned indexed documents using generic metadata", async () => { const document = { id: documentId, owner_id: userId, title: "Future Uploaded Protocol", file_name: "future-upload.pdf", - source_path: null, + source_path: "legacy/imports/rollback.pdf", import_batch_id: null, metadata: { existing: true }, }; @@ -1490,6 +1547,125 @@ describe("private document API access", () => { expect(client.calls.some((call) => call.table === "documents" && call.operation === "update")).toBe(false); }); + it("rolls back single-document queue mutation when full reindex job enqueue fails", async () => { + const document = { + id: documentId, + owner_id: userId, + title: "Rollback Protocol", + file_name: "rollback.pdf", + source_path: null, + import_batch_id: null, + status: "failed", + error_message: "older failure", + page_count: 12, + chunk_count: 34, + image_count: 2, + metadata: {}, + }; + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") return ok(document); + if (call.table === "import_batches") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "insert") return fail("job insert failed"); + if (call.table === "documents" && call.operation === "update") return ok([]); + return ok([]); + }); + mockRuntime(client); + const { POST } = await import("../src/app/api/documents/[id]/reindex/route"); + + const response = await POST( + authenticatedRequest(`/api/documents/${documentId}/reindex`, { + method: "POST", + body: JSON.stringify({ mode: "full" }), + }), + { params: Promise.resolve({ id: documentId }) }, + ); + const body = await payload(response); + const documentUpdates = client.calls.filter((call) => call.table === "documents" && call.operation === "update"); + + expect(response.status).toBe(400); + expect(body).toEqual({ error: "Request could not be completed." }); + expect(documentUpdates).toHaveLength(2); + expect(documentUpdates[0]?.updatePayload).toEqual({ + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + }); + expect(documentUpdates[1]?.updatePayload).toEqual({ + status: "failed", + error_message: "older failure", + page_count: 12, + chunk_count: 34, + image_count: 2, + }); + }); + + it("rolls back per-document queue mutation when bulk full reindex enqueue fails", async () => { + const document = { + id: documentId, + owner_id: userId, + title: "Bulk Rollback Protocol", + file_name: "bulk-rollback.pdf", + source_path: "legacy/imports/bulk-rollback.pdf", + import_batch_id: null, + status: "failed", + error_message: "older failure", + page_count: 3, + chunk_count: 8, + image_count: 1, + metadata: {}, + }; + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") return ok([document]); + if (call.table === "import_batches") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]); + if (call.table === "documents" && call.operation === "update") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "insert") return fail("bulk job insert failed"); + return ok([]); + }); + mockRuntime(client, { invalidateRagCachesForOwner: vi.fn() }); + const { POST } = await import("../src/app/api/documents/bulk/reindex/route"); + + const response = await POST( + authenticatedRequest("/api/documents/bulk/reindex", { + method: "POST", + body: JSON.stringify({ documentIds: [documentId], mode: "full" }), + }), + ); + const body = await payload(response); + const documentUpdates = client.calls.filter((call) => call.table === "documents" && call.operation === "update"); + + expect(response.status).toBe(200); + expect(body).toMatchObject({ + ok: false, + results: [ + { + documentId, + mode: "full", + ok: false, + error: "bulk job insert failed", + }, + ], + }); + expect(documentUpdates).toHaveLength(2); + expect(documentUpdates[0]?.updatePayload).toEqual({ + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + }); + expect(documentUpdates[1]?.updatePayload).toEqual({ + status: "failed", + error_message: "older failure", + page_count: 3, + chunk_count: 8, + image_count: 1, + }); + }); + it("cleans up uploaded storage when document insert fails", async () => { const client = createSupabaseMock((call) => call.table === "documents" && call.operation === "insert" ? fail("document insert failed") : ok([]), @@ -1536,6 +1712,15 @@ describe("private document API access", () => { expect(response.status).toBe(500); expect(client.storageMocks.remove).toHaveBeenCalledWith([uploadPath]); + expect( + client.calls.some( + (call) => + call.table === "documents" && + call.operation === "delete" && + call.filters.some((filter) => filter.column === "id" && typeof filter.value === "string") && + call.filters.some((filter) => filter.column === "owner_id" && filter.value === userId), + ), + ).toBe(true); }); it("does not return document details for an unowned document", async () => { diff --git a/tests/worker-visual-capture.test.ts b/tests/worker-visual-capture.test.ts index c910facc30..c882906ede 100644 --- a/tests/worker-visual-capture.test.ts +++ b/tests/worker-visual-capture.test.ts @@ -54,6 +54,12 @@ describe("worker visual capture hardening", () => { expect(workerSource).toContain("indexing_v3_agent_repair_reason: strictCompletionRepairReason"); }); + it("logs fallback import-batch update failures when RPC-based refresh is unavailable", () => { + expect(workerSource).toContain("Import batch status fallback update failed"); + expect(workerSource).toContain("update import batch status fallback"); + expect(workerSource).toContain("const { error: fallbackUpdateError } = await supabase"); + }); + it("invalidates stale image caption cache entries by policy, prompt, and context versions", () => { expect(workerSource).toContain('const imageCaptionCacheVersion = "clinical-image-caption-cache-v2"'); expect(workerSource).toContain('const visionClassificationPromptVersion = "clinical-image-classification-v1"'); diff --git a/worker/main.ts b/worker/main.ts index 4718f1b2ca..c2a3bc1351 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -171,7 +171,7 @@ async function updateBatch(batchId: string | null) { const failed = data.filter((job) => job.status === "failed").length; const status = terminalBatchStatus({ queued, processing, failed }); - await supabase + const { error: fallbackUpdateError } = await supabase .from("import_batches") .update({ status, @@ -179,6 +179,12 @@ async function updateBatch(batchId: string | null) { completed_at: status === "processing" ? null : new Date().toISOString(), }) .eq("id", batchId); + if (fallbackUpdateError) { + console.warn( + "Import batch status fallback update failed", + safeErrorLogDetails(supabaseStageError("update import batch status fallback", fallbackUpdateError)), + ); + } } async function completeJob(job: JobRow, stage: string) { From 470df91d009f915ca28a920fc5f8a08be8803cb9 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:34:39 +0800 Subject: [PATCH 2/5] test: align bug-hunter expectations with main error contract Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/private-access-routes.test.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index 54865f94bd..cdd52a174c 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -1066,8 +1066,9 @@ describe("private document API access", () => { }); it("rolls back the job retry when document queue status update fails", async () => { + const retryJobId = "99999999-9999-4999-8999-999999999999"; const previousJob = { - id: "job-1", + id: retryJobId, document_id: documentId, batch_id: null, status: "failed", @@ -1090,9 +1091,9 @@ describe("private document API access", () => { if (call.table === "ingestion_jobs" && call.operation === "update") { ingestionUpdateCount += 1; if (ingestionUpdateCount === 1) { - return ok({ id: "job-1", document_id: documentId, status: "pending" }); + return ok({ id: retryJobId, document_id: documentId, status: "pending" }); } - return ok({ id: "job-1" }); + return ok({ id: retryJobId }); } if (call.table === "documents" && call.operation === "update") return fail("documents update failed"); return ok([]); @@ -1100,12 +1101,12 @@ describe("private document API access", () => { mockRuntime(client); const { POST } = await import("../src/app/api/ingestion/jobs/[id]/retry/route"); - const response = await POST(authenticatedRequest(`/api/ingestion/jobs/job-1/retry`, { method: "POST" }), { - params: Promise.resolve({ id: "job-1" }), + const response = await POST(authenticatedRequest(`/api/ingestion/jobs/${retryJobId}/retry`, { method: "POST" }), { + params: Promise.resolve({ id: retryJobId }), }); - expect(response.status).toBe(400); - expect(String((await payload(response)).error)).toBe("Request could not be completed."); + expect(response.status).toBe(500); + expect(String((await payload(response)).error)).toBe("Request failed."); const jobUpdates = client.calls.filter((call) => call.table === "ingestion_jobs" && call.operation === "update"); expect(jobUpdates).toHaveLength(2); expect(jobUpdates[1]?.updatePayload).toEqual({ @@ -1583,8 +1584,8 @@ describe("private document API access", () => { const body = await payload(response); const documentUpdates = client.calls.filter((call) => call.table === "documents" && call.operation === "update"); - expect(response.status).toBe(400); - expect(body).toEqual({ error: "Request could not be completed." }); + expect(response.status).toBe(500); + expect(body).toEqual({ error: "Request failed." }); expect(documentUpdates).toHaveLength(2); expect(documentUpdates[0]?.updatePayload).toEqual({ status: "queued", From 214c6fe22b0bb94adb01dbe3a8e1deec90abad06 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:36:54 +0800 Subject: [PATCH 3/5] fix: harden rollback guards for review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/app/api/documents/[id]/reindex/route.ts | 25 ++- src/app/api/documents/bulk/reindex/route.ts | 27 +++- .../api/ingestion/jobs/[id]/retry/route.ts | 8 +- src/app/api/upload/route.ts | 21 ++- tests/private-access-routes.test.ts | 147 ++++++++++++++++++ 5 files changed, 211 insertions(+), 17 deletions(-) diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 3440dc21a7..c8b30479ae 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -209,13 +209,24 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: .single(); if (jobError) { - const { error: rollbackError } = await supabase - .from("documents") - .update(rollbackDocumentPayload) - .eq("id", id) - .eq("owner_id", user.id); - if (rollbackError) { - throw new Error(`Failed to enqueue reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); + const { data: competingJobs, error: competingJobsError } = await supabase + .from("ingestion_jobs") + .select("id") + .eq("document_id", id) + .in("status", ["pending", "processing"]) + .limit(1); + if (competingJobsError) { + throw new Error(`Failed to enqueue reindex job: ${jobError.message}; competing-job check failed: ${competingJobsError.message}`); + } + if ((competingJobs?.length ?? 0) === 0) { + const { error: rollbackError } = await supabase + .from("documents") + .update(rollbackDocumentPayload) + .eq("id", id) + .eq("owner_id", user.id); + if (rollbackError) { + throw new Error(`Failed to enqueue reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); + } } throw new Error(jobError.message); } diff --git a/src/app/api/documents/bulk/reindex/route.ts b/src/app/api/documents/bulk/reindex/route.ts index 9f2ae387ad..4050efbc1e 100644 --- a/src/app/api/documents/bulk/reindex/route.ts +++ b/src/app/api/documents/bulk/reindex/route.ts @@ -203,13 +203,26 @@ export async function POST(request: Request) { .select("id") .single(); if (jobError) { - const { error: rollbackError } = await supabase - .from("documents") - .update(rollbackDocumentPayload) - .eq("id", document.id) - .eq("owner_id", user.id); - if (rollbackError) { - throw new Error(`Failed to enqueue bulk reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); + const { data: competingJobs, error: competingJobsError } = await supabase + .from("ingestion_jobs") + .select("id") + .eq("document_id", document.id) + .in("status", ["pending", "processing"]) + .limit(1); + if (competingJobsError) { + throw new Error( + `Failed to enqueue bulk reindex job: ${jobError.message}; competing-job check failed: ${competingJobsError.message}`, + ); + } + if ((competingJobs?.length ?? 0) === 0) { + const { error: rollbackError } = await supabase + .from("documents") + .update(rollbackDocumentPayload) + .eq("id", document.id) + .eq("owner_id", user.id); + if (rollbackError) { + throw new Error(`Failed to enqueue bulk reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); + } } throw new Error(jobError.message); } diff --git a/src/app/api/ingestion/jobs/[id]/retry/route.ts b/src/app/api/ingestion/jobs/[id]/retry/route.ts index e1e52ccdee..31ba379c1c 100644 --- a/src/app/api/ingestion/jobs/[id]/retry/route.ts +++ b/src/app/api/ingestion/jobs/[id]/retry/route.ts @@ -98,7 +98,13 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: next_run_at: job.next_run_at, completed_at: job.completed_at, }) - .eq("id", id); + .eq("id", id) + .eq("status", "pending") + .eq("stage", "queued") + .eq("progress", 0) + .eq("attempt_count", 0) + .is("locked_at", null) + .is("locked_by", null); if (rollbackError) { throw new Error(`${documentError.message}; failed to roll back retried job state: ${rollbackError.message}`); } diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index 81119e3463..59bafda449 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -175,7 +175,18 @@ export async function POST(request: Request) { } catch (error) { if (insertedDocumentId && insertedDocumentOwnerId && supabase) { try { - await supabase.from("documents").delete().eq("id", insertedDocumentId).eq("owner_id", insertedDocumentOwnerId); + const { error: cleanupDeleteError } = await supabase + .from("documents") + .delete() + .eq("id", insertedDocumentId) + .eq("owner_id", insertedDocumentOwnerId); + if (cleanupDeleteError) { + logger.error("Upload cleanup failed; document row may be orphaned", { + documentId: insertedDocumentId, + ownerId: insertedDocumentOwnerId, + message: cleanupDeleteError.message, + }); + } } catch (cleanupError) { logger.error("Upload cleanup failed; document row may be orphaned", { documentId: insertedDocumentId, @@ -187,7 +198,13 @@ export async function POST(request: Request) { if (uploadedPath && supabase) { try { - await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).remove([uploadedPath]); + const { error: cleanupStorageError } = await supabase.storage.from(env.SUPABASE_DOCUMENT_BUCKET).remove([uploadedPath]); + if (cleanupStorageError) { + logger.error("Upload cleanup failed; storage object may be orphaned", { + storagePath: uploadedPath, + message: cleanupStorageError.message, + }); + } } catch (cleanupError) { // Cleanup is best-effort, but a silent failure leaves an orphaned storage // object. Record the path so it can be reconciled instead of dropping it. diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index cdd52a174c..c9008f778a 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -1603,6 +1603,57 @@ describe("private document API access", () => { }); }); + it("skips single-document rollback when a competing active job appears after the safety check", async () => { + const document = { + id: documentId, + owner_id: userId, + title: "Rollback Guard Protocol", + file_name: "rollback-guard.pdf", + source_path: null, + import_batch_id: null, + status: "failed", + error_message: "older failure", + page_count: 12, + chunk_count: 34, + image_count: 2, + metadata: {}, + }; + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") return ok(document); + if (call.table === "import_batches") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "select" && call.limitCount === 1) { + return ok([{ id: "competing-job" }]); + } + if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "insert") return fail("job insert failed"); + if (call.table === "documents" && call.operation === "update") return ok([]); + return ok([]); + }); + mockRuntime(client); + const { POST } = await import("../src/app/api/documents/[id]/reindex/route"); + + const response = await POST( + authenticatedRequest(`/api/documents/${documentId}/reindex`, { + method: "POST", + body: JSON.stringify({ mode: "full" }), + }), + { params: Promise.resolve({ id: documentId }) }, + ); + const body = await payload(response); + const documentUpdates = client.calls.filter((call) => call.table === "documents" && call.operation === "update"); + + expect(response.status).toBe(500); + expect(body).toEqual({ error: "Request failed." }); + expect(documentUpdates).toHaveLength(1); + expect(documentUpdates[0]?.updatePayload).toEqual({ + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + }); + }); + it("rolls back per-document queue mutation when bulk full reindex enqueue fails", async () => { const document = { id: documentId, @@ -1667,6 +1718,66 @@ describe("private document API access", () => { }); }); + it("skips bulk rollback when a competing active job appears after the safety check", async () => { + const document = { + id: documentId, + owner_id: userId, + title: "Bulk Rollback Guard Protocol", + file_name: "bulk-rollback-guard.pdf", + source_path: "legacy/imports/bulk-rollback-guard.pdf", + import_batch_id: null, + status: "failed", + error_message: "older failure", + page_count: 3, + chunk_count: 8, + image_count: 1, + metadata: {}, + }; + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "select") return ok([document]); + if (call.table === "import_batches") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "select" && call.limitCount === 1) { + return ok([{ id: "competing-job" }]); + } + if (call.table === "ingestion_jobs" && call.operation === "select") return ok([]); + if (call.table === "documents" && call.operation === "update") return ok([]); + if (call.table === "ingestion_jobs" && call.operation === "insert") return fail("bulk job insert failed"); + return ok([]); + }); + mockRuntime(client, { invalidateRagCachesForOwner: vi.fn() }); + const { POST } = await import("../src/app/api/documents/bulk/reindex/route"); + + const response = await POST( + authenticatedRequest("/api/documents/bulk/reindex", { + method: "POST", + body: JSON.stringify({ documentIds: [documentId], mode: "full" }), + }), + ); + const body = await payload(response); + const documentUpdates = client.calls.filter((call) => call.table === "documents" && call.operation === "update"); + + expect(response.status).toBe(200); + expect(body).toMatchObject({ + ok: false, + results: [ + { + documentId, + mode: "full", + ok: false, + error: "bulk job insert failed", + }, + ], + }); + expect(documentUpdates).toHaveLength(1); + expect(documentUpdates[0]?.updatePayload).toEqual({ + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + }); + }); + it("cleans up uploaded storage when document insert fails", async () => { const client = createSupabaseMock((call) => call.table === "documents" && call.operation === "insert" ? fail("document insert failed") : ok([]), @@ -1724,6 +1835,42 @@ describe("private document API access", () => { ).toBe(true); }); + it("still runs catch cleanup when upload cleanup calls return non-throwing errors", async () => { + const client = createSupabaseMock((call) => { + if (call.table === "documents" && call.operation === "insert") { + return ok({ id: documentId }); + } + if (call.table === "ingestion_jobs" && call.operation === "insert") { + return fail("job insert failed"); + } + if (call.table === "documents" && call.operation === "delete") { + return fail("document cleanup returned error"); + } + return ok([]); + }); + client.storageMocks.remove.mockResolvedValue({ + data: [], + error: { message: "storage cleanup returned error" }, + }); + mockRuntime(client); + const { POST } = await import("../src/app/api/upload/route"); + const formData = new FormData(); + formData.set("file", new File(["%PDF-1.7"], "guideline.pdf", { type: "application/pdf" })); + + const response = await POST( + authenticatedRequest("/api/upload", { + method: "POST", + body: formData, + }), + ); + const uploadPath = client.storageMocks.upload.mock.calls[0]?.[0] as string; + const documentDeletes = client.calls.filter((call) => call.table === "documents" && call.operation === "delete"); + + expect(response.status).toBe(500); + expect(documentDeletes.length).toBeGreaterThanOrEqual(2); + expect(client.storageMocks.remove).toHaveBeenCalledWith([uploadPath]); + }); + it("does not return document details for an unowned document", async () => { const client = createSupabaseMock(() => ok(null)); mockRuntime(client); From b65cfd8376bd1c77030fa3788a7cad5890d0cbeb Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:13:21 +0800 Subject: [PATCH 4/5] test: type remove storage mock as QueryError union Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/private-access-routes.test.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index 6336aae501..245404663c 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -175,10 +175,17 @@ function createSupabaseMock(resolve: QueryResolver = () => ok([])) { data: { signedUrl: `https://signed.local/${path}` }, error: null, })); - const remove = vi.fn(async (...args: [string[]]) => { - void args; - return { data: [], error: null }; - }); + const remove = vi.fn( + async ( + ...args: [string[]] + ): Promise<{ + data: string[] | null; + error: QueryError | null; + }> => { + void args; + return { data: [], error: null }; + }, + ); const storageFrom = vi.fn(() => ({ upload, createSignedUrl, remove })); const getUser = vi.fn(async (receivedToken?: string) => receivedToken === token From ab6d70b20f6c3c64d28769664ca249d44069688e Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 20:07:11 +0800 Subject: [PATCH 5/5] fix: fence stale-state rollbacks against concurrent queue mutations Address the open Codex review comments on #146: - retry route: next_run_at doubles as a per-request rollback fence, so a losing retry's rollback matches zero rows instead of reverting a concurrent retry's reset; the document re-queue also stamps updated_at - single/bulk reindex routes: queue-state writes stamp documents.updated_at with a per-request value and the enqueue-failure rollback matches on that stamp, turning it into one atomic conditional UPDATE; the competing-job SELECT remains only as a fast path - shared ingestionRollbackFenceStamp helper generates microsecond-unique timestamptz stamps - tests assert the fence filters and stamped payloads The preceding merge of origin/main adopts the current jsonError 5xx contract, aligning the routes with the rollback tests' 500 expectations. Co-Authored-By: Claude Fable 5 --- src/app/api/documents/[id]/reindex/route.ts | 28 ++++++++++++++++--- src/app/api/documents/bulk/reindex/route.ts | 26 ++++++++++++++--- .../api/ingestion/jobs/[id]/retry/route.ts | 17 +++++++++-- src/lib/ingestion-mutation-safety.ts | 13 +++++++++ tests/private-access-routes.test.ts | 26 ++++++++++++++++- 5 files changed, 98 insertions(+), 12 deletions(-) diff --git a/src/app/api/documents/[id]/reindex/route.ts b/src/app/api/documents/[id]/reindex/route.ts index 3113067480..cda3e65b1a 100644 --- a/src/app/api/documents/[id]/reindex/route.ts +++ b/src/app/api/documents/[id]/reindex/route.ts @@ -5,7 +5,11 @@ import { env, isDemoMode } from "@/lib/env"; import { upsertDocumentEnrichment } from "@/lib/document-enrichment"; import { upsertDocumentDeepMemory } from "@/lib/deep-memory"; import { jsonError } from "@/lib/http"; -import { checkIngestionMutationSafety, ingestionMutationSafetyPayload } from "@/lib/ingestion-mutation-safety"; +import { + checkIngestionMutationSafety, + ingestionMutationSafetyPayload, + ingestionRollbackFenceStamp, +} from "@/lib/ingestion-mutation-safety"; import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { committedIndexGeneration, @@ -176,6 +180,14 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: } const atomicReindex = isAtomicReindexCandidate(document); + // Rollback fence: the queue-state write stamps updated_at with a + // per-request value and the rollback below matches on that stamp, making + // it a single conditional UPDATE that is atomic server-side. An + // overlapping reindex/retry re-stamps the row before enqueueing its own + // job, so a stale rollback from this request matches zero rows instead of + // reverting the newer queue state. The competing-job SELECT below is only + // a cheap fast path; the fence is what closes the check-then-write race. + const rollbackFence = ingestionRollbackFenceStamp(); const rollbackDocumentPayload = atomicReindex ? { error_message: document.error_message ?? null } : { @@ -189,8 +201,15 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: .from("documents") .update( atomicReindex - ? { error_message: null } - : { status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 }, + ? { error_message: null, updated_at: rollbackFence } + : { + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + updated_at: rollbackFence, + }, ) .eq("id", id) .eq("owner_id", user.id); @@ -224,7 +243,8 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: .from("documents") .update(rollbackDocumentPayload) .eq("id", id) - .eq("owner_id", user.id); + .eq("owner_id", user.id) + .eq("updated_at", rollbackFence); if (rollbackError) { throw new Error(`Failed to enqueue reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); } diff --git a/src/app/api/documents/bulk/reindex/route.ts b/src/app/api/documents/bulk/reindex/route.ts index 109674d960..b4c107a8c9 100644 --- a/src/app/api/documents/bulk/reindex/route.ts +++ b/src/app/api/documents/bulk/reindex/route.ts @@ -5,7 +5,11 @@ import { upsertDocumentDeepMemory } from "@/lib/deep-memory"; import { upsertDocumentEnrichment } from "@/lib/document-enrichment"; import { env, isDemoMode } from "@/lib/env"; import { jsonError, PublicApiError } from "@/lib/http"; -import { checkIngestionMutationSafety, ingestionMutationSafetyPayload } from "@/lib/ingestion-mutation-safety"; +import { + checkIngestionMutationSafety, + ingestionMutationSafetyPayload, + ingestionRollbackFenceStamp, +} from "@/lib/ingestion-mutation-safety"; import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; import { invalidateRagCachesForOwner } from "@/lib/rag"; import { @@ -172,6 +176,12 @@ export async function POST(request: Request) { } const atomicReindex = isAtomicReindexCandidate(document); + // Rollback fence: same pattern as the single-document reindex route — + // the queue-state write stamps updated_at and the rollback matches on + // the stamp, so a stale rollback cannot revert a newer queue state + // written by an overlapping reindex/retry. The competing-job SELECT is + // only a fast path; the fence closes the check-then-write race. + const rollbackFence = ingestionRollbackFenceStamp(); const rollbackDocumentPayload = atomicReindex ? { error_message: document.error_message ?? null } : { @@ -185,8 +195,15 @@ export async function POST(request: Request) { .from("documents") .update( atomicReindex - ? { error_message: null } - : { status: "queued", error_message: null, page_count: 0, chunk_count: 0, image_count: 0 }, + ? { error_message: null, updated_at: rollbackFence } + : { + status: "queued", + error_message: null, + page_count: 0, + chunk_count: 0, + image_count: 0, + updated_at: rollbackFence, + }, ) .eq("id", document.id) .eq("owner_id", user.id); @@ -220,7 +237,8 @@ export async function POST(request: Request) { .from("documents") .update(rollbackDocumentPayload) .eq("id", document.id) - .eq("owner_id", user.id); + .eq("owner_id", user.id) + .eq("updated_at", rollbackFence); if (rollbackError) { throw new Error(`Failed to enqueue bulk reindex job: ${jobError.message}; rollback failed: ${rollbackError.message}`); } diff --git a/src/app/api/ingestion/jobs/[id]/retry/route.ts b/src/app/api/ingestion/jobs/[id]/retry/route.ts index 31ba379c1c..a10cc26746 100644 --- a/src/app/api/ingestion/jobs/[id]/retry/route.ts +++ b/src/app/api/ingestion/jobs/[id]/retry/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import { z } from "zod"; import { env, isDemoMode } from "@/lib/env"; import { jsonError } from "@/lib/http"; +import { ingestionRollbackFenceStamp } from "@/lib/ingestion-mutation-safety"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; import { parseRouteParams } from "@/lib/validation/params"; @@ -43,6 +44,13 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: // job is NOT processing, OR its lock is already stale, OR it has no lock. const staleThreshold = new Date(Date.now() - env.WORKER_STALE_AFTER_MINUTES * 60_000).toISOString(); + // Rollback fence: next_run_at doubles as a per-request stamp. Two + // overlapping retries write generically identical resets (pending/queued/ + // attempt 0), so the rollback below additionally matches on this exact + // value — a stale rollback from the losing request affects zero rows + // instead of reverting the winning request's reset. + const resetNextRunAt = ingestionRollbackFenceStamp(); + const { data, error } = await supabase .from("ingestion_jobs") .update({ @@ -54,7 +62,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: max_attempts: env.WORKER_MAX_ATTEMPTS, locked_at: null, locked_by: null, - next_run_at: new Date().toISOString(), + next_run_at: resetNextRunAt, completed_at: null, }) .eq("id", id) @@ -78,9 +86,11 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: // job start (worker/main.ts), so resetting before enqueue would leave a previously-good // clinical document with zero index if the worker never runs or fails permanently. We // only re-queue; the prior index stays live until the worker commits a fresh one. + // updated_at is stamped so the reindex routes' rollback fences (which + // match on documents.updated_at) can see this competing queue-state write. const { error: documentError } = await supabase .from("documents") - .update({ status: "queued", error_message: null }) + .update({ status: "queued", error_message: null, updated_at: ingestionRollbackFenceStamp() }) .eq("id", job.document_id) .eq("owner_id", user.id); if (documentError) { @@ -104,7 +114,8 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: .eq("progress", 0) .eq("attempt_count", 0) .is("locked_at", null) - .is("locked_by", null); + .is("locked_by", null) + .eq("next_run_at", resetNextRunAt); if (rollbackError) { throw new Error(`${documentError.message}; failed to roll back retried job state: ${rollbackError.message}`); } diff --git a/src/lib/ingestion-mutation-safety.ts b/src/lib/ingestion-mutation-safety.ts index 4bbf5bc532..45832429fc 100644 --- a/src/lib/ingestion-mutation-safety.ts +++ b/src/lib/ingestion-mutation-safety.ts @@ -64,6 +64,19 @@ function activeJobMessage(documentCount: number, staleCount: number) { : "One or more selected documents already have pending or processing indexing work."; } +// Rollback fence: a timestamptz value unique to this request. Queue-state +// writes stamp the row with it so the compensating rollback can run as a +// single conditional UPDATE (`.eq` on the stamp) — atomic server-side. If a +// competing request re-writes the row between our write and our rollback, the +// stamp no longer matches and the rollback affects zero rows instead of +// clobbering the newer queue state. JS Date carries only millisecond +// precision while timestamptz stores microseconds, so random microsecond +// digits keep two same-millisecond requests distinct. +export function ingestionRollbackFenceStamp(now = new Date()) { + const microseconds = String(Math.floor(Math.random() * 1000)).padStart(3, "0"); + return now.toISOString().replace("Z", `${microseconds}Z`); +} + export async function checkIngestionMutationSafety(args: { supabase: SupabaseAdminClient; documentIds: string[]; diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index fdd36ed2f1..e3b22007af 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -1068,7 +1068,12 @@ describe("private document API access", () => { expect(response.status).toBe(200); // IDX-H1: only re-queue; never zero the chunk/page counts here (the worker resets at start). - expect(documentUpdate?.updatePayload).toEqual({ status: "queued", error_message: null }); + // updated_at is the rollback-fence stamp shared with the reindex routes. + expect(documentUpdate?.updatePayload).toEqual({ + status: "queued", + error_message: null, + updated_at: expect.any(String), + }); expect(client.rpc).not.toHaveBeenCalled(); }); @@ -1128,6 +1133,12 @@ describe("private document API access", () => { next_run_at: previousJob.next_run_at, completed_at: previousJob.completed_at, }); + // Rollback fence: the rollback must be conditional on the exact + // next_run_at this reset wrote, so it cannot revert a concurrent retry's + // newer reset that re-wrote the same generic pending/queued fields. + const resetNextRunAt = (jobUpdates[0]?.updatePayload as { next_run_at?: string }).next_run_at; + expect(typeof resetNextRunAt).toBe("string"); + expect(jobUpdates[1]?.filters).toContainEqual({ column: "next_run_at", value: resetNextRunAt }); }); it("runs enrichment-only reindex for owned indexed documents using generic metadata", async () => { @@ -1600,6 +1611,7 @@ describe("private document API access", () => { page_count: 0, chunk_count: 0, image_count: 0, + updated_at: expect.any(String), }); expect(documentUpdates[1]?.updatePayload).toEqual({ status: "failed", @@ -1608,6 +1620,11 @@ describe("private document API access", () => { chunk_count: 34, image_count: 2, }); + // Rollback fence: the rollback must be conditional on the updated_at + // stamp the queue-state write set, so it is a single atomic UPDATE that + // cannot revert a newer queue state written by an overlapping request. + const fence = (documentUpdates[0]?.updatePayload as { updated_at?: string }).updated_at; + expect(documentUpdates[1]?.filters).toContainEqual({ column: "updated_at", value: fence }); }); it("skips single-document rollback when a competing active job appears after the safety check", async () => { @@ -1658,6 +1675,7 @@ describe("private document API access", () => { page_count: 0, chunk_count: 0, image_count: 0, + updated_at: expect.any(String), }); }); @@ -1715,6 +1733,7 @@ describe("private document API access", () => { page_count: 0, chunk_count: 0, image_count: 0, + updated_at: expect.any(String), }); expect(documentUpdates[1]?.updatePayload).toEqual({ status: "failed", @@ -1723,6 +1742,10 @@ describe("private document API access", () => { chunk_count: 8, image_count: 1, }); + // Rollback fence: same atomic-conditional guard as the single-document + // reindex route — the rollback matches on the stamp this request wrote. + const fence = (documentUpdates[0]?.updatePayload as { updated_at?: string }).updated_at; + expect(documentUpdates[1]?.filters).toContainEqual({ column: "updated_at", value: fence }); }); it("skips bulk rollback when a competing active job appears after the safety check", async () => { @@ -1782,6 +1805,7 @@ describe("private document API access", () => { page_count: 0, chunk_count: 0, image_count: 0, + updated_at: expect.any(String), }); });