diff --git a/package.json b/package.json index c690f8b7f3..5e0d118762 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "worker:once": "tsx worker/index.ts --once", "import:docs": "tsx scripts/import-documents.ts", "import:docs:20": "tsx scripts/import-documents.ts --queue-batch-size 20", + "measure:wrapped-dose-units": "tsx scripts/measure-wrapped-dose-prevalence.ts", "enrich:documents": "tsx scripts/enrich-documents.ts", "enrich:backfill": "tsx scripts/backfill-enrichment.ts", "classify:documents": "tsx scripts/classify-documents.ts", diff --git a/scripts/measure-wrapped-dose-prevalence.ts b/scripts/measure-wrapped-dose-prevalence.ts new file mode 100644 index 0000000000..1639925219 --- /dev/null +++ b/scripts/measure-wrapped-dose-prevalence.ts @@ -0,0 +1,140 @@ +import { loadEnvConfig } from "@next/env"; +import { loadAdminClient } from "./eval-utils"; + +// Must run before any import of @/lib/env (transitively via @/lib/chunking), +// which snapshots process.env at module load. countWrappedDoseUnitLines is +// therefore imported dynamically inside main(), after this call — the same +// deferral loadAdminClient uses for the admin client. +loadEnvConfig(process.cwd()); + +// Measures how many indexed pages carry a dose whose unit the pre-fix chunker +// deleted as short-line extraction debris ("12.5\nmg" -> the "mg" line dropped, +// indexing a unitless "12.5"). See the fix in src/lib/chunking.ts (PR #334). +// +// document_pages.text stores the RAW extracted page text (worker/main.ts writes +// cleanString(page.text), which only strips null bytes — removePageNoise runs +// later, inside buildChunks). So the wrapped unit is still present here and +// countWrappedDoseUnitLines reports exactly what the old chunker would have +// deleted from the corresponding chunk. This is an accurate, read-only measure +// that needs no PDF re-extraction and no re-index. +// +// Read-only: SELECTs document_pages/documents only; never writes. Usage: +// tsx scripts/measure-wrapped-dose-prevalence.ts [--limit N] [--top N] [--json] + +type PageRow = { + document_id: string; + page_number: number | null; + text: string | null; +}; + +type Args = { limit: number; top: number; json: boolean }; + +function parseArgs(): Args { + const argv = process.argv.slice(2); + const numberAfter = (flag: string, fallback: number) => { + const index = argv.indexOf(flag); + if (index === -1) return fallback; + const parsed = Number.parseInt(argv[index + 1] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; + }; + return { + limit: numberAfter("--limit", Number.POSITIVE_INFINITY), + top: numberAfter("--top", 15), + json: argv.includes("--json"), + }; +} + +async function main() { + const args = parseArgs(); + const { countWrappedDoseUnitLines } = await import("@/lib/chunking"); + const supabase = await loadAdminClient(); + + const pageSize = 1000; + let offset = 0; + let pagesScanned = 0; + let pagesAffected = 0; + let wrappedUnitTotal = 0; + const perDocument = new Map(); + + for (;;) { + if (pagesScanned >= args.limit) break; + const { data, error } = await supabase + .from("document_pages") + .select("document_id,page_number,text") + .order("document_id", { ascending: true }) + .order("page_number", { ascending: true }) + .range(offset, offset + pageSize - 1); + if (error) throw new Error(`document_pages read failed: ${error.message}`); + + const rows = (data ?? []) as PageRow[]; + if (rows.length === 0) break; + + for (const row of rows) { + if (pagesScanned >= args.limit) break; + pagesScanned += 1; + const count = countWrappedDoseUnitLines(row.text ?? ""); + if (count > 0) { + pagesAffected += 1; + wrappedUnitTotal += count; + perDocument.set(row.document_id, (perDocument.get(row.document_id) ?? 0) + count); + } + } + + if (rows.length < pageSize) break; + offset += pageSize; + } + + const affectedDocIds = [...perDocument.keys()]; + const titles = new Map(); + for (let index = 0; index < affectedDocIds.length; index += 500) { + const batch = affectedDocIds.slice(index, index + 500); + const { data, error } = await supabase.from("documents").select("id,title,file_name").in("id", batch); + if (error) throw new Error(`documents read failed: ${error.message}`); + for (const doc of data ?? []) { + titles.set(doc.id as string, (doc.title as string) || (doc.file_name as string) || (doc.id as string)); + } + } + + const topDocuments = [...perDocument.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, args.top) + .map(([id, count]) => ({ id, title: titles.get(id) ?? id, wrappedUnits: count })); + + const summary = { + pagesScanned, + pagesAffected, + documentsAffected: perDocument.size, + wrappedUnitTotal, + pageAffectedRate: pagesScanned > 0 ? Number((pagesAffected / pagesScanned).toFixed(4)) : 0, + topDocuments, + }; + + if (args.json) { + console.log(JSON.stringify(summary, null, 2)); + return; + } + + console.log(`Wrapped dose-unit prevalence (read-only, live corpus)`); + console.log(` pages scanned: ${summary.pagesScanned}`); + console.log(` pages affected: ${summary.pagesAffected} (${(summary.pageAffectedRate * 100).toFixed(2)}%)`); + console.log(` documents affected: ${summary.documentsAffected}`); + console.log(` wrapped units total: ${summary.wrappedUnitTotal}`); + if (topDocuments.length > 0) { + console.log(` most affected documents:`); + for (const doc of topDocuments) { + console.log(` ${String(doc.wrappedUnits).padStart(4)} ${doc.title}`); + } + } + if (summary.wrappedUnitTotal === 0) { + console.log(`\nNo stripped dose units found — a re-index would not recover any dose units.`); + } else { + console.log( + `\n${summary.wrappedUnitTotal} dose unit(s) across ${summary.documentsAffected} document(s) are indexed unitless; a re-index would recover them.`, + ); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 1611b5845c..40e44aed85 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -121,18 +121,27 @@ function buildRepeatedBoilerplateLines(inputs: ChunkInput[]) { } // Rejoin a wrapped dose unit ("12.5" / "mg" on consecutive lines) into a single -// line so the unit is not deleted as short-line extraction debris. Only merges -// when the previous line ends in a digit and is not a standalone page footer — -// a lone unit token with no preceding number stays subject to the noise filter. +// line so the unit is not deleted as short-line extraction debris. A lone unit +// token with no preceding number stays subject to the noise filter. +// +// A `line` is a wrapped dose unit continuing `previousLine` when the previous +// line ends in a digit and this line is a lone clinical unit token — the shape +// PDF extraction produces for "12.5\nmg". A standalone page footer preceding it +// (e.g. "Page 3 of 12") never counts, so a footer followed by a stray "mg" is +// left to the noise filter rather than being welded to the footer. +function isWrappedDoseUnitContinuation(previousLine: string | undefined, line: string) { + return Boolean( + previousLine && + /\d$/.test(previousLine) && + clinicalUnitLinePattern.test(line) && + !lineNoisePatterns.some((pattern) => pattern.test(previousLine)), + ); +} + function rejoinWrappedDoseUnits(lines: string[]) { return lines.reduce((kept, line) => { const previous = kept[kept.length - 1]; - if ( - previous && - /\d$/.test(previous) && - clinicalUnitLinePattern.test(line) && - !lineNoisePatterns.some((pattern) => pattern.test(previous)) - ) { + if (isWrappedDoseUnitContinuation(previous, line)) { kept[kept.length - 1] = `${previous} ${line}`; } else { kept.push(line); @@ -141,6 +150,21 @@ function rejoinWrappedDoseUnits(lines: string[]) { }, []); } +// How many wrapped dose units the rejoin would repair in `text`. Lets a corpus +// sample measure prevalence of the extraction bug without re-indexing: a +// positive count means the pre-fix chunker would have deleted that unit, +// indexing a unitless dose. Must run on raw extracted page text (the same input +// removePageNoise sees), since the deleted unit is not recoverable from stored +// chunks. Shares isWrappedDoseUnitContinuation with the fix so the two can't drift. +export function countWrappedDoseUnitLines(text: string) { + const lines = text.split(/\r?\n/).map((line) => line.trim()); + let count = 0; + for (let index = 1; index < lines.length; index += 1) { + if (isWrappedDoseUnitContinuation(lines[index - 1], lines[index])) count += 1; + } + return count; +} + function removePageNoise(text: string, repeatedBoilerplateLines = new Set()) { const lines = rejoinWrappedDoseUnits(text.split(/\r?\n/).map((line) => line.trim())); return lines diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 3a12d8879d..5ef9990b9c 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -6,6 +6,7 @@ import { buildImageTag, chunkContentKey, chunkTextWithOverlap, + countWrappedDoseUnitLines, } from "../src/lib/chunking"; describe("chunkTextWithOverlap", () => { @@ -75,6 +76,20 @@ describe("chunkTextWithOverlap", () => { expect(joined).not.toMatch(/\bmg\b/); }); + // countWrappedDoseUnitLines shares its predicate with the rejoin, so these + // cases mirror the rejoin tests above — it must count exactly the units the + // pre-fix chunker deleted. Used by scripts/measure-wrapped-dose-prevalence.ts + // to quantify the bug in the live corpus without a re-index. + it("counts wrapped dose units on raw page text and ignores non-bug shapes", () => { + expect(countWrappedDoseUnitLines("dose of\n12.5\nmg\nonce daily")).toBe(1); + expect(countWrappedDoseUnitLines("Thiamine\n300\nmg\ndaily. Fludrocortisone\n100\nmcg\nmane.")).toBe(2); + // No preceding number, and a page footer before the unit: neither counts. + expect(countWrappedDoseUnitLines("Withhold clozapine.\nmg\nrepeat")).toBe(0); + expect(countWrappedDoseUnitLines("Monitor levels.\nPage 3 of 12\nmg\nreview")).toBe(0); + // Already-inline "12.5 mg" is correct extraction, not a wrapped unit. + expect(countWrappedDoseUnitLines("Commence at 12.5 mg once daily.")).toBe(0); + }); + it("prefers paragraph boundaries before falling back to sentence windows", () => { const chunks = chunkTextWithOverlap("Heading\n\nFirst clinical paragraph.\n\nSecond clinical paragraph.", 32, 4);