From 8070da6f6e35b85cdca8688223b5964b639a9999 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:59:05 +0800 Subject: [PATCH 1/2] feat(chunking): measure wrapped-dose-unit prevalence in the live corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export countWrappedDoseUnitLines from chunking.ts (sharing the fix's isWrappedDoseUnitContinuation predicate so the two cannot drift) and add scripts/measure-wrapped-dose-prevalence.ts. document_pages.text retains the raw extracted page text (worker writes cleanString(page.text); removePageNoise runs later in buildChunks), so counting wrapped units on that column is an accurate, read-only measure of how many indexed doses lost their unit pre-fix — no PDF re-extraction or re-index required. This quantifies whether a re-index is worth the spend (PR #334 follow-up). Co-Authored-By: Claude Fable 5 --- package.json | 1 + scripts/measure-wrapped-dose-prevalence.ts | 136 +++++++++++++++++++++ src/lib/chunking.ts | 42 +++++-- tests/chunking.test.ts | 15 +++ 4 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 scripts/measure-wrapped-dose-prevalence.ts diff --git a/package.json b/package.json index c690f8b7f3..5e0d118762 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "worker:once": "tsx worker/index.ts --once", "import:docs": "tsx scripts/import-documents.ts", "import:docs:20": "tsx scripts/import-documents.ts --queue-batch-size 20", + "measure:wrapped-dose-units": "tsx scripts/measure-wrapped-dose-prevalence.ts", "enrich:documents": "tsx scripts/enrich-documents.ts", "enrich:backfill": "tsx scripts/backfill-enrichment.ts", "classify:documents": "tsx scripts/classify-documents.ts", diff --git a/scripts/measure-wrapped-dose-prevalence.ts b/scripts/measure-wrapped-dose-prevalence.ts new file mode 100644 index 0000000000..28dcf8f4f4 --- /dev/null +++ b/scripts/measure-wrapped-dose-prevalence.ts @@ -0,0 +1,136 @@ +import { loadEnvConfig } from "@next/env"; +import { countWrappedDoseUnitLines } from "@/lib/chunking"; +import { loadAdminClient } from "./eval-utils"; + +loadEnvConfig(process.cwd()); + +// Measures how many indexed pages carry a dose whose unit the pre-fix chunker +// deleted as short-line extraction debris ("12.5\nmg" -> the "mg" line dropped, +// indexing a unitless "12.5"). See the fix in src/lib/chunking.ts (PR #334). +// +// document_pages.text stores the RAW extracted page text (worker/main.ts writes +// cleanString(page.text), which only strips null bytes — removePageNoise runs +// later, inside buildChunks). So the wrapped unit is still present here and +// countWrappedDoseUnitLines reports exactly what the old chunker would have +// deleted from the corresponding chunk. This is an accurate, read-only measure +// that needs no PDF re-extraction and no re-index. +// +// Read-only: SELECTs document_pages/documents only; never writes. Usage: +// tsx scripts/measure-wrapped-dose-prevalence.ts [--limit N] [--top N] [--json] + +type PageRow = { + document_id: string; + page_number: number | null; + text: string | null; +}; + +type Args = { limit: number; top: number; json: boolean }; + +function parseArgs(): Args { + const argv = process.argv.slice(2); + const numberAfter = (flag: string, fallback: number) => { + const index = argv.indexOf(flag); + if (index === -1) return fallback; + const parsed = Number.parseInt(argv[index + 1] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; + }; + return { + limit: numberAfter("--limit", Number.POSITIVE_INFINITY), + top: numberAfter("--top", 15), + json: argv.includes("--json"), + }; +} + +async function main() { + const args = parseArgs(); + const supabase = await loadAdminClient(); + + const pageSize = 1000; + let offset = 0; + let pagesScanned = 0; + let pagesAffected = 0; + let wrappedUnitTotal = 0; + const perDocument = new Map(); + + for (;;) { + if (pagesScanned >= args.limit) break; + const { data, error } = await supabase + .from("document_pages") + .select("document_id,page_number,text") + .order("document_id", { ascending: true }) + .order("page_number", { ascending: true }) + .range(offset, offset + pageSize - 1); + if (error) throw new Error(`document_pages read failed: ${error.message}`); + + const rows = (data ?? []) as PageRow[]; + if (rows.length === 0) break; + + for (const row of rows) { + if (pagesScanned >= args.limit) break; + pagesScanned += 1; + const count = countWrappedDoseUnitLines(row.text ?? ""); + if (count > 0) { + pagesAffected += 1; + wrappedUnitTotal += count; + perDocument.set(row.document_id, (perDocument.get(row.document_id) ?? 0) + count); + } + } + + if (rows.length < pageSize) break; + offset += pageSize; + } + + const affectedDocIds = [...perDocument.keys()]; + const titles = new Map(); + for (let index = 0; index < affectedDocIds.length; index += 500) { + const batch = affectedDocIds.slice(index, index + 500); + const { data, error } = await supabase.from("documents").select("id,title,file_name").in("id", batch); + if (error) throw new Error(`documents read failed: ${error.message}`); + for (const doc of data ?? []) { + titles.set(doc.id as string, (doc.title as string) || (doc.file_name as string) || (doc.id as string)); + } + } + + const topDocuments = [...perDocument.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, args.top) + .map(([id, count]) => ({ id, title: titles.get(id) ?? id, wrappedUnits: count })); + + const summary = { + pagesScanned, + pagesAffected, + documentsAffected: perDocument.size, + wrappedUnitTotal, + pageAffectedRate: pagesScanned > 0 ? Number((pagesAffected / pagesScanned).toFixed(4)) : 0, + topDocuments, + }; + + if (args.json) { + console.log(JSON.stringify(summary, null, 2)); + return; + } + + console.log(`Wrapped dose-unit prevalence (read-only, live corpus)`); + console.log(` pages scanned: ${summary.pagesScanned}`); + console.log(` pages affected: ${summary.pagesAffected} (${(summary.pageAffectedRate * 100).toFixed(2)}%)`); + console.log(` documents affected: ${summary.documentsAffected}`); + console.log(` wrapped units total: ${summary.wrappedUnitTotal}`); + if (topDocuments.length > 0) { + console.log(` most affected documents:`); + for (const doc of topDocuments) { + console.log(` ${String(doc.wrappedUnits).padStart(4)} ${doc.title}`); + } + } + if (summary.wrappedUnitTotal === 0) { + console.log(`\nNo stripped dose units found — a re-index would not recover any dose units.`); + } else { + console.log( + `\n${summary.wrappedUnitTotal} dose unit(s) across ${summary.documentsAffected} document(s) are indexed unitless; a re-index would recover them.`, + ); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 1611b5845c..40e44aed85 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -121,18 +121,27 @@ function buildRepeatedBoilerplateLines(inputs: ChunkInput[]) { } // Rejoin a wrapped dose unit ("12.5" / "mg" on consecutive lines) into a single -// line so the unit is not deleted as short-line extraction debris. Only merges -// when the previous line ends in a digit and is not a standalone page footer — -// a lone unit token with no preceding number stays subject to the noise filter. +// line so the unit is not deleted as short-line extraction debris. A lone unit +// token with no preceding number stays subject to the noise filter. +// +// A `line` is a wrapped dose unit continuing `previousLine` when the previous +// line ends in a digit and this line is a lone clinical unit token — the shape +// PDF extraction produces for "12.5\nmg". A standalone page footer preceding it +// (e.g. "Page 3 of 12") never counts, so a footer followed by a stray "mg" is +// left to the noise filter rather than being welded to the footer. +function isWrappedDoseUnitContinuation(previousLine: string | undefined, line: string) { + return Boolean( + previousLine && + /\d$/.test(previousLine) && + clinicalUnitLinePattern.test(line) && + !lineNoisePatterns.some((pattern) => pattern.test(previousLine)), + ); +} + function rejoinWrappedDoseUnits(lines: string[]) { return lines.reduce((kept, line) => { const previous = kept[kept.length - 1]; - if ( - previous && - /\d$/.test(previous) && - clinicalUnitLinePattern.test(line) && - !lineNoisePatterns.some((pattern) => pattern.test(previous)) - ) { + if (isWrappedDoseUnitContinuation(previous, line)) { kept[kept.length - 1] = `${previous} ${line}`; } else { kept.push(line); @@ -141,6 +150,21 @@ function rejoinWrappedDoseUnits(lines: string[]) { }, []); } +// How many wrapped dose units the rejoin would repair in `text`. Lets a corpus +// sample measure prevalence of the extraction bug without re-indexing: a +// positive count means the pre-fix chunker would have deleted that unit, +// indexing a unitless dose. Must run on raw extracted page text (the same input +// removePageNoise sees), since the deleted unit is not recoverable from stored +// chunks. Shares isWrappedDoseUnitContinuation with the fix so the two can't drift. +export function countWrappedDoseUnitLines(text: string) { + const lines = text.split(/\r?\n/).map((line) => line.trim()); + let count = 0; + for (let index = 1; index < lines.length; index += 1) { + if (isWrappedDoseUnitContinuation(lines[index - 1], lines[index])) count += 1; + } + return count; +} + function removePageNoise(text: string, repeatedBoilerplateLines = new Set()) { const lines = rejoinWrappedDoseUnits(text.split(/\r?\n/).map((line) => line.trim())); return lines diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 3a12d8879d..5ef9990b9c 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -6,6 +6,7 @@ import { buildImageTag, chunkContentKey, chunkTextWithOverlap, + countWrappedDoseUnitLines, } from "../src/lib/chunking"; describe("chunkTextWithOverlap", () => { @@ -75,6 +76,20 @@ describe("chunkTextWithOverlap", () => { expect(joined).not.toMatch(/\bmg\b/); }); + // countWrappedDoseUnitLines shares its predicate with the rejoin, so these + // cases mirror the rejoin tests above — it must count exactly the units the + // pre-fix chunker deleted. Used by scripts/measure-wrapped-dose-prevalence.ts + // to quantify the bug in the live corpus without a re-index. + it("counts wrapped dose units on raw page text and ignores non-bug shapes", () => { + expect(countWrappedDoseUnitLines("dose of\n12.5\nmg\nonce daily")).toBe(1); + expect(countWrappedDoseUnitLines("Thiamine\n300\nmg\ndaily. Fludrocortisone\n100\nmcg\nmane.")).toBe(2); + // No preceding number, and a page footer before the unit: neither counts. + expect(countWrappedDoseUnitLines("Withhold clozapine.\nmg\nrepeat")).toBe(0); + expect(countWrappedDoseUnitLines("Monitor levels.\nPage 3 of 12\nmg\nreview")).toBe(0); + // Already-inline "12.5 mg" is correct extraction, not a wrapped unit. + expect(countWrappedDoseUnitLines("Commence at 12.5 mg once daily.")).toBe(0); + }); + it("prefers paragraph boundaries before falling back to sentence windows", () => { const chunks = chunkTextWithOverlap("Heading\n\nFirst clinical paragraph.\n\nSecond clinical paragraph.", 32, 4); From 043de065ed8364968aa5c13ab73521b263174c0e Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:02:27 +0800 Subject: [PATCH 2/2] fix(measure): defer chunking import until after loadEnvConfig @/lib/env snapshots process.env at module load, so importing chunking statically (before loadEnvConfig) left the admin client with empty Supabase config. Import countWrappedDoseUnitLines dynamically inside main(), matching loadAdminClient. Verified against the live corpus. Co-Authored-By: Claude Fable 5 --- scripts/measure-wrapped-dose-prevalence.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/measure-wrapped-dose-prevalence.ts b/scripts/measure-wrapped-dose-prevalence.ts index 28dcf8f4f4..1639925219 100644 --- a/scripts/measure-wrapped-dose-prevalence.ts +++ b/scripts/measure-wrapped-dose-prevalence.ts @@ -1,7 +1,10 @@ import { loadEnvConfig } from "@next/env"; -import { countWrappedDoseUnitLines } from "@/lib/chunking"; import { loadAdminClient } from "./eval-utils"; +// Must run before any import of @/lib/env (transitively via @/lib/chunking), +// which snapshots process.env at module load. countWrappedDoseUnitLines is +// therefore imported dynamically inside main(), after this call — the same +// deferral loadAdminClient uses for the admin client. loadEnvConfig(process.cwd()); // Measures how many indexed pages carry a dose whose unit the pre-fix chunker @@ -43,6 +46,7 @@ function parseArgs(): Args { async function main() { const args = parseArgs(); + const { countWrappedDoseUnitLines } = await import("@/lib/chunking"); const supabase = await loadAdminClient(); const pageSize = 1000;