From 6c273df6fa11d7d01fb9d413d0f3ad0177bffaee Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:04:53 +0800 Subject: [PATCH 1/2] fix(chunking): keep dose units wrapped onto their own line by PDF extraction PyMuPDF get_text(sort=True) splits a dose in a narrow table cell across lines ('12.5\nmg'); the <=2-char debris rule then deleted the unit line, indexing a unitless dose. removePageNoise now rejoins a unit-only line (mg, mcg, mL, IU, %, ...) to a preceding digit-ending line before filtering. A lone unit with no preceding number is still dropped, and page footers are never merged into. Co-Authored-By: Claude Fable 5 --- src/lib/chunking.ts | 32 +++++++++++++++++++++++++++++--- tests/chunking.test.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 651a61e14c..1611b5845c 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -26,6 +26,12 @@ const lineNoisePatterns: RegExp[] = [ /^\s*[-*_]{3,}\s*$/, /^\s*[\u25cf\u25e6\u2022]\s*$/, ]; +// PDF extraction wraps a dose across lines in narrow table cells — PyMuPDF's +// get_text("text", sort=True) emits each rendered line separately, so +// "12.5 mg" arrives as "12.5\nmg". The bare unit line is then <= 2 characters +// and looksLikeMetadataNoise would delete it, leaving a unitless dose in the +// indexed chunk. removePageNoise rejoins these lines before filtering. +const clinicalUnitLinePattern = /^(?:mg|mcg|µg|ug|g|kg|ml|l|iu|u|mmol|%)$/i; const maxImageContextItemsPerPage = 3; const highYieldSectionPattern = /\b(?:medicat|dose|dosage|dosing|administer|titrate|threshold|cut[\s-]?off|withhold|cease|stop|monitor|baseline|fbc|anc|neutrophil|level|risk|red flag|urgent|escalat|contraindicat|caution|toxicity|required|must|criteria|observation)\b/i; @@ -114,10 +120,30 @@ function buildRepeatedBoilerplateLines(inputs: ChunkInput[]) { return new Set([...counts.entries()].filter(([, count]) => count >= 2).map(([line]) => line)); } +// Rejoin a wrapped dose unit ("12.5" / "mg" on consecutive lines) into a single +// line so the unit is not deleted as short-line extraction debris. Only merges +// when the previous line ends in a digit and is not a standalone page footer — +// a lone unit token with no preceding number stays subject to the noise filter. +function rejoinWrappedDoseUnits(lines: string[]) { + return lines.reduce((kept, line) => { + const previous = kept[kept.length - 1]; + if ( + previous && + /\d$/.test(previous) && + clinicalUnitLinePattern.test(line) && + !lineNoisePatterns.some((pattern) => pattern.test(previous)) + ) { + kept[kept.length - 1] = `${previous} ${line}`; + } else { + kept.push(line); + } + return kept; + }, []); +} + function removePageNoise(text: string, repeatedBoilerplateLines = new Set()) { - return text - .split(/\r?\n/) - .map((line) => line.trim()) + const lines = rejoinWrappedDoseUnits(text.split(/\r?\n/).map((line) => line.trim())); + return lines .filter((line) => { if (line === "") return true; if (looksLikeMetadataNoise(line)) return false; diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index aeedf38fb3..3a12d8879d 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -44,6 +44,37 @@ describe("chunkTextWithOverlap", () => { expect(joined).not.toMatch(/Page 3 of 12/); }); + // PDF extraction (PyMuPDF get_text with sort=True) wraps a dose across lines + // in narrow table cells ("12.5\nmg"). The bare unit line is <= 2 chars, so the + // short-line debris rule used to delete it, indexing a unitless "12.5". + it("rejoins a dose unit wrapped onto its own line instead of deleting it as debris", () => { + const text = "Commence clozapine at a starting dose of\n12.5\nmg\nonce daily."; + const joined = chunkTextWithOverlap(text, 2000, 200).join(" "); + expect(joined).toContain("12.5 mg"); + }); + + it("rejoins wrapped units after bare-integer doses and longer unit tokens", () => { + const text = "Thiamine\n300\nmg\ndaily. Fludrocortisone\n100\nmcg\nmane."; + const joined = chunkTextWithOverlap(text, 2000, 200).join(" "); + expect(joined).toContain("300 mg"); + expect(joined).toContain("100 mcg"); + }); + + it("still drops a lone unit token with no preceding dose number", () => { + const text = "Withhold clozapine.\nmg\nRepeat the full blood count."; + const joined = chunkTextWithOverlap(text, 2000, 200).join(" "); + expect(joined).not.toMatch(/\bmg\b/); + expect(joined).toContain("Withhold clozapine."); + expect(joined).toContain("Repeat the full blood count."); + }); + + it("does not merge a unit token into a standalone page footer", () => { + const text = "Monitor lithium levels.\nPage 3 of 12\nmg\nReview renally."; + const joined = chunkTextWithOverlap(text, 2000, 200).join(" "); + expect(joined).not.toMatch(/Page 3 of 12/); + expect(joined).not.toMatch(/\bmg\b/); + }); + it("prefers paragraph boundaries before falling back to sentence windows", () => { const chunks = chunkTextWithOverlap("Heading\n\nFirst clinical paragraph.\n\nSecond clinical paragraph.", 32, 4); From 56f2f05d31afe98e9a0b6fc7dbf60d3d99f85372 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 7 Jul 2026 03:19:31 +0000 Subject: [PATCH 2/2] ci: retrigger checks after billing fix Co-authored-by: BigSimmo