fix(chunking): keep dose units wrapped onto their own line by PDF extraction

src/lib/chunking.ts

@@ -26,6 +26,12 @@ const lineNoisePatterns: RegExp[] = [
  /^\s*[-*_]{3,}\s*$/,
  /^\s*[\u25cf\u25e6\u2022]\s*$/,
 ];
+// PDF extraction wraps a dose across lines in narrow table cells — PyMuPDF's
+// get_text("text", sort=True) emits each rendered line separately, so
+// "12.5 mg" arrives as "12.5\nmg". The bare unit line is then <= 2 characters
+// and looksLikeMetadataNoise would delete it, leaving a unitless dose in the
+// indexed chunk. removePageNoise rejoins these lines before filtering.
+const clinicalUnitLinePattern = /^(?:mg|mcg|µg|ug|g|kg|ml|l|iu|u|mmol|%)$/i;
 const maxImageContextItemsPerPage = 3;
 const highYieldSectionPattern =
  /\b(?:medicat|dose|dosage|dosing|administer|titrate|threshold|cut[\s-]?off|withhold|cease|stop|monitor|baseline|fbc|anc|neutrophil|level|risk|red flag|urgent|escalat|contraindicat|caution|toxicity|required|must|criteria|observation)\b/i;
@@ -114,10 +120,30 @@ function buildRepeatedBoilerplateLines(inputs: ChunkInput[]) {
  return new Set([...counts.entries()].filter(([, count]) => count >= 2).map(([line]) => line));
 }
 
+// Rejoin a wrapped dose unit ("12.5" / "mg" on consecutive lines) into a single
+// line so the unit is not deleted as short-line extraction debris. Only merges
+// when the previous line ends in a digit and is not a standalone page footer —
+// a lone unit token with no preceding number stays subject to the noise filter.
+function rejoinWrappedDoseUnits(lines: string[]) {
+ return lines.reduce<string[]>((kept, line) => {
+ const previous = kept[kept.length - 1];
+ if (
+ previous &&
+ /\d$/.test(previous) &&
+ clinicalUnitLinePattern.test(line) &&
+ !lineNoisePatterns.some((pattern) => pattern.test(previous))
+ ) {
+ kept[kept.length - 1] = `${previous} ${line}`;
+ } else {
+ kept.push(line);
+ }
+ return kept;
+ }, []);
+}
+
 function removePageNoise(text: string, repeatedBoilerplateLines = new Set<string>()) {
- return text
- .split(/\r?\n/)
- .map((line) => line.trim())
+ const lines = rejoinWrappedDoseUnits(text.split(/\r?\n/).map((line) => line.trim()));
+ return lines
  .filter((line) => {
  if (line === "") return true;
  if (looksLikeMetadataNoise(line)) return false;

tests/chunking.test.ts

@@ -44,6 +44,37 @@ describe("chunkTextWithOverlap", () => {
  expect(joined).not.toMatch(/Page 3 of 12/);
  });
 
+ // PDF extraction (PyMuPDF get_text with sort=True) wraps a dose across lines
+ // in narrow table cells ("12.5\nmg"). The bare unit line is <= 2 chars, so the
+ // short-line debris rule used to delete it, indexing a unitless "12.5".
+ it("rejoins a dose unit wrapped onto its own line instead of deleting it as debris", () => {
+ const text = "Commence clozapine at a starting dose of\n12.5\nmg\nonce daily.";
+ const joined = chunkTextWithOverlap(text, 2000, 200).join(" ");
+ expect(joined).toContain("12.5 mg");
+ });
+
+ it("rejoins wrapped units after bare-integer doses and longer unit tokens", () => {
+ const text = "Thiamine\n300\nmg\ndaily. Fludrocortisone\n100\nmcg\nmane.";
+ const joined = chunkTextWithOverlap(text, 2000, 200).join(" ");
+ expect(joined).toContain("300 mg");
+ expect(joined).toContain("100 mcg");
+ });
+
+ it("still drops a lone unit token with no preceding dose number", () => {
+ const text = "Withhold clozapine.\nmg\nRepeat the full blood count.";
+ const joined = chunkTextWithOverlap(text, 2000, 200).join(" ");
+ expect(joined).not.toMatch(/\bmg\b/);
+ expect(joined).toContain("Withhold clozapine.");
+ expect(joined).toContain("Repeat the full blood count.");
+ });
+
+ it("does not merge a unit token into a standalone page footer", () => {
+ const text = "Monitor lithium levels.\nPage 3 of 12\nmg\nReview renally.";
+ const joined = chunkTextWithOverlap(text, 2000, 200).join(" ");
+ expect(joined).not.toMatch(/Page 3 of 12/);
+ expect(joined).not.toMatch(/\bmg\b/);
+ });
+
  it("prefers paragraph boundaries before falling back to sentence windows", () => {
  const chunks = chunkTextWithOverlap("Heading\n\nFirst clinical paragraph.\n\nSecond clinical paragraph.", 32, 4);