From 1f30ecce2e890bc51b82c56f757f58d1d214dac2 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:01:51 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix(display):=20polish=20extracted=20source?= =?UTF-8?q?=20snippets=20=E2=80=94=20strip=20PSPF=20banners,=20repair=20tr?= =?UTF-8?q?uncation,=20normalize=20bullets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source cards showed raw PDF-extraction artifacts: duplicated OFFICIAL classification banners glued to titles and body text, inline bullet and sub-bullet glyphs, snippets starting mid-clause or mid-numbered-list, and stored synopses cut mid-word before an ellipsis. - stripClassificationBanner: ALL-CAPS, line-anchored PSPF marking removal, wired into the low-yield noise strippers so every compact/snippet/synopsis surface inherits it; verbatim quotes keep banners by design (pinned by test) - sourceTextForCompactDisplay: inline bullets become '; ' separators; the PDF sub-bullet 'o' glyph is normalized conservatively (digit and single-capital guards); repairTruncatedCompactTail repairs stored mid-word cuts and never leaves meaning-inverting stubs (do not… -> …) - compactSourceSnippet: ellipsis-aware fragment pipeline, drops the card's own glued title (dropTitle), sheds orphaned closers and mid-list ordinals, starts at a real sentence when a partial first fragment has substance - cleanDisplayTitle/cleanCitationTitle: banner strip + missing space before acronym parentheticals (Guideline(EMHS) -> Guideline (EMHS)) - chunking: truncateAtWordBoundary replaces raw slices in synopsis/caption builders; banner-only lines dropped and per-sentence banner strip so new chunks store clean synopses (vectors untouched, no re-embed) Co-Authored-By: Claude Fable 5 --- docs/process-hardening.md | 1 + src/components/ClinicalDashboard.tsx | 2 +- .../clinical-dashboard/display-text.ts | 83 ++++++++++++-- src/lib/chunking.ts | 37 ++++++- src/lib/citations.ts | 11 +- src/lib/source-text-sanitizer.ts | 101 ++++++++++++++--- tests/chunking.test.ts | 39 +++++++ tests/display-text.test.ts | 88 ++++++++++++++- tests/rendered-text-formatting.test.ts | 9 ++ tests/source-text-sanitizer.test.ts | 102 ++++++++++++++++++ 10 files changed, 443 insertions(+), 30 deletions(-) diff --git a/docs/process-hardening.md b/docs/process-hardening.md index d41ba98b21..2bd6c110d0 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -46,6 +46,7 @@ This document turns the current process review into phased, durable repo practic - **Document-derived text must never be rendered raw.** Any value pulled from an ingested document — answer prose, exact quotes, source snippets, document titles, image captions, extracted table text — must be routed through a `source-text-sanitizer` (`src/lib/source-text-sanitizer.ts`) or `display-text` (`src/components/clinical-dashboard/display-text.ts`) helper before it reaches JSX. Verbatim quotes use `sourceTextForVerbatimQuote`; titles use `cleanDisplayTitle`; snippets/captions use `sourceTextForCompactDisplay`. - `normalizeExtractedGlyphs` is the shared, lossless glyph-repair primitive (ligatures, soft hyphens, zero-width/control chars). It is wired into the base `compactWhitespace`/`readableWhitespace` cleaners and into ingestion (`buildChunks`), so every formatter and newly-indexed chunk inherits it. It must never strip clinical meaning (numbers, units, dose strings, comparison symbols, hyphens, or legitimate bullet structure). It deliberately does **not** rejoin line-break hyphenation — a soft-wrap hyphen is indistinguishable from a real compound hyphen (`low-dose`, `twice-daily`), so fusing would corrupt clinical compounds and verbatim quotes. - `tests/rendered-text-formatting.test.ts` is a static guard that fails if a known content surface reintroduces a raw interpolation. Extend it when adding new document-derived render surfaces. +- **Compact snippet polish** (2026-07-02): `stripClassificationBanner` removes PSPF protective-marking banners ("OFFICIAL", "OFFICIAL: Sensitive" — ALL-CAPS, line-anchored only) from compact/snippet/synopsis/title surfaces and from newly-built synopses; verbatim quotes keep banners by design. Inline bullet glyphs in compact previews become `"; "` separators (the `readableTableRows` joiner). `repairTruncatedCompactTail` repairs stored mid-word truncations ("where poss..."); ingestion now truncates synopses at word boundaries (`truncateAtWordBoundary` in `chunking.ts`), so this repair mainly serves pre-fix stored rows. - **Static UI copy** (headings, empty states, error/toast messages, placeholders, starter prompts) lives in `src/lib/ui-copy.ts`, alongside `app-modes.ts` (mode labels) and `source-metadata.ts` (status labels). Do not hardcode new visible chrome copy inline. - `scripts/backfill-text-normalization.ts` cleans already-stored `document_chunks` text in place using the same primitive. It is dry-run by default, requires `--write --confirm` to mutate, writes a revertible JSON backup first, and **never re-embeds** — existing vectors are frozen, so retrieval is unchanged by construction. diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 7276ea2614..bff397a83d 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -2466,7 +2466,7 @@ function RenderModelSourceList({
{sources.map((source, index) => { const metadata = normalizeSourceMetadata(source.sourceMetadata); - const snippet = compactSourceSnippet(source.snippet ?? ""); + const snippet = compactSourceSnippet(source.snippet ?? "", { dropTitle: source.title }); const openLabel = `Open source ${index + 1}: ${cleanDisplayTitle(source.title)}${query ? ` for ${query}` : ""}`; return (
diff --git a/src/components/clinical-dashboard/display-text.ts b/src/components/clinical-dashboard/display-text.ts index a0ad3ff748..804c78c5e2 100644 --- a/src/components/clinical-dashboard/display-text.ts +++ b/src/components/clinical-dashboard/display-text.ts @@ -3,6 +3,7 @@ import { sourceTextForCompactDisplay, sourceTextForClinicalProse, sourceTextForClinicalProsePreservingBreaks, + stripClassificationBanner, } from "@/lib/source-text-sanitizer"; import { polishClinicalAnswerProse } from "@/lib/rag-answer-text"; import type { SearchResult } from "@/lib/types"; @@ -86,20 +87,69 @@ export function sourceSnippetKey(value: string) { .slice(0, 160); } -export function compactSourceSnippet(value: string) { +// Strips a leading duplicate of the card's own visible title (extraction often +// glues the running header onto the body text). Matches word-by-word so +// punctuation/spacing differences ("Guideline(EMHS)" vs "Guideline (EMHS)") +// don't defeat it, but requires a non-space separator after the title so a +// sentence that legitimately starts with the title words as its grammatical +// subject is never cut. +function stripLeadingTitleDuplicate(text: string, title: string) { + const titleWords = title.toLowerCase().match(/[a-z0-9]+/g) ?? []; + if (titleWords.length < 2) return text; + let matched = 0; + let end = 0; + for (const wordMatch of text.matchAll(/[A-Za-z0-9]+/g)) { + const gap = text.slice(end, wordMatch.index); + if (!/^[\s\-–—:,()&/'"·]*$/.test(gap)) break; + if (wordMatch[0].toLowerCase() !== titleWords[matched]) break; + matched += 1; + end = wordMatch.index + wordMatch[0].length; + if (matched === titleWords.length) break; + } + if (matched !== titleWords.length) return text; + // Allow closers left over from a parenthetical title ("… Guideline (EMHS)") + // before the separator, but the separator itself is mandatory. + const separator = text.slice(end).match(/^[\s)\]]*[;:·\-–—]+\s*/); + if (!separator) return text; + const rest = text.slice(end + separator[0].length).trim(); + return rest.length >= 20 ? rest : text; +} + +export type CompactSourceSnippetOptions = { + // The card's visible title, so a glued duplicate of it at the head of the + // snippet can be dropped. + dropTitle?: string; +}; + +export function compactSourceSnippet(value: string, options: CompactSourceSnippetOptions = {}) { const normalized = sanitizeDisplayText(value, { minLength: 10, minTokens: 2, compactSource: true }); if (!normalized) return ""; - const fragments = normalized + // A trailing ellipsis (stored truncation) must come off before the sentence + // split, otherwise the cut tail masquerades as a complete final sentence. + const truncatedTail = /(?:\.{3}|…)\s*$/.test(normalized); + let body = normalized.replace(/\s*(?:\.{3}|…)\s*$/, ""); + if (options.dropTitle) { + body = stripLeadingTitleDuplicate(body, cleanDisplayTitle(options.dropTitle)); + } + if (!body) return ""; + const fragments = body .replace( /\b(?:source excerpt|relevant excerpt|source text|table text|clinical table|image caption|caption)\s*[:=-]\s*/gi, " ", ) - .match(/[^.!?]+[.!?]?/g) ?? [normalized]; + .match(/[^.!?]+[.!?]?/g) ?? [body]; const seen = new Set(); const selected: string[] = []; for (const fragment of fragments) { - const cleaned = fragment.replace(/\s+/g, " ").trim(); + let cleaned = fragment.replace(/\s+/g, " ").trim(); + if (selected.length === 0) { + // The head of the snippet can be cut mid-structure by chunking: shed + // orphaned closers (")." "]:") and a mid-list ordinal ("2. MO to + // check…" keeps its content, loses the navigational marker; "1." is a + // genuine list start and is kept). + cleaned = cleaned.replace(/^[)\]}»”’]+[.,;:]?\s*/, "").replace(/^(?:[2-9]|1\d)[.)]\s+(?=["'(]?[A-Z])/, ""); + } if (!cleaned || cleaned.length < 10) continue; if (/^(?:source|citation|document|file|filename|chunk|page|image|provenance)\b/i.test(cleaned)) continue; const key = sourceSnippetKey(cleaned); @@ -109,7 +159,24 @@ export function compactSourceSnippet(value: string) { if (selected.length >= 4) break; } - return truncateWords((selected.length ? selected : [normalized]).join(" "), 90); + if (!selected.length) selected.push(body.replace(/\s+/g, " ").trim()); + // A first fragment starting lowercase is a mid-clause continuation. When the + // rest of the snippet has real sentence starts and enough substance, start + // there instead — the missing clause head could carry a negation. When the + // partial fragment is all we have, keep every word and mark the continuation + // honestly rather than fabricating a sentence start by capitalizing. + if (selected.length > 1 && /^[a-z]/.test(selected[0])) { + const rest = selected.slice(1); + if (rest.some((fragment) => /^["'(]?[A-Z0-9]/.test(fragment)) && rest.join(" ").length >= 40) { + selected.shift(); + } + } + const leadingContinuation = /^[a-z]/.test(selected[0]); + + let text = truncateWords(selected.join(" "), 90); + if (leadingContinuation) text = `… ${text}`; + if (truncatedTail && !/(?:\.{3}|…)$/.test(text)) text = `${text} …`; + return text; } export function compactTableFact(fact: NonNullable[number]) { @@ -149,9 +216,13 @@ export function sanitizeAnswerDisplayText(value: string, options: DisplayTextSan } export function cleanDisplayTitle(title: string) { - return normalizeExtractedGlyphs(title ?? "") + return stripClassificationBanner(normalizeExtractedGlyphs(title ?? "")) .replace(/^Synthetic /, "") .replace(/\.pdf$/i, "") + // Missing space before an acronym-like parenthetical: "Guideline(EMHS)" → + // "Guideline (EMHS)". Requires 2+ leading capitals inside the parens so + // "guideline(s)" and "dose(mg)" stay untouched. + .replace(/([A-Za-z])\((?=[A-Z]{2}[^)]*\))/g, "$1 (") .replace(/\s+/g, " ") .trim(); } diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 8c5d32057b..52b2ba53e6 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -1,6 +1,6 @@ import { env } from "@/lib/env"; import { sourceSpanForText } from "@/lib/source-spans"; -import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer"; +import { normalizeExtractedGlyphs, stripClassificationBanner } from "@/lib/source-text-sanitizer"; import type { ChunkInput, DocumentChunk } from "@/lib/types"; const sentenceBoundary = /(?<=[.!?])\s+/; @@ -52,9 +52,16 @@ function normalizeLookupText(value: string) { .trim(); } +// A line that is nothing but a PSPF protective marking ("OFFICIAL", +// "OFFICIAL: Sensitive") — running headers stamped on every page. +function isClassificationBannerLine(line: string) { + return Boolean(line.trim()) && !stripClassificationBanner(line).trim(); +} + function looksLikeMetadataNoise(line: string) { if (!line || line.length <= 2) return true; if (/^\d+$/.test(line)) return true; + if (isClassificationBannerLine(line)) return true; if (metadataNoisePatterns.some((pattern) => pattern.test(line))) return true; if (lineNoisePatterns.some((pattern) => pattern.test(line))) return true; return false; @@ -337,12 +344,29 @@ function chunkTextBySentence(clean: string, chunkSize: number, overlap: number) return chunks; } +// Word-safe truncation for stored display text (synopses, caption snippets). +// A raw slice cut mid-word ("where poss...") and every surface downstream +// inherited the artifact; cutting at a clause boundary when one lands late +// enough, otherwise at the last word boundary, keeps the stored tail readable. +function truncateAtWordBoundary(value: string, limit: number) { + if (value.length <= limit) return value; + const window = value.slice(0, limit - 3); + const clauseCut = Math.max(window.lastIndexOf(". "), window.lastIndexOf("; "), window.lastIndexOf(": ")); + const wordCut = window.lastIndexOf(" "); + const cut = clauseCut >= limit * 0.6 ? clauseCut + 1 : wordCut > 0 ? wordCut : window.length; + const trimmed = window + .slice(0, cut) + .replace(/[\s,;:([{\-–—]+$/, "") + .trim(); + return trimmed ? `${trimmed}...` : `${window.trim()}...`; +} + function compactImageText(value: string | null | undefined, limit = 420) { const text = String(value ?? "") .replace(/\s+/g, " ") .trim(); if (!text) return ""; - return text.length > limit ? `${text.slice(0, limit - 3).trim()}...` : text; + return truncateAtWordBoundary(text, limit); } function compactSynopsisText(value: string | null | undefined, limit = 720) { @@ -351,13 +375,16 @@ function compactSynopsisText(value: string | null | undefined, limit = 720) { .replace(/\[\[IMAGE_DATA_OMITTED\]\][\s\S]*?\[\[\/IMAGE_DATA_OMITTED\]\]/g, " "); const sentences = withoutImageTags .split(/(?<=[.!?])\s+|\n+/) - .map((sentence) => sentence.replace(/\s+/g, " ").trim()) + // Extraction glues the protective-marking header onto body sentences + // ("OFFICIAL: OFFICIAL Lithium Therapy …"); shed the banner prefix so it + // never enters the stored synopsis. + .map((sentence) => stripClassificationBanner(sentence.replace(/\s+/g, " ").trim()).trim()) .filter((sentence) => sentence.length >= 12 && !boilerplateSynopsisPattern.test(sentence)); const highYieldSentences = sentences.filter((sentence) => highYieldSectionPattern.test(sentence)); const selected = (highYieldSentences.length ? highYieldSentences : sentences).slice(0, 4).join(" "); const compact = selected.replace(/\s+/g, " ").trim(); if (!compact) return ""; - return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact; + return truncateAtWordBoundary(compact, limit); } function buildRetrievalSynopsis(args: { @@ -376,7 +403,7 @@ function buildRetrievalSynopsis(args: { .filter(Boolean) .join(" | "); const facts = compactSynopsisText(args.content); - return [prefix, facts].filter(Boolean).join(" | ").slice(0, 900); + return truncateAtWordBoundary([prefix, facts].filter(Boolean).join(" | "), 900); } export function buildImageTag(image: { diff --git a/src/lib/citations.ts b/src/lib/citations.ts index f08abefdb6..aedb64831d 100644 --- a/src/lib/citations.ts +++ b/src/lib/citations.ts @@ -1,12 +1,13 @@ -import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer"; +import { normalizeExtractedGlyphs, stripClassificationBanner } from "@/lib/source-text-sanitizer"; import type { Citation, SearchResult } from "@/lib/types"; // Citation titles come straight from document extraction, so repair glyph -// artifacts (ligatures, soft hyphens, control chars) and drop the synthetic -// prefix before they reach any label — keeps mobile/compact labels consistent -// with the cleaned titles rendered elsewhere (cleanDisplayTitle). +// artifacts (ligatures, soft hyphens, control chars), drop protective-marking +// banners ("OFFICIAL:"), and drop the synthetic prefix before they reach any +// label — keeps mobile/compact labels consistent with the cleaned titles +// rendered elsewhere (cleanDisplayTitle). function cleanCitationTitle(value: string) { - return normalizeExtractedGlyphs(value) + return stripClassificationBanner(normalizeExtractedGlyphs(value)) .replace(/^Synthetic\s+/i, "") .replace(/\s+/g, " ") .trim(); diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts index 086a2534a2..2d388d4d10 100644 --- a/src/lib/source-text-sanitizer.ts +++ b/src/lib/source-text-sanitizer.ts @@ -24,6 +24,18 @@ const sourceTitleWithCodePattern = new RegExp( ); const sourceControlLinePattern = /\b(?:uncontrolled when printed|document control|document owner|authoris(?:ed|ation)|authorised by|published date|effective from|review date|version\s+\d+|amendment|supporting information|relevant standards|references)\b/i; +// PSPF protective-marking banners stamped on WA Health PDFs ("OFFICIAL", +// "OFFICIAL: Sensitive"). Case-sensitive ALL-CAPS tokens only, anchored to a +// line start, so clinical prose ("the official guideline") and title-case +// names ("Official Visitors Scheme") can never match. Extraction often glues +// the running header onto body text, sometimes twice ("OFFICIAL: OFFICIAL …"), +// hence the {1,3} repetition. +const classificationMarkerSource = String.raw`(?:UNOFFICIAL|OFFICIAL(?:\s*:\s*Sensitive)?|SENSITIVE|PROTECTED)`; +const classificationBannerLinePattern = new RegExp(String.raw`^\s*(?:${classificationMarkerSource}\s*:?\s*){1,3}$`); +const leadingClassificationBannerPattern = new RegExp( + String.raw`^(?:[ \t]*${classificationMarkerSource}(?=[\s:])(?:[ \t]*:[ \t]*|[ \t]+)){1,3}`, + "gm", +); const clinicalSignalPattern = /\b(?:administer|anc|assess|baseline|blood|cease|contraindicat|dose|dosing|ecg|escalat|fbc|level|mg|mcg|mmol|monitor|neutrophil|prescrib|review|risk|symptom|threshold|titrate|toxicity|urgent|withhold|wbc)\b/i; const provenancePhrasePattern = @@ -88,6 +100,19 @@ export function normalizeExtractedGlyphs(value: string) { return out; } +// Removes PSPF protective-marking banners: lines that are nothing but the +// marking ("OFFICIAL", "OFFICIAL: OFFICIAL") and line-leading marking prefixes +// glued onto content ("OFFICIAL: OFFICIAL Lithium Therapy - …" → "Lithium +// Therapy - …"). Never touches the marker words mid-sentence. Idempotent. +export function stripClassificationBanner(value: string) { + if (!value) return value; + return value + .split("\n") + .filter((line) => !classificationBannerLinePattern.test(line)) + .join("\n") + .replace(leadingClassificationBannerPattern, ""); +} + function compactWhitespace(value: string) { return normalizeExtractedGlyphs(value).replace(/\s+/g, " ").trim(); } @@ -189,6 +214,7 @@ function stripLowYieldLines(value: string) { .filter((line) => { const normalized = compactWhitespace(line); if (!normalized) return true; + if (classificationBannerLinePattern.test(normalized)) return false; const isControlLine = sourceControlLinePattern.test(normalized); const hasSourceMarker = sourceDocumentCodeTestPattern.test(normalized) || pageBoilerplateTestPattern.test(normalized); @@ -202,6 +228,7 @@ function stripLowYieldLines(value: string) { export function stripLowYieldSourceNoise(text: string) { return stripLowYieldLines(text) + .replace(leadingClassificationBannerPattern, "") .replace(internalImageTokenPattern, " ") .replace(sourceTitleWithCodePattern, " ") .replace(sourceDocumentCodePattern, " ") @@ -211,7 +238,11 @@ export function stripLowYieldSourceNoise(text: string) { .replace(genericReferencePattern, "") .replace(/\b(?:chunk|similarity)\s+\d+(?:\.\d+)?\b/gi, " ") .replace(/\s+([,.;:])/g, "$1") - .replace(/(?:\.\s*){2,}/g, ". "); + // Collapse doubled dots mid-text, but keep a trailing "..." — that is a + // stored truncation marker repairTruncatedCompactTail needs to see. The + // lookahead excludes dots so backtracking can't shave a trailing ellipsis + // down to ".." and still match. + .replace(/(?:\.\s*){2,}(?=[^.\s])/g, ". "); } export function lowYieldSourceNoiseScore(text: string) { @@ -373,18 +404,64 @@ export function sourceTextForDisplayPreservingBreaks(text: string) { ); } +// Words that must not be left dangling before an ellipsis: connectors, +// subordinators, and meaning-inverting auxiliaries/negations ("do not…" must +// never become "do…"). Broader than truncateWords' connector list because here +// the preceding token was presumed partial and already dropped. +const unsafeTruncationTailPattern = + /^(?:or|and|to|with|of|for|the|a|an|until|than|in|on|at|by|not|no|never|nor|do|does|did|is|are|was|were|be|been|being|has|have|had|if|unless|except|without|must|should|shall|may|might|can|cannot|could|will|would|that|which|who|whom|whose|where|when|while|because|since|but|so|then|as|per|via|from|into|onto|during|before|after|between|below|above|under|over|their|its|this|these|those|any|all|each)$/i; + +// Repairs text whose stored form was cut mid-word before an ellipsis was glued +// on (the pre-fix retrieval_synopsis truncation: "where poss..."). The final +// token is presumed partial and dropped unless it ends at a natural boundary, +// then the tail walks back past connectors/negations and bare numbers so the +// preview never ends on a misleading stub. Emits " …" (space + ellipsis) — +// which is also the already-repaired marker that keeps this idempotent. +export function repairTruncatedCompactTail(value: string) { + if (!value || /\s…$/.test(value)) return value; + const match = value.match(/^([\s\S]*?)\s*(?:\.{3}|…)\s*$/); + if (!match) return value; + const words = match[1].trim().split(/\s+/).filter(Boolean); + if (!words.length) return ""; + const last = words[words.length - 1]; + if (/[A-Za-z0-9]$/.test(last)) words.pop(); + // Draining to empty is deliberate: an all-function-word stub ("do not…") + // is worse than no preview at all. + while ( + words.length > 0 && + (unsafeTruncationTailPattern.test(words[words.length - 1]) || + /^[<>≤≥~]?\d[\d.,–—-]*$/.test(words[words.length - 1])) + ) { + words.pop(); + } + return words.length ? `${words.join(" ")} …` : ""; +} + export function sourceTextForCompactDisplay(text: string) { - return readableWhitespace( - sourceTextForDisplayPreservingBreaks(text) - .replace( - /(?:^|\n)\s*(?:source|sources|citation|citations|document|file|filename|chunk|page|image|provenance|retrieved|indexed)\s*(?:id|ids|index|number|path)?\s*[:#=-]\s*[^\n]+/gi, - " ", - ) - .replace(/\b(?:clinical table|table text|accessible table|image caption|caption|excerpt)\s*[:=-]\s*/gi, " ") - .replace(/\b(?:source|chunk|document|image)\s*(?:id|index)?\s*[:#=-]?\s*[a-z0-9_-]{8,}\b/gi, " ") - .replace(/\bpage\s*(?:number)?\s*[:#=-]?\s*(?:n\/a|\d+(?:\s*[-,]\s*\d+)*)\b/gi, " ") - .replace(/\bchunk\s*(?:index)?\s*[:#=-]?\s*\d+\b/gi, " ") - .replace(/\s+([,.;:])/g, "$1"), + return repairTruncatedCompactTail( + readableWhitespace( + sourceTextForDisplayPreservingBreaks(text) + .replace( + /(?:^|\n)\s*(?:source|sources|citation|citations|document|file|filename|chunk|page|image|provenance|retrieved|indexed)\s*(?:id|ids|index|number|path)?\s*[:#=-]\s*[^\n]+/gi, + " ", + ) + .replace(/\b(?:clinical table|table text|accessible table|image caption|caption|excerpt)\s*[:=-]\s*/gi, " ") + .replace(/\b(?:source|chunk|document|image)\s*(?:id|index)?\s*[:#=-]?\s*[a-z0-9_-]{8,}\b/gi, " ") + .replace(/\bpage\s*(?:number)?\s*[:#=-]?\s*(?:n\/a|\d+(?:\s*[-,]\s*\d+)*)\b/gi, " ") + .replace(/\bchunk\s*(?:index)?\s*[:#=-]?\s*\d+\b/gi, " ") + // Bullet glyphs in a compact one-line preview become "; " separators + // (the same joiner readableTableRows uses); a leading bullet is + // dropped outright. Hyphen bullets are left alone — indistinguishable + // from compound hyphens and ranges. + .replace(/^\s*[•◦▪‣●]+\s*/, "") + .replace(/\s*[•◦▪‣●]+\s*/g, "; ") + // PDF sub-bullet glyph rendered as a bare lowercase "o" between words: + // only when whitespace-delimited, not after a digit ("37 o C" stays), + // and followed by a capitalized token of 2+ chars ("o C" stays). + .replace(/(?<=[^\d\s]\s)o(?=\s+(?:[A-Z][a-z0-9]|[A-Z]{2,}))/g, ";") + .replace(/\s+([,.;:])/g, "$1") + .replace(/;(?:\s*;)+/g, ";"), + ), ); } diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index f9c5a8ecb0..32c6568ed0 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -267,6 +267,45 @@ describe("section-aware chunking groundwork", () => { expect(content).toContain("Review lithium levels"); }); + it("drops protective-marking banner lines from chunk content", () => { + const chunks = buildChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: "OFFICIAL: Sensitive\n\nLithium Monitoring\n\nCheck renal function before starting lithium therapy.", + metadata: {}, + }, + ]); + + const content = chunks.map((chunk) => chunk.content).join("\n"); + expect(content).not.toContain("OFFICIAL"); + expect(content).toContain("Check renal function"); + }); + + it("keeps the retrieval synopsis banner-free and never cut mid-word", () => { + const longSentence = + "Monitor lithium dose thresholds carefully and escalate abnormal results quickly because delayed review of toxicity increases the risk of renal impairment and neurotoxicity across the whole treatment pathway for every patient cohort."; + const pageText = [ + "OFFICIAL: OFFICIAL Lithium Therapy - dose guidance requires monitoring.", + longSentence, + longSentence.replace("cohort", "cohort again"), + longSentence.replace("cohort", "cohort as well"), + ].join(" "); + + const chunks = buildChunks([{ documentId: "doc-1", pageNumber: 1, pageText, metadata: {} }]); + const synopsis = chunks[0]?.retrieval_synopsis ?? ""; + + expect(synopsis).not.toContain("OFFICIAL"); + expect(synopsis).toContain("Lithium Therapy - dose guidance requires monitoring"); + // The 720-char cap must land on a word boundary: the token before the + // ellipsis has to be a complete word from the source text. + expect(synopsis.endsWith("...")).toBe(true); + const lastWord = (synopsis.slice(0, -3).trim().split(/\s+/).pop() ?? "").replace(/[.,;:]+$/, ""); + const sourceWords = new Set(pageText.split(/\s+/).map((word) => word.replace(/[.,;:]+$/, ""))); + expect(lastWord.length).toBeGreaterThan(0); + expect(sourceWords.has(lastWord)).toBe(true); + }); + it("keeps repeated clinical chunks on different pages instead of document-wide deduping them", () => { const repeatedMonitoringText = "Clozapine monitoring table\n\nANC threshold 0.5 x 10^9/L: withhold clozapine and repeat FBC daily."; diff --git a/tests/display-text.test.ts b/tests/display-text.test.ts index 34b1f3bc9c..82a7d6d64f 100644 --- a/tests/display-text.test.ts +++ b/tests/display-text.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { sanitizeAnswerDisplayText, truncateWords } from "../src/components/clinical-dashboard/display-text"; +import { + cleanDisplayTitle, + compactSourceSnippet, + sanitizeAnswerDisplayText, + truncateWords, +} from "../src/components/clinical-dashboard/display-text"; describe("clinical dashboard display text", () => { it("polishes cached generated answer prose before rendering", () => { @@ -34,4 +39,85 @@ describe("clinical dashboard display text", () => { expect(result.endsWith("until...")).toBe(false); }); }); + + describe("compactSourceSnippet", () => { + it("keeps a lone mid-clause fragment verbatim and marks the continuation honestly", () => { + const snippet = compactSourceSnippet( + "combination with lithium may lead to serotonin toxicity • Concurrent antipsychotic medications o Rapid dose increase of lithium and antipsychotics together may increase risk of neurotoxicity", + ); + + expect(snippet.startsWith("… combination with lithium")).toBe(true); + expect(snippet).toContain("toxicity; Concurrent"); + expect(snippet).toContain("medications; Rapid"); + }); + + it("starts at the first real sentence when a partial first fragment has substantial follow-on", () => { + expect( + compactSourceSnippet( + "tion of therapy requires care. Monitor serum lithium levels weekly until stable. Check renal function and thyroid function every six months.", + ), + ).toBe( + "Monitor serum lithium levels weekly until stable. Check renal function and thyroid function every six months.", + ); + }); + + it("sheds a mid-list ordinal at the snippet head but keeps a genuine list start", () => { + expect(compactSourceSnippet("2) MO to check the serum lithium level and renal function as soon as possible")).toBe( + "MO to check the serum lithium level and renal function as soon as possible", + ); + expect(compactSourceSnippet("1) Confirm baseline ECG before starting lithium therapy")).toBe( + "1) Confirm baseline ECG before starting lithium therapy", + ); + }); + + it("cleans the mid-numbered-list stored synopsis without losing content or the truncation marker", () => { + const stored = + 'Pharmacist or Medical Officer (MO) to withhold lithium (noting "W" on WA Hospital Medication Chart [HMC]). 2. MO to check the serum lithium level (note time of last dose) and renal function as soon as possible from th...'; + + const snippet = compactSourceSnippet(stored); + + expect(snippet.startsWith("Pharmacist or Medical Officer (MO) to withhold lithium")).toBe(true); + expect(snippet).toContain("MO to check the serum lithium level"); + expect(snippet).not.toContain("from th"); + expect(snippet.endsWith("…")).toBe(true); + }); + + it("drops a glued duplicate of the card title but never a sentence that starts with it", () => { + expect( + compactSourceSnippet( + "Lithium Clinical Guideline (EMHS) - NSAIDs such as ibuprofen can reduce lithium clearance and increase toxicity risk substantially", + { dropTitle: "Lithium Clinical Guideline(EMHS)" }, + ), + ).toBe("NSAIDs such as ibuprofen can reduce lithium clearance and increase toxicity risk substantially"); + + expect( + compactSourceSnippet("Lithium levels should be checked weekly after any dose change or interacting medicine", { + dropTitle: "Lithium levels", + }), + ).toBe("Lithium levels should be checked weekly after any dose change or interacting medicine"); + }); + + it("repairs a stored mid-word truncation and still ends with an ellipsis", () => { + expect( + compactSourceSnippet("Monitor renal function every three months and review lithium dose where poss..."), + ).toBe("Monitor renal function every three months and review lithium dose …"); + }); + }); + + describe("cleanDisplayTitle", () => { + it("inserts the missing space before an acronym parenthetical", () => { + expect(cleanDisplayTitle("Lithium Clinical Guideline(EMHS)")).toBe("Lithium Clinical Guideline (EMHS)"); + }); + + it("leaves lowercase and unit parentheticals untouched", () => { + expect(cleanDisplayTitle("guideline(s) update")).toBe("guideline(s) update"); + expect(cleanDisplayTitle("dose(mg) chart")).toBe("dose(mg) chart"); + }); + + it("strips a protective-marking banner and the pdf extension from titles", () => { + expect(cleanDisplayTitle("OFFICIAL: Lithium Clinical Guideline(EMHS).pdf")).toBe( + "Lithium Clinical Guideline (EMHS)", + ); + }); + }); }); diff --git a/tests/rendered-text-formatting.test.ts b/tests/rendered-text-formatting.test.ts index 1a5d241e4d..f123d6d87a 100644 --- a/tests/rendered-text-formatting.test.ts +++ b/tests/rendered-text-formatting.test.ts @@ -32,6 +32,15 @@ describe("document-derived text must route through a formatter", () => { expect(dashboard).toContain("sourceTextForCompactDisplay(item.tableTextSnippet)"); }); + it("renders source-card snippets through compactSourceSnippet with the card title deduped", () => { + expect(dashboard).not.toMatch(/(? { + expect(dashboard).toContain("sourceTextForCompactDisplay(row.quote || row.source.snippet"); + }); + it("renders document-viewer image captions through a formatter, never raw", () => { expect(documentViewer).not.toMatch(/\{image\.caption\}/); expect(documentViewer).toContain("sourceTextForCompactDisplay(image.caption)"); diff --git a/tests/source-text-sanitizer.test.ts b/tests/source-text-sanitizer.test.ts index 81dfed6f59..646b07f8f3 100644 --- a/tests/source-text-sanitizer.test.ts +++ b/tests/source-text-sanitizer.test.ts @@ -5,12 +5,15 @@ import { isLowYieldClinicalText, lowYieldSourceNoiseScore, normalizeExtractedGlyphs, + repairTruncatedCompactTail, sourceTextForClinicalProse, + sourceTextForCompactDisplay, sourceTextForDisplay, sourceTextForDocumentViewer, sourceTextForIndexedPage, sourceTextForModel, sourceTextForVerbatimQuote, + stripClassificationBanner, } from "../src/lib/source-text-sanitizer"; describe("source text sanitizer", () => { @@ -224,4 +227,103 @@ describe("sourceTextForVerbatimQuote", () => { expect(sourceTextForDisplay(quote)).not.toContain("IMAGE_DATA_OMITTED"); expect(sourceTextForDocumentViewer(quote)).not.toContain("IMAGE_DATA_OMITTED"); }); + + it("keeps protective-marking banners verbatim in exact quotes", () => { + // Quotes must never be rewritten — even for boilerplate. Banner removal is + // a display/synopsis concern only. + const quote = "OFFICIAL: OFFICIAL Lithium Therapy - Initiation and Continuation • NSAIDs can reduce clearance."; + + const cleaned = sourceTextForVerbatimQuote(quote); + + expect(cleaned).toContain("OFFICIAL: OFFICIAL"); + expect(cleaned).toContain("•"); + }); +}); + +describe("stripClassificationBanner", () => { + it("strips a leading PSPF marking, including the doubled extraction form", () => { + expect(stripClassificationBanner("OFFICIAL: Lithium Therapy - Initiation and Continuation")).toBe( + "Lithium Therapy - Initiation and Continuation", + ); + expect(stripClassificationBanner("OFFICIAL: OFFICIAL Lithium Therapy - Initiation and Continuation")).toBe( + "Lithium Therapy - Initiation and Continuation", + ); + expect(stripClassificationBanner("OFFICIAL: Sensitive Withhold lithium and recheck the level.")).toBe( + "Withhold lithium and recheck the level.", + ); + }); + + it("removes banner-only lines from multi-line text", () => { + expect(stripClassificationBanner("OFFICIAL\nMonitor lithium levels weekly.")).toBe( + "Monitor lithium levels weekly.", + ); + expect(stripClassificationBanner("OFFICIAL: Sensitive\nCheck renal function.")).toBe("Check renal function."); + }); + + it("never touches the marker words in prose, title case, or as a prefix of longer words", () => { + expect(stripClassificationBanner("the official guideline recommends monitoring")).toBe( + "the official guideline recommends monitoring", + ); + expect(stripClassificationBanner("Official Visitors Scheme referral process")).toBe( + "Official Visitors Scheme referral process", + ); + expect(stripClassificationBanner("OFFICIALLY sanctioned pathway")).toBe("OFFICIALLY sanctioned pathway"); + }); + + it("is idempotent", () => { + const once = stripClassificationBanner("OFFICIAL: OFFICIAL Lithium Therapy - dose guidance"); + expect(stripClassificationBanner(once)).toBe(once); + }); +}); + +describe("repairTruncatedCompactTail", () => { + it("drops the presumed-partial final token behind a glued ellipsis", () => { + expect(repairTruncatedCompactTail("Avoid the combination where poss...")).toBe("Avoid the combination …"); + expect(repairTruncatedCompactTail("check the level as soon as possible from th…")).toBe( + "check the level as soon as possible …", + ); + }); + + it("never leaves a meaning-inverting or dangling stub before the ellipsis", () => { + expect(repairTruncatedCompactTail("withhold lithium and do not...")).toBe("withhold lithium …"); + expect(repairTruncatedCompactTail("keep the dose below 1.5...")).toBe("keep the dose …"); + expect(repairTruncatedCompactTail("do not...")).toBe(""); + }); + + it("leaves text without a trailing ellipsis unchanged and is idempotent", () => { + expect(repairTruncatedCompactTail("Avoid the combination where possible.")).toBe( + "Avoid the combination where possible.", + ); + const once = repairTruncatedCompactTail("Avoid the combination where poss..."); + expect(repairTruncatedCompactTail(once)).toBe(once); + }); +}); + +describe("sourceTextForCompactDisplay snippet polish", () => { + it("cleans the banner + glued-title + bullet + truncated-tail artifact end to end", () => { + const stored = + "OFFICIAL: OFFICIAL Lithium Therapy - Initiation and Continuation • NSAIDs: (e.g. ibuprofen) can reduce lithium clearance and therefore increase lithium levels and the risk of toxicity. Avoid the combination where poss..."; + + const cleaned = sourceTextForCompactDisplay(stored); + + expect(cleaned).not.toContain("OFFICIAL"); + expect(cleaned).toContain("Continuation; NSAIDs:"); + expect(cleaned).toContain("can reduce lithium clearance"); + expect(cleaned).not.toContain("poss"); + expect(cleaned).toMatch(/combination …$/); + }); + + it("converts inline bullets and the PDF sub-bullet 'o' glyph into readable separators", () => { + const stored = + "combination with lithium may lead to serotonin toxicity • Concurrent antipsychotic medications o Rapid dose increase of lithium and antipsychotics"; + + expect(sourceTextForCompactDisplay(stored)).toBe( + "combination with lithium may lead to serotonin toxicity; Concurrent antipsychotic medications; Rapid dose increase of lithium and antipsychotics", + ); + }); + + it("leaves a temperature-style ' o ' glyph and lowercase follow-ons untouched", () => { + expect(sourceTextForCompactDisplay("Store below 37 o C at all times")).toBe("Store below 37 o C at all times"); + expect(sourceTextForCompactDisplay("blood group o positive result")).toBe("blood group o positive result"); + }); }); From 0056245ae830bedbb325321d3410b03b91513472 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:41:37 +0800 Subject: [PATCH 2/3] feat(backfill): polish stored retrieval synopses (banner strip + tail repair) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the in-place text-normalization backfill so retrieval_synopsis rows stored before the snippet-polish fix get the same repairs at rest: polishStoredSynopsis applies glyph repair, per-segment protective-marking banner removal (the synopsis prefix format puts the banner mid-string), and truncated-tail repair. content/section_heading keep glyph-only normalization — they feed verbatim quotes and span matching and must not be rewritten. Dry-run default, mandatory backup, and no-re-embed guarantees unchanged; synopsis-only changes now surface in the dry-run samples. Co-Authored-By: Claude Fable 5 --- scripts/backfill-text-normalization.ts | 43 ++++++++++++++++---------- src/lib/source-text-sanitizer.ts | 13 ++++++++ tests/source-text-sanitizer.test.ts | 22 +++++++++++++ 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/scripts/backfill-text-normalization.ts b/scripts/backfill-text-normalization.ts index e14be79c93..f569b92eb3 100644 --- a/scripts/backfill-text-normalization.ts +++ b/scripts/backfill-text-normalization.ts @@ -3,9 +3,13 @@ * * Applies the shared, lossless `normalizeExtractedGlyphs` transform to the stored * `document_chunks.content` (and `section_heading`) of already-indexed documents. - * Only rows whose text actually changes are updated. Embeddings are NEVER - * recomputed, so vector/semantic retrieval is unchanged by construction; the - * generated `search_tsv` column refreshes automatically and can only improve. + * `retrieval_synopsis` additionally gets `polishStoredSynopsis` (protective-marking + * banner removal + truncated-tail repair) — the synopsis is a derived display/ + * retrieval summary, never quoted verbatim, so the stronger polish is safe there + * and only there. Only rows whose text actually changes are updated. Embeddings + * are NEVER recomputed, so vector/semantic retrieval is unchanged by + * construction; the generated `search_tsv` column refreshes automatically and + * can only improve. * * Safety: * - Confirms the Supabase project before writing. @@ -109,12 +113,13 @@ const PAGE_SIZE = 1000; async function main() { const args = parseArgs(process.argv.slice(2)); - const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs }, projectModule] = await Promise.all([ - import("@/lib/env"), - import("@/lib/supabase/admin"), - import("@/lib/source-text-sanitizer"), - import("@/lib/supabase/project"), - ]); + const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs, polishStoredSynopsis }, projectModule] = + await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("@/lib/source-text-sanitizer"), + import("@/lib/supabase/project"), + ]); requireServerEnv(); const { checkSupabaseProjectConfig, expectedSupabaseProject, formatSupabaseProjectCheck } = projectModule; @@ -167,7 +172,7 @@ async function main() { const newHeading = row.section_heading == null ? row.section_heading : normalizeExtractedGlyphs(row.section_heading); const newSynopsis = - row.retrieval_synopsis == null ? row.retrieval_synopsis : normalizeExtractedGlyphs(row.retrieval_synopsis); + row.retrieval_synopsis == null ? row.retrieval_synopsis : polishStoredSynopsis(row.retrieval_synopsis); const contentChanged = newContent !== row.content; const headingChanged = newHeading !== row.section_heading; const synopsisChanged = newSynopsis !== row.retrieval_synopsis; @@ -189,12 +194,18 @@ async function main() { new_retrieval_synopsis: newSynopsis, }); } - if (contentChanged && sampleDiffs.length < 8) { - sampleDiffs.push({ - id: row.id, - before: (row.content ?? "").slice(0, 160), - after: (newContent ?? "").slice(0, 160), - }); + if ((contentChanged || synopsisChanged) && sampleDiffs.length < 8) { + // Prefer showing the content diff; fall back to the synopsis diff for + // synopsis-only rows so the dry run still demonstrates the change. + sampleDiffs.push( + contentChanged + ? { id: row.id, before: (row.content ?? "").slice(0, 160), after: (newContent ?? "").slice(0, 160) } + : { + id: row.id, + before: (row.retrieval_synopsis ?? "").slice(0, 160), + after: (newSynopsis ?? "").slice(0, 160), + }, + ); } } diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts index 2d388d4d10..37127da1c9 100644 --- a/src/lib/source-text-sanitizer.ts +++ b/src/lib/source-text-sanitizer.ts @@ -437,6 +437,19 @@ export function repairTruncatedCompactTail(value: string) { return words.length ? `${words.join(" ")} …` : ""; } +// Repairs a stored retrieval_synopsis in place (backfill path): glyph repair, +// protective-marking banner removal per " | "-delimited segment (the synopsis +// prefix format puts the banner mid-string, after "Section: … | Page: N | "), +// and truncated-tail repair. Newly-built synopses already get all of this at +// ingestion; this exists for rows stored before the fix. Idempotent. +export function polishStoredSynopsis(value: string) { + const segments = normalizeExtractedGlyphs(value) + .split(/\s*\|\s*/) + .map((segment) => stripClassificationBanner(segment).replace(/\s+/g, " ").trim()) + .filter(Boolean); + return repairTruncatedCompactTail(segments.join(" | ")); +} + export function sourceTextForCompactDisplay(text: string) { return repairTruncatedCompactTail( readableWhitespace( diff --git a/tests/source-text-sanitizer.test.ts b/tests/source-text-sanitizer.test.ts index 646b07f8f3..3d68953d81 100644 --- a/tests/source-text-sanitizer.test.ts +++ b/tests/source-text-sanitizer.test.ts @@ -5,6 +5,7 @@ import { isLowYieldClinicalText, lowYieldSourceNoiseScore, normalizeExtractedGlyphs, + polishStoredSynopsis, repairTruncatedCompactTail, sourceTextForClinicalProse, sourceTextForCompactDisplay, @@ -299,6 +300,27 @@ describe("repairTruncatedCompactTail", () => { }); }); +describe("polishStoredSynopsis", () => { + it("strips a banner glued after the synopsis prefix and repairs the truncated tail", () => { + const stored = + "Section: Interactions | Page: 4 | OFFICIAL: OFFICIAL Lithium Therapy - dose guidance • avoid NSAIDs where poss..."; + + expect(polishStoredSynopsis(stored)).toBe( + "Section: Interactions | Page: 4 | Lithium Therapy - dose guidance • avoid NSAIDs …", + ); + }); + + it("returns an already-clean synopsis unchanged and is idempotent", () => { + const clean = "Section: Dosing | Page: 2 | Monitor lithium levels weekly after any dose change."; + expect(polishStoredSynopsis(clean)).toBe(clean); + + const once = polishStoredSynopsis( + "Section: Interactions | Page: 4 | OFFICIAL: OFFICIAL Lithium Therapy - dose guidance • avoid NSAIDs where poss...", + ); + expect(polishStoredSynopsis(once)).toBe(once); + }); +}); + describe("sourceTextForCompactDisplay snippet polish", () => { it("cleans the banner + glued-title + bullet + truncated-tail artifact end to end", () => { const stored = From 55e6fea67b6d8acc103cf8641edbb9a611c5e8c5 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:43:16 +0800 Subject: [PATCH 3/3] style: prettier formatting for snippet-polish files Co-Authored-By: Claude Fable 5 --- scripts/backfill-text-normalization.ts | 18 ++++++---- .../clinical-dashboard/display-text.ts | 20 ++++++----- src/lib/source-text-sanitizer.ts | 34 ++++++++++--------- tests/display-text.test.ts | 6 ++-- 4 files changed, 43 insertions(+), 35 deletions(-) diff --git a/scripts/backfill-text-normalization.ts b/scripts/backfill-text-normalization.ts index f569b92eb3..b2c8368533 100644 --- a/scripts/backfill-text-normalization.ts +++ b/scripts/backfill-text-normalization.ts @@ -113,13 +113,17 @@ const PAGE_SIZE = 1000; async function main() { const args = parseArgs(process.argv.slice(2)); - const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs, polishStoredSynopsis }, projectModule] = - await Promise.all([ - import("@/lib/env"), - import("@/lib/supabase/admin"), - import("@/lib/source-text-sanitizer"), - import("@/lib/supabase/project"), - ]); + const [ + { requireServerEnv }, + { createAdminClient }, + { normalizeExtractedGlyphs, polishStoredSynopsis }, + projectModule, + ] = await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("@/lib/source-text-sanitizer"), + import("@/lib/supabase/project"), + ]); requireServerEnv(); const { checkSupabaseProjectConfig, expectedSupabaseProject, formatSupabaseProjectCheck } = projectModule; diff --git a/src/components/clinical-dashboard/display-text.ts b/src/components/clinical-dashboard/display-text.ts index 804c78c5e2..c599b295b6 100644 --- a/src/components/clinical-dashboard/display-text.ts +++ b/src/components/clinical-dashboard/display-text.ts @@ -216,15 +216,17 @@ export function sanitizeAnswerDisplayText(value: string, options: DisplayTextSan } export function cleanDisplayTitle(title: string) { - return stripClassificationBanner(normalizeExtractedGlyphs(title ?? "")) - .replace(/^Synthetic /, "") - .replace(/\.pdf$/i, "") - // Missing space before an acronym-like parenthetical: "Guideline(EMHS)" → - // "Guideline (EMHS)". Requires 2+ leading capitals inside the parens so - // "guideline(s)" and "dose(mg)" stay untouched. - .replace(/([A-Za-z])\((?=[A-Z]{2}[^)]*\))/g, "$1 (") - .replace(/\s+/g, " ") - .trim(); + return ( + stripClassificationBanner(normalizeExtractedGlyphs(title ?? "")) + .replace(/^Synthetic /, "") + .replace(/\.pdf$/i, "") + // Missing space before an acronym-like parenthetical: "Guideline(EMHS)" → + // "Guideline (EMHS)". Requires 2+ leading capitals inside the parens so + // "guideline(s)" and "dose(mg)" stay untouched. + .replace(/([A-Za-z])\((?=[A-Z]{2}[^)]*\))/g, "$1 (") + .replace(/\s+/g, " ") + .trim() + ); } export function sourceDisplayTitle(source: SearchResult) { diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts index 37127da1c9..2ce60d27d4 100644 --- a/src/lib/source-text-sanitizer.ts +++ b/src/lib/source-text-sanitizer.ts @@ -227,22 +227,24 @@ function stripLowYieldLines(value: string) { } export function stripLowYieldSourceNoise(text: string) { - return stripLowYieldLines(text) - .replace(leadingClassificationBannerPattern, "") - .replace(internalImageTokenPattern, " ") - .replace(sourceTitleWithCodePattern, " ") - .replace(sourceDocumentCodePattern, " ") - .replace(pageBoilerplatePattern, " ") - .replace(provenancePhrasePattern, " ") - .replace(evidenceLabelPattern, " ") - .replace(genericReferencePattern, "") - .replace(/\b(?:chunk|similarity)\s+\d+(?:\.\d+)?\b/gi, " ") - .replace(/\s+([,.;:])/g, "$1") - // Collapse doubled dots mid-text, but keep a trailing "..." — that is a - // stored truncation marker repairTruncatedCompactTail needs to see. The - // lookahead excludes dots so backtracking can't shave a trailing ellipsis - // down to ".." and still match. - .replace(/(?:\.\s*){2,}(?=[^.\s])/g, ". "); + return ( + stripLowYieldLines(text) + .replace(leadingClassificationBannerPattern, "") + .replace(internalImageTokenPattern, " ") + .replace(sourceTitleWithCodePattern, " ") + .replace(sourceDocumentCodePattern, " ") + .replace(pageBoilerplatePattern, " ") + .replace(provenancePhrasePattern, " ") + .replace(evidenceLabelPattern, " ") + .replace(genericReferencePattern, "") + .replace(/\b(?:chunk|similarity)\s+\d+(?:\.\d+)?\b/gi, " ") + .replace(/\s+([,.;:])/g, "$1") + // Collapse doubled dots mid-text, but keep a trailing "..." — that is a + // stored truncation marker repairTruncatedCompactTail needs to see. The + // lookahead excludes dots so backtracking can't shave a trailing ellipsis + // down to ".." and still match. + .replace(/(?:\.\s*){2,}(?=[^.\s])/g, ". ") + ); } export function lowYieldSourceNoiseScore(text: string) { diff --git a/tests/display-text.test.ts b/tests/display-text.test.ts index 82a7d6d64f..f5119d299b 100644 --- a/tests/display-text.test.ts +++ b/tests/display-text.test.ts @@ -62,9 +62,9 @@ describe("clinical dashboard display text", () => { }); it("sheds a mid-list ordinal at the snippet head but keeps a genuine list start", () => { - expect(compactSourceSnippet("2) MO to check the serum lithium level and renal function as soon as possible")).toBe( - "MO to check the serum lithium level and renal function as soon as possible", - ); + expect( + compactSourceSnippet("2) MO to check the serum lithium level and renal function as soon as possible"), + ).toBe("MO to check the serum lithium level and renal function as soon as possible"); expect(compactSourceSnippet("1) Confirm baseline ECG before starting lithium therapy")).toBe( "1) Confirm baseline ECG before starting lithium therapy", );