From a17bc92059b52884e271a402c8aff7f33ebed24c Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:42:34 +0800 Subject: [PATCH] feat: overhaul document viewer page (smart summary, badges, flattened evidence, flowing source text) - Add display-time smart summary formatter (document-summary-formatting.ts): strips glued PDF-header boilerplate, dedupes repeated passages, sections inline numbered headings, repairs mid-word truncated tails. Works on all stored summaries without re-indexing. - Add summary badge system (document-summary-badges.ts): safety-relevant labels + detected phrases render as tone-ordered ClinicalBadge cluster; new flags registered in the colour-coding catalogue; DocumentTagCloud colours unified with the canonical semantic-tone system app-wide. - Flatten pinned source evidence card: single quiet panel, no accent left borders/ring/solid header bar; excerpts flow via flowIndexedText. - Fix indexed source text: merge soft-wrap continuation blocks, recognise multi-level numbered headings, render as flowing typography instead of per-line bordered cards. - Remove meta-only Document details card: warnings become an InlineNotice at the sidebar top, index metadata demoted to a collapsed "Indexing details" disclosure, tables/diagrams count folded into that section. - Route the Overview hero through the formatter (was leaking boilerplate). - Extend demo data with labels/summary/indexHealth so the page is fully previewable in demo mode; add vitest suites for formatter + badges and a ui-smoke spec for the redesigned sidebar. Co-Authored-By: Claude Fable 5 --- src/components/DocumentTagCloud.tsx | 26 +- src/components/DocumentViewer.tsx | 380 +++++++++++++--------- src/lib/demo-data.ts | 170 +++++++++- src/lib/document-summary-badges.ts | 134 ++++++++ src/lib/document-summary-formatting.ts | 370 +++++++++++++++++++++ src/lib/indexed-source-formatting.ts | 58 +++- src/lib/semantic-flags.ts | 59 ++++ src/lib/source-text-sanitizer.ts | 11 + tests/demo-data.test.ts | 16 + tests/document-summary-badges.test.ts | 88 +++++ tests/document-summary-formatting.test.ts | 140 ++++++++ tests/indexed-source-formatting.test.ts | 90 ++++- tests/ui-smoke.spec.ts | 27 ++ 13 files changed, 1391 insertions(+), 178 deletions(-) create mode 100644 src/lib/document-summary-badges.ts create mode 100644 src/lib/document-summary-formatting.ts create mode 100644 tests/document-summary-badges.test.ts create mode 100644 tests/document-summary-formatting.test.ts diff --git a/src/components/DocumentTagCloud.tsx b/src/components/DocumentTagCloud.tsx index 84e4a7151e..8f8069809c 100644 --- a/src/components/DocumentTagCloud.tsx +++ b/src/components/DocumentTagCloud.tsx @@ -8,7 +8,9 @@ import { type SmartDocumentTag, type SmartDocumentTagGroup, } from "@/lib/document-tags"; +import { documentTagGroupTone } from "@/lib/document-summary-badges"; import type { DocumentLabel } from "@/lib/types"; +import { clinicalBadgeToneClass } from "@/components/clinical-dashboard/clinical-badge"; import { cn } from "@/components/ui-primitives"; type DocumentTagCloudProps = { @@ -40,22 +42,12 @@ const groupIcon: Record = { Manual: Sparkles, }; -const groupTone: Record = { - Site: "border-[color:var(--border-lux)] bg-[color:var(--surface-subtle)] text-[color:var(--text-muted)]", - Medication: "border-[color:var(--primary)]/30 bg-[color:var(--primary-soft)]/45 text-[color:var(--primary)]", - Risk: "border-[color:var(--warning)]/30 bg-[color:var(--warning-soft)]/50 text-[color:var(--warning)]", - Workflow: "border-[color:var(--info)]/30 bg-[color:var(--info-soft)]/50 text-[color:var(--info)]", - Topic: "border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] text-[color:var(--text-muted)]", - Population: "border-[color:var(--success)]/25 bg-[color:var(--success-soft)]/40 text-[color:var(--success)]", - Setting: "border-[color:var(--border-lux)] bg-[color:var(--surface-subtle)] text-[color:var(--text-muted)]", - Service: "border-[color:var(--primary)]/20 bg-[color:var(--surface-raised)] text-[color:var(--primary)]", - "Document type": "border-[color:var(--border-lux)] bg-[color:var(--surface-raised)] text-[color:var(--text-muted)]", - "Clinical action": "border-[color:var(--info)]/30 bg-[color:var(--info-soft)]/50 text-[color:var(--info)]", - "Care phase": "border-[color:var(--success)]/25 bg-[color:var(--success-soft)]/40 text-[color:var(--success)]", - "Document intent": "border-[color:var(--primary)]/20 bg-[color:var(--surface-raised)] text-[color:var(--primary)]", - "Content feature": "border-[color:var(--border-lux)] bg-[color:var(--surface-subtle)] text-[color:var(--text-muted)]", - Manual: "border-[color:var(--primary)]/35 bg-[color:var(--primary-soft)] text-[color:var(--primary)]", -}; +// Chip colours come from the canonical semantic-tone system (see +// /reference/colour-coding) via the shared group→tone map, so document tag +// chips read the same as every other clinical badge in the app. +function groupToneClass(group: SmartDocumentTagGroup): string { + return clinicalBadgeToneClass(documentTagGroupTone[group]); +} function confidenceTitle(tag: SmartDocumentTag) { return `${tag.group}: ${tag.source} tag, ${Math.round(tag.confidence * 100)}% confidence`; @@ -76,7 +68,7 @@ function DocumentTagChip({ const tagClassName = cn( "inline-flex max-w-full items-center gap-1 rounded-md border font-semibold shadow-[var(--shadow-inset)]", compact ? "min-h-6 px-2 text-3xs" : "min-h-7 px-2 text-2xs", - groupTone[tag.group], + groupToneClass(tag.group), tag.queryMatched && "ring-2 ring-[color:var(--focus)]/25", selected && "ring-2 ring-[color:var(--primary)]/35", onTagClick && diff --git a/src/components/DocumentViewer.tsx b/src/components/DocumentViewer.tsx index d7ab5ef237..8d25da14e8 100644 --- a/src/components/DocumentViewer.tsx +++ b/src/components/DocumentViewer.tsx @@ -32,7 +32,7 @@ import { Trash2, X, } from "lucide-react"; -import { type FormEvent, useEffect, useRef, useState } from "react"; +import { type FormEvent, useEffect, useMemo, useRef, useState } from "react"; import { AccessibleTable } from "@/components/AccessibleTable"; import { documentDisplayTitle } from "@/components/DocumentOrganizationBadges"; import { @@ -50,11 +50,12 @@ import { appBackdrop, clinicalDivider, cn, - evidenceSurface, + codeText, eyebrowText, fieldControl, fieldLabel, floatingControl, + InlineNotice, LoadingPanel, panel, PanelHeading, @@ -64,6 +65,7 @@ import { textMuted, toolbarButton, } from "@/components/ui-primitives"; +import { BadgeCluster } from "@/components/clinical-dashboard/clinical-badge"; import { clearCachedSignedUrl, getCachedSignedUrl, setCachedSignedUrl } from "@/lib/signed-url-cache"; import { readLocalProjectIdentity, unsafeLocalProjectMessage } from "@/lib/local-project-identity"; import { formatClinicalDate } from "@/lib/source-metadata"; @@ -88,7 +90,12 @@ import { sourceTextForIndexedPage, } from "@/lib/source-text-sanitizer"; import { smartEvidenceTags } from "@/lib/evidence-tags"; -import { parseIndexedSourceText } from "@/lib/indexed-source-formatting"; +import { flowIndexedText, parseIndexedSourceText } from "@/lib/indexed-source-formatting"; +import { + formatDocumentSummary, + type FormattedDocumentSummary as FormattedDocumentSummaryModel, +} from "@/lib/document-summary-formatting"; +import { buildDocumentSummaryBadges } from "@/lib/document-summary-badges"; type PageRow = { id: string; @@ -264,6 +271,75 @@ function ClinicalSummaryProfile({ profile }: { profile: ClinicalDocumentSummaryP ); } +// Structured renderer for the raw stored summary text, sharing +// ClinicalSummaryProfile's visual language (lead paragraph, eyebrow section +// headings, accent-dot bullets). Collapsed by default with an explicit +// "Show full summary" toggle so nothing is silently hidden. +const collapsedSummarySectionCap = 4; +const collapsedSummaryItemCap = 5; + +function FormattedHighYieldSummary({ formatted }: { formatted: FormattedDocumentSummaryModel }) { + const [expanded, setExpanded] = useState(false); + if (formatted.isEmpty) return null; + + const visibleSections = expanded + ? formatted.sections + : formatted.sections + .slice(0, collapsedSummarySectionCap) + .map((section) => ({ ...section, items: section.items.slice(0, collapsedSummaryItemCap) })); + const totalItems = formatted.sections.reduce((count, section) => count + section.items.length, 0); + const visibleItems = visibleSections.reduce((count, section) => count + section.items.length, 0); + const hasOverflow = totalItems > visibleItems || formatted.sections.length > visibleSections.length; + + return ( +
+ {formatted.lead ? ( +

+ +

+ ) : null} + {visibleSections.map((section, index) => ( +
0) && "border-t border-[color:var(--border)] pt-3")} + > +

+ {section.heading ?? "Key points"} +

+
    + {section.items.map((item, itemIndex) => ( +
  • +
    • + ))} +
+
+ ))} + {hasOverflow || expanded ? ( + + ) : null} + {formatted.truncatedTail ? ( +

+ Summary trimmed at indexing — open the source PDF for full detail. +

+ ) : null} +
+ ); +} + function looksLikeTableText(value?: string | null) { return Boolean(value?.includes("|") && value.split("|").filter((cell) => cell.trim()).length >= 3); } @@ -560,7 +636,7 @@ function DocumentViewerAnchors({ {anchor.label} @@ -582,7 +658,7 @@ function PinnedSourceEvidence({ compact?: boolean; sectionId?: "source-evidence" | "source-evidence-rail"; }) { - const displayContent = chunk ? sourceTextForDocumentViewer(chunk.content) : ""; + const displayContent = chunk ? flowIndexedText(sourceTextForDocumentViewer(chunk.content)) : ""; const previewLimit = compact ? 220 : 300; const [expandedChunkId, setExpandedChunkId] = useState(null); const isLong = displayContent.length > previewLimit; @@ -622,7 +698,7 @@ function PinnedSourceEvidence({
{loading ? ( @@ -630,58 +706,45 @@ function PinnedSourceEvidence({ ) : chunk ? (
-
-
-

- - Highlighted source passage -

- {chunkMeta ?

{chunkMeta}

: null} - {chunk.section_heading && ( -

{chunk.section_heading}

- )} -
-
-
-

- Excerpt -

-

- {visibleContent || "No displayable clinical text was available for this indexed passage."} +

+

+ + Highlighted source passage

+ {chunkMeta ?

{chunkMeta}

: null} +
+ {chunk.section_heading && ( +

{chunk.section_heading}

+ )} +
+ {visibleContent || "No displayable clinical text was available for this indexed passage."}
-
-
- - - Open source - - {compact && isLong ? ( - - ) : null} -
- {compact ? ( -

- Full indexed page text remains available in the source text section. -

+
+ + + Open source + + {compact && isLong ? ( + ) : null}
+ {compact ? ( +

+ Full indexed page text remains available in the source text section. +

+ ) : null}
) : ( -

+

Open a citation from an answer to see the exact indexed passage.

)} @@ -704,21 +767,15 @@ function IndexedSourceText({ } return ( -
+
{blocks.map((block) => { if (block.type === "heading") { return block.level === "title" ? ( -

+

{block.text}

) : ( -

+

{block.text}

); @@ -729,12 +786,12 @@ function IndexedSourceText({
    {block.items.map((item, index) => ( -
  • +
  • {item}
    • ))} @@ -758,10 +815,7 @@ function IndexedSourceText({ return (

    {block.text}

    @@ -1018,10 +1072,12 @@ function IndexedTextPanel({ Excerpt

    {normalizedSearch ? ( -

    +

    @@ -1696,11 +1752,13 @@ function compactDocumentType(document: ClinicalDocument) { function documentOverviewText(document: ClinicalDocument) { const profile = document.summary?.clinical_specifics?.profile; + // The stored raw summary opens with PDF-header boilerplate on many live + // documents, so route it through the smart formatter and show its lead + // sentences instead of the raw string. + const formattedSummary = profile?.overview ? null : formatDocumentSummary(document.summary?.summary); const overview = profile?.overview ? cleanClinicalSummaryText(profile.overview) - : document.summary?.summary - ? cleanClinicalSummaryText(document.summary.summary) - : ""; + : (formattedSummary?.lead ?? formattedSummary?.sections[0]?.items.join(" ") ?? ""); if (overview && !/source-backed review/i.test(overview)) return overview; return "A clear overview of this document, useful pages, and source PDF access."; } @@ -2341,7 +2399,18 @@ export function DocumentViewer({ const selectedChunk = chunkId ? chunks.find((chunk) => chunk.id === chunkId) : undefined; const { clinicalImages, auditImages } = partitionViewerImages(images); const generatedSummaryText = summary ? cleanClinicalSummaryText(summary.answer) : ""; - const usefulPageCount = usefulDocumentPages(initialPage, pages).length || 1; + const storedSummaryText = document?.summary?.summary ?? null; + const documentLabels = document?.labels; + const formattedStoredSummary = useMemo(() => formatDocumentSummary(storedSummaryText), [storedSummaryText]); + const summaryBadges = useMemo( + () => buildDocumentSummaryBadges({ labels: documentLabels, summaryText: storedSummaryText }), + [documentLabels, storedSummaryText], + ); + const indexWarnings = Array.isArray(indexHealth?.warnings) + ? indexHealth.warnings.map((warning) => String(warning)).filter(Boolean) + : typeof indexHealth?.warnings === "string" && indexHealth.warnings + ? [indexHealth.warnings] + : []; useEffect(() => { if (!chunkId || loadingDocument) return; const target = window.document.querySelector(`[data-source-chunk-id="${CSS.escape(chunkId)}"]`); @@ -2810,6 +2879,17 @@ export function DocumentViewer({

)} - + {document.labels?.length ? ( +
+

Browse by tag

+ +
+ ) : null} {canUsePrivateApis ? (
@@ -2951,7 +2989,13 @@ export function DocumentViewer({
{canUsePrivateApis && tableFacts.length ? ( @@ -2990,6 +3034,28 @@ export function DocumentViewer({ ) : null}
+ + {indexHealth ? ( +
+ Indexing details +
+
+
Extraction
+
{indexHealth.extractionQuality ?? "unknown"}
+
+
+
Index version
+
+ {indexHealth.indexVersion ?? "unknown"} +
+
+
+
Indexed
+
{indexHealth.indexedAt ?? "not recorded"}
+
+
+
+ ) : null} {readyDocument ? ( diff --git a/src/lib/demo-data.ts b/src/lib/demo-data.ts index 9929c17894..8f929b6fb2 100644 --- a/src/lib/demo-data.ts +++ b/src/lib/demo-data.ts @@ -2,6 +2,8 @@ import type { ChunkImage, ClinicalDocument, DocumentBreakdown, + DocumentLabel, + DocumentSummary, IngestionJob, RagAnswer, SearchResult, @@ -286,6 +288,157 @@ export const demoJobs: IngestionJob[] = demoDocuments.map((document, index) => ( updated_at: now, })); +// Labels + stored summaries let the document viewer's high-yield summary, +// badge cluster, and tag cloud render in demo mode. The lithium summary +// deliberately reproduces the stored-summary failure modes (glued header +// boilerplate, repeated passages, an inline numbered heading, and a mid-word +// truncated tail) that the display-time formatter repairs. +export const demoDocumentLabels: DocumentLabel[] = [ + { + id: "88888888-8888-4888-8888-888888888801", + document_id: demoDocuments[0].id, + label: "fiona stanley hospital", + label_type: "site", + source: "generated", + confidence: 0.94, + }, + { + id: "88888888-8888-4888-8888-888888888802", + document_id: demoDocuments[0].id, + label: "lithium", + label_type: "medication", + source: "generated", + confidence: 0.97, + }, + { + id: "88888888-8888-4888-8888-888888888803", + document_id: demoDocuments[0].id, + label: "mood stabilisers", + label_type: "medication", + source: "generated", + confidence: 0.82, + }, + { + id: "88888888-8888-4888-8888-888888888804", + document_id: demoDocuments[0].id, + label: "toxicity risk", + label_type: "risk", + source: "generated", + confidence: 0.9, + }, + { + id: "88888888-8888-4888-8888-888888888805", + document_id: demoDocuments[0].id, + label: "prescribing", + label_type: "workflow", + source: "generated", + confidence: 0.86, + }, + { + id: "88888888-8888-4888-8888-888888888806", + document_id: demoDocuments[0].id, + label: "contains monitoring schedule", + label_type: "content_feature", + source: "generated", + confidence: 0.8, + }, + { + id: "88888888-8888-4888-8888-888888888807", + document_id: demoDocuments[1].id, + label: "clozapine", + label_type: "medication", + source: "generated", + confidence: 0.97, + }, + { + id: "88888888-8888-4888-8888-888888888808", + document_id: demoDocuments[1].id, + label: "agranulocytosis risk", + label_type: "risk", + source: "generated", + confidence: 0.88, + }, + { + id: "88888888-8888-4888-8888-888888888809", + document_id: demoDocuments[2].id, + label: "risk assessment", + label_type: "workflow", + source: "generated", + confidence: 0.85, + }, +]; + +const demoProfileItem = (text: string, pages: number[]) => ({ + text, + source_chunk_ids: [], + source_image_ids: [], + pages, + evidence_type: "text" as const, + support: "direct" as const, +}); + +export const demoDocumentSummaries: DocumentSummary[] = [ + { + id: "99999999-9999-4999-8999-999999999901", + document_id: demoDocuments[0].id, + summary: + "OFFICIAL Guideline Lithium Therapy- Initiation and Continuation Reference #: FSFHG-HW-GUI-0017 Scope Site " + + "Service/Department/Unit Disciplines Fiona Stanley Hospital Hospital Wide Medical, Nursing, Pharmacy " + + "Fremantle Hospital Lithium is a high-risk medication with a narrow therapeutic index. Careful patient " + + "selection and monitoring is required to minimise the risk of lithium toxicity. 1. Introduction Hospital " + + "Wide Medical, Nursing, Pharmacy Fremantle Hospital Lithium is a high-risk medication with a narrow " + + "therapeutic index. Careful patient selection and monitoring is required to minimise the risk of lithium " + + "toxicity. 1. Introduction In this synthetic protocol, lithium levels are checked 5 to 7 days after " + + "initiation or dose change, then repeated until stable. After stability the sample schedule uses lithium " + + "levels every 3 months, renal and thyroid tests every 6 months, and calcium annually. The therapeutic " + + "effect occurs gradually and may take up to three weeks. Escalate review for vomiting, diarrhoea, " + + "dehydration, acute kidney injury, new NSAID/ACE inhibitor/diuretic exposure, tremor, confusion, or " + + "ataxia. therapeutic effect occurs gradually and may take up to three weeks. Lithium is a narro", + clinical_specifics: {}, + source_chunk_ids: [], + source_image_ids: [], + model: null, + }, + { + id: "99999999-9999-4999-8999-999999999902", + document_id: demoDocuments[1].id, + summary: + "Synthetic clozapine monitoring summary covering FBC/ANC checks, myocarditis screening, metabolic " + + "monitoring, and constipation prevention.", + clinical_specifics: { + profile: { + overview: + "Synthetic clozapine monitoring protocol emphasising FBC/ANC monitoring, myocarditis symptom screening, metabolic monitoring, and constipation prevention. Demo only, not clinical guidance.", + applies_to: [demoProfileItem("Adults prescribed clozapine in the synthetic shared-care demo pathway.", [1])], + key_clinical_actions: [ + demoProfileItem("Record baseline FBC/ANC, weight, lipids, glucose/HbA1c, and bowel history.", [1]), + demoProfileItem("Screen for myocarditis symptoms during initiation.", [1]), + ], + medication_dose_monitoring: [ + demoProfileItem("Continue scheduled FBC/ANC monitoring per the synthetic protocol table.", [2]), + ], + thresholds_timing: [], + escalation_risk_warnings: [ + demoProfileItem( + "Urgent review for fever, chest pain, dyspnoea, tachycardia, marked sedation, seizures, or severe constipation.", + [1], + ), + ], + required_forms_documentation: [], + not_covered: [], + important_tables_images: [ + demoProfileItem("Monitoring domains table across baseline, initiation, and ongoing care.", [2]), + ], + best_questions: [], + source_quality_notes: [], + }, + }, + source_chunk_ids: [], + source_image_ids: [], + model: null, + }, +]; + export function getDemoDocument(id: string) { return demoDocuments.find((document) => document.id === id) ?? null; } @@ -295,6 +448,8 @@ export function getDemoDocumentPayload(id: string, chunkId?: string | null) { if (!document) return null; const pages = demoPages.filter((page) => page.document_id === id); const images = demoImages.filter((image) => image.document_id === id); + const labels = demoDocumentLabels.filter((label) => label.document_id === id); + const summary = demoDocumentSummaries.find((row) => row.document_id === id) ?? null; const chunks = demoChunks .filter((chunk) => chunk.document_id === id) .filter((chunk) => !chunkId || chunk.id === chunkId) @@ -308,7 +463,20 @@ export function getDemoDocumentPayload(id: string, chunkId?: string | null) { image_ids: chunk.image_ids, })); - return { document, pages, images, chunks }; + // Shape mirrors the live GET /api/documents/[id] response (labels + summary + // joined onto the document, indexHealth synthesized from metadata). + return { + document: { ...document, labels, summary }, + pages, + images, + chunks, + indexHealth: { + extractionQuality: syntheticMetadata.extraction_quality, + indexedAt: syntheticMetadata.indexed_at, + indexVersion: "rag-deep-memory-v1", + warnings: [], + }, + }; } const queryTerms: Record = { diff --git a/src/lib/document-summary-badges.ts b/src/lib/document-summary-badges.ts new file mode 100644 index 0000000000..d77dc8abe7 --- /dev/null +++ b/src/lib/document-summary-badges.ts @@ -0,0 +1,134 @@ +// Badge derivation for the document viewer's high-yield summary card. +// +// Framework-free (mirrors medication-badges.ts): data in, ClinicalBadge-shaped +// items out. Two inputs feed the cluster — safety-relevant document labels +// (Risk / Medication / Clinical action) and phrases detected in the stored +// summary text itself ("narrow therapeutic index", "Schedule 8", …). Tone +// rules follow the badge governance in docs/clinical-badge-system-guide.md: +// danger is reserved for true contraindications; regulatory/caution signals +// (S8, high-risk, toxicity) are warnings. + +import { buildSmartDocumentTags, type SmartDocumentTagGroup } from "@/lib/document-tags"; +import { sortBySemanticTonePriority, type SemanticIconKey, type SemanticTone } from "@/lib/semantic-tone"; +import type { DocumentLabel } from "@/lib/types"; + +export type DocumentSummaryBadge = { + id: string; + label: string; + tone: SemanticTone; + iconKey?: SemanticIconKey; +}; + +type DocumentLabelLike = Pick; + +// Canonical tone for each smart-tag group. Shared with DocumentTagCloud so the +// tag chips and the summary badge cluster speak the same colour language. +export const documentTagGroupTone: Record = { + Risk: "warning", + Medication: "clinical", + "Clinical action": "clinical", + Workflow: "info", + Manual: "success", + Site: "neutral", + Topic: "neutral", + Population: "neutral", + Setting: "neutral", + Service: "neutral", + "Document type": "neutral", + "Care phase": "neutral", + "Document intent": "neutral", + "Content feature": "neutral", +}; + +// Groups whose labels are important enough to promote into the badge cluster; +// the rest stay in the browse-by-tag cloud. +const badgeWorthyGroups = new Set(["Risk", "Medication", "Clinical action"]); + +type SummaryPhraseRule = { + id: string; + pattern: RegExp; + label: string; + tone: SemanticTone; + iconKey?: SemanticIconKey; +}; + +// Phrase catalogue over the stored summary text. Every rule here is also +// registered in SEMANTIC_FLAG_CATALOGUE (document domain) so the +// /reference/colour-coding legend stays complete. +const summaryPhraseRules: SummaryPhraseRule[] = [ + { id: "summary-contraindication", pattern: /contraindicat/i, label: "Contraindications", tone: "danger" }, + { + id: "summary-narrow-therapeutic-index", + pattern: /narrow therapeutic (?:index|window|range)/i, + label: "Narrow therapeutic index", + tone: "warning", + }, + { + id: "summary-high-risk-medication", + pattern: /high[- ]?risk medication/i, + label: "High-risk medication", + tone: "warning", + }, + { + id: "summary-controlled-drug", + pattern: /\bschedule\s*8\b|\bS8\b/i, + label: "Schedule 8", + tone: "warning", + iconKey: "controlled", + }, + { id: "summary-toxicity", pattern: /\btoxic(?:ity)?\b/i, label: "Toxicity risk", tone: "warning" }, + { + id: "summary-escalation", + pattern: /\b(?:escalat\w*|urgent(?:ly)? review\w*|emergency)\b/i, + label: "Escalation criteria", + tone: "warning", + }, + { + id: "summary-pregnancy", + pattern: /\b(?:pregnan\w*|lactation|breastfeed\w*)\b/i, + label: "Pregnancy & lactation", + tone: "warning", + }, + { + id: "summary-monitoring", + pattern: /\b(?:monitor(?:ing|ed)?|serum levels?|blood tests?|fbc|anc|ecg|qtc)\b/i, + label: "Monitoring required", + tone: "info", + }, +]; + +export function buildDocumentSummaryBadges({ + labels, + summaryText, + limit = 8, +}: { + labels?: DocumentLabelLike[] | null; + summaryText?: string | null; + limit?: number; +}): DocumentSummaryBadge[] { + const badges: DocumentSummaryBadge[] = []; + const seenIds = new Set(); + const seenLabels = new Set(); + + const push = (badge: DocumentSummaryBadge) => { + const labelKey = badge.label.toLowerCase(); + if (seenIds.has(badge.id) || seenLabels.has(labelKey)) return; + seenIds.add(badge.id); + seenLabels.add(labelKey); + badges.push(badge); + }; + + for (const tag of buildSmartDocumentTags(labels, { includeManualGroup: false })) { + if (!badgeWorthyGroups.has(tag.group)) continue; + push({ id: `label-${tag.key}`, label: tag.label, tone: documentTagGroupTone[tag.group] }); + } + + if (summaryText) { + for (const rule of summaryPhraseRules) { + if (!rule.pattern.test(summaryText)) continue; + push({ id: rule.id, label: rule.label, tone: rule.tone, iconKey: rule.iconKey }); + } + } + + return sortBySemanticTonePriority(badges).slice(0, Math.max(0, limit)); +} diff --git a/src/lib/document-summary-formatting.ts b/src/lib/document-summary-formatting.ts new file mode 100644 index 0000000000..6685374f98 --- /dev/null +++ b/src/lib/document-summary-formatting.ts @@ -0,0 +1,370 @@ +// Client-side "smart summary" formatter for indexing-time document summaries. +// +// Stored `document_summaries.summary` text is LLM-generated at indexing but +// frequently carries PDF-header debris: protective-marking banners glued to a +// document title ("OFFICIAL Guideline Lithium Therapy … Reference #: …"), +// Scope/Site/Disciplines runs, sentences repeated 2-3x, inline numbered +// headings ("1. Introduction"), and a mid-word truncated tail. ~2000 live +// documents already have these summaries stored, so the repair has to happen +// at display time — this module turns the raw string into a structured, +// readable display model without re-indexing and without dropping clinical +// content (same generous keep-bias as source-text-sanitizer's H2 rules). + +import { + cleanClinicalSummaryText, + hasClinicalContentSignal, + repairTruncatedCompactTail, +} from "@/lib/source-text-sanitizer"; + +export type DocumentSummarySection = { + id: string; + /** null = un-headed "key points" run before/without any detected heading. */ + heading: string | null; + items: string[]; +}; + +export type FormattedDocumentSummary = { + /** Short plain-language opener (first 1-2 sentences). */ + lead: string | null; + sections: DocumentSummarySection[]; + /** A mid-word trailing fragment was repaired or removed. */ + truncatedTail: boolean; + isEmpty: boolean; +}; + +const EMPTY_SUMMARY: FormattedDocumentSummary = { + lead: null, + sections: [], + truncatedTail: false, + isEmpty: true, +}; + +// Connector words allowed inside a Title-Case run (document titles, headings). +const titleConnectorPattern = /^(?:and|of|the|for|in|to|with|a|an|or|on|at|&)$/i; + +// Boilerplate markers that may open a stored summary. Split into two tiers: +// openers can strip from the very start of a segment; gated markers only strip +// once an opener has already matched, so a genuine sentence like "Fremantle +// Hospital provides…" is never beheaded. +const openerMarkerPatterns: RegExp[] = [ + // Protective-marking banners glued inline ("OFFICIAL", "OFFICIAL: Sensitive"). + /^(?:UNOFFICIAL|OFFICIAL(?:\s*:\s*Sensitive)?|SENSITIVE|PROTECTED)\b:?\s*/, + // Document-type word immediately followed by a Title-Case run (a title), not + // by ordinary prose ("Guideline recommendations include…" is left alone). + /^(?:Clinical\s+)?(?:Guideline|Procedure|Protocol|Policy|Standard|Form)\b:?\s+(?=[A-Z0-9])/, + /^Reference\s*(?:#|No\.?|Number)?\s*:?\s*[A-Za-z0-9][A-Za-z0-9/-]{3,}\s*/i, + /^(?:Scope|Applicability|Audience|Target\s+audience)\b:?\s*/, + /^Service\/Department\/Unit\b:?\s*/i, + /^Disciplines?\b:?\s*(?=[A-Z])/, + /^Hospital[- ]Wide\b:?\s*/i, +]; + +const gatedMarkerPatterns: RegExp[] = [ + /^Site\b:?\s*(?=[A-Z])/, + // Hospital / health-service proper names ("Fiona Stanley Hospital"). + /^(?:[A-Z][A-Za-z'-]+\s+){1,3}(?:Hospitals?|Health\s+Service|Health\s+Campus)\b\s*/, + // Discipline lists ("Medical, Nursing, Pharmacy"). + /^(?:Medical|Nursing|Midwifery|Pharmacy|Medicine|Dental|Allied\s+Health)(?:\s*[,&]\s*(?:Medical|Nursing|Midwifery|Pharmacy|Medicine|Dental|Allied\s+Health))*\b,?\s*/, +]; + +// A Title-Case run that leads straight into a boilerplate marker is the glued +// document title ("Lithium Therapy - Initiation and Continuation Reference #:"). +const bridgeToMarkerPattern = + /^[A-Z][^.!?]{0,140}?(?=(?:Reference\s*(?:#|No\.?|Number)?\s*:|Scope\b|Service\/Department\/Unit\b|Disciplines?\b|Hospital[- ]Wide\b))/; + +function startsWithBoilerplateMarker(value: string) { + return ( + openerMarkerPatterns.some((pattern) => pattern.test(value)) || + gatedMarkerPatterns.some((pattern) => pattern.test(value)) + ); +} + +// Strips a *leading* run of document-header boilerplate from a summary segment. +// Only ever eats from the front; never drops text carrying clinical signal +// (thresholds, action verbs) and reverts entirely if it would leave nothing. +// Idempotent: a stripped segment no longer starts with any marker. +export function stripSummaryBoilerplate(text: string): string { + const input = text.trimStart(); + let out = input; + let consumedMarker = false; + let guard = 0; + + stripping: while (out && guard < 40 && input.length - out.length <= 600) { + guard += 1; + for (const pattern of openerMarkerPatterns) { + const match = out.match(pattern); + if (match && match[0]) { + out = out.slice(match[0].length).trimStart(); + consumedMarker = true; + continue stripping; + } + } + if (consumedMarker) { + for (const pattern of gatedMarkerPatterns) { + const match = out.match(pattern); + if (match && match[0]) { + out = out.slice(match[0].length).trimStart(); + continue stripping; + } + } + } + + const bridge = out.match(bridgeToMarkerPattern); + if (bridge && bridge[0] && !hasClinicalContentSignal(bridge[0])) { + out = out.slice(bridge[0].length).trimStart(); + consumedMarker = true; + continue; + } + + if (consumedMarker) { + // Leftover proper-noun run (site names, discipline words). Consume one + // Title-Case token at a time, but stop at the token that starts a real + // clause — a Title-Case word followed by lowercase prose ("Lithium is…"). + const token = out.match(/^([A-Z][A-Za-z'()/-]*)([,&/]\s*|\s+|-\s*)/); + if (token) { + const rest = out.slice(token[0].length); + const nextIsTitleCase = /^(?:[A-Z]|(?:and|of|the|for|&)\s+[A-Z])/.test(rest); + if (nextIsTitleCase && !hasClinicalContentSignal(token[1])) { + out = rest.trimStart(); + continue; + } + } + } + break; + } + + // Safety: if stripping consumed essentially everything, the heuristics were + // wrong for this text — keep the original rather than losing content. + if (out.length < 40 && input.length >= 80) return input; + return out; +} + +// Sentence split that survives abbreviations, initials, and numbered +// cross-references ("section 1.9. Therapeutic…"). Lowercase continuations are +// split too: stored summaries drop capitals when passages are glued together. +const sentenceSplitPattern = + /(?<=[.!?])(? sentence.trim()) + .filter(Boolean); +} + +function normalizeSentenceKey(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +// Drops exact repeats and long (>=40-char key) containment repeats, keeping the +// first occurrence — stored summaries frequently repeat whole passages 2-3x. +export function dedupeSummarySentences(sentences: string[]): string[] { + const kept: string[] = []; + const keys: string[] = []; + for (const sentence of sentences) { + const key = normalizeSentenceKey(sentence); + if (!key) continue; + const isDuplicate = keys.some( + (existing) => + existing === key || + (key.length >= 40 && existing.includes(key)) || + (existing.length >= 40 && key.includes(existing)), + ); + if (isDuplicate) continue; + kept.push(sentence); + keys.push(key); + } + return kept; +} + +// Inline numbered headings ("1. Introduction", "2.7. Dosage (as lithium +// carbonate)") glued into the flowing summary text. Only at a sentence +// boundary, and never after cross-reference words ("refer to section 1.9."). +const inlineHeadingPattern = + /(?<=^|[.!?:]\s{1,3})(? 0) { + consumed.push(connector[1]); + rest = rest.slice(connector[0].length); + continue; + } + const token = rest.match(tokenPattern); + if (!token) break; + consumed.push(token[1]); + rest = rest.slice(token[0].length); + } + + // Give the clause subject back to the body: "… Monitoring Serum lithium + // levels should …" keeps "Serum" with the sentence, not the heading. When + // that empties the heading ("3. Lithium toxicity risk increases…"), the + // caller treats the segment as ordinary text rather than mangling it. + if (consumed.length >= 1 && /^[a-z]/.test(rest)) { + const last = consumed[consumed.length - 1]; + if (/^[A-Z]/.test(last)) { + consumed.pop(); + rest = `${last} ${rest}`; + } + } + + // Trim trailing connectors left dangling by the give-back. + while (consumed.length && titleConnectorPattern.test(consumed[consumed.length - 1])) { + rest = `${consumed.pop()} ${rest}`; + } + + return { heading: consumed.join(" ").trim(), remainder: rest.trim() }; +} + +function sectionIdFrom(heading: string | null, index: number) { + if (!heading) return `summary-section-${index}`; + const slug = heading + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48); + return slug ? `summary-${slug}` : `summary-section-${index}`; +} + +type RawSection = { heading: string | null; text: string }; + +function splitIntoRawSections(text: string): RawSection[] { + const sections: RawSection[] = []; + let lastIndex = 0; + let currentHeading: string | null = null; + + for (const match of text.matchAll(inlineHeadingPattern)) { + const matchIndex = match.index ?? 0; + if (matchIndex < lastIndex) continue; // Inside heading text already consumed. + const after = text.slice(matchIndex + match[0].length); + const { heading, remainder } = extractHeadingText(after); + + if (!heading) continue; // Not a confident heading — leave the text in place. + + sections.push({ heading: currentHeading, text: text.slice(lastIndex, matchIndex).trim() }); + currentHeading = heading; + // The input is whitespace-collapsed, so the remainder (including any + // clause-subject give-back) is an exact suffix of `after`. + lastIndex = matchIndex + match[0].length + (after.length - remainder.length); + } + sections.push({ heading: currentHeading, text: text.slice(lastIndex).trim() }); + + return sections.filter((section) => section.heading !== null || section.text.length > 0); +} + +export function formatDocumentSummary(raw: string | null | undefined): FormattedDocumentSummary { + if (!raw || !raw.trim()) return EMPTY_SUMMARY; + + // Reuse the house sanitizer first (glyph repair, protective markings, source + // codes, label noise), then flatten to a single line for sentence work. + const cleaned = cleanClinicalSummaryText(raw).replace(/\s+/g, " ").trim(); + if (!cleaned) return EMPTY_SUMMARY; + + const stripped = stripSummaryBoilerplate(cleaned); + const rawSections = splitIntoRawSections(stripped); + + // Assemble sections with global sentence dedupe (repeats cross section + // boundaries in stored summaries) and per-section boilerplate stripping + // (running headers re-glue after inline headings). + const seenKeys: string[] = []; + const seenHeadings = new Map(); + const orderedSections: DocumentSummarySection[] = []; + let truncatedTail = false; + + const keepNewSentences = (text: string) => { + const sentences = dedupeSummarySentences(splitSentences(text)); + const fresh: string[] = []; + for (const sentence of sentences) { + const key = normalizeSentenceKey(sentence); + if (!key) continue; + const isDuplicate = seenKeys.some( + (existing) => + existing === key || + (key.length >= 40 && existing.includes(key)) || + (existing.length >= 40 && key.includes(existing)), + ); + if (isDuplicate) continue; + fresh.push(sentence); + seenKeys.push(key); + } + return fresh; + }; + + for (const rawSection of rawSections) { + const body = rawSection.heading === null ? rawSection.text : stripSummaryBoilerplate(rawSection.text); + const items = keepNewSentences(body); + if (!items.length && !rawSection.heading) continue; + + const headingKey = rawSection.heading ? normalizeSentenceKey(rawSection.heading) : null; + if (headingKey && seenHeadings.has(headingKey)) { + const existing = seenHeadings.get(headingKey)!; + existing.items.push(...items); + continue; + } + + const section: DocumentSummarySection = { + id: sectionIdFrom(rawSection.heading, orderedSections.length), + heading: rawSection.heading, + items, + }; + orderedSections.push(section); + if (headingKey) seenHeadings.set(headingKey, section); + } + + // Repair or drop a mid-word truncated tail on the very last item. + for (let index = orderedSections.length - 1; index >= 0; index -= 1) { + const items = orderedSections[index].items; + if (!items.length) continue; + const last = items[items.length - 1]; + const endsWithEllipsis = /(?:\.{3}|…)\s*$/.test(last); + const endsCleanly = /[.!?:;)\]"']$/.test(last.trim()); + if (endsCleanly && !endsWithEllipsis) break; + const repaired = repairTruncatedCompactTail(endsWithEllipsis ? last : `${last} ...`); + truncatedTail = true; + if (repaired && repaired.split(/\s+/).length >= 5) { + items[items.length - 1] = repaired; + } else { + items.pop(); + } + break; + } + + const sections = orderedSections.filter((section) => section.items.length > 0); + if (!sections.length) return { ...EMPTY_SUMMARY, truncatedTail }; + + // Lead: first 1-2 sentences of the first un-headed section. + let lead: string | null = null; + if (sections[0].heading === null) { + const first = sections[0]; + const leadItems: string[] = []; + while (first.items.length && leadItems.length < 2 && leadItems.join(" ").length + first.items[0].length <= 300) { + leadItems.push(first.items.shift()!); + } + if (!leadItems.length && first.items.length) leadItems.push(first.items.shift()!); + lead = leadItems.join(" ") || null; + } + + const finalSections = sections.filter((section) => section.items.length > 0); + return { + lead, + sections: finalSections, + truncatedTail, + isEmpty: !lead && finalSections.length === 0, + }; +} diff --git a/src/lib/indexed-source-formatting.ts b/src/lib/indexed-source-formatting.ts index c814023564..e49e54c92b 100644 --- a/src/lib/indexed-source-formatting.ts +++ b/src/lib/indexed-source-formatting.ts @@ -13,7 +13,10 @@ function isPageFooter(line: string) { } function isNumberedHeading(line: string) { - return /^\d{1,2}\.\s+\S/.test(line.trim()) && line.trim().length <= 96; + // Supports multi-level numbering ("2.7. Dosage", "3.1.2 Monitoring") as well + // as top-level "9. Polypharmacy". A bare number ("12 hours") is not a + // heading — top-level numbering must carry its dot. + return /^\d{1,2}(?:(?:\.\d{1,2})+\.?|\.)\s+\S/.test(line.trim()) && line.trim().length <= 96; } function isLikelyTitle(rawLine: string, line: string, index: number) { @@ -149,6 +152,57 @@ function paragraphFrom(lines: string[]) { return compactInline(lines.join(" ")); } +// PDF extraction inserts blank lines at soft wraps, so a wrapped sentence +// arrives as separate blocks ("• NSAIDs: … and therefore" / "increase lithium +// levels …"). This post-pass re-joins those continuations so text flows: +// a paragraph starting lowercase (or following an unterminated paragraph) +// continues the previous block; after a list it continues the last bullet. +// Headings and tables are hard boundaries. +export function mergeContinuationBlocks(blocks: IndexedTextBlock[]): IndexedTextBlock[] { + const merged: IndexedTextBlock[] = []; + + for (const block of blocks) { + const previous = merged[merged.length - 1]; + if (block.type === "paragraph" && previous) { + const startsAsContinuation = /^[a-z(]/.test(block.text); + if (previous.type === "paragraph" && (startsAsContinuation || !/[.!?:;]$/.test(previous.text))) { + previous.text = `${previous.text} ${block.text}`; + continue; + } + if (previous.type === "list" && startsAsContinuation && previous.items.length > 0) { + previous.items[previous.items.length - 1] = `${previous.items[previous.items.length - 1]} ${block.text}`; + continue; + } + } + if (block.type === "list" && previous?.type === "list") { + previous.items.push(...block.items); + continue; + } + merged.push( + block.type === "paragraph" ? { ...block } : block.type === "list" ? { ...block, items: [...block.items] } : block, + ); + } + + return merged; +} + +// Excerpt display helper: raw chunk text keeps hard wraps from extraction. +// Newlines become spaces so sentences flow; blank-line runs survive as +// paragraph breaks unless they sit mid-sentence (next line starts lowercase), +// which is how extraction separates soft-wrapped continuations. +export function flowIndexedText(text: string): string { + return text + .replace(/\r/g, "\n") + .replace(/[ \t]+\n[ \t]*/g, "\n") + .replace(/\n+/g, (run: string, offset: number, full: string) => { + if (run.length === 1) return " "; + const next = full.charAt(offset + run.length); + return /[a-z(]/.test(next) ? " " : "\n\n"; + }) + .replace(/[ \t]{2,}/g, " ") + .trim(); +} + export function parseIndexedSourceText(text: string): IndexedTextBlock[] { const rawLines = text .replace(/\r/g, "\n") @@ -219,5 +273,5 @@ export function parseIndexedSourceText(text: string): IndexedTextBlock[] { blocks.push({ type: "paragraph", id: `paragraph:${index}:${paragraph.slice(0, 24)}`, text: paragraph }); } - return blocks; + return mergeContinuationBlocks(blocks); } diff --git a/src/lib/semantic-flags.ts b/src/lib/semantic-flags.ts index d35f9f1c8c..4a923e1384 100644 --- a/src/lib/semantic-flags.ts +++ b/src/lib/semantic-flags.ts @@ -192,6 +192,65 @@ export const SEMANTIC_FLAG_CATALOGUE: SemanticFlagDef[] = [ tone: "danger", meaning: "Source is out of date — do not rely on it.", }, + // High-yield summary badges (document viewer): derived from safety-relevant + // labels and phrases detected in the stored summary text. + { + id: "doc-summary-contraindication", + domain: "document", + label: "Contraindications", + tone: "danger", + meaning: "The document describes contraindications — read before acting.", + }, + { + id: "doc-summary-narrow-ti", + domain: "document", + label: "Narrow therapeutic index", + tone: "warning", + meaning: "Covers a narrow therapeutic index medicine.", + }, + { + id: "doc-summary-high-risk-med", + domain: "document", + label: "High-risk medication", + tone: "warning", + meaning: "Covers a locally designated high-risk medication.", + }, + { + id: "doc-summary-controlled", + domain: "document", + label: "Schedule 8", + tone: "warning", + meaning: "Covers a controlled (Schedule 8) drug. Regulatory, not a clinical stop.", + iconKey: "controlled", + }, + { + id: "doc-summary-toxicity", + domain: "document", + label: "Toxicity risk", + tone: "warning", + meaning: "Toxicity risks are described in the source.", + }, + { + id: "doc-summary-escalation", + domain: "document", + label: "Escalation criteria", + tone: "warning", + meaning: "Contains escalation / urgent-review criteria.", + }, + { + id: "doc-summary-pregnancy", + domain: "document", + label: "Pregnancy & lactation", + tone: "warning", + meaning: "Contains pregnancy / lactation guidance.", + }, + { + id: "doc-summary-monitoring", + domain: "document", + label: "Monitoring required", + tone: "info", + meaning: "Contains monitoring requirements (levels, bloods, ECG).", + }, // Evidence & retrieval { diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts index cceebaccb1..00cbf924f2 100644 --- a/src/lib/source-text-sanitizer.ts +++ b/src/lib/source-text-sanitizer.ts @@ -242,6 +242,17 @@ function stripLowYieldLines(value: string) { .join("\n"); } +// Shared with the document-summary formatter so its boilerplate stripping can +// reuse the exact control-line vocabulary and the H2 keep-bias signals above +// instead of duplicating the regexes. +export function isDocumentControlLine(value: string) { + return sourceControlLinePattern.test(value); +} + +export function hasClinicalContentSignal(value: string) { + return clinicalSignalPattern.test(value) || clinicalThresholdSignalPattern.test(value); +} + export function stripLowYieldSourceNoise(text: string) { return ( stripLowYieldLines(text) diff --git a/tests/demo-data.test.ts b/tests/demo-data.test.ts index 677f6f6d16..70d570bfc1 100644 --- a/tests/demo-data.test.ts +++ b/tests/demo-data.test.ts @@ -82,6 +82,22 @@ describe("demo data mode", () => { expect(payload?.chunks.length).toBeGreaterThan(0); expect(payload?.images[0].caption).toContain("monitoring table"); }); + + it("joins labels, stored summary, and index health onto the viewer payload like the live API", () => { + const lithium = getDemoDocumentPayload(demoDocuments[0].id); + expect(lithium?.document.labels?.length).toBeGreaterThan(0); + expect(lithium?.document.labels?.some((label) => label.label_type === "medication")).toBe(true); + // Deliberately messy stored summary so the display-time formatter is exercised end-to-end. + expect(lithium?.document.summary?.summary).toContain("Reference #"); + expect(lithium?.document.summary?.summary).toContain("narrow therapeutic index"); + expect(lithium?.indexHealth).toMatchObject({ + extractionQuality: "good", + indexVersion: "rag-deep-memory-v1", + }); + + const clozapine = getDemoDocumentPayload(demoDocuments[1].id); + expect(clozapine?.document.summary?.clinical_specifics?.profile?.overview).toContain("clozapine"); + }); }); // Class-level guard so a future route cannot reintroduce the /api/search/universal leak: demo diff --git a/tests/document-summary-badges.test.ts b/tests/document-summary-badges.test.ts new file mode 100644 index 0000000000..5f7e1bf425 --- /dev/null +++ b/tests/document-summary-badges.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { buildDocumentSummaryBadges, documentTagGroupTone } from "@/lib/document-summary-badges"; +import { SEMANTIC_TONE_PRIORITY } from "@/lib/semantic-tone"; +import type { DocumentLabel } from "@/lib/types"; + +function label(overrides: Partial): DocumentLabel { + return { + id: overrides.id ?? `${overrides.label ?? "label"}-id`, + document_id: overrides.document_id ?? "doc-1", + label: overrides.label ?? "monitoring", + label_type: overrides.label_type ?? "topic", + source: overrides.source ?? "generated", + confidence: overrides.confidence ?? 0.8, + ...overrides, + }; +} + +describe("documentTagGroupTone", () => { + it("maps safety and action groups onto the canonical tones", () => { + expect(documentTagGroupTone.Risk).toBe("warning"); + expect(documentTagGroupTone.Medication).toBe("clinical"); + expect(documentTagGroupTone["Clinical action"]).toBe("clinical"); + expect(documentTagGroupTone.Workflow).toBe("info"); + expect(documentTagGroupTone.Manual).toBe("success"); + expect(documentTagGroupTone.Site).toBe("neutral"); + expect(documentTagGroupTone.Topic).toBe("neutral"); + }); +}); + +describe("buildDocumentSummaryBadges", () => { + it("promotes safety-relevant labels and detected phrases, ordered by tone priority", () => { + const badges = buildDocumentSummaryBadges({ + labels: [ + label({ label: "lithium", label_type: "medication", confidence: 0.9 }), + label({ label: "toxicity risk", label_type: "risk", confidence: 0.85 }), + label({ label: "prescribing", label_type: "workflow", confidence: 0.8 }), + label({ label: "fiona stanley hospital", label_type: "site", confidence: 0.9 }), + ], + summaryText: + "Lithium is a high-risk medication with a narrow therapeutic index. Serum levels require monitoring.", + }); + + const labels_ = badges.map((badge) => badge.label); + expect(labels_).toContain("Lithium"); + expect(labels_).toContain("Narrow therapeutic index"); + expect(labels_).toContain("High-risk medication"); + expect(labels_).toContain("Monitoring required"); + // Workflow and site labels stay in the tag cloud, not the badge cluster. + expect(labels_).not.toContain("Prescribing"); + expect(labels_.join(" ")).not.toMatch(/Stanley/); + + // Ordered by descending tone priority: warnings before clinical before info. + const priorities = badges.map((badge) => SEMANTIC_TONE_PRIORITY[badge.tone]); + expect(priorities).toEqual([...priorities].sort((a, b) => b - a)); + }); + + it("reserves danger for contraindications and gives Schedule 8 the lock icon", () => { + const badges = buildDocumentSummaryBadges({ + summaryText: "Contraindicated in severe renal impairment. This Schedule 8 medicine requires monitoring.", + }); + const contraindication = badges.find((badge) => badge.label === "Contraindications"); + const controlled = badges.find((badge) => badge.label === "Schedule 8"); + expect(contraindication?.tone).toBe("danger"); + expect(controlled?.tone).toBe("warning"); + expect(controlled?.iconKey).toBe("controlled"); + expect(badges[0]).toBe(contraindication); + }); + + it("deduplicates equivalent label- and phrase-derived badges by display label", () => { + const badges = buildDocumentSummaryBadges({ + labels: [label({ label: "high-risk medication", label_type: "risk", confidence: 0.9 })], + summaryText: "This is a high-risk medication.", + }); + expect(badges.filter((badge) => badge.label.toLowerCase() === "high-risk medication")).toHaveLength(1); + }); + + it("applies the limit and handles empty input", () => { + const badges = buildDocumentSummaryBadges({ + summaryText: + "Contraindicated in pregnancy. Schedule 8. Toxicity and escalation criteria apply; urgent review. " + + "Narrow therapeutic index, high-risk medication, monitoring required.", + limit: 3, + }); + expect(badges).toHaveLength(3); + expect(buildDocumentSummaryBadges({})).toEqual([]); + expect(buildDocumentSummaryBadges({ labels: null, summaryText: null })).toEqual([]); + }); +}); diff --git a/tests/document-summary-formatting.test.ts b/tests/document-summary-formatting.test.ts new file mode 100644 index 0000000000..26d514693f --- /dev/null +++ b/tests/document-summary-formatting.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import { + dedupeSummarySentences, + formatDocumentSummary, + stripSummaryBoilerplate, +} from "../src/lib/document-summary-formatting"; + +// Verbatim shape of a real stored summary: protective marking + document title +// + reference code + Scope/Site/Disciplines run glued ahead of the content, +// passages repeated, inline numbered headings, mid-word truncated tail. +const messyLithiumSummary = + "OFFICIAL Guideline Lithium Therapy- Initiation and Continuation Reference #: FSFHG-HW-GUI-0017 Scope Site " + + "Service/Department/Unit Disciplines Fiona Stanley Hospital Hospital Wide Medical, Nursing, Pharmacy " + + "Fremantle Hospital Lithium is a high-risk medication with a narrow therapeutic index. Careful patient " + + "selection and monitoring is required to minimise the risk of lithium toxicity. 1. Introduction Hospital " + + "Wide Medical, Nursing, Pharmacy Fremantle Hospital Lithium is a high-risk medication with a narrow " + + "therapeutic index. Careful patient selection and monitoring is required to minimise the risk of lithium " + + "toxicity. 1. Introduction Lithium has an established role in the treatment of a number of psychiatric " + + "conditions, including prophylaxis of bipolar disorder (BD), acute mania and treatment-resistant depression. " + + "The therapeutic effect occurs gradually and may take up to three weeks. Lithium is a narrow therapeutic " + + "index drug. It is handled by the body in a similar way to sodium; most risk factors for toxicity relate to " + + "changes in sodium levels and fluid status. therapeutic effect occurs gradually and may take up to three " + + "weeks. Lithium is a narro"; + +describe("stripSummaryBoilerplate", () => { + it("removes the glued document-header run while keeping the first clinical sentence", () => { + const stripped = stripSummaryBoilerplate( + "Guideline Lithium Therapy- Initiation and Continuation Reference #: FSFHG- Scope Site " + + "Service/Department/Unit Disciplines Fiona Stanley Hospital Hospital Wide Medical, Nursing, Pharmacy " + + "Fremantle Hospital Lithium is a high-risk medication with a narrow therapeutic index.", + ); + expect(stripped).toBe("Lithium is a high-risk medication with a narrow therapeutic index."); + }); + + it("is idempotent", () => { + const once = stripSummaryBoilerplate( + "OFFICIAL Guideline Falls Prevention Reference #: ABCD-1234 Scope Site All adult inpatients must have a falls risk assessment.", + ); + expect(stripSummaryBoilerplate(once)).toBe(once); + }); + + it("never strips sentences carrying clinical thresholds or actions", () => { + const clinical = "Withhold lithium if the level is above 1.2 mmol/L and review within 24 hours."; + expect(stripSummaryBoilerplate(clinical)).toBe(clinical); + // A title-cased clinical opener is not consumed as a proper-noun run. + const serumOpener = "Scope Serum Lithium Levels must be checked 12 hours post dose."; + expect(stripSummaryBoilerplate(serumOpener)).toContain("must be checked 12 hours post dose"); + }); + + it("leaves ordinary prose that merely mentions a document type untouched", () => { + const prose = "Guideline recommendations include gradual titration and regular monitoring."; + expect(stripSummaryBoilerplate(prose)).toBe(prose); + }); + + it("reverts rather than stripping a summary down to nothing", () => { + const allBoilerplate = + "Guideline North Metropolitan Health Service Community Directory And Contact Register Of Sites"; + expect(stripSummaryBoilerplate(allBoilerplate)).toBe(allBoilerplate); + }); +}); + +describe("dedupeSummarySentences", () => { + it("drops exact and containment repeats, keeping the first occurrence", () => { + const deduped = dedupeSummarySentences([ + "Lithium is a high-risk medication with a narrow therapeutic index.", + "Careful patient selection and monitoring is required to minimise the risk of toxicity.", + "Lithium is a high-risk medication with a narrow therapeutic index.", + "careful patient selection and monitoring is required to minimise the risk of toxicity", + ]); + expect(deduped).toEqual([ + "Lithium is a high-risk medication with a narrow therapeutic index.", + "Careful patient selection and monitoring is required to minimise the risk of toxicity.", + ]); + }); + + it("keeps short sentences that merely share a prefix", () => { + const deduped = dedupeSummarySentences(["Monitor sodium.", "Monitor sodium and fluid status closely."]); + expect(deduped).toHaveLength(2); + }); +}); + +describe("formatDocumentSummary", () => { + it("turns the messy stored summary into a structured, deduplicated model", () => { + const formatted = formatDocumentSummary(messyLithiumSummary); + + expect(formatted.isEmpty).toBe(false); + expect(formatted.lead).toContain("Lithium is a high-risk medication"); + + const allText = [formatted.lead, ...formatted.sections.flatMap((section) => [section.heading, ...section.items])] + .filter(Boolean) + .join(" "); + + // Boilerplate gone. + expect(allText).not.toContain("Reference #"); + expect(allText).not.toContain("FSFHG"); + expect(allText).not.toContain("Service/Department/Unit"); + expect(allText).not.toContain("Hospital Wide"); + expect(allText).not.toMatch(/^OFFICIAL/); + + // Repeats collapsed to a single occurrence. + expect(allText.match(/high-risk medication with a narrow therapeutic index/g)).toHaveLength(1); + expect(allText.match(/Careful patient selection/g)).toHaveLength(1); + + // Inline numbered heading became a real (merged) section. + const introSections = formatted.sections.filter((section) => section.heading === "Introduction"); + expect(introSections).toHaveLength(1); + expect(introSections[0].items.join(" ")).toContain("established role in the treatment"); + + // Mid-word truncated tail removed and flagged. + expect(allText).not.toMatch(/narro$/); + expect(formatted.truncatedTail).toBe(true); + }); + + it("handles empty and null input", () => { + expect(formatDocumentSummary(null).isEmpty).toBe(true); + expect(formatDocumentSummary(" ").isEmpty).toBe(true); + expect(formatDocumentSummary(undefined).sections).toEqual([]); + }); + + it("keeps a clean summary intact as lead plus key points", () => { + const formatted = formatDocumentSummary( + "This guideline covers clozapine initiation. Baseline FBC must be obtained before the first dose. " + + "Weekly monitoring continues for 18 weeks.", + ); + expect(formatted.lead).toBe( + "This guideline covers clozapine initiation. Baseline FBC must be obtained before the first dose.", + ); + expect(formatted.sections).toHaveLength(1); + expect(formatted.sections[0].heading).toBeNull(); + expect(formatted.sections[0].items).toEqual(["Weekly monitoring continues for 18 weeks."]); + expect(formatted.truncatedTail).toBe(false); + }); + + it("does not treat numbered cross-references as headings", () => { + const formatted = formatDocumentSummary( + "Doses are titrated based on serum lithium levels (refer to section 1.9. Therapeutic Drug Monitoring), tolerability and clinical response.", + ); + expect(formatted.sections.every((section) => section.heading === null)).toBe(true); + }); +}); diff --git a/tests/indexed-source-formatting.test.ts b/tests/indexed-source-formatting.test.ts index 4ba4ed9dc2..0ba80740e0 100644 --- a/tests/indexed-source-formatting.test.ts +++ b/tests/indexed-source-formatting.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { parseIndexedSourceText } from "../src/lib/indexed-source-formatting"; +import { + flowIndexedText, + mergeContinuationBlocks, + parseIndexedSourceText, + type IndexedTextBlock, +} from "../src/lib/indexed-source-formatting"; describe("indexed source formatting", () => { it("turns raw PDF page extraction into headings, paragraphs, lists, and tables", () => { @@ -73,4 +78,87 @@ The Clozapine monitoring protocol must be followed if a patient's blood test is }); expect(JSON.stringify(blocks)).not.toContain("Page 8 of 15"); }); + + it("re-joins soft-wrap continuations that extraction split with blank lines", () => { + const blocks = parseIndexedSourceText(` +Lithium Therapy - Initiation and Continuation + +• NSAIDs: (e.g. ibuprofen) can reduce lithium clearance and therefore + +increase lithium levels and the risk of toxicity. Avoid the combination where possible. + +• Serotonergic drugs: Lithium can contribute to serotonin toxicity, therefore + +patients who are prescribed combinations of serotonergic drugs should be closely monitored + +2.7. Dosage (as lithium carbonate) + +Doses should be individualised depending on indication and patient risk + +factors i.e. weight, comorbidities (e.g. renal impairment) and concomitant medications. +`); + + const list = blocks.find((block) => block.type === "list"); + expect(list).toMatchObject({ + type: "list", + items: [ + "NSAIDs: (e.g. ibuprofen) can reduce lithium clearance and therefore increase lithium levels and the risk of toxicity. Avoid the combination where possible.", + "Serotonergic drugs: Lithium can contribute to serotonin toxicity, therefore patients who are prescribed combinations of serotonergic drugs should be closely monitored", + ], + }); + + // Multi-level numbered heading is recognised and blocks merging across it. + expect(blocks).toContainEqual( + expect.objectContaining({ type: "heading", level: "section", text: "2.7. Dosage (as lithium carbonate)" }), + ); + expect(blocks).toContainEqual( + expect.objectContaining({ + type: "paragraph", + text: "Doses should be individualised depending on indication and patient risk factors i.e. weight, comorbidities (e.g. renal impairment) and concomitant medications.", + }), + ); + }); + + it("merges unterminated paragraphs but never merges across headings or tables", () => { + const heading: IndexedTextBlock = { type: "heading", id: "h", text: "1. Scope", level: "section" }; + const merged = mergeContinuationBlocks([ + { type: "paragraph", id: "a", text: "Monitoring must continue until" }, + { type: "paragraph", id: "b", text: "the level stabilises." }, + heading, + { type: "paragraph", id: "c", text: "applies to all adult inpatients." }, + ]); + expect(merged).toEqual([ + { type: "paragraph", id: "a", text: "Monitoring must continue until the level stabilises." }, + heading, + { type: "paragraph", id: "c", text: "applies to all adult inpatients." }, + ]); + }); + + it("keeps genuinely separate sentences as separate paragraphs", () => { + const merged = mergeContinuationBlocks([ + { type: "paragraph", id: "a", text: "Reduce doses in the elderly." }, + { type: "paragraph", id: "b", text: "Twice daily dosing should be spaced by 12 hours." }, + ]); + expect(merged).toHaveLength(2); + }); +}); + +describe("flowIndexedText", () => { + it("flows hard-wrapped excerpt text into readable sentences", () => { + const flowed = flowIndexedText( + "NSAIDs: (e.g. ibuprofen) can reduce lithium clearance and therefore\nincrease lithium levels and the risk of toxicity. Avoid the combination where\npossible. Low dose aspirin is safe to use.", + ); + expect(flowed).toBe( + "NSAIDs: (e.g. ibuprofen) can reduce lithium clearance and therefore increase lithium levels and the risk of toxicity. Avoid the combination where possible. Low dose aspirin is safe to use.", + ); + }); + + it("keeps paragraph breaks but heals blank lines that split a sentence", () => { + expect(flowIndexedText("First paragraph ends here.\n\nSecond paragraph starts here.")).toBe( + "First paragraph ends here.\n\nSecond paragraph starts here.", + ); + expect(flowIndexedText("can reduce lithium clearance and therefore\n\nincrease lithium levels.")).toBe( + "can reduce lithium clearance and therefore increase lithium levels.", + ); + }); }); diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index 78331466e2..1415c71531 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -2231,6 +2231,33 @@ test.describe("Clinical KB UI smoke coverage", () => { await expectNoPageHorizontalOverflow(page); }); + test("document viewer smart summary is structured with badges and demoted indexing details", async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await mockDemoApi(page); + await gotoApp(page, "/documents/11111111-1111-4111-8111-111111111111?page=1"); + + const summaryCard = page.getByTestId("high-yield-summary"); + await expect(summaryCard).toBeVisible(); + // Smart summary: badge cluster from labels + detected phrases, structured + // sections, and no document-header boilerplate leaking through. + await expect(summaryCard.getByText("Narrow therapeutic index", { exact: true })).toBeVisible(); + await expect(summaryCard.getByTestId("formatted-high-yield-summary")).toBeVisible(); + await expect(summaryCard).not.toContainText("Reference #"); + await expect(summaryCard).not.toContainText("Service/Department/Unit"); + + // The old meta-only "Document details" card is gone; indexing metadata is + // demoted behind a collapsed disclosure at the bottom of the sidebar. + await expect(page.getByText("Document details", { exact: true })).toHaveCount(0); + const indexingDetails = page.getByTestId("indexing-details"); + await expect(indexingDetails).toBeVisible(); + await expect(indexingDetails.getByText("rag-deep-memory-v1")).toBeHidden(); + await indexingDetails.getByText("Indexing details", { exact: true }).click(); + await expect(indexingDetails.getByText("rag-deep-memory-v1")).toBeVisible(); + + await expectDomIntegrity(page); + await expectNoPageHorizontalOverflow(page); + }); + test("phone universal header fully hides while scrolling dashboard main on phones", async ({ page }) => { await page.setViewportSize({ width: 390, height: 844 }); await gotoApp(page, "/?mode=answer");