fix(display): polish extracted source snippets — strip PSPF banners, repair truncation, normalize bullets

docs/process-hardening.md

@@ -46,6 +46,7 @@ This document turns the current process review into phased, durable repo practic
 - **Document-derived text must never be rendered raw.** Any value pulled from an ingested document — answer prose, exact quotes, source snippets, document titles, image captions, extracted table text — must be routed through a `source-text-sanitizer` (`src/lib/source-text-sanitizer.ts`) or `display-text` (`src/components/clinical-dashboard/display-text.ts`) helper before it reaches JSX. Verbatim quotes use `sourceTextForVerbatimQuote`; titles use `cleanDisplayTitle`; snippets/captions use `sourceTextForCompactDisplay`.
 - `normalizeExtractedGlyphs` is the shared, lossless glyph-repair primitive (ligatures, soft hyphens, zero-width/control chars). It is wired into the base `compactWhitespace`/`readableWhitespace` cleaners and into ingestion (`buildChunks`), so every formatter and newly-indexed chunk inherits it. It must never strip clinical meaning (numbers, units, dose strings, comparison symbols, hyphens, or legitimate bullet structure). It deliberately does **not** rejoin line-break hyphenation — a soft-wrap hyphen is indistinguishable from a real compound hyphen (`low-dose`, `twice-daily`), so fusing would corrupt clinical compounds and verbatim quotes.
 - `tests/rendered-text-formatting.test.ts` is a static guard that fails if a known content surface reintroduces a raw interpolation. Extend it when adding new document-derived render surfaces.
+- **Compact snippet polish** (2026-07-02): `stripClassificationBanner` removes PSPF protective-marking banners ("OFFICIAL", "OFFICIAL: Sensitive" — ALL-CAPS, line-anchored only) from compact/snippet/synopsis/title surfaces and from newly-built synopses; verbatim quotes keep banners by design. Inline bullet glyphs in compact previews become `"; "` separators (the `readableTableRows` joiner). `repairTruncatedCompactTail` repairs stored mid-word truncations ("where poss..."); ingestion now truncates synopses at word boundaries (`truncateAtWordBoundary` in `chunking.ts`), so this repair mainly serves pre-fix stored rows.
 - **Static UI copy** (headings, empty states, error/toast messages, placeholders, starter prompts) lives in `src/lib/ui-copy.ts`, alongside `app-modes.ts` (mode labels) and `source-metadata.ts` (status labels). Do not hardcode new visible chrome copy inline.
 - `scripts/backfill-text-normalization.ts` cleans already-stored `document_chunks` text in place using the same primitive. It is dry-run by default, requires `--write --confirm` to mutate, writes a revertible JSON backup first, and **never re-embeds** — existing vectors are frozen, so retrieval is unchanged by construction.
 

scripts/backfill-text-normalization.ts

@@ -3,9 +3,13 @@
  *
  * Applies the shared, lossless `normalizeExtractedGlyphs` transform to the stored
  * `document_chunks.content` (and `section_heading`) of already-indexed documents.
- * Only rows whose text actually changes are updated. Embeddings are NEVER
- * recomputed, so vector/semantic retrieval is unchanged by construction; the
- * generated `search_tsv` column refreshes automatically and can only improve.
+ * `retrieval_synopsis` additionally gets `polishStoredSynopsis` (protective-marking
+ * banner removal + truncated-tail repair) — the synopsis is a derived display/
+ * retrieval summary, never quoted verbatim, so the stronger polish is safe there
+ * and only there. Only rows whose text actually changes are updated. Embeddings
+ * are NEVER recomputed, so vector/semantic retrieval is unchanged by
+ * construction; the generated `search_tsv` column refreshes automatically and
+ * can only improve.
  *
  * Safety:
  * - Confirms the Supabase project before writing.
@@ -109,7 +113,12 @@ const PAGE_SIZE = 1000;
 async function main() {
  const args = parseArgs(process.argv.slice(2));
 
- const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs }, projectModule] = await Promise.all([
+ const [
+ { requireServerEnv },
+ { createAdminClient },
+ { normalizeExtractedGlyphs, polishStoredSynopsis },
+ projectModule,
+ ] = await Promise.all([
  import("@/lib/env"),
  import("@/lib/supabase/admin"),
  import("@/lib/source-text-sanitizer"),
@@ -167,7 +176,7 @@ async function main() {
  const newHeading =
  row.section_heading == null ? row.section_heading : normalizeExtractedGlyphs(row.section_heading);
  const newSynopsis =
- row.retrieval_synopsis == null ? row.retrieval_synopsis : normalizeExtractedGlyphs(row.retrieval_synopsis);
+ row.retrieval_synopsis == null ? row.retrieval_synopsis : polishStoredSynopsis(row.retrieval_synopsis);
  const contentChanged = newContent !== row.content;
  const headingChanged = newHeading !== row.section_heading;
  const synopsisChanged = newSynopsis !== row.retrieval_synopsis;
@@ -189,12 +198,18 @@ async function main() {
  new_retrieval_synopsis: newSynopsis,
  });
  }
- if (contentChanged && sampleDiffs.length < 8) {
- sampleDiffs.push({
- id: row.id,
- before: (row.content ?? "").slice(0, 160),
- after: (newContent ?? "").slice(0, 160),
- });
+ if ((contentChanged || synopsisChanged) && sampleDiffs.length < 8) {
+ // Prefer showing the content diff; fall back to the synopsis diff for
+ // synopsis-only rows so the dry run still demonstrates the change.
+ sampleDiffs.push(
+ contentChanged
+ ? { id: row.id, before: (row.content ?? "").slice(0, 160), after: (newContent ?? "").slice(0, 160) }
+ : {
+ id: row.id,
+ before: (row.retrieval_synopsis ?? "").slice(0, 160),
+ after: (newSynopsis ?? "").slice(0, 160),
+ },
+ );
  }
  }
 

src/components/ClinicalDashboard.tsx

@@ -2466,7 +2466,7 @@ function RenderModelSourceList({
  <div className="space-y-3">
  {sources.map((source, index) => {
  const metadata = normalizeSourceMetadata(source.sourceMetadata);
- const snippet = compactSourceSnippet(source.snippet ?? "");
+ const snippet = compactSourceSnippet(source.snippet ?? "", { dropTitle: source.title });
  const openLabel = `Open source ${index + 1}: ${cleanDisplayTitle(source.title)}${query ? ` for ${query}` : ""}`;
  return (
  <article key={`${source.id}:${source.href}`} className={cn(sourceCard, "overflow-hidden p-0")}>

src/components/clinical-dashboard/display-text.ts

@@ -3,6 +3,7 @@ import {
  sourceTextForCompactDisplay,
  sourceTextForClinicalProse,
  sourceTextForClinicalProsePreservingBreaks,
+ stripClassificationBanner,
 } from "@/lib/source-text-sanitizer";
 import { polishClinicalAnswerProse } from "@/lib/rag-answer-text";
 import type { SearchResult } from "@/lib/types";
@@ -86,20 +87,69 @@ export function sourceSnippetKey(value: string) {
  .slice(0, 160);
 }
 
-export function compactSourceSnippet(value: string) {
+// Strips a leading duplicate of the card's own visible title (extraction often
+// glues the running header onto the body text). Matches word-by-word so
+// punctuation/spacing differences ("Guideline(EMHS)" vs "Guideline (EMHS)")
+// don't defeat it, but requires a non-space separator after the title so a
+// sentence that legitimately starts with the title words as its grammatical
+// subject is never cut.
+function stripLeadingTitleDuplicate(text: string, title: string) {
+ const titleWords = title.toLowerCase().match(/[a-z0-9]+/g) ?? [];
+ if (titleWords.length < 2) return text;
+ let matched = 0;
+ let end = 0;
+ for (const wordMatch of text.matchAll(/[A-Za-z0-9]+/g)) {
+ const gap = text.slice(end, wordMatch.index);
+ if (!/^[\s\-–—:,()&/'"·]*$/.test(gap)) break;
+ if (wordMatch[0].toLowerCase() !== titleWords[matched]) break;
+ matched += 1;
+ end = wordMatch.index + wordMatch[0].length;
+ if (matched === titleWords.length) break;
+ }
+ if (matched !== titleWords.length) return text;
+ // Allow closers left over from a parenthetical title ("… Guideline (EMHS)")
+ // before the separator, but the separator itself is mandatory.
+ const separator = text.slice(end).match(/^[\s)\]]*[;:·\-–—]+\s*/);
+ if (!separator) return text;
+ const rest = text.slice(end + separator[0].length).trim();
+ return rest.length >= 20 ? rest : text;
+}
+
+export type CompactSourceSnippetOptions = {
+ // The card's visible title, so a glued duplicate of it at the head of the
+ // snippet can be dropped.
+ dropTitle?: string;
+};
+
+export function compactSourceSnippet(value: string, options: CompactSourceSnippetOptions = {}) {
  const normalized = sanitizeDisplayText(value, { minLength: 10, minTokens: 2, compactSource: true });
  if (!normalized) return "";
- const fragments = normalized
+ // A trailing ellipsis (stored truncation) must come off before the sentence
+ // split, otherwise the cut tail masquerades as a complete final sentence.
+ const truncatedTail = /(?:\.{3}|…)\s*$/.test(normalized);
+ let body = normalized.replace(/\s*(?:\.{3}|…)\s*$/, "");
+ if (options.dropTitle) {
+ body = stripLeadingTitleDuplicate(body, cleanDisplayTitle(options.dropTitle));
+ }
+ if (!body) return "";
+ const fragments = body
  .replace(
  /\b(?:source excerpt|relevant excerpt|source text|table text|clinical table|image caption|caption)\s*[:=-]\s*/gi,
  " ",
  )
- .match(/[^.!?]+[.!?]?/g) ?? [normalized];
+ .match(/[^.!?]+[.!?]?/g) ?? [body];
  const seen = new Set<string>();
  const selected: string[] = [];
 
  for (const fragment of fragments) {
- const cleaned = fragment.replace(/\s+/g, " ").trim();
+ let cleaned = fragment.replace(/\s+/g, " ").trim();
+ if (selected.length === 0) {
+ // The head of the snippet can be cut mid-structure by chunking: shed
+ // orphaned closers (")." "]:") and a mid-list ordinal ("2. MO to
+ // check…" keeps its content, loses the navigational marker; "1." is a
+ // genuine list start and is kept).
+ cleaned = cleaned.replace(/^[)\]}»”’]+[.,;:]?\s*/, "").replace(/^(?:[2-9]|1\d)[.)]\s+(?=["'(]?[A-Z])/, "");
+ }
  if (!cleaned || cleaned.length < 10) continue;
  if (/^(?:source|citation|document|file|filename|chunk|page|image|provenance)\b/i.test(cleaned)) continue;
  const key = sourceSnippetKey(cleaned);
@@ -109,7 +159,24 @@ export function compactSourceSnippet(value: string) {
  if (selected.length >= 4) break;
  }
 
- return truncateWords((selected.length ? selected : [normalized]).join(" "), 90);
+ if (!selected.length) selected.push(body.replace(/\s+/g, " ").trim());
+ // A first fragment starting lowercase is a mid-clause continuation. When the
+ // rest of the snippet has real sentence starts and enough substance, start
+ // there instead — the missing clause head could carry a negation. When the
+ // partial fragment is all we have, keep every word and mark the continuation
+ // honestly rather than fabricating a sentence start by capitalizing.
+ if (selected.length > 1 && /^[a-z]/.test(selected[0])) {
+ const rest = selected.slice(1);
+ if (rest.some((fragment) => /^["'(]?[A-Z0-9]/.test(fragment)) && rest.join(" ").length >= 40) {
+ selected.shift();
+ }
+ }
+ const leadingContinuation = /^[a-z]/.test(selected[0]);
+
+ let text = truncateWords(selected.join(" "), 90);
+ if (leadingContinuation) text = `… ${text}`;
+ if (truncatedTail && !/(?:\.{3}|…)$/.test(text)) text = `${text} …`;
+ return text;
 }
 
 export function compactTableFact(fact: NonNullable<SearchResult["table_facts"]>[number]) {
@@ -149,11 +216,17 @@ export function sanitizeAnswerDisplayText(value: string, options: DisplayTextSan
 }
 
 export function cleanDisplayTitle(title: string) {
- return normalizeExtractedGlyphs(title ?? "")
- .replace(/^Synthetic /, "")
- .replace(/\.pdf$/i, "")
- .replace(/\s+/g, " ")
- .trim();
+ return (
+ stripClassificationBanner(normalizeExtractedGlyphs(title ?? ""))
+ .replace(/^Synthetic /, "")
+ .replace(/\.pdf$/i, "")
+ // Missing space before an acronym-like parenthetical: "Guideline(EMHS)" →
+ // "Guideline (EMHS)". Requires 2+ leading capitals inside the parens so
+ // "guideline(s)" and "dose(mg)" stay untouched.
+ .replace(/([A-Za-z])\((?=[A-Z]{2}[^)]*\))/g, "$1 (")
+ .replace(/\s+/g, " ")
+ .trim()
+ );
 }
 
 export function sourceDisplayTitle(source: SearchResult) {

src/lib/chunking.ts

@@ -1,6 +1,6 @@
 import { env } from "@/lib/env";
 import { sourceSpanForText } from "@/lib/source-spans";
-import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer";
+import { normalizeExtractedGlyphs, stripClassificationBanner } from "@/lib/source-text-sanitizer";
 import type { ChunkInput, DocumentChunk } from "@/lib/types";
 
 const sentenceBoundary = /(?<=[.!?])\s+/;
@@ -52,9 +52,16 @@ function normalizeLookupText(value: string) {
  .trim();
 }
 
+// A line that is nothing but a PSPF protective marking ("OFFICIAL",
+// "OFFICIAL: Sensitive") — running headers stamped on every page.
+function isClassificationBannerLine(line: string) {
+ return Boolean(line.trim()) && !stripClassificationBanner(line).trim();
+}
+
 function looksLikeMetadataNoise(line: string) {
  if (!line || line.length <= 2) return true;
  if (/^\d+$/.test(line)) return true;
+ if (isClassificationBannerLine(line)) return true;
  if (metadataNoisePatterns.some((pattern) => pattern.test(line))) return true;
  if (lineNoisePatterns.some((pattern) => pattern.test(line))) return true;
  return false;
@@ -337,12 +344,29 @@ function chunkTextBySentence(clean: string, chunkSize: number, overlap: number)
  return chunks;
 }
 
+// Word-safe truncation for stored display text (synopses, caption snippets).
+// A raw slice cut mid-word ("where poss...") and every surface downstream
+// inherited the artifact; cutting at a clause boundary when one lands late
+// enough, otherwise at the last word boundary, keeps the stored tail readable.
+function truncateAtWordBoundary(value: string, limit: number) {
+ if (value.length <= limit) return value;
+ const window = value.slice(0, limit - 3);
+ const clauseCut = Math.max(window.lastIndexOf(". "), window.lastIndexOf("; "), window.lastIndexOf(": "));
+ const wordCut = window.lastIndexOf(" ");
+ const cut = clauseCut >= limit * 0.6 ? clauseCut + 1 : wordCut > 0 ? wordCut : window.length;
+ const trimmed = window
+ .slice(0, cut)
+ .replace(/[\s,;:([{\-–—]+$/, "")
+ .trim();
+ return trimmed ? `${trimmed}...` : `${window.trim()}...`;
+}
+
 function compactImageText(value: string | null | undefined, limit = 420) {
  const text = String(value ?? "")
  .replace(/\s+/g, " ")
  .trim();
  if (!text) return "";
- return text.length > limit ? `${text.slice(0, limit - 3).trim()}...` : text;
+ return truncateAtWordBoundary(text, limit);
 }
 
 function compactSynopsisText(value: string | null | undefined, limit = 720) {
@@ -351,13 +375,16 @@ function compactSynopsisText(value: string | null | undefined, limit = 720) {
  .replace(/\[\[IMAGE_DATA_OMITTED\]\][\s\S]*?\[\[\/IMAGE_DATA_OMITTED\]\]/g, " ");
  const sentences = withoutImageTags
  .split(/(?<=[.!?])\s+|\n+/)
- .map((sentence) => sentence.replace(/\s+/g, " ").trim())
+ // Extraction glues the protective-marking header onto body sentences
+ // ("OFFICIAL: OFFICIAL Lithium Therapy …"); shed the banner prefix so it
+ // never enters the stored synopsis.
+ .map((sentence) => stripClassificationBanner(sentence.replace(/\s+/g, " ").trim()).trim())
  .filter((sentence) => sentence.length >= 12 && !boilerplateSynopsisPattern.test(sentence));
  const highYieldSentences = sentences.filter((sentence) => highYieldSectionPattern.test(sentence));
  const selected = (highYieldSentences.length ? highYieldSentences : sentences).slice(0, 4).join(" ");
  const compact = selected.replace(/\s+/g, " ").trim();
  if (!compact) return "";
- return compact.length > limit ? `${compact.slice(0, limit - 3).trim()}...` : compact;
+ return truncateAtWordBoundary(compact, limit);
 }
 
 function buildRetrievalSynopsis(args: {
@@ -376,7 +403,7 @@ function buildRetrievalSynopsis(args: {
  .filter(Boolean)
  .join(" | ");
  const facts = compactSynopsisText(args.content);
- return [prefix, facts].filter(Boolean).join(" | ").slice(0, 900);
+ return truncateAtWordBoundary([prefix, facts].filter(Boolean).join(" | "), 900);
 }
 
 export function buildImageTag(image: {

src/lib/citations.ts

@@ -1,12 +1,13 @@
-import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer";
+import { normalizeExtractedGlyphs, stripClassificationBanner } from "@/lib/source-text-sanitizer";
 import type { Citation, SearchResult } from "@/lib/types";
 
 // Citation titles come straight from document extraction, so repair glyph
-// artifacts (ligatures, soft hyphens, control chars) and drop the synthetic
-// prefix before they reach any label — keeps mobile/compact labels consistent
-// with the cleaned titles rendered elsewhere (cleanDisplayTitle).
+// artifacts (ligatures, soft hyphens, control chars), drop protective-marking
+// banners ("OFFICIAL:"), and drop the synthetic prefix before they reach any
+// label — keeps mobile/compact labels consistent with the cleaned titles
+// rendered elsewhere (cleanDisplayTitle).
 function cleanCitationTitle(value: string) {
- return normalizeExtractedGlyphs(value)
+ return stripClassificationBanner(normalizeExtractedGlyphs(value))
  .replace(/^Synthetic\s+/i, "")
  .replace(/\s+/g, " ")
  .trim();