From c09f8081c2e1847da98daf50bc7226954b17080e Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:29:06 +0800 Subject: [PATCH 1/4] fix(text): repair extraction glyph artifacts app-wide and route all document-derived text through formatters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add normalizeExtractedGlyphs (ligatures, soft hyphens, zero-width/control chars; deliberately no line-break de-hyphenation — fusing would corrupt clinical compounds like low-dose) wired into the base compactWhitespace/ readableWhitespace cleaners so every formatter inherits it - Fix bypass surfaces: verbatim quote body + clipboard, citation labels (formatCitationLabel/formatCompactCitationLabel now glyph-clean titles), source titles, visual-evidence labels, table snippets, image captions/alt, evidence-map details, AccessibleTable cells/captions - Drop unknown-filler segments from provenance joins while always keeping clinical governance warnings (source-metadata.ts, ui-primitives.tsx) - Normalize glyphs at ingestion (buildChunks) with source-span offsets kept consistent via a single normalized page text - Add scripts/backfill-text-normalization.ts: in-place, no-re-embed, dry-run by default, revertible JSON backup, project-guarded - Centralize static UI copy in src/lib/ui-copy.ts (empty states, errors, starter prompts) for the dashboard and document manager - Add tests/rendered-text-formatting.test.ts guard against new raw interpolations; document conventions in docs/process-hardening.md Co-Authored-By: Claude Fable 5 --- docs/process-hardening.md | 8 + package.json | 1 + scripts/backfill-text-normalization.ts | 244 ++++++++++++++++++ src/components/AccessibleTable.tsx | 3 +- src/components/ClinicalDashboard.tsx | 109 ++++---- src/components/DocumentManagementActions.tsx | 2 + src/components/DocumentViewer.tsx | 14 +- .../DocumentManagerPanel.tsx | 18 +- .../clinical-dashboard/answer-status.tsx | 33 ++- .../clinical-dashboard/display-text.ts | 10 +- .../master-search-header.tsx | 5 +- src/components/ui-primitives.tsx | 10 +- src/lib/chunking.ts | 13 +- src/lib/citations.ts | 16 +- src/lib/source-metadata.ts | 21 +- src/lib/source-text-sanitizer.ts | 45 +++- src/lib/ui-copy.ts | 100 +++++++ tests/citations.test.ts | 14 + tests/rendered-text-formatting.test.ts | 58 +++++ tests/source-metadata.test.ts | 25 ++ tests/source-text-sanitizer.test.ts | 47 ++++ 21 files changed, 698 insertions(+), 98 deletions(-) create mode 100644 scripts/backfill-text-normalization.ts create mode 100644 src/lib/ui-copy.ts create mode 100644 tests/rendered-text-formatting.test.ts diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 843bd1499b..c6a869af16 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -41,6 +41,14 @@ This document turns the current process review into phased, durable repo practic - Add explicit review ownership for clinical source governance, outdated-source handling, incident review, and decommission decisions. - Record production-readiness outcomes in release notes whenever clinical workflow, source governance, privacy, or deployment assumptions change. +## Text formatting and copy conventions + +- **Document-derived text must never be rendered raw.** Any value pulled from an ingested document — answer prose, exact quotes, source snippets, document titles, image captions, extracted table text — must be routed through a `source-text-sanitizer` (`src/lib/source-text-sanitizer.ts`) or `display-text` (`src/components/clinical-dashboard/display-text.ts`) helper before it reaches JSX. Verbatim quotes use `sourceTextForVerbatimQuote`; titles use `cleanDisplayTitle`; snippets/captions use `sourceTextForCompactDisplay`. +- `normalizeExtractedGlyphs` is the shared, lossless glyph-repair primitive (ligatures, soft hyphens, zero-width/control chars). It is wired into the base `compactWhitespace`/`readableWhitespace` cleaners and into ingestion (`buildChunks`), so every formatter and newly-indexed chunk inherits it. It must never strip clinical meaning (numbers, units, dose strings, comparison symbols, hyphens, or legitimate bullet structure). It deliberately does **not** rejoin line-break hyphenation — a soft-wrap hyphen is indistinguishable from a real compound hyphen (`low-dose`, `twice-daily`), so fusing would corrupt clinical compounds and verbatim quotes. +- `tests/rendered-text-formatting.test.ts` is a static guard that fails if a known content surface reintroduces a raw interpolation. Extend it when adding new document-derived render surfaces. +- **Static UI copy** (headings, empty states, error/toast messages, placeholders, starter prompts) lives in `src/lib/ui-copy.ts`, alongside `app-modes.ts` (mode labels) and `source-metadata.ts` (status labels). Do not hardcode new visible chrome copy inline. +- `scripts/backfill-text-normalization.ts` cleans already-stored `document_chunks` text in place using the same primitive. It is dry-run by default, requires `--write --confirm` to mutate, writes a revertible JSON backup first, and **never re-embeds** — existing vectors are frozen, so retrieval is unchanged by construction. + ## Known limits - Chromium UI coverage is active in CI on all branches; Firefox and WebKit run in the gated release-browser CI job and remain available locally through `npm run test:e2e` and `npm run verify:release`. diff --git a/package.json b/package.json index c55c1ad54f..61c66a64e1 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "tags:backfill": "tsx scripts/backfill-document-tags.ts", "index:backfill": "tsx scripts/backfill-smart-index.ts", "visual:backfill": "tsx scripts/backfill-visual-intelligence.ts", + "backfill:text-normalization": "tsx scripts/backfill-text-normalization.ts", "check:supabase-project": "tsx scripts/check-supabase-project.ts", "check:indexing": "tsx scripts/check-indexing.ts", "recover:ingestion": "tsx scripts/recover-ingestion-queue.ts", diff --git a/scripts/backfill-text-normalization.ts b/scripts/backfill-text-normalization.ts new file mode 100644 index 0000000000..32bfb9d5db --- /dev/null +++ b/scripts/backfill-text-normalization.ts @@ -0,0 +1,244 @@ +/** + * In-place text-normalization backfill (no re-index, no re-embed). + * + * Applies the shared, lossless `normalizeExtractedGlyphs` transform to the stored + * `document_chunks.content` (and `section_heading`) of already-indexed documents. + * Only rows whose text actually changes are updated. Embeddings are NEVER + * recomputed, so vector/semantic retrieval is unchanged by construction; the + * generated `search_tsv` column refreshes automatically and can only improve. + * + * Safety: + * - Confirms the Supabase project before writing. + * - Dry-run by default; requires BOTH --write and --confirm to mutate. + * - Writes a JSON backup of every changed row (id, old content/heading) before + * updating, so the change is fully revertible. + * + * Usage: + * npm run backfill:text-normalization # dry-run, all indexed docs + * npm run backfill:text-normalization -- --limit 50 # dry-run sample + * npm run backfill:text-normalization -- --document-id # single document + * npm run backfill:text-normalization -- --write --confirm # apply (with backup) + */ + +import { mkdirSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { loadEnvConfig } from "@next/env"; + +loadEnvConfig(process.cwd()); + +type Args = { + limit: number; + documentId?: string; + write: boolean; + confirm: boolean; + backup: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { limit: 0, write: false, confirm: false, backup: true }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--write") { + args.write = true; + continue; + } + if (token === "--confirm") { + args.confirm = true; + continue; + } + if (token === "--no-backup") { + args.backup = false; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + index += 1; + if (token === "--limit") args.limit = Number.parseInt(value, 10); + if (token === "--document-id") args.documentId = value; + } + if (args.limit && (!Number.isInteger(args.limit) || args.limit <= 0)) { + throw new Error("--limit must be a positive integer."); + } + return args; +} + +type ChunkRow = { + id: string; + document_id: string; + content: string | null; + section_heading: string | null; +}; + +type ChangedRow = { + id: string; + document_id: string; + content_changed: boolean; + heading_changed: boolean; + old_content: string | null; + new_content: string | null; + old_section_heading: string | null; + new_section_heading: string | null; +}; + +type SelectResult = Promise<{ data: ChunkRow[] | null; error: { message: string } | null }> & { + eq: (column: string, value: unknown) => SelectResult; + order: (column: string, opts: { ascending: boolean }) => SelectResult; + range: (from: number, to: number) => SelectResult; +}; + +type AdminClient = { + from: (table: string) => { + select: (columns: string) => SelectResult; + update: (patch: Record) => { + eq: (column: string, value: unknown) => Promise<{ error: { message: string } | null }>; + }; + }; +}; + +const PAGE_SIZE = 1000; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs }, projectModule] = await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("@/lib/source-text-sanitizer"), + import("@/lib/supabase/project"), + ]); + requireServerEnv(); + + const { checkSupabaseProjectConfig, expectedSupabaseProject, formatSupabaseProjectCheck } = projectModule; + const projectCheck = checkSupabaseProjectConfig( + { + NEXT_PUBLIC_SUPABASE_URL: process.env.NEXT_PUBLIC_SUPABASE_URL, + SUPABASE_PROJECT_REF: process.env.SUPABASE_PROJECT_REF, + SUPABASE_PROJECT_NAME: process.env.SUPABASE_PROJECT_NAME, + }, + { requireMetadata: false }, + ); + if (projectCheck.status === "missing" || projectCheck.status === "mismatch") { + console.error(formatSupabaseProjectCheck(projectCheck)); + throw new Error(`Refusing to run: Supabase project is not ${expectedSupabaseProject.name}.`); + } + + const supabase = createAdminClient() as unknown as AdminClient; + + const writeEnabled = args.write && args.confirm; + if (args.write && !args.confirm) console.log("WRITE requested without --confirm; staying in dry-run mode."); + console.log( + `Text-normalization backfill ${writeEnabled ? "WRITE" : "DRY-RUN"} against ${expectedSupabaseProject.name} (${expectedSupabaseProject.ref})`, + ); + + let changed: ChangedRow[] = []; + const sampleDiffs: Array<{ id: string; before: string; after: string }> = []; + let scanned = 0; + let offset = 0; + + for (;;) { + let query = supabase + .from("document_chunks") + .select("id,document_id,content,section_heading") + .order("id", { ascending: true }) + .range(offset, offset + PAGE_SIZE - 1); + if (args.documentId) query = query.eq("document_id", args.documentId); + + const { data, error } = (await query) as { data: ChunkRow[] | null; error: { message: string } | null }; + if (error) throw new Error(error.message); + const rows = data ?? []; + if (rows.length === 0) break; + + for (const row of rows) { + scanned += 1; + const newContent = row.content == null ? row.content : normalizeExtractedGlyphs(row.content); + const newHeading = row.section_heading == null ? row.section_heading : normalizeExtractedGlyphs(row.section_heading); + const contentChanged = newContent !== row.content; + const headingChanged = newHeading !== row.section_heading; + if (!contentChanged && !headingChanged) continue; + + changed.push({ + id: row.id, + document_id: row.document_id, + content_changed: contentChanged, + heading_changed: headingChanged, + old_content: row.content, + new_content: newContent, + old_section_heading: row.section_heading, + new_section_heading: newHeading, + }); + if (contentChanged && sampleDiffs.length < 8) { + sampleDiffs.push({ + id: row.id, + before: (row.content ?? "").slice(0, 160), + after: (newContent ?? "").slice(0, 160), + }); + } + } + + offset += rows.length; + if (rows.length < PAGE_SIZE) break; + if (args.limit && changed.length >= args.limit) break; + } + + // --limit bounds the number of rows we will WRITE, not just how far we page. + if (args.limit && changed.length > args.limit) changed = changed.slice(0, args.limit); + + console.log(`Scanned ${scanned} chunks; ${changed.length} would change${args.limit ? ` (capped at --limit ${args.limit})` : ""}.`); + for (const diff of sampleDiffs) { + console.log(`\n chunk ${diff.id}`); + console.log(` before: ${JSON.stringify(diff.before)}`); + console.log(` after: ${JSON.stringify(diff.after)}`); + } + + if (changed.length === 0) { + console.log("\nNothing to update."); + return; + } + + if (!writeEnabled) { + console.log("\nDRY-RUN complete. Re-run with --write --confirm to apply (a JSON backup is written first)."); + return; + } + + if (args.backup) { + const backupDir = resolve(process.cwd(), "output", "backfills"); + mkdirSync(backupDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const backupPath = resolve(backupDir, `text-normalization-${stamp}.json`); + writeFileSync( + backupPath, + JSON.stringify( + changed.map((row) => ({ + id: row.id, + document_id: row.document_id, + old_content: row.old_content, + old_section_heading: row.old_section_heading, + })), + null, + 2, + ), + "utf8", + ); + console.log(`\nBackup of ${changed.length} rows written to ${backupPath}`); + } + + let updated = 0; + for (const row of changed) { + const patch: Record = {}; + if (row.content_changed) patch.content = row.new_content; + if (row.heading_changed) patch.section_heading = row.new_section_heading; + const { error } = (await supabase.from("document_chunks").update(patch).eq("id", row.id)) as { + error: { message: string } | null; + }; + if (error) throw new Error(`Update failed for chunk ${row.id}: ${error.message}`); + updated += 1; + if (updated % 250 === 0) console.log(` updated ${updated}/${changed.length}`); + } + + console.log(`\nDone. Updated ${updated} chunks in place (no embeddings changed).`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/components/AccessibleTable.tsx b/src/components/AccessibleTable.tsx index ea7e9a671a..bd92d19be8 100644 --- a/src/components/AccessibleTable.tsx +++ b/src/components/AccessibleTable.tsx @@ -14,6 +14,7 @@ import { } from "react"; import { cn, textMuted } from "@/components/ui-primitives"; import { normalizeAccessibleTable, type NormalizedAccessibleTable } from "@/lib/accessible-table-normalization"; +import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer"; const tableExpandMediaQuery = "(max-width: 768px), ((max-width: 1023px) and (hover: none) and (pointer: coarse))"; const metadataHeaderPattern = /^(?:source|sources|support|pages?|chunk|file|document|citation|citations|provenance)$/i; @@ -39,7 +40,7 @@ function parseMarkdownTable(markdown?: string | null) { } function cleanClinicalTableText(value: string) { - return value + return normalizeExtractedGlyphs(value) .replace(metadataCellPattern, "") .replace(fileNamePattern, "") .replace(/\b(?:direct|partial|nearby|unsupported|source-linked)\s+support\b/gi, "") diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 498b610d4c..95b0a9c873 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -145,11 +145,13 @@ import { UtilityDrawer, } from "@/components/clinical-dashboard/dashboard-shell"; import { + cleanDisplayTitle, compactSourceSnippet, sanitizeAnswerDisplayText, sanitizeDisplayText, } from "@/components/clinical-dashboard/display-text"; import { MasterSearchHeader } from "@/components/clinical-dashboard/master-search-header"; +import { emptyStates, errorCopy } from "@/lib/ui-copy"; import { MedicationPrescribingWorkspace } from "@/components/clinical-dashboard/medication-prescribing-workspace"; import { ApplicationsLauncherWorkspace, applicationsLauncherItemCount } from "@/components/applications-launcher-page"; import { @@ -189,7 +191,12 @@ import { } from "@/lib/app-modes"; import { buildAnswerRenderModel, type AnswerRenderModel, type SourceLink } from "@/lib/answer-render-policy"; import { SourceActionRow, sourceResultHref } from "@/components/clinical-dashboard/source-actions"; -import { clinicalProseUsefulness, sourceTextForCompactDisplay } from "@/lib/source-text-sanitizer"; +import { + clinicalProseUsefulness, + normalizeExtractedGlyphs, + sourceTextForCompactDisplay, + sourceTextForVerbatimQuote, +} from "@/lib/source-text-sanitizer"; import { groupSourceGovernanceWarnings, type SourceGovernanceWarning } from "@/lib/source-governance"; import { smartEvidenceTags } from "@/lib/evidence-tags"; import { @@ -781,12 +788,12 @@ function SourcePreviewContent({ data-testid="source-capsule-preview-row" className="grid min-h-[44px] grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2 rounded-md border border-[color:var(--border)] bg-[color:var(--surface)] px-2.5 py-2 text-left transition hover:border-[color:var(--primary)]/45 hover:bg-[color:var(--surface-subtle)] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)]" role="listitem" - aria-label={`Open source ${source.title}, page ${source.pageNumber ?? "not available"}`} + aria-label={`Open source ${cleanDisplayTitle(source.title)}, page ${source.pageNumber ?? "not available"}`} >