diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 1f92998921..4b7d68ce1d 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -41,6 +41,14 @@ This document turns the current process review into phased, durable repo practic - Add explicit review ownership for clinical source governance, outdated-source handling, incident review, and decommission decisions. - Record production-readiness outcomes in release notes whenever clinical workflow, source governance, privacy, or deployment assumptions change. +## Text formatting and copy conventions + +- **Document-derived text must never be rendered raw.** Any value pulled from an ingested document — answer prose, exact quotes, source snippets, document titles, image captions, extracted table text — must be routed through a `source-text-sanitizer` (`src/lib/source-text-sanitizer.ts`) or `display-text` (`src/components/clinical-dashboard/display-text.ts`) helper before it reaches JSX. Verbatim quotes use `sourceTextForVerbatimQuote`; titles use `cleanDisplayTitle`; snippets/captions use `sourceTextForCompactDisplay`. +- `normalizeExtractedGlyphs` is the shared, lossless glyph-repair primitive (ligatures, soft hyphens, zero-width/control chars). It is wired into the base `compactWhitespace`/`readableWhitespace` cleaners and into ingestion (`buildChunks`), so every formatter and newly-indexed chunk inherits it. It must never strip clinical meaning (numbers, units, dose strings, comparison symbols, hyphens, or legitimate bullet structure). It deliberately does **not** rejoin line-break hyphenation — a soft-wrap hyphen is indistinguishable from a real compound hyphen (`low-dose`, `twice-daily`), so fusing would corrupt clinical compounds and verbatim quotes. +- `tests/rendered-text-formatting.test.ts` is a static guard that fails if a known content surface reintroduces a raw interpolation. Extend it when adding new document-derived render surfaces. +- **Static UI copy** (headings, empty states, error/toast messages, placeholders, starter prompts) lives in `src/lib/ui-copy.ts`, alongside `app-modes.ts` (mode labels) and `source-metadata.ts` (status labels). Do not hardcode new visible chrome copy inline. +- `scripts/backfill-text-normalization.ts` cleans already-stored `document_chunks` text in place using the same primitive. It is dry-run by default, requires `--write --confirm` to mutate, writes a revertible JSON backup first, and **never re-embeds** — existing vectors are frozen, so retrieval is unchanged by construction. + ## Known limits - Chromium UI coverage is active in CI on all branches; Firefox and WebKit run in the gated release-browser CI job and remain available locally through `npm run test:e2e` and `npm run verify:release`. diff --git a/package.json b/package.json index b8a99eaecb..cbcf4f82f5 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "tags:backfill": "tsx scripts/backfill-document-tags.ts", "index:backfill": "tsx scripts/backfill-smart-index.ts", "visual:backfill": "tsx scripts/backfill-visual-intelligence.ts", + "backfill:text-normalization": "tsx scripts/backfill-text-normalization.ts", "check:supabase-project": "tsx scripts/check-supabase-project.ts", "check:indexing": "tsx scripts/check-indexing.ts", "recover:ingestion": "tsx scripts/recover-ingestion-queue.ts", diff --git a/scripts/backfill-text-normalization.ts b/scripts/backfill-text-normalization.ts new file mode 100644 index 0000000000..979b040b4a --- /dev/null +++ b/scripts/backfill-text-normalization.ts @@ -0,0 +1,263 @@ +/** + * In-place text-normalization backfill (no re-index, no re-embed). + * + * Applies the shared, lossless `normalizeExtractedGlyphs` transform to the stored + * `document_chunks.content` (and `section_heading`) of already-indexed documents. + * Only rows whose text actually changes are updated. Embeddings are NEVER + * recomputed, so vector/semantic retrieval is unchanged by construction; the + * generated `search_tsv` column refreshes automatically and can only improve. + * + * Safety: + * - Confirms the Supabase project before writing. + * - Dry-run by default; requires BOTH --write and --confirm to mutate. + * - Writes a JSON backup of every changed row (id, old content/heading) before + * updating, so the change is fully revertible. + * + * Usage: + * npm run backfill:text-normalization # dry-run, all indexed docs + * npm run backfill:text-normalization -- --limit 50 # dry-run sample + * npm run backfill:text-normalization -- --document-id # single document + * npm run backfill:text-normalization -- --write --confirm # apply (with backup) + */ + +import { mkdirSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { loadEnvConfig } from "@next/env"; + +loadEnvConfig(process.cwd()); + +type Args = { + limit: number; + documentId?: string; + write: boolean; + confirm: boolean; +}; + +// The backup is deliberately NOT optional: every write run must leave a +// revertible artifact, so there is no --no-backup escape hatch. +function parseArgs(argv: string[]): Args { + const args: Args = { limit: 0, write: false, confirm: false }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--write") { + args.write = true; + continue; + } + if (token === "--confirm") { + args.confirm = true; + continue; + } + if (token !== "--limit" && token !== "--document-id") { + throw new Error(`Unknown flag: ${token}`); + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + index += 1; + if (token === "--limit") { + const parsed = Number.parseInt(value, 10); + // Validate at parse time: NaN and 0 are falsy, so a later truthiness + // check would silently treat "--limit foo" / "--limit 0" as unlimited. + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`--limit must be a positive integer (got "${value}").`); + } + args.limit = parsed; + } + if (token === "--document-id") args.documentId = value; + } + return args; +} + +type ChunkRow = { + id: string; + document_id: string; + content: string | null; + section_heading: string | null; + retrieval_synopsis: string | null; +}; + +type ChangedRow = { + id: string; + document_id: string; + content_changed: boolean; + heading_changed: boolean; + synopsis_changed: boolean; + old_content: string | null; + new_content: string | null; + old_section_heading: string | null; + new_section_heading: string | null; + old_retrieval_synopsis: string | null; + new_retrieval_synopsis: string | null; +}; + +type SelectResult = Promise<{ data: ChunkRow[] | null; error: { message: string } | null }> & { + eq: (column: string, value: unknown) => SelectResult; + order: (column: string, opts: { ascending: boolean }) => SelectResult; + range: (from: number, to: number) => SelectResult; +}; + +type AdminClient = { + from: (table: string) => { + select: (columns: string) => SelectResult; + update: (patch: Record) => { + eq: (column: string, value: unknown) => Promise<{ error: { message: string } | null }>; + }; + }; +}; + +const PAGE_SIZE = 1000; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs }, projectModule] = await Promise.all([ + import("@/lib/env"), + import("@/lib/supabase/admin"), + import("@/lib/source-text-sanitizer"), + import("@/lib/supabase/project"), + ]); + requireServerEnv(); + + const { checkSupabaseProjectConfig, expectedSupabaseProject, formatSupabaseProjectCheck } = projectModule; + const projectCheck = checkSupabaseProjectConfig( + { + NEXT_PUBLIC_SUPABASE_URL: process.env.NEXT_PUBLIC_SUPABASE_URL, + SUPABASE_PROJECT_REF: process.env.SUPABASE_PROJECT_REF, + SUPABASE_PROJECT_NAME: process.env.SUPABASE_PROJECT_NAME, + }, + { requireMetadata: false }, + ); + if (projectCheck.status === "missing" || projectCheck.status === "mismatch") { + console.error(formatSupabaseProjectCheck(projectCheck)); + throw new Error(`Refusing to run: Supabase project is not ${expectedSupabaseProject.name}.`); + } + + const supabase = createAdminClient() as unknown as AdminClient; + + const writeEnabled = args.write && args.confirm; + if (args.write && !args.confirm) console.log("WRITE requested without --confirm; staying in dry-run mode."); + console.log( + `Text-normalization backfill ${writeEnabled ? "WRITE" : "DRY-RUN"} against ${expectedSupabaseProject.name} (${expectedSupabaseProject.ref})`, + ); + + let changed: ChangedRow[] = []; + const sampleDiffs: Array<{ id: string; before: string; after: string }> = []; + let scanned = 0; + let offset = 0; + + for (;;) { + let query = supabase + .from("document_chunks") + .select("id,document_id,content,section_heading,retrieval_synopsis") + .order("id", { ascending: true }) + .range(offset, offset + PAGE_SIZE - 1); + if (args.documentId) query = query.eq("document_id", args.documentId); + + const { data, error } = (await query) as { data: ChunkRow[] | null; error: { message: string } | null }; + if (error) throw new Error(error.message); + const rows = data ?? []; + if (rows.length === 0) break; + + for (const row of rows) { + scanned += 1; + const newContent = row.content == null ? row.content : normalizeExtractedGlyphs(row.content); + const newHeading = + row.section_heading == null ? row.section_heading : normalizeExtractedGlyphs(row.section_heading); + const newSynopsis = + row.retrieval_synopsis == null ? row.retrieval_synopsis : normalizeExtractedGlyphs(row.retrieval_synopsis); + const contentChanged = newContent !== row.content; + const headingChanged = newHeading !== row.section_heading; + const synopsisChanged = newSynopsis !== row.retrieval_synopsis; + if (!contentChanged && !headingChanged && !synopsisChanged) continue; + + changed.push({ + id: row.id, + document_id: row.document_id, + content_changed: contentChanged, + heading_changed: headingChanged, + synopsis_changed: synopsisChanged, + old_content: row.content, + new_content: newContent, + old_section_heading: row.section_heading, + new_section_heading: newHeading, + old_retrieval_synopsis: row.retrieval_synopsis, + new_retrieval_synopsis: newSynopsis, + }); + if (contentChanged && sampleDiffs.length < 8) { + sampleDiffs.push({ + id: row.id, + before: (row.content ?? "").slice(0, 160), + after: (newContent ?? "").slice(0, 160), + }); + } + } + + offset += rows.length; + if (rows.length < PAGE_SIZE) break; + if (args.limit && changed.length >= args.limit) break; + } + + // --limit bounds the number of rows we will WRITE, not just how far we page. + if (args.limit && changed.length > args.limit) changed = changed.slice(0, args.limit); + + console.log( + `Scanned ${scanned} chunks; ${changed.length} would change${args.limit ? ` (capped at --limit ${args.limit})` : ""}.`, + ); + for (const diff of sampleDiffs) { + console.log(`\n chunk ${diff.id}`); + console.log(` before: ${JSON.stringify(diff.before)}`); + console.log(` after: ${JSON.stringify(diff.after)}`); + } + + if (changed.length === 0) { + console.log("\nNothing to update."); + return; + } + + if (!writeEnabled) { + console.log("\nDRY-RUN complete. Re-run with --write --confirm to apply (a JSON backup is written first)."); + return; + } + + // Mandatory revertible backup before any write — there is no opt-out. + const backupDir = resolve(process.cwd(), "output", "backfills"); + mkdirSync(backupDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const backupPath = resolve(backupDir, `text-normalization-${stamp}.json`); + writeFileSync( + backupPath, + JSON.stringify( + changed.map((row) => ({ + id: row.id, + document_id: row.document_id, + old_content: row.old_content, + old_section_heading: row.old_section_heading, + old_retrieval_synopsis: row.old_retrieval_synopsis, + })), + null, + 2, + ), + "utf8", + ); + console.log(`\nBackup of ${changed.length} rows written to ${backupPath}`); + + let updated = 0; + for (const row of changed) { + const patch: Record = {}; + if (row.content_changed) patch.content = row.new_content; + if (row.heading_changed) patch.section_heading = row.new_section_heading; + if (row.synopsis_changed) patch.retrieval_synopsis = row.new_retrieval_synopsis; + const { error } = (await supabase.from("document_chunks").update(patch).eq("id", row.id)) as { + error: { message: string } | null; + }; + if (error) throw new Error(`Update failed for chunk ${row.id}: ${error.message}`); + updated += 1; + if (updated % 250 === 0) console.log(` updated ${updated}/${changed.length}`); + } + + console.log(`\nDone. Updated ${updated} chunks in place (no embeddings changed).`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/components/AccessibleTable.tsx b/src/components/AccessibleTable.tsx index c80334b376..340d005f85 100644 --- a/src/components/AccessibleTable.tsx +++ b/src/components/AccessibleTable.tsx @@ -14,6 +14,7 @@ import { } from "react"; import { cn, textMuted } from "@/components/ui-primitives"; import { normalizeAccessibleTable, type NormalizedAccessibleTable } from "@/lib/accessible-table-normalization"; +import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer"; const tableExpandMediaQuery = "(max-width: 768px), ((max-width: 1023px) and (hover: none) and (pointer: coarse))"; const metadataHeaderPattern = /^(?:source|sources|support|pages?|chunk|file|document|citation|citations|provenance)$/i; @@ -39,7 +40,7 @@ function parseMarkdownTable(markdown?: string | null) { } function cleanClinicalTableText(value: string) { - return value + return normalizeExtractedGlyphs(value) .replace(metadataCellPattern, "") .replace(fileNamePattern, "") .replace(/\b(?:direct|partial|nearby|unsupported|source-linked)\s+support\b/gi, "") diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 0c2d7a83ab..6211a879e1 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -147,11 +147,13 @@ import { UtilityDrawer, } from "@/components/clinical-dashboard/dashboard-shell"; import { + cleanDisplayTitle, compactSourceSnippet, sanitizeAnswerDisplayText, sanitizeDisplayText, } from "@/components/clinical-dashboard/display-text"; import { MasterSearchHeader } from "@/components/clinical-dashboard/master-search-header"; +import { emptyStates, errorCopy } from "@/lib/ui-copy"; import { DifferentialsHome } from "@/components/clinical-dashboard/differentials-home"; import { FavouritesHub } from "@/components/clinical-dashboard/favourites-hub"; import { MedicationPrescribingWorkspace } from "@/components/clinical-dashboard/medication-prescribing-workspace"; @@ -198,7 +200,12 @@ import { searchFormRecords } from "@/lib/forms"; import { searchServiceRecords } from "@/lib/services"; import { buildAnswerRenderModel, type AnswerRenderModel, type SourceLink } from "@/lib/answer-render-policy"; import { SourceActionRow, sourceResultHref } from "@/components/clinical-dashboard/source-actions"; -import { clinicalProseUsefulness, sourceTextForCompactDisplay } from "@/lib/source-text-sanitizer"; +import { + clinicalProseUsefulness, + normalizeExtractedGlyphs, + sourceTextForCompactDisplay, + sourceTextForVerbatimQuote, +} from "@/lib/source-text-sanitizer"; import { groupSourceGovernanceWarnings, type SourceGovernanceWarning } from "@/lib/source-governance"; import { smartEvidenceTags } from "@/lib/evidence-tags"; import { @@ -803,12 +810,12 @@ function SourcePreviewContent({ data-testid="source-capsule-preview-row" className="grid min-h-[44px] grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-2 rounded-md border border-[color:var(--border)] bg-[color:var(--surface)] px-2.5 py-2 text-left transition hover:border-[color:var(--primary)]/45 hover:bg-[color:var(--surface-subtle)] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)]" role="listitem" - aria-label={`Open source ${source.title}, page ${source.pageNumber ?? "not available"}`} + aria-label={`Open source ${cleanDisplayTitle(source.title)}, page ${source.pageNumber ?? "not available"}`} >