fix(text): repair extraction glyph artifacts app-wide and route all document-derived text through formatters

docs/process-hardening.md

@@ -41,6 +41,14 @@ This document turns the current process review into phased, durable repo practic
 - Add explicit review ownership for clinical source governance, outdated-source handling, incident review, and decommission decisions.
 - Record production-readiness outcomes in release notes whenever clinical workflow, source governance, privacy, or deployment assumptions change.
 
+## Text formatting and copy conventions
+
+- **Document-derived text must never be rendered raw.** Any value pulled from an ingested document — answer prose, exact quotes, source snippets, document titles, image captions, extracted table text — must be routed through a `source-text-sanitizer` (`src/lib/source-text-sanitizer.ts`) or `display-text` (`src/components/clinical-dashboard/display-text.ts`) helper before it reaches JSX. Verbatim quotes use `sourceTextForVerbatimQuote`; titles use `cleanDisplayTitle`; snippets/captions use `sourceTextForCompactDisplay`.
+- `normalizeExtractedGlyphs` is the shared, lossless glyph-repair primitive (ligatures, soft hyphens, zero-width/control chars). It is wired into the base `compactWhitespace`/`readableWhitespace` cleaners and into ingestion (`buildChunks`), so every formatter and newly-indexed chunk inherits it. It must never strip clinical meaning (numbers, units, dose strings, comparison symbols, hyphens, or legitimate bullet structure). It deliberately does **not** rejoin line-break hyphenation — a soft-wrap hyphen is indistinguishable from a real compound hyphen (`low-dose`, `twice-daily`), so fusing would corrupt clinical compounds and verbatim quotes.
+- `tests/rendered-text-formatting.test.ts` is a static guard that fails if a known content surface reintroduces a raw interpolation. Extend it when adding new document-derived render surfaces.
+- **Static UI copy** (headings, empty states, error/toast messages, placeholders, starter prompts) lives in `src/lib/ui-copy.ts`, alongside `app-modes.ts` (mode labels) and `source-metadata.ts` (status labels). Do not hardcode new visible chrome copy inline.
+- `scripts/backfill-text-normalization.ts` cleans already-stored `document_chunks` text in place using the same primitive. It is dry-run by default, requires `--write --confirm` to mutate, writes a revertible JSON backup first, and **never re-embeds** — existing vectors are frozen, so retrieval is unchanged by construction.
+
 ## Known limits
 
 - Chromium UI coverage is active in CI on all branches; Firefox and WebKit run in the gated release-browser CI job and remain available locally through `npm run test:e2e` and `npm run verify:release`.

package.json

@@ -44,6 +44,7 @@
  "tags:backfill": "tsx scripts/backfill-document-tags.ts",
  "index:backfill": "tsx scripts/backfill-smart-index.ts",
  "visual:backfill": "tsx scripts/backfill-visual-intelligence.ts",
+ "backfill:text-normalization": "tsx scripts/backfill-text-normalization.ts",
  "check:supabase-project": "tsx scripts/check-supabase-project.ts",
  "check:indexing": "tsx scripts/check-indexing.ts",
  "recover:ingestion": "tsx scripts/recover-ingestion-queue.ts",

scripts/backfill-text-normalization.ts

@@ -0,0 +1,263 @@
+/**
+ * In-place text-normalization backfill (no re-index, no re-embed).
+ *
+ * Applies the shared, lossless `normalizeExtractedGlyphs` transform to the stored
+ * `document_chunks.content` (and `section_heading`) of already-indexed documents.
+ * Only rows whose text actually changes are updated. Embeddings are NEVER
+ * recomputed, so vector/semantic retrieval is unchanged by construction; the
+ * generated `search_tsv` column refreshes automatically and can only improve.
+ *
+ * Safety:
+ * - Confirms the Supabase project before writing.
+ * - Dry-run by default; requires BOTH --write and --confirm to mutate.
+ * - Writes a JSON backup of every changed row (id, old content/heading) before
+ * updating, so the change is fully revertible.
+ *
+ * Usage:
+ * npm run backfill:text-normalization # dry-run, all indexed docs
+ * npm run backfill:text-normalization -- --limit 50 # dry-run sample
+ * npm run backfill:text-normalization -- --document-id <id> # single document
+ * npm run backfill:text-normalization -- --write --confirm # apply (with backup)
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { loadEnvConfig } from "@next/env";
+
+loadEnvConfig(process.cwd());
+
+type Args = {
+ limit: number;
+ documentId?: string;
+ write: boolean;
+ confirm: boolean;
+};
+
+// The backup is deliberately NOT optional: every write run must leave a
+// revertible artifact, so there is no --no-backup escape hatch.
+function parseArgs(argv: string[]): Args {
+ const args: Args = { limit: 0, write: false, confirm: false };
+ for (let index = 0; index < argv.length; index += 1) {
+ const token = argv[index];
+ if (token === "--write") {
+ args.write = true;
+ continue;
+ }
+ if (token === "--confirm") {
+ args.confirm = true;
+ continue;
+ }
+ if (token !== "--limit" && token !== "--document-id") {
+ throw new Error(`Unknown flag: ${token}`);
+ }
+ const value = argv[index + 1];
+ if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
+ index += 1;
+ if (token === "--limit") {
+ const parsed = Number.parseInt(value, 10);
+ // Validate at parse time: NaN and 0 are falsy, so a later truthiness
+ // check would silently treat "--limit foo" / "--limit 0" as unlimited.
+ if (!Number.isInteger(parsed) || parsed <= 0) {
+ throw new Error(`--limit must be a positive integer (got "${value}").`);
+ }
+ args.limit = parsed;
+ }
+ if (token === "--document-id") args.documentId = value;
+ }
+ return args;
+}
+
+type ChunkRow = {
+ id: string;
+ document_id: string;
+ content: string | null;
+ section_heading: string | null;
+ retrieval_synopsis: string | null;
+};
+
+type ChangedRow = {
+ id: string;
+ document_id: string;
+ content_changed: boolean;
+ heading_changed: boolean;
+ synopsis_changed: boolean;
+ old_content: string | null;
+ new_content: string | null;
+ old_section_heading: string | null;
+ new_section_heading: string | null;
+ old_retrieval_synopsis: string | null;
+ new_retrieval_synopsis: string | null;
+};
+
+type SelectResult = Promise<{ data: ChunkRow[] | null; error: { message: string } | null }> & {
+ eq: (column: string, value: unknown) => SelectResult;
+ order: (column: string, opts: { ascending: boolean }) => SelectResult;
+ range: (from: number, to: number) => SelectResult;
+};
+
+type AdminClient = {
+ from: (table: string) => {
+ select: (columns: string) => SelectResult;
+ update: (patch: Record<string, string | null>) => {
+ eq: (column: string, value: unknown) => Promise<{ error: { message: string } | null }>;
+ };
+ };
+};
+
+const PAGE_SIZE = 1000;
+
+async function main() {
+ const args = parseArgs(process.argv.slice(2));
+
+ const [{ requireServerEnv }, { createAdminClient }, { normalizeExtractedGlyphs }, projectModule] = await Promise.all([
+ import("@/lib/env"),
+ import("@/lib/supabase/admin"),
+ import("@/lib/source-text-sanitizer"),
+ import("@/lib/supabase/project"),
+ ]);
+ requireServerEnv();
+
+ const { checkSupabaseProjectConfig, expectedSupabaseProject, formatSupabaseProjectCheck } = projectModule;
+ const projectCheck = checkSupabaseProjectConfig(
+ {
+ NEXT_PUBLIC_SUPABASE_URL: process.env.NEXT_PUBLIC_SUPABASE_URL,
+ SUPABASE_PROJECT_REF: process.env.SUPABASE_PROJECT_REF,
+ SUPABASE_PROJECT_NAME: process.env.SUPABASE_PROJECT_NAME,
+ },
+ { requireMetadata: false },
+ );
+ if (projectCheck.status === "missing" || projectCheck.status === "mismatch") {
+ console.error(formatSupabaseProjectCheck(projectCheck));
+ throw new Error(`Refusing to run: Supabase project is not ${expectedSupabaseProject.name}.`);
+ }
+
+ const supabase = createAdminClient() as unknown as AdminClient;
+
+ const writeEnabled = args.write && args.confirm;
+ if (args.write && !args.confirm) console.log("WRITE requested without --confirm; staying in dry-run mode.");
+ console.log(
+ `Text-normalization backfill ${writeEnabled ? "WRITE" : "DRY-RUN"} against ${expectedSupabaseProject.name} (${expectedSupabaseProject.ref})`,
+ );
+
+ let changed: ChangedRow[] = [];
+ const sampleDiffs: Array<{ id: string; before: string; after: string }> = [];
+ let scanned = 0;
+ let offset = 0;
+
+ for (;;) {
+ let query = supabase
+ .from("document_chunks")
+ .select("id,document_id,content,section_heading,retrieval_synopsis")
+ .order("id", { ascending: true })
+ .range(offset, offset + PAGE_SIZE - 1);
+ if (args.documentId) query = query.eq("document_id", args.documentId);
+
+ const { data, error } = (await query) as { data: ChunkRow[] | null; error: { message: string } | null };
+ if (error) throw new Error(error.message);
+ const rows = data ?? [];
+ if (rows.length === 0) break;
+
+ for (const row of rows) {
+ scanned += 1;
+ const newContent = row.content == null ? row.content : normalizeExtractedGlyphs(row.content);
+ const newHeading =
+ row.section_heading == null ? row.section_heading : normalizeExtractedGlyphs(row.section_heading);
+ const newSynopsis =
+ row.retrieval_synopsis == null ? row.retrieval_synopsis : normalizeExtractedGlyphs(row.retrieval_synopsis);
+ const contentChanged = newContent !== row.content;
+ const headingChanged = newHeading !== row.section_heading;
+ const synopsisChanged = newSynopsis !== row.retrieval_synopsis;
+ if (!contentChanged && !headingChanged && !synopsisChanged) continue;
+
+ changed.push({
+ id: row.id,
+ document_id: row.document_id,
+ content_changed: contentChanged,
+ heading_changed: headingChanged,
+ synopsis_changed: synopsisChanged,
+ old_content: row.content,
+ new_content: newContent,
+ old_section_heading: row.section_heading,
+ new_section_heading: newHeading,
+ old_retrieval_synopsis: row.retrieval_synopsis,
+ new_retrieval_synopsis: newSynopsis,
+ });
+ if (contentChanged && sampleDiffs.length < 8) {
+ sampleDiffs.push({
+ id: row.id,
+ before: (row.content ?? "").slice(0, 160),
+ after: (newContent ?? "").slice(0, 160),
+ });
+ }
+ }
+
+ offset += rows.length;
+ if (rows.length < PAGE_SIZE) break;
+ if (args.limit && changed.length >= args.limit) break;
+ }
+
+ // --limit bounds the number of rows we will WRITE, not just how far we page.
+ if (args.limit && changed.length > args.limit) changed = changed.slice(0, args.limit);
+
+ console.log(
+ `Scanned ${scanned} chunks; ${changed.length} would change${args.limit ? ` (capped at --limit ${args.limit})` : ""}.`,
+ );
+ for (const diff of sampleDiffs) {
+ console.log(`\n chunk ${diff.id}`);
+ console.log(` before: ${JSON.stringify(diff.before)}`);
+ console.log(` after: ${JSON.stringify(diff.after)}`);
+ }
+
+ if (changed.length === 0) {
+ console.log("\nNothing to update.");
+ return;
+ }
+
+ if (!writeEnabled) {
+ console.log("\nDRY-RUN complete. Re-run with --write --confirm to apply (a JSON backup is written first).");
+ return;
+ }
+
+ // Mandatory revertible backup before any write — there is no opt-out.
+ const backupDir = resolve(process.cwd(), "output", "backfills");
+ mkdirSync(backupDir, { recursive: true });
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+ const backupPath = resolve(backupDir, `text-normalization-${stamp}.json`);
+ writeFileSync(
+ backupPath,
+ JSON.stringify(
+ changed.map((row) => ({
+ id: row.id,
+ document_id: row.document_id,
+ old_content: row.old_content,
+ old_section_heading: row.old_section_heading,
+ old_retrieval_synopsis: row.old_retrieval_synopsis,
+ })),
+ null,
+ 2,
+ ),
+ "utf8",
+ );
+ console.log(`\nBackup of ${changed.length} rows written to ${backupPath}`);
+
+ let updated = 0;
+ for (const row of changed) {
+ const patch: Record<string, string | null> = {};
+ if (row.content_changed) patch.content = row.new_content;
+ if (row.heading_changed) patch.section_heading = row.new_section_heading;
+ if (row.synopsis_changed) patch.retrieval_synopsis = row.new_retrieval_synopsis;
+ const { error } = (await supabase.from("document_chunks").update(patch).eq("id", row.id)) as {
+ error: { message: string } | null;
+ };
+ if (error) throw new Error(`Update failed for chunk ${row.id}: ${error.message}`);
+ updated += 1;
+ if (updated % 250 === 0) console.log(` updated ${updated}/${changed.length}`);
+ }
+
+ console.log(`\nDone. Updated ${updated} chunks in place (no embeddings changed).`);
+}
+
+main().catch((error) => {
+ console.error(error instanceof Error ? error.message : error);
+ process.exit(1);
+});

src/components/AccessibleTable.tsx

@@ -14,6 +14,7 @@ import {
 } from "react";
 import { cn, textMuted } from "@/components/ui-primitives";
 import { normalizeAccessibleTable, type NormalizedAccessibleTable } from "@/lib/accessible-table-normalization";
+import { normalizeExtractedGlyphs } from "@/lib/source-text-sanitizer";
 
 const tableExpandMediaQuery = "(max-width: 768px), ((max-width: 1023px) and (hover: none) and (pointer: coarse))";
 const metadataHeaderPattern = /^(?:source|sources|support|pages?|chunk|file|document|citation|citations|provenance)$/i;
@@ -39,7 +40,7 @@ function parseMarkdownTable(markdown?: string | null) {
 }
 
 function cleanClinicalTableText(value: string) {
- return value
+ return normalizeExtractedGlyphs(value)
  .replace(metadataCellPattern, "")
  .replace(fileNamePattern, "")
  .replace(/\b(?:direct|partial|nearby|unsupported|source-linked)\s+support\b/gi, "")