diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh new file mode 100755 index 0000000000..863c7f7452 --- /dev/null +++ b/.claude/hooks/session-start.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# SessionStart hook for Claude Code on the web. +# The app is engine-strict on Node 24.x / npm 11.x, but web containers ship an +# older Node on PATH, so nothing installs or runs until Node 24 is present. +# Installs Node 24 into $HOME/.node24 (cached with the container), exposes it +# via $CLAUDE_ENV_FILE, and installs npm dependencies. +set -euo pipefail + +if [ "${CLAUDE_CODE_REMOTE:-}" != "true" ]; then + exit 0 +fi + +NODE_VERSION="24.13.0" +NODE_HOME="$HOME/.node24" +NODE_BIN="$NODE_HOME/node-v${NODE_VERSION}-linux-x64/bin" + +current_major="$(node -v 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/' || echo 0)" +if [ "$current_major" != "24" ] && [ ! -x "$NODE_BIN/node" ]; then + echo "[session-start] Installing Node ${NODE_VERSION} (found v${current_major:-none})" + mkdir -p "$NODE_HOME" + curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + | tar -xJ -C "$NODE_HOME" +fi + +if [ -x "$NODE_BIN/node" ]; then + export PATH="$NODE_BIN:$PATH" + echo "export PATH=\"$NODE_BIN:\$PATH\"" >> "$CLAUDE_ENV_FILE" +fi + +echo "[session-start] Using node $(node -v) / npm $(npm -v)" + +cd "$CLAUDE_PROJECT_DIR" +# npm ci keeps the lockfile untouched (npm install rewrites peer/optional +# metadata and dirties the worktree); skip entirely when the cached container +# already has node_modules. +if [ ! -d node_modules ]; then + npm ci --no-audit --no-fund + echo "[session-start] Dependencies installed" +else + echo "[session-start] node_modules already present, skipping install" +fi diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000000..e06b0338e2 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + } + ] + } + ] + } +} diff --git a/.prettierignore b/.prettierignore index aefbc2e0b0..2c40481a2c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -16,3 +16,4 @@ scratch/ # Generated by `supabase gen types`; keep the generator's formatting so # regeneration stays churn-free. src/lib/supabase/database.types.ts +supabase/drift-manifest.json diff --git a/data/differentials-snapshot.json b/data/differentials-snapshot.json index d425a138cf..a680edf127 100644 --- a/data/differentials-snapshot.json +++ b/data/differentials-snapshot.json @@ -27730,13 +27730,6 @@ } ], "presets": [ - { - "id": "scenario-presets", - "query": "# Scenario Presets", - "signals": [], - "entryIds": [], - "presentationSlugs": [] - }, { "id": "1-older-adult-acute-confusion", "query": "older adult acute confusion", @@ -27788,14 +27781,6 @@ } ], "redFlagFlows": [ - { - "id": "red-flag-flows", - "title": "# Red Flag Flows", - "entryId": "", - "presentationSlug": "red-flag-flows", - "bedsideQuestions": "", - "keyRedFlags": "" - }, { "id": "1-suicide", "title": "1. Suicide", @@ -27873,15 +27858,7 @@ "medicine": ["medication", "iatrogenic", "toxicity"], "qt": ["qtc", "arrhythmia", "toxicity"], "catatonia": ["mutism", "shutdown", "stupor"], - "field": ["weight"], - "presentation": ["2.6"], - "clinicalhinge": ["2.2"], - "mustnotmiss": ["2.0"], - "immediateactions": ["1.7"], - "mimics": ["1.5"], - "investigations": ["1.2"], - "tags": ["1.1"], - "optiontext": ["1.0"] + "field": ["weight"] }, "governance": { "version": "v10", diff --git a/docs/archive/operator-decisions-2026-07-06.md b/docs/archive/operator-decisions-2026-07-06.md new file mode 100644 index 0000000000..46572cec40 --- /dev/null +++ b/docs/archive/operator-decisions-2026-07-06.md @@ -0,0 +1,27 @@ +# Operator decisions — 2026-07-06 + +Approvals granted during the repository-review follow-up session. **No live actions were taken from this checklist** — the authoring environment had no authenticated Supabase connection. Any live-connected session (or the operator) can execute these without re-asking. + +## Pending live migrations — APPROVED to apply + +**Decision:** The operator explicitly approved applying both pending migrations to the live `Clinical KB Database` project (`sjrfecxgysukkwxsowpy`). + +1. **M13** `supabase/migrations/20260702000000_commit_generation_preserve_legacy_artifacts.sql` — prerequisite for reindex commits safely purging legacy NULL-generation rows. +2. `supabase/migrations/20260703030000_reconcile_storage_cleanup_jobs_indexes.sql` — drops the legacy auto-named indexes and (re)creates the intended named/partial indexes to match `supabase/schema.sql`. (This is the migration the debt log marked "apply to live only with explicit approval" — that approval is now recorded here.) + +**Post-apply verification (required):** + +```bash +npm run check:m13-migration +npm run reindex:health +npm run check:indexing +npm run check:supabase-project +``` + +## Edge function deploy — APPROVED + +**Decision:** Deploy `supabase/functions/indexing-v3-agent` so the JSONB status-RPC parsing is live (follow-up noted in `docs/process-hardening.md` "Live database drift reconciliation (2026-07-05)"). + +## Context + +Approval came alongside the operator green-lighting the two structural efforts (finish the ClinicalDashboard admin cutover; decompose `src/lib/rag.ts`) tracked in `docs/process-hardening.md`. diff --git a/docs/codebase-index.md b/docs/codebase-index.md index 31ab8aee5e..bc47dabb47 100644 --- a/docs/codebase-index.md +++ b/docs/codebase-index.md @@ -250,6 +250,16 @@ Golden retrieval fixture: `scripts/fixtures/rag-retrieval-golden.json` - Registry modes: services, forms, medications, differentials - Demo mode: synthetic data when Supabase unavailable (`demo-data.ts`, `isDemoMode()` in `env.ts`) +### Global search composer placement rules + +One shared composer (`master-search-header.tsx`) serves every mode. Placement: + +- **Mode homes** (`/services`, `/forms`, `/favourites`, `/differentials`, `/applications`, and dashboard homes): inline in the hero via the `mode-home-composer-slot` portal, on phone and tablet+ alike. +- **Result and detail views**: fixed bottom dock on phone (compact variant on submitted searches), sticky top from `sm` up. +- **Results routing**: standalone routes own their submitted searches via `?q=…&run=1` (`/services` → `ServicesNavigatorPage`, `/forms` → `FormsSearchResultsPage`, `/differentials` → `DifferentialsHome` results view, `/favourites` filters the command library in place). Answer, Documents, and Prescribing submitted searches render inside `ClinicalDashboard` — intentional, since they need retrieval/answer state. `/?mode=favourites` redirects to `/favourites`; `/?mode=differentials` redirects to `/differentials`. +- **Intentionally composer-free routes**: `/differentials/presentations/*` (comparison workflow owns its chrome), `/documents/[id]` viewer (has its own in-document ask composer), `/documents/source/*` (document flow owns mobile chrome). Do not re-flag these in search-consistency audits. +- **Local filter fields** (sidebar "Search chats", document drawer "Find a document"/"Find a source PDF") are scoped filters, not global search; they share the `fieldControlWithIcon`/`fieldIcon` primitives. + --- ## Key config files diff --git a/docs/database-drift-detection.md b/docs/database-drift-detection.md new file mode 100644 index 0000000000..956cd5567c --- /dev/null +++ b/docs/database-drift-detection.md @@ -0,0 +1,134 @@ +# Database drift detection (`npm run check:drift`) + +Last updated: 2026-07-07 + +This repo's worst operational incidents were live-vs-repo schema drift: hybrid +retrieval RPCs silently broken on live for an unknown period, and migrations +recorded as applied whose objects were absent. `search_schema_health()` guards +a curated subset (signatures, 22 required indexes, execution smoke). +`check:drift` generalizes that into a full-inventory comparison of **every** +application-owned object against `supabase/schema.sql`. + +## How it works + +Three committed artifacts: + +| Artifact | Role | +| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `supabase/migrations/20260706200000_schema_drift_snapshot.sql` | `public.schema_drift_snapshot()` — service-role-only RPC returning the normalized live inventory (also declared in `supabase/schema.sql`; a test enforces byte parity). | +| `supabase/drift-manifest.json` | The expected state: the same snapshot captured from a **from-scratch replay of `supabase/schema.sql`** into a disposable `supabase/postgres` Docker container (`npm run drift:manifest`). Embeds the sha256 of the schema.sql it came from. | +| `supabase/drift-allowlist.json` | Known, documented divergence (each entry has a `reason`). Reported as warnings; anything not listed fails the check. | + +`npm run check:drift` (needs live service-role env) verifies the project ref, +fails fast if the manifest is stale, calls the RPC, diffs, applies the +allowlist, and exits 1 on unallowlisted divergence. The offline half runs in +`tests/drift-detection.test.ts` under `verify:cheap`: manifest freshness +(sha256), migration↔schema.sql parity for the snapshot function, allowlist +hygiene, and unit tests of the comparison engine. + +Inventory coverage: functions (comment/whitespace-stripped `pg_get_functiondef` +md5 + sorted ACLs), indexes (normalized `pg_get_indexdef`), RLS policies +(permissive/roles/cmd/qual/with_check), table shapes (columns sorted by name, +RLS flags, reloptions, ACLs), constraints, triggers, views, extensions, and +storage bucket rows + storage.objects policies. + +### Noise sources handled by design + +- **Whitespace/comments in function bodies** — `prosrc` is stored verbatim, so + migration text vs schema.sql text differ trivially; both are stripped before + hashing (the same trick `20260701140631` used to validate byte-equivalence). +- **Rendering search_path** — the snapshot pins `search_path = ''` so + `pg_get_expr`/`pg_get_indexdef`/policy quals render fully qualified and + identically on live and replay. +- **Column ordinal drift** — live tables grew via `ALTER TABLE ADD COLUMN`; + columns compare sorted by name, not `attnum`. +- **ACL append order** — aclitem arrays are sorted. +- **Duplicate migration-history versions** — history is _not_ compared at all; + the check compares actual object state (history presence proved unreliable: + see `20260703030000` below). +- **Platform-provisioned extensions** (pg_net, pgsodium, pgmq, …) — extra live + extensions are informational; missing schema.sql-declared ones fail. +- **Legacy index names** — `alias` allowlist entries assert the live database + carries the _identical_ name-stripped index definition under a legacy name + (the machine-checked version of `search_schema_health()`'s `index_aliases`). + +### Workflow + +- Change `supabase/schema.sql` → run `npm run drift:manifest` (Docker) in the + same PR. The freshness test fails otherwise. This also continuously proves + schema.sql replays from scratch — which it did **not** before 2026-07-07 + (`document_index_units` was declared after its first validating reference). +- Live drifts (check:drift red) → either codify live state (migration + + schema.sql + manifest regen) or fix live **through an approved migration**. + Never raw SQL against live; that is how this incident class started. +- New known-divergence → allowlist entry with a reason and a backlog line here. +- After the pending-migration backlog lands, delete the matching allowlist + entries; check:drift reports stale entries so they cannot silently linger. + +## 2026-07-07 baseline audit (three-way: live vs schema.sql vs migration chain) + +Both repo lineages were replayed into scratch containers and compared with the +live inventory. 166 divergent keys, fully classified: + +- **Reconciled in this PR (schema.sql/migrations only, live untouched):** + replay-order fix; `20260707000000_codify_live_observed_drift.sql` codifying + 15 live-only columns (`document_images` ×7, `document_index_quality` ×6, + `ingestion_job_stages` ×2 — worker-written, branch DBs broke without them), + 3 `content_not_blank` NOT VALID checks, autovacuum reloptions on 5 RAG + tables, `content_hash` nullability alignment, 4 live-only functions + (`set_owner_id_from_auth_uid` + rag_queries/misses triggers, + `purge_expired_rag_queries`, `correct_clinical_query_terms`, + `invoke_ingestion_worker`) and 2 ACL tightenings; schema.sql function/policy + text realigned to the migration-chain truth for `analyze_rag_tables`, + `claim_indexing_v3_agent_jobs`, `is_committed_artifact_generation(uuid,jsonb)`, + `match_document_memory_cards_hybrid`, and 6 owner-read policies (operand + order only). +- **Allowlisted (124 entries)** — see `supabase/drift-allowlist.json`; backlog + below. + +## Reconciliation backlog + +Ordered; each item removes allowlist entries when it lands. Items touching the +live project need explicit operator approval. + +1. **Apply the pending migrations** (`supabase db push` after review): + `20260705210000` (owner-sentinel rewrite — 8 live function bodies stale), + `20260706010000` (search_schema_health M13 guard), `20260706130000` + (embedding-fields-text sentinel), `20260706200000` (drift snapshot RPC — + required before check:drift can run at all), `20260707000000` (codification + wave; no-op on live by construction). +2. **`20260703030000` is recorded as applied but its effects are absent on + live** (storage_cleanup_jobs still has the legacy index names and the + non-partial status index). Recorded-but-ineffective is the exact original + incident class, recurring. Re-apply its idempotent statements under a new + version with approval; do not trust history presence. +3. **Codify the remaining live-only functions**: `get_visual_evidence_cards`, + `repair_enrichment_quality_batch`, `run_all_visual_eval_cases`, + `run_visual_eval_case` (same pattern as `20260707000000`). +4. **Authenticated-grant posture decision**: live revoked the authenticated + Data API grants on 17 tables (fail-closed hardening; the owner-read RLS + policies are currently dead on live) while schema.sql still declares them. + Either codify the revokes (schema.sql + tests + migration) or restore the + grants live. +5. **PUBLIC-execute revokes**: 4 security-invoker functions retain default + PUBLIC execute on live (`detect_legacy_ivfflat_indexes`, + `document_summary_text`, `search_document_chunks`, + `set_document_embedding_field_content_hash`). +6. **`document_label_metadata` direction**: schema.sql is AHEAD (hidden-label + filtering added without a migration). Ship the migration or revert. +7. **Index estate**: rename 10 legacy-named live indexes to schema.sql names; + decide the 24 schema.sql-declared indexes absent on live (recreate vs + remove — includes `documents_search_idx`, `document_chunks_anchor_idx`, + `documents_owner_content_hash_unique_idx`); drop ~45 live-only duplicate + indexes after `pg_stat_user_indexes` scan verification; reshape 3 + (`import_batches_status_created_idx`, `ingestion_jobs_document_status_idx`, + `ingestion_jobs_status_next_run_idx`). +8. **Constraints**: add `ingestion_job_stages_job_id_fkey` to live; align the + `rag_visual_eval_*` document FK definitions. +9. **`invoke_ingestion_worker`** hardcodes the project URL — migrate to the + GUC pattern (`20260702160000` precedent). +10. **Migration-chain fidelity** (affects Supabase Preview/branches, not + live): 13 keys where the chain diverges from schema.sql — buckets are only + created by schema.sql, `documents`/`ingestion_jobs` updated_at trigger + variants, post-legacy-drop embedding-fields index set, + `document_chunks_content_trgm_idx` shape, `rag_visual_eval_*` shapes. diff --git a/docs/disaster-recovery-runbook.md b/docs/disaster-recovery-runbook.md new file mode 100644 index 0000000000..05943ef56a --- /dev/null +++ b/docs/disaster-recovery-runbook.md @@ -0,0 +1,126 @@ +# Disaster recovery runbook — Clinical KB Database + +Last rehearsed: 2026-07-07 (schema restore rehearsal against a local Supabase +Postgres container; the live project `sjrfecxgysukkwxsowpy` was read-only +throughout). + +## What a recovery is made of + +A full recovery of this system has four independent layers. **Only the first +is covered by the repo**; the rest are Supabase-platform or operator state: + +| Layer | Source of truth | RPO | RTO (measured/estimated) | +| --------------------------------------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Schema (tables, RPCs, indexes, RLS, buckets config) | `supabase/schema.sql` in git | 0 (git) | **~19 s** into a local container (measured, repeatedly); minutes on a hosted project | +| Data (documents, 69k+ chunks, embeddings, jobs, logs) | Supabase managed backups | Daily backups → up to 24 h; with PITR enabled → ~2 min granularity. **Check the dashboard (Database → Backups) for the actual plan setting.** | Platform restore; not directly measurable without executing one. For the ~8.6 GB database expect tens of minutes to hours. Practice target: restore to a **new** project, never in place. | +| Storage objects (`clinical-documents`, `clinical-images` files) | Supabase Storage (S3) — separate from DB backups | Platform-managed | Bucket **rows** are recreated by schema.sql; the **files** are not in a DB restore. A DB-only restore leaves `documents.storage_path` pointing at objects that must still exist in Storage. | +| Config & secrets | Nowhere in the repo (deliberate) | n/a | Manual re-entry, see the checklist below | + +## Schema restore procedure (rehearsed) + +Works on any machine with Docker; identical mechanics on a fresh hosted +project via psql. This is exactly what `npm run drift:manifest` automates +(container start → scaffold → replay → snapshot → destroy), so **schema +restorability is re-proven every time the drift manifest is regenerated**. + +```sh +# 1. Scratch Supabase Postgres matching live (17.6.1.127) +docker run -d --name kb-restore -e POSTGRES_PASSWORD=postgres -p 56543:5432 supabase/postgres:17.6.1.127 +docker exec kb-restore pg_isready -U postgres # wait until ready + +# 2. Storage scaffold (bare image ships an empty storage schema; the hosted +# platform provisions the real one). Run as supabase_admin. +docker cp scripts/sql/drift-replay-scaffold.sql kb-restore:/tmp/scaffold.sql +docker exec kb-restore psql -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -f /tmp/scaffold.sql + +# 3. Replay the canonical schema as postgres (matches how live is administered) +docker cp supabase/schema.sql kb-restore:/tmp/schema.sql +docker exec kb-restore psql -U postgres -d postgres -v ON_ERROR_STOP=1 -f /tmp/schema.sql + +# 4. Verify +docker exec kb-restore psql -U postgres -d postgres -tAc "select (public.search_schema_health())->>'ok'" # must print: true +``` + +On a hosted target, prefer the migration chain (`supabase db push` / +`supabase db reset --linked` semantics); measured chain replay: **48 s** for +102 migrations into the same container. Note the chain has known fidelity gaps +vs schema.sql (buckets, two triggers, a handful of indexes — item 10 in +[database-drift-detection.md](database-drift-detection.md#reconciliation-backlog)). + +## Retrieval verification on the restored copy (rehearsed 2026-07-07) + +1. `search_schema_health()` → `ok: true, missing: []` on the restored copy. +2. `schema_drift_snapshot()` → full inventory captured; this became + `supabase/drift-manifest.json`. +3. Seeded a synthetic document + chunk + embedding field + index unit + memory + card (1536-dim unit vectors) and invoked the full retrieval surface: + `match_document_chunks_hybrid`, `match_document_embedding_fields_hybrid`, + `match_document_index_units_hybrid`, `match_document_memory_cards_hybrid`, + `match_document_chunks_text`, `match_documents_for_query`. All returned the + seeded rows (similarity 1.000 on the identity vector, hybrid score 0.760) — + the lexical, vector, and hybrid scoring paths all execute correctly on a + from-scratch restore. +4. **Golden retrieval eval (`npm run eval:retrieval:quality`) cannot run + against a schema-only restore** — it needs the real ~2,065-document corpus + plus `OPENAI_API_KEY` and service-role env. After a _data_ restore, point + `.env.local` at the restored project (update `NEXT_PUBLIC_SUPABASE_URL`, + `SUPABASE_PROJECT_REF`, keys), run `npm run check:supabase-project`, then + `npm run eval:retrieval:quality` and require the 23/23 pass before serving + traffic. This is the acceptance test for a real recovery. + +## What did NOT survive the schema restore (verified) + +Everything below must come from a data restore or manual re-entry — plan for +it before you need it: + +- **All data** — documents, chunks, embeddings (re-embedding the corpus from + scratch costs real OpenAI spend and hours of worker time; the backup is the + only cheap path), jobs, caches, logs, registry records. +- **Storage object files** — schema.sql recreates the two private bucket rows + (`clinical-documents`, `clinical-images`) but no files. +- **Auth users** — `auth.users` is platform state; owner-scoped rows restored + from backup reference user ids that must exist again (same project restore + preserves them; a new project does not). +- **12 platform-provisioned extensions** (pg_net, pgsodium, pgmq, pg_cron, + pg_graphql, vault, …) — present on hosted projects, absent in the bare + image; schema.sql only declares vector/pg_trgm/uuid-ossp. +- **pg_cron schedules** — the invoked functions (`invoke_ingestion_worker`, + `invoke_indexing_v3_agent`) are codified, but the `cron.schedule(...)` rows + themselves are live-only. After restore, re-create the cron jobs. +- **Vault secrets** — `cron_ingestion_jwt` (and any siblings) must be re-added + before the cron→edge-function chain works. +- **Custom GUCs** — `20260702160000` reads the agent URL from a database GUC; + re-set it (`alter database ... set ...`) on the restored project. +- **Edge functions** — deploy `indexing-v3-agent` (and the ingestion worker + function) separately via the CLI. +- **Dashboard config** — auth providers (magic link, Google/Microsoft SSO + redirect URLs), connection-pool caps (the documented 10-connection auth cap + is dashboard-only), API keys (publishable + service role are per-project; + every consumer needs the new values), `E2E_USER_*` test accounts. +- **Role settings** — e.g. `alter role authenticator set +idle_in_transaction_session_timeout` (in the migration chain, so a chain + replay restores it; a schema.sql-only replay does not). + +## Rehearsal findings that changed the repo + +- `supabase/schema.sql` did **not** replay from scratch before 2026-07-07 + (`document_index_units` was created ~900 lines after its first validating + reference — the strict-gate view). Fixed by reordering; now continuously + re-proven by `npm run drift:manifest` + the manifest freshness test. +- The bare `supabase/postgres` image lacks the storage schema objects; the + committed scaffold `scripts/sql/drift-replay-scaffold.sql` fills the gap. +- The first migration (`20260527000000`) dies at its storage-policy section + without that scaffold — the chain is **not** self-sufficient on a bare + Postgres either. +- Worker-written columns existed only on live (see + `20260707000000_codify_live_observed_drift.sql`): a branch/preview database + restored from the repo would have broken ingestion writes. Codified. + +## Standing cadence + +- Every `drift:manifest` regeneration = a schema-restore rehearsal (free). +- Run `npm run check:drift` after any live apply and on the operational + cadence alongside `check:indexing`. +- Re-rehearse the **data** layer (platform restore to a scratch project + + golden eval) before any risky bulk operation (re-index, mass migration), and + record the measured restore time here. diff --git a/docs/ingestion-state-machine.md b/docs/ingestion-state-machine.md new file mode 100644 index 0000000000..a51bf8f300 --- /dev/null +++ b/docs/ingestion-state-machine.md @@ -0,0 +1,455 @@ +# Ingestion state machine — documents × ingestion_jobs × index generations + +Phase-1 deliverable of the ingestion-concurrency/scale review (2026-07-07, branch +`claude/ingestion-concurrency-scale`). Documents every legal state of the ingestion +pipeline, which writer may perform each transition, what happens on a crash +between any two steps, and the verified concurrency violations. Fixes are +deliberately held until the db-reliability branch merges (phase 3). + +**Method.** Seven scoped race-hunter agents (one per writer × transition group) +plus direct analysis produced a registry of claimed races; every claim was then +re-derived from the code by an independent adversarial verifier agent, which +had to state the exact reaching schedule (or the killing guard) for each. Two +facts were additionally checked against the live database catalog: +`ingestion_job_stages` has no `job_id → ingestion_jobs` FK on live (schema.sql +declares one), and the live `claim_indexing_v3_agent_jobs` has the seed-insert +and documents-join (schema.sql's copy has neither). Verdicts: 24/24 claims +confirmed (5 narrowed, 0 refuted). Seven violations are **deterministic** — no +concurrency required at all. + +Companion docs: `docs/audit/repo-audit-2026-07-01.md` (M9/M11/M13 proved this +bug class live), `docs/scale-readiness-review.md` (phase 2), +`docs/reindex-runbook.md`. + +## 1. Entities and state columns + +### documents (one row per document) + +| Column | Values / meaning | +| ------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `status` | `queued` → `processing` → `indexed` \| `failed`. `queued` set by upload/reindex/retry routes and recovery; `processing` only by the local worker on the non-atomic path; `indexed` by `commit_document_index_generation`; `failed` by `fail_or_retry_ingestion_job`. | +| `page_count`, `chunk_count`, `image_count` | Denormalized counts, written at commit **from client-side values**; zeroed by the non-atomic reindex enqueue and by recovery resets. | +| `error_message` | Cleared at claim/enqueue, set by fail path. | +| `metadata.index_generation_id` | **The committed-generation pointer.** Artifact rows whose generation differs are invisible to readers (`is_committed_artifact_generation`) and are deleted by the commit RPC / abandoned-generation cleanup. | +| `metadata.enrichment_status` | `pending` \| `processing` \| `completed` \| `failed` \| `needs_enrichment_artifacts` (dual-written with `indexing_v3_agent_jobs.enrichment_status`). | +| `metadata.indexing_v3_agent_*` | Legacy JSONB mirror of the agent job row: `status`, `locked_by`, `locked_at`, `next_run_at`, `attempt_count`, `deferral_count`, `last_error`. Dual-written by `claim_indexing_v3_agent_jobs`, the edge agent, and `complete_strict_enrichment_job`. The live claim RPC also **seeds** job rows from this mirror. | +| `updated_at` | Doubles as the **rollback fence** for queue-state writes (`ingestionRollbackFenceStamp`, microsecond-salted). No generic BEFORE UPDATE trigger on this table. | + +### ingestion_jobs (many rows per document; core pipeline queue) + +| Column | Values / meaning | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `status` | `pending` → `processing` → `completed` \| `failed`. `failed`+retry goes back to `pending`. No unique constraint prevents multiple open jobs per document. | +| `stage`, `progress` | Human-readable progress; drives UI. | +| `attempt_count` / `max_attempts` | Incremented **at claim** by `claim_ingestion_jobs`; reset to 0 by the retry route and queue recovery. | +| `locked_at`, `locked_by` | Lease. Set once at claim. **There is no heartbeat** — a running worker never refreshes `locked_at`; the lease only ages. Staleness (45 min default) is therefore _runtime ceiling_, not liveness. | +| `next_run_at` | Retry backoff (`nextRetryAt`, exp backoff capped 30 min). Also used as a per-request fence stamp by the retry route. | +| `completed_at` | Terminal timestamp. | + +### indexing_v3_agent_jobs (exactly one row per document; enrichment queue) + +| Column | Values / meaning | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `status` | `pending` \| `processing` \| `completed` \| `failed` \| `needs_enrichment_artifacts`. `completed` and `needs_enrichment_artifacts` are **never re-claimed**. | +| `enrichment_status` | Same domain; claim filter accepts (`pending`,`failed`,`processing`). | +| `attempt_count` / `max_attempts` | Incremented at claim (including claims that return nothing). | +| `locked_by`, `locked_at` | Lease, 45-min staleness, no heartbeat. Edge-function wall-clock is far below 45 min, so agent-vs-agent same-job overlap requires a crashed prior invocation. | +| `next_run_at` | Deferral / retry schedule (deferral ladder capped by `INDEXING_V3_MAX_DEFERRALS`). | + +Rows are seeded lazily _inside_ the live `claim_indexing_v3_agent_jobs` from +`documents.metadata.indexing_v3_agent_status` (`status='indexed'` docs only), +`on conflict (document_id) do nothing`. + +### Index artifact tables + +| Table | Generation column | Notes | +| ---------------------------------------------------------------------------------------------------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `document_pages` | none | Replaced wholesale inside the commit RPC; `unique (document_id, page_number)`. | +| `document_chunks` | typed `index_generation_id` | Uniqueness key includes the generation (`document_id, index_generation_id, chunk_index`) — so two concurrent builds **never collide** on chunks; there is no unique-constraint safety net against a double-build. | +| `document_images` | typed + `metadata->>'index_generation_id'` fallback | Storage path embeds the generation: `{owner}/images/{doc}/{generation}/image-N.ext`. | +| `document_sections` | typed + metadata fallback | `unique (document_id, section_index)` — the collision surface between concurrent enrichment writers. | +| `document_table_facts`, `document_embedding_fields`, `document_index_units`, `document_memory_cards` | typed + metadata fallback | Edge-agent rows carry `metadata.generated_by='indexing-v3-agent'` and **no generation id** (NULL generation = always visible). No unique constraint on `document_index_units`. | +| `document_index_quality` | none (1 row/doc) | Upserted by worker commit, edge agent, and strict-completion RPC (monotonic `greatest()` merges). | +| `document_summaries`, `document_labels` | none | Shared between edge agent, worker inline enrichment, and route enrichment. | + +### Storage + ledgers + +- Document bucket: source file at `documents.storage_path` (uploaded before the + document row exists). +- Image bucket: per-generation prefixes. **The only ledger writer is the DELETE + route** (`storage_cleanup_jobs`); `scripts/cleanup-storage.ts` drains the + ledger (`status in ('pending','failed')`); nothing lists bucket prefixes. +- `ingestion_job_stages`: append-only stage log written by the edge agent. + Live has only the `document_id → documents` FK; schema.sql additionally + declares `job_id → ingestion_jobs(id)` (drift — see R24e). + +## 2. Writers + +| Writer | Identity | Touches | +| --------------------------------------------------------------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **W1 — local worker** (`worker/main.ts`) | service-role PostgREST | `claim_ingestion_jobs`, `reset_document_index`, all artifact inserts, image-bucket uploads, `commit_document_index_generation`, `complete_ingestion_job` / `fail_or_retry_ingestion_job`, `complete_strict_enrichment_job`, unconditional `documents` updates, inline enrichment (deep-memory) when `WORKER_INLINE_ENRICHMENT`. | +| **W2 — edge agent** (`supabase/functions/indexing-v3-agent/index.ts`) | direct Postgres (postgres.js, pool max 4) | `claim_indexing_v3_agent_jobs`, delete+insert cycles on embedding fields / index units / memory cards / sections / labels / summaries, `document_images.metadata` patches, `update_indexing_v3_agent_job_status` (keyed by document_id, no lock-holder check), `complete_strict_enrichment_job`, jsonb-merge patches of `documents.metadata`. | +| **W3 — API routes** (`src/app/api/...`) | service-role PostgREST | upload; reindex single/bulk (fence-stamped queue-state write + job insert); **reindex `mode:'enrichment'`** (runs deep-memory/enrichment in-route with **no job row and no fence**); retry (guarded job reset + **unguarded document status write**); delete single (guard → enumerate → ledger → cascade → storage remove); rename / bulk metadata edit (read-modify-write of `documents.metadata`). | +| **W4 — ops** (scripts + SQL) | service-role | `scripts/recover-ingestion-queue.ts` + `scripts/reindex.ts` (supersede/retry plans, `reset_document_index`, attempt_count=0 re-pends), `cleanup_abandoned_document_index_generations`, `scripts/cleanup-storage.ts` (ledger janitor), cron `invoke_indexing_v3_agent`. | + +## 3. Legal composite states + +`D:` = `documents.status`, `J:` = newest `ingestion_jobs` row, `A:` = the +`indexing_v3_agent_jobs` row, `G:` = `metadata.index_generation_id`. + +| # | State | Legal because | +| --- | ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| S0 | no rows | pre-upload. | +| S1 | storage object only | upload crash window U1; invisible; leaks. | +| S2 | `D:queued`, `J:pending`, `G:∅` | fresh upload awaiting first claim. | +| S3 | `D:processing`, `J:processing(locked)`, `G:∅` | first index build; index tables empty (reset ran). | +| S4 | `D:queued`/`processing`, `J:processing`, staged rows `G'≠G` | mid-build; staged generation invisible to readers. | +| S5 | `D:indexed`, `G:G'`, `J:processing` | commit done, completion RPC not yet run. | +| S6 | `D:indexed`, `G:G'`, `J:completed`, `A:pending` | steady state awaiting enrichment. | +| S7 | `D:indexed`, `A:processing(locked)` | agent enriching; generation-less agent artifacts appear incrementally (each row is visible the moment it is inserted — enrichment is **not** generation-fenced). | +| S8 | `D:indexed`, `A:completed`, `enrichment_status:completed` | fully enriched (strict gate passed). | +| S9 | `D:indexed`, `A:needs_enrichment_artifacts` | deferral ladder exhausted; terminal — never re-claimed. | +| S10 | `D:failed`, `J:failed` | terminal failure. | +| S11 | `D:indexed`, `J:pending` (reindex) | atomic reindex queued; old generation stays live until the new commit swaps `G`. | +| S12 | `D:queued`, `J:pending`, `G:G₀`, counts zeroed | non-atomic reindex of a failed/queued doc; old artifacts (if any) present until the worker's reset. | +| S13 | `D:indexed`, `J:failed(stage='needs recovery after partial index write')` | duplicate-key partial write routed to manual recovery. | + +Everything else observed is a violation (§6). + +## 4. Transition table (writer × transition) + +| Transition | Guard / mechanism | Allowed writer(s) | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | +| T1 upload: storage put → `documents` insert (`queued`) → `ingestion_jobs` insert (`pending`) | `upsert:false` storage put; compensating document delete on job-insert failure (error path only) | W3 | +| T2 claim: `J:pending→processing`, `attempt_count++`, lease set | `claim_ingestion_jobs`: `FOR UPDATE … SKIP LOCKED` on job+document rows; excludes docs with a _fresh_ processing sibling; reclaims stale (>45 min) processing jobs — including live ones (no heartbeat) | W1 | +| T3 reset (non-atomic path): delete **all** index rows incl. pages, generation-blind | `reset_document_index`; runs whenever claim-time `D≠indexed` | W1, W4 (recovery/reindex scripts) | +| T4 build: storage image uploads + staged artifact inserts under new `G'` | none (rows invisible via generation predicate; storage uploads unledgered) | W1 | +| T5 commit: `D→indexed`, `G:=G'`, pages replaced, other-generation rows deleted (legacy NULL rows preserved unless replaced), **counts written from client-side values** | `commit_document_index_generation` (single transaction, 180 s timeout) | W1 | +| T6 final metadata write: full `documents.metadata` **replace** built from claim-time snapshot | unconditional (`worker/main.ts:1783`, also the commit RPC's `p_metadata` replace and the strict-blocked variant at 1796) | W1 (violation source: R3/R5) | +| T7 complete: `J→completed`, all pending/processing/failed siblings force-completed, batch refresh | `complete_ingestion_job` (no `locked_by` fence) | W1 | +| T8 fail/retry: `J→failed` or `→pending`+`next_run_at`, locks nulled; `D→failed`/`queued`/`indexed` per claim-time flag | `fail_or_retry_ingestion_job` (unconditional, no fence) | W1 | +| T9 agent claim: seed job rows from metadata mirror, `A:→processing`, `attempt_count++`, jsonb metadata patch | `claim_indexing_v3_agent_jobs` (SKIP LOCKED; live joins `d.status='indexed'`) | W2 | +| T10 enrichment writes: delete+insert cycles per artifact family (agent scopes deletes to its own `generated_by`; worker/route deep-memory deletes are **unscoped**) | per-family; only the agent's visual deletes are transactional | W2, W1 (inline), W3 (`mode:'enrichment'`) | +| T11 agent complete: strict gate → metadata merge + `A→completed` | `complete_strict_enrichment_job` (`FOR UPDATE` on the document row — the only lock any enrichment writer takes) | W2, W1 | +| T12 agent defer/fail: metadata merge + `update_indexing_v3_agent_job_status` | keyed by `document_id`, no lock-holder check | W2 | +| T13 reindex enqueue: fence-stamped queue-state write + `J:pending` insert; compensating fence rollback (error path only) | `checkIngestionMutationSafety` (advisory 409; reads `ingestion_jobs` only) | W3 | +| T14 retry: conditional job reset (refuses fresh `processing` locks; accepts failed **and completed** jobs) + unconditional `D→queued` | single conditional UPDATE + fence for the job row; **no guard on the document write** | W3 | +| T15 delete: active-jobs guard → enumerate paths → ledger insert → late re-guard → trace cleanup → cascade delete → storage remove | M9 fix in place; both checks advisory; abort paths leave the ledger row `failed` **with paths populated** | W3 | +| T16 queue recovery: supersede (indexed + chunk_count>0 + J≠completed — includes `pending`) or retry-with-reset (everything else incl. plain `pending` and stale-alive `processing`) | `buildIngestionRecoveryPlan`; plan applied after an unbounded interactive confirm with no re-check; writes have no status guards; `attempt_count=0` | W4 | +| T17 abandoned-generation cleanup: delete artifact rows with `G≠committed` | job-existence guard **only in the candidate CTE**; counts/deletes re-check generation only, each on a fresh snapshot | W4 | +| T18 ledger janitor: remove storage objects listed in `storage_cleanup_jobs` rows with `status in ('pending','failed')` | no check that the document still exists | W4 | + +## 5. Crash-window analysis + +Crash = process death / network loss between two adjacent steps. Because there +is no lease heartbeat, every worker window below also has a "slow, not crashed" +variant after 45 minutes. + +**Upload (W3)** — R14 + +- U1: after storage put, before `documents` insert → orphaned storage object; + no ledger, no sweeper. A failed compensating remove is likewise only logged. +- U2: after `documents` insert, before job insert (crash, or rollback-delete + failure) → `D:queued` with no job — invisible to the claim RPC _and_ to + queue recovery (both scan `ingestion_jobs`). Only `reindex.ts`'s + incomplete-documents counter hints at it. + +**Worker (W1)** + +- C1: crash after claim → job `processing` until stale reclaim; attempt burned. +- C2 (non-atomic only): crash after reset → document has zero index rows and + `D:processing` until reclaim. Steady-state docs are safe (atomic path skips + T3) — but see R15 for how the retry route re-opens this for indexed docs. +- C3: crash mid-T4 → staged rows (cleaned by T17) + storage objects under the + never-committed generation prefix, permanently stranded (R12 family). +- C4: crash between T5 and T7 → S5; stale reclaim re-runs as atomic reindex; + converges at the cost of a full rebuild + another stranded storage generation. +- C5: crash between T5 and T6 → commit metadata already carries + `indexing_v3_agent_status='pending'`; agent path still triggers. Converges. +- C6: crash inside T8's PostgREST fallback between document and job updates → + `D:failed` with `J:processing(stale)`; reclaim/recovery converge it. + +**Edge agent (W2)** + +- E1: crash after T9 → `A:processing`; stale reclaim after 45 min; benign. +- E2: crash (or OpenAI failure) mid-T10 → the family being rebuilt was already + deleted; family stays empty until a later pass — and if attempts exhaust or + the deferral ladder terminates, **permanently** (R24a). +- E3: crash between T11's document update and the job-status RPC → metadata + says completed, job row processing; next pass hits the gate-complete fast + path and re-completes. Converges. + +**API routes (W3)** + +- R1w: reindex enqueue crash between queue-state write and job insert → + `D:queued`, counts zeroed, no job (non-indexed docs only; atomic variant + touches only `error_message`). Narrowed: usually an old failed job still + exists for recovery to find; truly stuck only when no job rows remain (R18). +- R2w: retry crash between job reset and document update → `J:pending` with + `D:failed`; worker fixes status at claim. Converges. +- R3w: delete crash after cascade, before storage removal → ledger row records + what to remove; recoverable **but see R11** — the same ledger row becomes a + loaded gun if the delete aborted instead of crashing. + +## 6. Verified violations (adversarial verifier verdicts; 0 refuted) + +Classes: [DATA-LOSS] destroys committed clinical data or storage; +[SILENT-CORRUPTION] wrong/partial data served with no error; [AVAILABILITY] +stuck pipeline states; [STORAGE-LEAK]; [OPS-CHURN]; [FRESH-ENV-ONLY]. +**DET** = deterministic, no concurrency required. + +### Tier 1 — destroys good data + +**R11 [DATA-LOSS, DET] — Aborted DELETE poisons the cleanup ledger; the janitor destroys a LIVE document's storage.** +The DELETE route creates the `storage_cleanup_jobs` row (with the live doc's +source-PDF + image paths) _before_ the late re-check; every abort path (late +re-check 409, trace-cleanup failure, DB-delete failure) marks it `failed` +without clearing the paths (`route.ts:545-593`; `updateStorageCleanupJob` never +touches paths). `scripts/cleanup-storage.ts:69` selects `status in +('pending','failed')` and never checks the document still exists — although the +FK is `on delete set null`, so a non-null `document_id` _proves_ the doc is +alive. One transient error + one routine janitor run permanently deletes a live +document's PDF and images. A janitor run can also consume the `pending` row +concurrently with an in-flight DELETE that later aborts. + +**R15 [DATA-LOSS, DET] — Retrying a failed (or completed) job of an indexed document destroys its live committed index.** +The retry route's guard only rejects fresh-processing jobs, then unconditionally +sets `documents.status='queued'` (`retry/route.ts:69, 91-95`). The next claim +sees `queued` → non-atomic → `reset_document_index` deletes the entire live +index at job start, hours before any replacement commit; a second failure +leaves `failed` with zero index. The IDX-H1 comment's promise ("the prior index +stays live until the worker commits") is defeated by this route's own status +write. Defeats the atomic-reindex design for exactly the docs it protects. + +**R3 [DATA-LOSS] — Double-build metadata repoint, then cleanup deletes the survivors.** +Under R1 (stale reclaim of a live >45-min job), the loser's post-commit +full-replace metadata write (`worker/main.ts:1783`, containing its own +generation id) can land after the winner's commit → the pointer names a +deleted generation → every row fails the committed-generation predicate → +document "indexed" but retrieval-empty; `cleanup_abandoned_document_index_generations` +then deletes the surviving winner rows permanently. + +**R24d [DATA-LOSS/SILENT-CORRUPTION] — Route-enrichment vs agent: "completed/good" documents with zero artifacts; no repair path exists.** +Reindex `mode:'enrichment'` creates no job row and its safety check reads only +`ingestion_jobs`, so it runs freely against a live agent pass. Deep-memory +deletes ALL index units/cards/sections unscoped (`deep-memory.ts:700-706`); +document-enrichment deletes all generated labels; the agent's own-scoped +deletes can't see the route's rows. Confirmed end states: duplicate +`section_summary` index units (no unique constraint), memory cards detached by +`section_id ON DELETE SET NULL`, dangling jsonb section references — and the +strict gate can complete (quality force-promoted to `good`) immediately before +a late delete-all lands whose re-insert then fails → completed/good document +with **zero** enrichment artifacts. `repair_strict_enrichment_gate_batch` +exists in schema but is invoked by nothing. + +### Tier 2 — silently corrupts committed indexes + +**R4 / R19 / R23 [SILENT-CORRUPTION] — Generation-blind resets amputate a live build; commit stamps full client-side counts over the partial row set.** +Three writers run `reset_document_index` (or generation-mismatch deletes) +against a document another worker is actively building: a reclaiming worker +(R4, needs R1), queue recovery retrying a stale-but-alive or merely-`pending` +job (R19 — `buildIngestionRecoveryPlan` treats plain `pending` as +retry-with-reset, and **runbook line 48 directs operators to run recovery in +exactly this state**; `reindex.ts` then spawns `worker:once` itself), and +`cleanup_abandoned_document_index_generations` (R23 — the job-existence guard +lives only in the candidate CTE; the 7 counts + 7 deletes re-check generation +only, each on a fresh READ COMMITTED snapshot, so a reindex claimed after +selection has its staged rows deleted table-by-table, leaving internally +inconsistent survivors). In all three the original worker's commit succeeds +with `p_chunk_count = chunks.length` computed client-side +(`worker/main.ts:1665-1674`) — indexed document, inflated counts, an arbitrary +prefix of rows missing, no error, no flag; `reindex-health` counts documents +and jobs, never chunks. + +**R2 / R5 / R6 / R8 [SILENT-CORRUPTION/OPS-CHURN] — No write is fenced by `locked_by`; the writer set is multi-master after any 45-min job.** +`complete_ingestion_job`, `fail_or_retry_ingestion_job`, `updateJob`, +`updateDocument`, and the commit RPC all write by id with no lease check. +Confirmed consequences under R1/R8: a zombie's failure flips a freshly-indexed +document to `failed` (with a message directing the operator to run the recovery +script — see R19) and a completed job to failed/pending (R2); the worker's +full-replace metadata writes erase concurrent bulk metadata edits, renames, and +agent state patches (R5 — three replace sites: commit `p_metadata`, line 1783, +line 1796); a loser's retryable failure re-pends the job and NULLs the +winner's live lock, inviting a third concurrent worker, or resurrects a +completed job into a zombie re-ingest (R6); an attempt-exhausted stale sibling +neither blocks nor ranks, so a pending sibling is claimed alongside the old +holder, whose row the first finisher force-completes and the loser then +resurrects (R8). + +**R24a [SILENT-CORRUPTION] — Agent artifact rebuilds delete before the OpenAI call; terminal jobs are never re-claimed.** +All four families (core embedding fields, memory cards, section index units, +visual) delete agent-generated rows, then call OpenAI, then insert row-by-row +(only the visual _deletes_ are transactional). A persistent OpenAI outage +across the retry budget, or a deferral-ladder exhaustion, leaves families +empty **permanently**: `failed` (attempts) and `needs_enrichment_artifacts` +are both excluded from claim eligibility forever. Narrowed: deletes are scoped +to `generated_by='indexing-v3-agent'`, which for agent-enriched documents is +the entire family. + +**R24c [SILENT-CORRUPTION] — Live seed-insert makes worker-inline-enrichment vs agent overlap deterministic.** +The worker's core commit writes `indexing_v3_agent_status='pending'` _before_ +its multi-minute inline enrichment; any cron tick in that window seeds+claims +the job and runs concurrently: unique `(document_id, section_index)` collisions +fail the worker's enrichment, section deletes FK-break the agent's card +inserts, and the worker's metadata replace overwrites the agent's completion → +`completed` job row with a `pending` metadata mirror that nothing reconciles; +heuristic agent artifacts (first-1800-char summary, chunk-per-section) can +permanently displace LLM enrichment. + +**R10 [SILENT-CORRUPTION, narrowed] — Claim-time `atomicReindex` stamps `indexed` onto an emptied document.** +An atomic-claim worker failing after a composed reset (R15 → sibling +non-atomic claim → reset) writes `status='indexed'` unconditionally → ghost +document: `indexed`, zero chunks. Recovery's supersede branch misses it +(`chunk_count>0` gate); its retry branch eventually catches the leftover job. + +### Tier 3 — operator tooling (all confirmed against the runbook's actual guidance: it never says to quiesce workers) + +**R20 [DATA-LOSS window] — Recovery's interactive confirm is an unbounded plan→apply TOCTOU.** +The plan is computed, the script blocks on `confirm(...)`, then applies the +stale plan verbatim with no status re-checks: a document whose job completed +during the wait gets its **freshly committed healthy index reset** (doc → +queued, counts zeroed) and its completed job flipped to pending, +`attempt_count=0`. + +**R22 [OPS-CHURN, DET] — Recovery supersede silently cancels every queued reindex of an indexed document.** +Plan rule `indexed && chunk_count>0 && status!='completed'` → supersede +includes plain `pending` jobs, so a routine `recover:ingestion --apply` marks +legitimately queued reindex jobs "completed / superseded by successful index". +The reindex never happens; nothing reports it. + +**R21 [SILENT-CORRUPTION] — Recovery's `attempt_count=0` re-pend on the same row a live worker holds → instant double-build.** +The anti-double-claim guard checks only _sibling_ rows, so the reset row is +immediately claimable — no further 45-minute wait; `reindex.ts` runs +`worker:once` right afterwards, orchestrating the race single-handedly. The +`attempt_count=0` overwrite also defeats `max_attempts` indefinitely for +poison documents. + +**R9 / R7 [AVAILABILITY] — Batches that never complete.** +R9: the batch's last two jobs completing in overlapping transactions both +compute `processing` from pre-commit snapshots; the second write is a classic +lost update → all jobs terminal, batch `processing` forever (reachable from a +single worker's `Promise.all`). R7: a zombie's stale-snapshot retry re-pends a +job whose DB `attempt_count` already equals `max_attempts` → permanently +unclaimable pending job pinning its batch. Both ops-recoverable. + +### Tier 4 — leaks, strandings, drift + +**R12 [STORAGE-LEAK, DET] — Every successful reindex permanently strands the previous generation's image objects.** +Commit deletes old `document_images` rows only; the delete route enumerates +current rows; no ledger entry, sweeper, or prefix listing ever sees the old +`{owner}/images/{doc}/{oldGen}/` objects again. Crash/double-build variants +(C3, R3) strand additional generations. Unbounded bucket growth. + +**R13 [STORAGE-LEAK, narrowed] — Delete TOCTOU beyond M9.** +A reindex enqueued+claimed between the late re-check and the cascade delete +puts a worker mid-build on a deleted document: silent 0-row updates, new- +generation storage uploads no ledger references, FK failure ends the run, and +the reindex caller holds a 201 for a ghost job. Narrow window (worker poll must +land inside the trace-cleanup seconds). + +**R14 [STORAGE-LEAK/AVAILABILITY, DET on crash] — Upload durability gaps.** +Storage-put orphans (no ledger), unledgered compensating-remove failures, and +the `queued`-doc-with-no-job zombie invisible to claim and recovery. + +**R16 [AVAILABILITY, narrowed] — Retry's late document write → sticky `queued` indexed doc.** +Requires the route to stall between its two writes for a full claim+build; +when it lands, all future reindexes of that doc take the destructive +non-atomic path (feeds R15's damage without another retry). + +**R17 [OPS-CHURN] — Concurrent reindex POSTs.** Duplicate open jobs (no unique +constraint); benign via completion-time supersede except when the first job +exceeds 45 min (dual ingest via R8) or exhausts attempts (redundant rebuild). + +**R18 [AVAILABILITY, narrowed, DET on crash] — Reindex enqueue crash strands `queued`+zeroed-counts docs** — truly stuck only when no other job rows remain for recovery to find. + +**R24b [AVAILABILITY] — Agent mid-batch failure strands the rest of the batch.** +`markJobFailure` on a cascade-deleted document throws inside the loop's catch +(job-status RPC returns `ok:false` → throw), abandoning the remaining claimed +jobs locked for 45 min with attempts burned; three such events make innocent +jobs permanently unclaimable. `update_indexing_v3_agent_job_status` is keyed by +`document_id` with no lock-holder check. + +**R24e / R1 drift [FRESH-ENV-ONLY] — schema.sql diverges from live in two load-bearing places.** +(a) schema.sql declares `ingestion_job_stages.job_id → ingestion_jobs(id)`; +the agent inserts `indexing_v3_agent_jobs.id` → on any schema.sql-provisioned +environment every needs-work agent run dies at its first `stageStart` (FK 23503) and burns attempts to terminal `failed`. Live has no such FK (verified +against the live catalog). (b) schema.sql's `claim_indexing_v3_agent_jobs` +lacks the seed-insert and the `d.status='indexed'` join that live has +(migration 20260705230000): fresh environments never seed agent jobs +(enrichment silently inert) and claim rows for mid-reindex docs that are then +never returned, burning attempts invisibly. Tests and scratch environments are +validating a different state machine than production runs. + +**R1 [enabler] — No lease heartbeat.** Confirmed root enabler for R2-R8: any +job >45 min is reclaimed while its worker is alive, with `WORKER_STALE_AFTER_MINUTES` +acting as a hard runtime ceiling rather than a liveness signal. On its own it +costs double compute; composed, it produces everything in Tier 2. + +## 7. Non-violations checked and cleared + +- Retry-vs-claim job-row clobber (IDX-C3/B6): the conditional-UPDATE guard is + sound _for the job row_ — a fresh `processing` lock refuses the reset. (The + route's **document** write is R15/R16 — the guard never covered it.) +- Reindex rollback fence (`ingestionRollbackFenceStamp`): microsecond-salted + `updated_at` match makes stale rollbacks no-ops on the error path. Cleared + (crash path is R18, a durability gap, not a race). +- Commit RPC legacy-artifact preservation (M13 fix): replacement-exists + predicates verified in both the RPC and the client fallback. Cleared. +- `complete_strict_enrichment_job` takes `FOR UPDATE` on the document row — + serializes W1-inline vs W2 _completion_ (not artifact writes). Cleared. +- Same-instant double _claim_ of one job: `FOR UPDATE SKIP LOCKED` is correct. + Cleared (the reclaim-while-alive case is R1, a lease-liveness problem). +- Upload duplicate-content race: closed by the partial unique index + + `duplicateUploadResponse`. Cleared. +- M9's original window (path enumeration vs pending job): the added `pending` + guard + late re-check work as designed; what survives is R11/R13 around them. + +## 8. Fix backlog (phase 3 — HELD until db-reliability merges) + +Ranked by damage-per-effort; smallest safe change each; migrations as committed +files only (never raw SQL against live; pause and confirm before applying any +migration to the live project). + +1. **R11**: janitor guard — skip `storage_cleanup_jobs` rows whose + `document_id` still resolves (FK is `on delete set null`, so non-null id ⇒ + live doc); clear paths (or use a distinct `aborted` status) on the DELETE + route's abort paths. +2. **R15/R16**: retry route — stop demoting indexed documents: only set + `status='queued'` when the document is not `indexed` (single conditional + UPDATE), and reject retries of `completed` jobs. +3. **R1/R2 root fix**: lease heartbeat — refresh `locked_at` (guarded by + `locked_by = workerId`) inside `updateJobProgress`; add + `and locked_by = p_worker_id` to `complete_ingestion_job` / + `fail_or_retry_ingestion_job`; worker aborts the job when a lease write + matches 0 rows. Kills R3-R8 as a class. +4. **R5**: replace the three full-metadata writes with server-side jsonb + merges scoped to worker-owned keys (or fence on `updated_at`). +5. **R19/R20/R21/R22**: recovery hardening — re-select plan state after + `confirm`; status-guarded conditional updates (`.eq('status', expected)`); + don't reset `attempt_count` to 0 on rows with a live lock; exclude + `pending` jobs younger than the plan snapshot from supersede. +6. **R24d/R24c**: give route-enrichment and worker-inline enrichment a gate on + `indexing_v3_agent_jobs` (extend `checkIngestionMutationSafety`), or take + the document `FOR UPDATE` around family rebuilds; scope deep-memory deletes + to its own `generated_by`. +7. **R12 (+R13/R14/C3)**: storage reconciliation — write `storage_cleanup_jobs` + rows for superseded generations at commit time (the RPC knows both + generation ids), and a periodic prefix-vs-committed-generation sweep. +8. **R23**: repeat the job-existence guard inside cleanup's delete predicates. +9. **R9**: recompute batch counts after acquiring the `import_batches` row + lock (`select … for update` then count, or make the UPDATE self-computing). +10. **R24a**: stage-then-swap for agent families (insert new tag, delete old + tag after success); add a re-open path for `needs_enrichment_artifacts`. +11. **R24b**: wrap `markJobFailure` in its own try/catch inside the batch loop. +12. **R24e + claim-RPC drift**: re-sync schema.sql with live (drop the + job_id FK or repoint it; adopt the migration's claim RPC body); add a + `search_schema_health` check for both. +13. **R7**: clamp `attempt_count` writes (`least(attempt_count, max_attempts)`) + or have the claim filter accept `attempt_count <= max_attempts` for + `pending` rows re-pended by the fail path. +14. **R17**: partial unique index on `ingestion_jobs(document_id) where status +in ('pending','processing')` — also structurally closes R13's enqueue arm + and simplifies M9's guards. + +Verification gates for phase 3 (per instruction): `npm run verify:cheap`, +`npm run check:indexing`, `npm run reindex:health`. diff --git a/docs/process-hardening.md b/docs/process-hardening.md index dc4f3eedb1..938bfacec3 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -47,7 +47,7 @@ This document turns the current process review into phased, durable repo practic All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went from ~8.8k → ~4.7k lines and now holds the main `ClinicalDashboard` orchestrator, its data/state hooks, and the deferred admin surfaces only. The 6 extracted modules live in `src/components/clinical-dashboard/`: `auth-panel`, `answer-content`, `evidence-panels`, `output-panel`, `visual-evidence`, `document-results` (+ the shared `use-mobile-preview-sheet` hook and `display-text` helpers). The barrel `index.ts` was intentionally not extended. -**Out of scope (deferred admin surfaces, optional later pass):** `DocumentDrawer`, `SettingsDialog`, `ToolsHub`, `MobileSectionFab`, `DocumentLabelReviewPanel`/`DocumentTagQualityPanel`/`DocumentIndexRepairPanel` and their label helpers (`tagQualityTone`, `labelTierTone`, `documentLabelTypeOptions`) all remain in `ClinicalDashboard.tsx`. +**Deferred admin surfaces: DONE (2026-07-06).** `DocumentDrawer` + the label panels/helpers had already landed via #250/#251 (`document-admin.tsx`, wired). The remaining half-wired state was finished on `claude/dashboard-decomp-final`: `ToolsHub` + `MobileSectionFab` cut over to `dashboard-nav.tsx` and `SettingsDialog` (+ its 7 `Settings*` helpers) to `settings-dialog.tsx`. **Both #250 sibling files had drifted from the live monolith** (settings-dialog.tsx predated the auth-email sign-in flow; dashboard-nav.tsx had stale colour tokens and prop lists) and were regenerated verbatim from the current monolith blocks before wiring — reusable lesson: an orphaned prepared module is stale the moment the monolith copy keeps evolving; always re-diff before cutover. `global-mockup-search-shell.tsx` now imports `SettingsDialog` from the module. The dead `document-admin/` **directory** (shadowed by the live `document-admin.tsx` file in module resolution, imported by nothing) was deleted. Monolith: 4,373 → ~3,450 lines (orchestrator + data/state hooks + small render/stream helpers). Live-surface `data-testid`/`aria-label` corpus verified byte-identical across both moves. ## Phase 4 - Release maturity @@ -105,6 +105,13 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went - **Edge function follow-up:** deploy `indexing-v3-agent` after merge so JSONB status RPC parsing is live. - **Operator-only:** publishable key rotation (`docs/archive/operator-decisions-2026-07-04.md`). +## Full-inventory drift detection & DR rehearsal (2026-07-07) + +- `npm run check:drift` generalizes the `search_schema_health()` single-hash approach into a full-inventory comparison of every live function (normalized `pg_get_functiondef` hash + ACLs), index, RLS policy, table shape, constraint, trigger, view, and storage bucket against `supabase/schema.sql`. Expected state is `supabase/drift-manifest.json` (generated by `npm run drift:manifest`, which replays schema.sql from scratch into a Docker container — so schema.sql replayability is re-proven on every regeneration); known divergence lives in `supabase/drift-allowlist.json` with per-entry reasons. Offline halves (manifest freshness, migration↔schema.sql snapshot parity, allowlist hygiene, engine unit tests) run in `verify:cheap` via `tests/drift-detection.test.ts`. See [docs/database-drift-detection.md](database-drift-detection.md). +- The 2026-07-07 three-way audit (live vs schema.sql replay vs migration-chain replay) found 166 divergent keys: worker-written columns existing only on live (codified by `20260707000000`), schema.sql not replayable from scratch (fixed), pending migrations explaining 11 function-body drifts, `20260703030000` **recorded as applied but ineffective on live**, live-revoked authenticated grants, and a large legacy index estate. Full classified backlog in the drift doc. +- DR rehearsal completed: schema restore ≈ 19 s to a local container, `search_schema_health()` ok and all four hybrid RPCs proven on the restored copy with seeded vectors; measured RPO/RTO and the did-not-survive list are in [docs/disaster-recovery-runbook.md](disaster-recovery-runbook.md). Expand/contract policy for retrieval tables added to [docs/supabase-migration-reconciliation.md](supabase-migration-reconciliation.md). +- Migration `20260706200000_schema_drift_snapshot.sql` (the snapshot RPC) and `20260707000000_codify_live_observed_drift.sql` are **prepared but NOT applied to live** — apply with the other pending migrations on the next approved `supabase db push`; `check:drift` cannot run against live until the RPC lands. + ## PR merge gate: tiered CI + required checks (2026-07-02) - CI is now two parallel PR jobs instead of one serial 6-7 minute job: `verify` (runtime alignment, edge typecheck, CI-safe production readiness, lint, typecheck, unit tests with coverage gate, build — ~3 min) and `ui-smoke` (Chromium Playwright smoke against its own dev server — ~4.5 min). Wall-clock PR feedback drops to the slower of the two, and a flaky smoke rerun no longer repeats lint/typecheck/tests/build. @@ -191,3 +198,15 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went - **Shipped on `claude/search-cross-mode-links-qscj1n`:** post-answer "Also in your library" strip (`src/lib/cross-mode-links.ts` + `CrossModeLinksStrip`), thread-wide entity fallback, word-boundary field matching in `rankCatalogRecords` (substring hits like "renal" inside "adrenaline" no longer count as name/title matches), `fields=index` slim mode on `/api/medications`, cross-mode click telemetry via `/api/search/interaction` (`crossMode` target, `metadata.interaction: "cross_mode_link_open"`), the same strip on documents-mode results, and answer `crossModes` command-surface parity. - **Verification debt:** `npm run verify:release` (and its governance/eval gates) has not been run for this workstream — the authoring environment has no live Supabase/OpenAI keys. Run it from a secrets-equipped environment after merge; the cross-mode surface itself is additive/navigational, so `verify:cheap` + `verify:ui` are the load-bearing local gates. - **Telemetry note:** cross-mode clicks write `rag_query_misses` rows with `clicked_document_id: null` and the target mode/slug in `metadata`; retrieval-quality reviews that aggregate misses by document should filter on `metadata.interaction`. + +## Answer-thread Back button: URL and visible answer can disagree (2026-07-06) + +- **Behaviour:** inside an Answer thread, browser Back changes the URL (`/?mode=answer&q=A&run=1` ← `...q=B&run=1`) but the rendered answer/thread does not change. Two guards produce this: the auto-run effect skips when `run=1` is already present, and the answer view early-returns when an answer is already on screen (`ClinicalDashboard`). This is thread persistence by design, not an accident — clearing the thread on Back would destroy in-progress clinical context. +- **Open product decision:** either (a) accept that the URL is not a faithful pointer to the visible answer inside a thread (current state), or (b) make Back restore the previous question's answer from thread history. Option (b) needs answer-thread state keyed by URL and re-render on `popstate`/param change; it is a deliberate feature, not a bug fix. +- **Guardrail until decided:** do not "fix" the skipped auto-run on Back in isolation — re-running the search on Back would clobber the visible thread with a regenerated (and possibly different) answer, which is worse than either deliberate option. + +## Audit re-triage & property-test suite (2026-07-06) + +- **Full re-triage of the 2026-07-01 audit's H1–H4 and M1–M17 against current main: all 21 are closed.** The 2026-07-02 remediation branch landed (H1/H2/H4+M8+M16, M1–M12, M14, M15, M17 all carry audit-referencing fixes verified in code, not just comments); PR #278 added the P0 carry-over (including the M9 DELETE TOCTOU re-check); M13's migration (`20260702000000`) is in-tree (live-apply verification needs `npm run check:m13-migration` with live keys). H3 is closed by supersession: PR #118 removed the governance penalties it complained about entirely (measured, "do not reintroduce" comment in `retrieval-selection.ts`); the residual `Math.max` floor only affects intent boosts, and un-flooring it was already tried and withdrawn (compounding across selection passes — audit R1). Do not re-open H3 without the golden retrieval eval. +- **New fast-check property suite** (`tests/property-*.test.ts`) pins the text-processing core's clinical invariants over generated inputs: unit-bearing numeric tokens survive sanitization; chunking terminates (including `CHUNK_OVERLAP >= CHUNK_SIZE`, M17) and neither loses nor invents content; table normalization stays rectangular, preserves numeric values, and the low-confidence caveat survives both clipboard export paths (H4). The suite immediately caught a live line-level H2 residue in `stripLowYieldLines` (control markers glued to threshold sentences deleted the whole line), fixed in the same PR. +- **Eval debt (blocking merge of the sanitizer fix, not development):** the environment had no live Supabase/OpenAI keys, so `npm run eval:quality -- --rag-only` (answer-path gate for the sanitizer change) could not run. Run it before merging `claude/audit-sweep-property-tests`. No retrieval-selection/ranking files were touched, so the golden retrieval eval is not required for this branch. diff --git a/docs/rag-hybrid-findings-and-todo.md b/docs/rag-hybrid-findings-and-todo.md index 048e44b7bd..66fb26b91f 100644 --- a/docs/rag-hybrid-findings-and-todo.md +++ b/docs/rag-hybrid-findings-and-todo.md @@ -250,41 +250,85 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows ## Follow-ups filed 2026-07-06 (universal-search workstream) -17. ⏳ **Alias promotion pipeline is blocked by privacy redaction.** `rag_query_misses` rows store - hashed/redacted queries with empty `candidate_aliases`, so hardcoded `synonymGroups` / - `domainAliasGroups` / special-case rewrites in `src/lib/clinical-search.ts` cannot be replaced - with data-driven `rag_aliases` rows until a privacy-safe candidate-alias capture is designed. +17. 🔶 **Alias promotion pipeline is blocked by privacy redaction — PARTIALLY UNBLOCKED + (2026-07-06).** Weak-search misses now store `queryVocabularyAliasesForStorage(query)` as + `candidate_aliases` when raw retention is off: only canonical terms from the curated + clinical vocabulary that the query MATCHED are persisted (output text comes from the fixed + vocabulary table, never the raw query, so RET-H4 holds). Remaining: terms OUTSIDE the + curated vocabulary still cannot be captured without a privacy review; promotion tooling + from `candidate_aliases` → `rag_aliases` is still manual. 18. ⏳ **`document_index_units` vector recall** — no HNSW index (dropped 2026-07-02) and hosted Supabase denies `ALTER FUNCTION … SET hnsw.ef_search` for the `language sql` hybrid RPCs, so - only `match_document_memory_cards_hybrid` pins `ef_search=100`. Quantify the recall impact - before reintroducing an index. -19. ⏳ **Demo fallback can mask live retrieval failures in non-prod.** `/api/search` and - `/api/answer` silently swap in demo data on Supabase errors outside production (only an - `X-Clinical-KB-Fallback` header signals it). Proposal: surface a warning in - `check:production-readiness` output and/or a visible dev-mode banner rather than changing - the fallback behaviour. -20. ⏳ **Automated guard for governance-weighting regressions.** The 23/23 → 16/23 golden-set - regression class (governance metadata weighting selection ordering) is only guarded by the - manual PR checklist because `eval:retrieval:quality` needs live keys. Investigate a - keys-free structural test (e.g. assert selection sort inputs exclude governance fields). -21. ⏳ **Recalibrate gates for synthetic text-only similarity (RC9 residual) — audited 2026-07-07, - scope reduced.** Full consumer audit on `claude/retrieval-correctness`: the headline - `least(0.95, 0.56 + text_rank*0.39)` proxy NO LONGER EXISTS — `match_document_chunks_text` - already returns `similarity = 0` with hybrid capped at 0.5 and the lexical signal isolated in - `lexical_score` (codified in schema.sql with the "do not fabricate" comment). What remains - synthetic are three app-side fabricators, all tagged `similarity_origin: "synthetic_text"`: - the document-lookup fast path (0.58 + documentScore, hybrid ≤ 0.94), memory-card chunk loader - (0.58 + confidence·0.28, hybrid ≤ 0.89), and table-fact signal matches. Their consumers: + only `match_document_memory_cards_hybrid` pins `ef_search=100`. Concrete measurement plan + (needs live keys, ~1 hour): run `eval:retrieval:quality` twice with `--force-embedding` + (bypasses lexical fast paths, exercising vectors directly) — once as-is and once after + `create index concurrently` on `document_index_units.embedding` in a Supabase branch — and + compare doc-recall@5 + p90 latency. If recall gain < 1 case, close as not-worth-4.4GB. The + ef_search half can be retested via the plpgsql-wrapper trick that memory_cards already uses + (wrap the `language sql` RPC in a plpgsql shim that SETs it). +19. ✅ **Demo fallback can mask live retrieval failures in non-prod — DONE (2026-07-06).** + `nonProductionSupabaseDemoFallbackReason` (the shared choke point for /api/search, + /api/answer, and /api/answer/stream) now emits a loud `console.warn` naming the env vars to + check whenever the non-prod demo fallback fires; behaviour and the + `X-Clinical-KB-Fallback` header are unchanged. A visible dev-mode banner remains optional. +20. ✅ **Automated guard for governance-weighting regressions — ALREADY COVERED.** A keys-free + structural test exists: `tests/retrieval-selection.test.ts` ("keeps relevance ordering and + does not let source-governance metadata reorder selection") asserts a higher-relevance + `review_due`/`unverified` source outranks a lower-relevance `current`/`reviewed` one. The + manual golden-eval checklist remains the live backstop; no further action. +21. 🔶 **Recalibrate gates for synthetic text-only similarity (RC9 residual) — DATA NOW + FLOWING (2026-07-06); audited 2026-07-07, scope reduced.** `synthetic_similarity_count` and + `text_or_relaxation_used` are now persisted into `rag_retrieval_logs.metadata` (they were + computed but dropped by the telemetry whitelist in /api/search). Once ~2 weeks of live rows + exist, recalibrate `evaluateEvidenceCoverageGate` / text-fast-path thresholds against real + cosine distributions: query `metadata->>'synthetic_similarity_count'` joined to `is_miss` to + see how often synthetic scores cross the 0.58/0.62 gates on misses vs hits. Full consumer + audit on `claude/retrieval-correctness`: the headline `least(0.95, 0.56 + text_rank*0.39)` + proxy NO LONGER EXISTS — `match_document_chunks_text` already returns `similarity = 0` with + hybrid capped at 0.5 and the lexical signal isolated in `lexical_score` (codified in + schema.sql with the "do not fabricate" comment). What remains synthetic are three app-side + fabricators, all tagged `similarity_origin: "synthetic_text"`: the document-lookup fast path + (0.58 + documentScore, hybrid ≤ 0.94), memory-card chunk loader (0.58 + confidence·0.28, + hybrid ≤ 0.89), and table-fact signal matches. Their consumers: `evaluateEvidenceCoverageGate` / `shouldReturnTextFastPath` / `chooseAnswerRoute` / `shouldUseExtractiveAnswer` (thresholds 0.32–0.76 — the fabricated 0.58 floor is deliberately load-bearing there, always paired with structural checks like `directTitleSupport`; re-gating them on native signals is the deferred recalibration and must not be attempted without the telemetry distributions), `buildRetrievalDiagnostics` - (topScore < 0.5 weak gate — floor also load-bearing), and `deriveConfidence`. **Fixed now:** - `deriveConfidence` no longer lets a fabricated 0.82+ mint a "high" answer-confidence label — - "high" requires a genuine-cosine citation; synthetic-origin evidence caps at "medium" - (strictly tightening, ordering/routing untouched, unit-tested in tests/rag-score.test.ts). + (topScore < 0.5 weak gate — floor also load-bearing), and `deriveConfidence`. **Fixed + (2026-07-07):** `deriveConfidence` no longer lets a fabricated 0.82+ mint a "high" + answer-confidence label — "high" requires a genuine-cosine citation; synthetic-origin + evidence caps at "medium" (strictly tightening, ordering/routing untouched, unit-tested in + tests/rag-score.test.ts). 22. ⏳ **Registry-to-corpus embedding (universal search Phase 5).** Medications/services/forms/ differentials are federated into `/api/search/universal` but are not retrieval-corpus - entities, so Answer mode cannot cite them. If product wants that: env-flagged ingestion, - golden-eval + invented-term controls first (depends on 17 for alias hygiene). + entities, so Answer mode cannot cite them. Concrete implementation spec (in order): + 1. Flag `RAG_REGISTRY_CORPUS_EMBEDDING` (default off) in `src/lib/env.ts`. + 2. Ingestion script `scripts/embed-registry-records.ts`: map each registry record to a + synthetic "document" (`metadata.source_kind = 'registry_record'`, title = record title, + one chunk per record from the record's search text, embedded with the standard + `text-embedding-3-small` path) so the existing chunk pipeline/RPCs need no schema change. + 3. Re-embed on registry edit: hook `ensureRegistrySeeded` / record-update routes to enqueue + re-embedding for the changed slug only. + 4. Answer-surface labelling: `sourceGovernanceWarnings` must label registry-backed + citations distinctly (registry records are curated summaries, not source documents). + 5. Gates before enabling anywhere real: `eval:retrieval:quality` 23/23 with the flag ON, + plus invented-term controls ("florbizone syndrome management") still refusing — registry + rows must not become a fabrication surface for unsupported topics. +23. ⏳ **Finding #11 full fix (RAG optimisation Phase 2)** — the classifier-verdict memo (shipped + 2026-07-06) makes zero-result behaviour deterministic per query but does not close the gap: + the deterministic analyzer still cannot tell in-corpus topics from out-of-corpus ones. + Phase-2 spec stands (corpus-grounded relevance: IDF/corpus-frequency weighting of query + terms + data-driven vocabulary), with the added prerequisite that item 17's vocabulary + capture now supplies real miss data to seed the vocabulary from. +24. ⏳ **OCR dropped-letter corruption in table index units** — no reliable detector exists (82% + false positives; guard reverted). Next viable angle: dictionary-based repair at INGESTION + (compare table-cell tokens against the document's own clean chunk text — "p ycho ocial" + aligns to "psychosocial" within the same page's raw text) rather than heuristic detection at + query time. Scope to `worker/` table extraction; requires the Python OCR stack to test. +25. ⏳ **Retrieval latency p90 ~8.6s (local)** — remaining sequential layers after the 2026-07-01 + parallelisation. Cheapest next step (measure first): overlap `embedTextWithTelemetry` with + the text fast path unconditionally (today preload only fires when `shouldPreloadEmbedding`), + and collapse the repeated `attachDocumentRankingMetadata` calls to one batched fetch per + request. Both are perf-only; gate with the golden eval unchanged + p90 from + `rag_retrieval_logs` before/after. diff --git a/docs/scale-readiness-review.md b/docs/scale-readiness-review.md new file mode 100644 index 0000000000..39e95891db --- /dev/null +++ b/docs/scale-readiness-review.md @@ -0,0 +1,209 @@ +# Scale-readiness review — retrieval RPCs and ingestion queries at 10× corpus + +Phase-2 deliverable of the ingestion-concurrency/scale review (2026-07-07, branch +`claude/ingestion-concurrency-scale`). Question: what breaks, and in what order, +when the corpus grows from ~2k documents to ~20k? + +Method: read-only against the live `Clinical KB Database` project +(`sjrfecxgysukkwxsowpy`) via `explain_retrieval_rpc(p_analyze=true)` and direct +`EXPLAIN (ANALYZE)` of the hybrid RPCs using a stored chunk embedding as the +query vector (local `npm run profile:retrieval` was not runnable on this machine +— no Supabase credentials present — so its underlying RPC was invoked directly; +same measurements). One `SET LOCAL hnsw.ef_search` experiment ran inside a +rolled-back transaction; nothing on the live project was modified. + +## 1. Live baseline (2026-07-07) + +### Corpus + +| Table | Rows | Total size (heap) | 10× projection | +| ------------------------- | --------------------- | ----------------- | -------------- | +| documents | 2,065 (all `indexed`) | 27 MB | ~20k | +| document_chunks | 69,334 | 1.62 GB (124 MB) | ~700k / ~16 GB | +| document_index_units | 111,991 | 1.16 GB (159 MB) | ~1.1M / ~12 GB | +| document_embedding_fields | 215,072 | 3.92 GB (558 MB) | ~2.2M / ~39 GB | +| document_memory_cards | 53,041 | 1.02 GB (84 MB) | ~530k / ~10 GB | +| document_table_facts | 34,795 | 76 MB | ~350k | +| document_pages / images | 27,416 / 12,202 | 51 / 38 MB | ~270k / ~120k | + +~7.8 GB of RAG tables today (indexes dominate — embedding_fields carries +~3.4 GB of index for 558 MB of heap). 10× ≈ **75–80 GB**. + +### Instance + +`shared_buffers` 256 MB, `effective_cache_size` 768 MB, `work_mem` 3.5 MB, +`max_connections` 60, `random_page_cost` 1.1, `jit` off. pgvector **0.8.0**; +HNSW indexes `m=24, ef_construction=128` on chunks / embedding_fields / +memory_cards; `hnsw.ef_search` at the **default 40** (no RPC or role sets it); +`hnsw.iterative_scan` at the default **off**. + +### Measured RPC latencies (warm-ish, single caller, live corpus) + +| RPC | Time | Notes | +| -------------------------------------- | ------------ | -------------------------------------------- | +| match_document_lookup_chunks_text | 9 ms | fine | +| match_documents_for_query | 52 ms | label/summary joins + per-row `similarity()` | +| match_document_chunks_hybrid | 141 ms | two arms + fusion | +| match_document_chunks_text | 301 ms | tsv + trgm fallbacks | +| match_document_index_units_hybrid | 444 ms | **text-gated only — see F2** | +| match_document_embedding_fields_hybrid | 520 ms | heaviest vector table, disk-bound | +| match_document_memory_cards_hybrid_v2 | 687 ms | | +| match_document_table_facts_text | **6,750 ms** | **unindexable trigram OR — see F1** | + +### Per-request fan-out (src/lib/rag.ts, cold cache worst case) + +Up to 3 query variants (`maxTextRpcQueryVariants=3`) each for +`match_document_chunks_text`, `match_documents_for_query`, and +`match_document_table_facts_text` (all three variant sets run under +`Promise.all`), plus the parallel hybrid trio (embedding fields, index units, +chunks hybrid), plus lookup-chunks and memory-cards calls and OR-relaxation / +trigram-correction retries when strict matching comes back weak or empty. +**A single cold request can issue 10–14 RPCs, of which the table-facts trio +alone is ~3 × 6.75 s of parallel DB CPU.** + +## 2. Findings (ranked) + +### F1 — `match_document_table_facts_text` is already pathological: 6.75 s at 2k docs, ~linear in corpus (CRITICAL, live today) + +The candidate predicate ends in +`or similarity(lower(coalesce(table_title,'')||' '||row_label||' '||clinical_parameter||' '||threshold_value||' '||action), query) >= 0.18` +(schema.sql:3087-3100). This disjunct is unindexable twice over: the expression +concatenates **five** columns while the trigram GIN index covers **three** +(`document_table_facts_title_row_param_trgm_idx`), and `similarity(x,y) >= k` +never uses a GIN index anyway (only the `%` operator with +`pg_trgm.similarity_threshold` does). The OR therefore forces a full scan of +all 35k rows computing per-row `similarity()` plus two `regexp_split_to_array` +calls in the rank expression. Measured 6.75 s; the answer path calls it up to +three times in parallel (rag.ts:2955-2961). At 10× (~350k rows): **~60-70 s per +call** — the RPC is effectively down, and three concurrent copies of it will +monopolize the connection pool. Mitigation (smallest safe): rewrite the fuzzy +disjunct to `lower(...3-column expr...) % query.normalized` so it matches the +existing index expression and uses the `%` operator (set +`pg_trgm.similarity_threshold` locally), or gate the trigram arm behind +"tsv/terms arms returned nothing". Verify with the same explain harness. + +### F2 — Index-unit "hybrid" retrieval has no vector arm at all; the missing HNSW index is being masked (HIGH, correctness-at-scale) + +`document_index_units` (112k rows, the visual/enrichment evidence table) has +**no vector index** — 14 btree/GIN indexes, zero HNSW. The RPC compensates by +gating candidates on text only: `where (search_tsv @@ tsq or normalized_terms +&& terms) order by text_rank desc limit 72` (schema.sql:4074-4076), computing +embedding similarity only as a **re-score** of those 72 text hits. Consequences: +(a) a query phrased differently from the unit's stored vocabulary can never +reach index units via semantics — vector recall is structurally zero for this +artifact class; (b) the `normalized_terms && terms` arm splits the raw query on +non-alphanumerics with no stopword filtering, so at 10× a query containing one +common clinical token ANDs `ts_rank_cd` + array-overlap over tens of thousands +of rows before the sort (444 ms today, roughly linear growth). Mitigation: +add an HNSW index on `document_index_units.embedding` (concurrently; ~1.1M +rows at 10× is fine for HNSW) and give the RPC a real vector arm mirroring +`match_document_chunks_hybrid`; keep the text arm as-is. + +### F3 — `ef_search` 40 silently caps every vector arm below its own LIMIT (HIGH, recall) + +The chunks-hybrid vector arm asks for `limit greatest(match_count*6, 48)` = 72 +candidates and embedding-fields asks for 48, but pgvector returns at most +`hnsw.ef_search` = 40 tuples per scan with `iterative_scan` off. Measured live: +the raw chunks vector arm returned **exactly 40 rows** against LIMIT 72; with +`SET LOCAL hnsw.ef_search = 200` (rolled back) the same query returned 72. +Today's effect is a mildly starved fusion pool. At 10× it compounds: the 40 +nearest neighbors are drawn from a 700k-chunk graph and then post-filtered by +`d.status='indexed'`, owner scope, and committed-generation checks — every +filtered-out neighbor is a permanently lost candidate, and multi-tenant owner +filtering makes the loss systematic for small tenants (their rows are a thin +slice of the graph). Mitigation (choose one, eval-gated): +`set local hnsw.ef_search` inside the hybrid RPCs to ≥ the arm LIMIT (cheap, +surgical), or enable `hnsw.iterative_scan = relaxed_order` (pgvector 0.8 +feature, purpose-built for filtered HNSW). + +### F4 — The whole vector working set already exceeds RAM by ~10×; at 10× corpus it exceeds it by ~100× (HIGH, latency/cost) + +256 MB `shared_buffers` / 768 MB `effective_cache_size` against ~5 GB of HNSW +indexes today explains the measured 440-690 ms hybrid RPCs (disk-bound graph +traversal). At 10× (~50 GB of vector indexes) every HNSW hop is a random read; +p95s move to seconds and the three-variant fan-out multiplies it. Mitigations +in order of leverage: (1) prune `document_embedding_fields` — at 215k rows it +is 3× larger than the chunk table it decorates and 3.9 GB of the 7.8 GB total; +half-precision (`halfvec`) or dropping low-value field types would halve the +hot set; (2) instance upgrade so the chunk + embedding-field HNSW indexes fit +in cache; (3) collapse the five per-artifact vector searches per query into +fewer arms (see F6). + +### F5 — `match_documents_for_query` recomputes unindexed tsvectors and trigram similarity per label/summary row (MEDIUM) + +The document-gate RPC computes `to_tsvector('english', l.label)` and +`similarity(lower(...), query)` inline for every candidate label and summary +(schema.sql:2637-2672) — no expression index exists for either. 52 ms today at +2k docs × labels; linear in both document count and label count, so ~0.5-1 s at +10× sitting on the answer path's critical prefix (it gates lookup-first +retrieval). Mitigation: precomputed `search_tsv` on document_labels (indexed) +and `%` instead of `similarity() >=`. + +### F6 — Connection/latency budget: 10-14 RPCs per cold request against `max_connections` 60 (MEDIUM, throughput ceiling) + +Each RPC is a separate PostgREST round-trip holding a pooled connection for its +full runtime. Today a cold clinical query consumes roughly 9-10 s of aggregate +DB time (dominated by F1); a dozen concurrent cold users would saturate the +60-connection budget even before 10×. The worker and edge agent (pool max 4) +share the same instance. Mitigations: fix F1/F2/F5 first (they are the +long-pole holders); then merge the three text-variant calls per RPC into one +RPC taking `text[]` of variants (one round-trip, one scan with +`websearch_to_tsquery` per variant unioned server-side); consider a single +`retrieve_all(query, embedding)` orchestrator RPC to collapse the hybrid trio. + +### F7 — Statistics churn from `analyze_rag_tables` after every job (LOW) + +The worker runs `ANALYZE` over six RAG tables after each completed job, +throttled to 45 s (worker/main.ts:334-343). At 10× ingestion volume this is a +steady background full-table sampling load on multi-GB tables and will start +appearing in p95s. Autovacuum's `autovacuum_analyze_scale_factor` handles this +fine at scale; the manual sweep should become conditional (row-delta threshold) +or scoped to the tables the job actually grew. + +### F8 — `cleanup_abandoned_document_index_generations` scans every artifact table by generation-mismatch with no supporting index (LOW, ops) + +Candidate selection unions seven `table × documents` joins filtering on +`index_generation_id::text is distinct from metadata->>'index_generation_id'` +— none of the partial indexes cover that predicate shape, so each run is seven +full scans (fine at 76 MB-1.6 GB, minutes at 10×, all inside one transaction +with a 180 s statement timeout that it will start hitting). Mitigation: chunked +per-document invocation (the `p_document_id` parameter already exists) driven +from the ops script, or a partial index on `(document_id) where +index_generation_id is not null`. + +## 3. What holds up fine + +- `match_document_chunks_text` (301 ms) and `match_document_lookup_chunks_text` + (9 ms): GIN `search_tsv` arms with bounded candidate LIMITs; growth is in + ts_rank sort width, roughly logarithmic-ish in practice. Acceptable at 10×. +- HNSW build parameters (`m=24, ef_construction=128`) are sensible for 1M-row + tables; no rebuild needed for 10× — the problem is search-time `ef_search` + (F3) and cache (F4), not graph quality. +- `claim_ingestion_jobs` / `claim_indexing_v3_agent_jobs`: SKIP LOCKED on small + hot tables with partial indexes — queue mechanics scale fine; the concurrency + bugs are in the lease semantics (phase 1 doc), not the query plans. +- Commit RPC's per-document deletes are all `document_id`-indexed; 180 s + statement timeout has ample headroom at 10× per-document sizes. + +## 4. Ranked mitigation list + +1. **F1**: make the table-facts trigram disjunct indexable (`%` operator + + 3-column expression matching the existing index) or short-circuit it. + Unbreaks the worst RPC today; mandatory before any growth. +2. **F3**: `set local hnsw.ef_search = greatest(match_count*6, 64)` (or enable + iterative scan) inside the three vector-arm RPCs. One-line recall fix, + eval-gate with `npm run eval:retrieval` content_mrr@10. +3. **F2**: HNSW index on `document_index_units.embedding` + real vector arm in + its RPC. Restores semantic recall for the visual/enrichment evidence class. +4. **F5**: indexed `search_tsv` for document_labels; `%` for fuzzy matching in + `match_documents_for_query`. +5. **F6**: variant-array RPCs (3 round-trips → 1) and, later, a consolidated + retrieval orchestrator RPC. +6. **F4**: shrink `document_embedding_fields` (field-type audit / halfvec) and + size the instance so chunk+field HNSW fit `effective_cache_size` before 10×. +7. **F7/F8**: conditional `analyze_rag_tables`; chunked abandoned-generation + cleanup. + +All changes above are eval-gated config/SQL work; none change ranking semantics +except F2/F3, which must show a content_mrr@10 non-regression on the golden set +before defaults change (per the repo's reindex-eval-gate convention). diff --git a/docs/supabase-migration-reconciliation.md b/docs/supabase-migration-reconciliation.md index e6c51c7fa0..5d3b675c52 100644 --- a/docs/supabase-migration-reconciliation.md +++ b/docs/supabase-migration-reconciliation.md @@ -1,6 +1,6 @@ # Supabase Migration Reconciliation -Last reviewed: 2026-07-05 +Last reviewed: 2026-07-07 Target project: Clinical KB Database (`sjrfecxgysukkwxsowpy`) @@ -11,6 +11,62 @@ Target project: Clinical KB Database (`sjrfecxgysukkwxsowpy`) - Use `supabase migration repair --linked --status applied ` only when live database evidence proves the migration effect already exists. - Leave other local-only migrations unrepaired until their effects are verified or deliberately applied. - Run `npx supabase migration list --linked` at apply/reconcile time; do not rely on a frozen “aligned through” snapshot in this doc alone. +- **History presence is not effect presence.** `20260703030000` is recorded as applied on live while its index changes are absent. After every apply, verify object state with `npm run check:drift` (and `search_schema_health()`), not the history table. +- Any PR that changes `supabase/schema.sql` regenerates `supabase/drift-manifest.json` in the same PR (`npm run drift:manifest`, Docker required); `tests/drift-detection.test.ts` fails otherwise. This doubles as a from-scratch replay proof of schema.sql. + +## Expand/contract policy for retrieval tables + +Applies to anything touching `documents`, `document_chunks`, +`document_embedding_fields`, `document_index_units`, `document_memory_cards`, +`document_table_facts`, embedding columns, or the RPCs that read them. Written +after the 2026-07-07 DR rehearsal +([disaster-recovery-runbook.md](disaster-recovery-runbook.md)), which showed +(a) live carried worker-written columns that existed in no repo lineage, so a +restored/branch database silently broke ingestion, and (b) a "recorded as +applied" migration whose effects never landed. + +**Expand phase (additive, ships first):** + +- New columns are `add column if not exists`, nullable or defaulted — the RAG + tables have 69k–215k rows; a table rewrite (non-constant default, type + change) needs an explicit lock/duration plan in the migration header. +- New/changed RPC behaviour ships as a side-by-side version + (`match_document_memory_cards_hybrid_v2` precedent) with the old RPC left + callable until the app is fully cut over; grants replicated explicitly + (`revoke ... from public, anon, authenticated; grant ... to service_role`). +- New constraints on populated tables use `NOT VALID` now + `VALIDATE +CONSTRAINT` in a later migration (the live `*_content_not_blank` checks are + the precedent). +- Index replacements create the new index first (on live, prefer + `CONCURRENTLY` run manually outside the transaction — CLI migrations are + transactional); the old index is NOT dropped in the same migration. +- Embedding columns: `vector(N)` is coupled to `EMBEDDING_DIMENSIONS` and the + worker's startup check — a dimension change is a re-index project with the + reindex-eval gate, never a plain migration. +- The same PR updates `supabase/schema.sql`, regenerates the drift manifest, + and (for anything behaviour-adjacent) passes `npm run +eval:retrieval:quality` per the standing merge gate. + +**Verify phase (between expand and contract):** + +- Apply to live only through the linked workflow with explicit approval, then + immediately run `npm run check:drift` — the applied objects must match the + manifest (this is what catches recorded-but-ineffective applies). +- Run `search_schema_health()` / `npm run check:indexing` and the golden + retrieval eval against live before relying on the new path. +- Dual-read/dual-write windows (old + new column/RPC) stay until the eval and + telemetry confirm the new path. + +**Contract phase (destructive, ships last and separately):** + +- Drops (columns, old RPC versions, superseded indexes) go in their own + migration, at least one release after expand, never bundled with it. +- Before contracting: a fresh backup/PITR point exists, `pg_stat_user_indexes` + scan evidence for index drops (the `20260702014803` discipline), and + check:drift green so the pre-contract state is fully accounted for. +- Rollback plan is written in the migration header (what to recreate, from + where) — after contract, rollback means restore-from-backup for data-bearing + drops, so say so explicitly. ## Verified Applied (through June 2026) @@ -45,6 +101,17 @@ The repo also includes additional July 2026 migrations beyond the June checkpoin Live-only drift, duplicate migration-version churn, and outstanding follow-up debts are tracked in the **Retrieval RPC drift & indexing hygiene** section of [`docs/process-hardening.md`](process-hardening.md). Treat that section as the operational supplement to this reconciliation doc. +**2026-07-07 full-inventory audit:** the standing drift check +([database-drift-detection.md](database-drift-detection.md)) measured live +against both repo lineages. Pending on live as of the audit: `20260705210000` +(owner-sentinel — 8 function bodies), `20260706010000` (M13 guard), +`20260706130000`, plus the new `20260706200000` (drift snapshot RPC) and +`20260707000000` (codification wave, no-op on live). `20260703030000` is +recorded in live history but its effects are absent — repair by re-applying +its statements under a new version, with approval. The complete reconciliation +backlog (index estate, grant posture, remaining live-only functions) lives in +the drift doc. + Before applying pending migrations to live: 1. Run `npx supabase migration list --linked` and confirm local vs remote alignment. diff --git a/package-lock.json b/package-lock.json index 594ea5d86d..64a8061f17 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.39.4", "eslint-config-next": "16.2.10", + "fast-check": "^4.8.0", "playwright": "^1.61.1", "prettier": "^3.9.4", "tailwindcss": "^4.3.1", @@ -5428,6 +5429,29 @@ "node": ">=12.0.0" } }, + "node_modules/fast-check": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.8.0.tgz", + "integrity": "sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fast-csv": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", @@ -8252,6 +8276,23 @@ "node": ">=6" } }, + "node_modules/pure-rand": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.1.tgz", + "integrity": "sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", diff --git a/package.json b/package.json index 8dd106ce11..c690f8b7f3 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,9 @@ "workflow:deps": "node ../.local-dev/workflow-deps.mjs", "workflow:clean-state": "node ../.local-dev/workflow-clean-state.mjs", "workflow:export": "node ../.local-dev/workflow-export.mjs", - "workflow:handoff": "node ../.local-dev/workflow-handoff.mjs" + "workflow:handoff": "node ../.local-dev/workflow-handoff.mjs", + "check:drift": "tsx scripts/check-drift.ts", + "drift:manifest": "tsx scripts/generate-drift-manifest.ts" }, "dependencies": { "@next/env": "16.2.10", @@ -132,6 +134,7 @@ "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.39.4", "eslint-config-next": "16.2.10", + "fast-check": "^4.8.0", "playwright": "^1.61.1", "prettier": "^3.9.4", "tailwindcss": "^4.3.1", diff --git a/public/llms.txt b/public/llms.txt index c2ec0686b7..a908f6dc0f 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -5,7 +5,7 @@ Purpose: Clinical Guide is a local clinical knowledge-base interface for searchi Agent / codebase orientation: docs/codebase-index.md (module map, APIs, Supabase, worker). Route index: docs/site-map.md. Key routes: -- / opens the main dashboard. Use ?mode=answer, ?mode=documents, ?mode=tools, ?mode=favourites, ?mode=differentials, or ?mode=prescribing to choose the workspace. +- / opens the main dashboard. Use ?mode=answer, ?mode=documents, ?mode=tools, ?mode=differentials, or ?mode=prescribing to choose the workspace. ?mode=favourites redirects to /favourites. - /documents/search opens the documents search command centre after submitting a documents-mode query. - /documents/:id opens an indexed source document. - /services opens source-backed service records. diff --git a/scripts/capture-chrome-parity.ts b/scripts/capture-chrome-parity.ts index 567a937173..e2dcee26c9 100644 --- a/scripts/capture-chrome-parity.ts +++ b/scripts/capture-chrome-parity.ts @@ -90,16 +90,16 @@ const selectorGroups: Array<{ key: string; selector: string; pseudo?: string }> type Snapshot = Record>; async function mockApis(page: Page) { - await page.route("**/api/setup-status**", async (route) => { + await page.route("**/api/setup-status**", async (route: Route) => { await route.fulfill({ json: { demoMode: true, checks: readySetupChecks } }); }); - await page.route(/\/api\/documents\/[0-9a-f-]+(?:\?.*)?$/, async (route) => { + await page.route(/\/api\/documents\/[0-9a-f-]+(?:\?.*)?$/, async (route: Route) => { const id = new URL(route.request().url()).pathname.split("/").pop() ?? ""; const payload = getDemoDocumentPayload(id); if (payload) await route.fulfill({ json: payload }); else await route.fulfill({ status: 404, json: { error: "not found" } }); }); - await page.route(/\/api\/documents(?:\?.*)?$/, async (route) => { + await page.route(/\/api\/documents(?:\?.*)?$/, async (route: Route) => { await route.fulfill({ json: { documents: demoDocuments, diff --git a/scripts/check-drift.ts b/scripts/check-drift.ts new file mode 100644 index 0000000000..965dc064b8 --- /dev/null +++ b/scripts/check-drift.ts @@ -0,0 +1,312 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { loadEnvConfig } from "@next/env"; + +loadEnvConfig(process.cwd()); + +/** + * check:drift — compares the live database's full schema inventory against the + * committed expectation derived from supabase/schema.sql. + * + * Expected side: supabase/drift-manifest.json, generated by replaying + * supabase/schema.sql into a scratch Supabase Postgres container + * (`npm run drift:manifest`). The manifest embeds the sha256 of the schema.sql + * it was generated from, so a stale manifest fails fast here (and offline in + * tests/drift-detection.test.ts) instead of producing phantom drift. + * + * Live side: public.schema_drift_snapshot() (migration + * 20260706200000_schema_drift_snapshot.sql), a service-role-only RPC returning + * the same normalized inventory the manifest holds. + * + * Known, documented divergence is carried in supabase/drift-allowlist.json — + * every entry needs a reason and is reported as a warning, never silently + * dropped. Anything not allowlisted exits 1. See docs/database-drift-detection.md. + */ + +type SnapshotObject = Record; +type Snapshot = Record; + +type AllowlistEntry = { + category: string; + kind: "missing_live" | "unexpected_live" | "mismatch" | "alias"; + key: string; + live_key?: string; + reason: string; +}; + +type Finding = { + category: string; + kind: "missing_live" | "unexpected_live" | "mismatch"; + key: string; + detail?: string; +}; + +const read = (relative: string) => readFileSync(new URL(`../${relative}`, import.meta.url), "utf8"); + +export function normalizedSchemaSha256(schemaSqlText: string) { + return createHash("sha256").update(schemaSqlText.replace(/\r\n/g, "\n")).digest("hex"); +} + +// category -> unique key extractor. Must stay aligned with the RPC's shape. +const categoryKeys: Record string> = { + extensions: (o) => String(o.name), + tables: (o) => String(o.name), + views: (o) => String(o.name), + functions: (o) => String(o.signature), + indexes: (o) => String(o.name), + policies: (o) => `${o.schema}.${o.table}.${o.name}`, + constraints: (o) => `${o.table}.${o.name}`, + triggers: (o) => `${o.table}.${o.name}`, + storage_buckets: (o) => String(o.id), +}; + +// Fields that identify content per category (compared as canonical JSON). +const comparedFields: Record = { + extensions: ["schema"], + tables: ["rls_enabled", "rls_forced", "reloptions", "acl", "columns"], + views: ["def_hash"], + functions: ["def_hash", "acl"], + indexes: ["table", "def_hash"], + policies: ["permissive", "roles", "cmd", "qual", "with_check"], + constraints: ["def"], + triggers: ["def"], + storage_buckets: ["public", "file_size_limit", "allowed_mime_types"], +}; + +function canonical(value: unknown): string { + return JSON.stringify(value, (_key, v) => + v && typeof v === "object" && !Array.isArray(v) + ? Object.fromEntries( + Object.keys(v as Record) + .sort() + .map((k) => [k, (v as Record)[k]]), + ) + : v, + ); +} + +function indexObjects(category: string, snapshot: Snapshot): Map { + const list = snapshot[category]; + const keyOf = categoryKeys[category]; + const map = new Map(); + if (!Array.isArray(list)) return map; + for (const raw of list) { + if (raw && typeof raw === "object") { + const o = raw as SnapshotObject; + map.set(keyOf(o), o); + } + } + return map; +} + +function contentOf(category: string, o: SnapshotObject): string { + return canonical(Object.fromEntries(comparedFields[category].map((f) => [f, o[f] ?? null]))); +} + +function fieldDiff(category: string, expected: SnapshotObject, actual: SnapshotObject): string { + const parts: string[] = []; + for (const field of comparedFields[category]) { + const e = canonical(expected[field] ?? null); + const a = canonical(actual[field] ?? null); + if (e !== a) { + const clip = (s: string) => (s.length > 240 ? `${s.slice(0, 240)}…` : s); + parts.push(`${field}: manifest=${clip(e)} live=${clip(a)}`); + } + } + return parts.join("; "); +} + +export type DriftComparison = { + findings: Finding[]; + allowed: { entry: AllowlistEntry; finding: Finding }[]; + staleEntries: AllowlistEntry[]; + infos: string[]; +}; + +export function compareDriftSnapshots( + expected: Snapshot, + live: Snapshot, + allowlist: AllowlistEntry[], +): DriftComparison { + const findings: Finding[] = []; + const infos: string[] = []; + + for (const category of Object.keys(categoryKeys)) { + const expectedMap = indexObjects(category, expected); + const liveMap = indexObjects(category, live); + + for (const [key, expectedObject] of expectedMap) { + const liveObject = liveMap.get(key); + if (!liveObject) { + findings.push({ category, kind: "missing_live", key, detail: contentOf(category, expectedObject) }); + continue; + } + if (contentOf(category, expectedObject) !== contentOf(category, liveObject)) { + findings.push({ category, kind: "mismatch", key, detail: fieldDiff(category, expectedObject, liveObject) }); + } + } + + for (const key of liveMap.keys()) { + if (expectedMap.has(key)) continue; + // Hosted platform provisions extensions (pg_net, pgsodium, …) that a + // schema.sql replay never creates; extra live extensions are informational. + if (category === "extensions") { + infos.push(`extra live extension (platform-provisioned): ${key}`); + continue; + } + findings.push({ category, kind: "unexpected_live", key }); + } + } + + // Apply the allowlist. Alias entries assert the live database carries the + // same index under a legacy name; they consume both the missing_live finding + // for the manifest name and the unexpected_live finding for the legacy name. + const allowed: { entry: AllowlistEntry; finding: Finding }[] = []; + const remaining: Finding[] = []; + const usedEntries = new Set(); + + const liveIndexes = indexObjects("indexes", live); + const expectedIndexes = indexObjects("indexes", expected); + + const matchesAlias = (entry: AllowlistEntry, finding: Finding): boolean => { + if (entry.category !== "indexes" || finding.category !== "indexes") return false; + if (finding.kind === "missing_live" && finding.key === entry.key) { + const manifestIdx = expectedIndexes.get(entry.key); + const liveIdx = entry.live_key ? liveIndexes.get(entry.live_key) : undefined; + if (!manifestIdx || !liveIdx) return false; + const stripName = (o: SnapshotObject) => + String(o.def ?? "") + .split(String(o.name)) + .join("@NAME@") + .replace(/\s+/g, ""); + return manifestIdx.table === liveIdx.table && stripName(manifestIdx) === stripName(liveIdx); + } + return finding.kind === "unexpected_live" && finding.key === entry.live_key; + }; + + for (const finding of findings) { + const entry = allowlist.find( + (candidate) => + (candidate.kind === "alias" && matchesAlias(candidate, finding)) || + (candidate.kind === finding.kind && candidate.category === finding.category && candidate.key === finding.key), + ); + if (entry) { + usedEntries.add(entry); + allowed.push({ entry, finding }); + } else { + remaining.push(finding); + } + } + + return { + findings: remaining, + allowed, + staleEntries: allowlist.filter((entry) => !usedEntries.has(entry)), + infos, + }; +} + +async function main() { + const [{ requireServerEnv }, { createAdminClient }, { checkSupabaseProjectConfig, formatSupabaseProjectCheck }] = + await Promise.all([import("@/lib/env"), import("@/lib/supabase/admin"), import("@/lib/supabase/project")]); + + requireServerEnv(); + const projectCheck = checkSupabaseProjectConfig( + { + NEXT_PUBLIC_SUPABASE_URL: process.env.NEXT_PUBLIC_SUPABASE_URL, + SUPABASE_PROJECT_REF: process.env.SUPABASE_PROJECT_REF, + SUPABASE_PROJECT_NAME: process.env.SUPABASE_PROJECT_NAME, + }, + { requireMetadata: false }, + ); + if (projectCheck.status === "missing" || projectCheck.status === "mismatch") { + throw new Error(formatSupabaseProjectCheck(projectCheck)); + } + + const manifestFile = JSON.parse(read("supabase/drift-manifest.json")) as { + schema_sha256: string; + generated_at: string; + postgres_image: string; + snapshot: Snapshot; + }; + const allowlistFile = JSON.parse(read("supabase/drift-allowlist.json")) as { entries: AllowlistEntry[] }; + const allowlist = allowlistFile.entries ?? []; + + const schemaSha = normalizedSchemaSha256(read("supabase/schema.sql")); + if (schemaSha !== manifestFile.schema_sha256) { + throw new Error( + `supabase/drift-manifest.json is stale: it was generated from a different supabase/schema.sql ` + + `(manifest ${manifestFile.schema_sha256.slice(0, 12)}…, current ${schemaSha.slice(0, 12)}…). ` + + `Regenerate with: npm run drift:manifest`, + ); + } + + const supabase = createAdminClient(); + const { data: liveSnapshot, error } = await supabase.rpc("schema_drift_snapshot" as never); + if (error) { + const message = String(error.message ?? error); + if (/could not find the function|schema cache|PGRST202/i.test(message)) { + throw new Error( + `schema_drift_snapshot() is not available on the live project. Apply migration ` + + `20260706200000_schema_drift_snapshot.sql through the normal linked migration workflow first. (${message})`, + ); + } + throw new Error(`schema_drift_snapshot RPC failed: ${message}`); + } + if (!liveSnapshot || typeof liveSnapshot !== "object") { + throw new Error("schema_drift_snapshot returned an unexpected payload"); + } + const live = liveSnapshot as Snapshot; + const expected = manifestFile.snapshot; + + const { findings: remaining, allowed, staleEntries, infos } = compareDriftSnapshots(expected, live, allowlist); + + console.log(`Drift manifest: generated ${manifestFile.generated_at} from schema.sql ${schemaSha.slice(0, 12)}…`); + console.log( + `Compared ${Object.keys(categoryKeys) + .map((category) => `${indexObjects(category, expected).size} ${category}`) + .join(", ")} against live.`, + ); + for (const info of infos) console.log(` info: ${info}`); + if (allowed.length > 0) { + console.log(`\nAllowlisted divergence (${allowed.length}) — known and documented, not failures:`); + for (const { entry, finding } of allowed) { + console.log(` ~ [${finding.category}] ${finding.kind} ${finding.key} — ${entry.reason}`); + } + } + if (staleEntries.length > 0) { + console.log(`\nStale allowlist entries (${staleEntries.length}) — no longer matching, remove them:`); + for (const entry of staleEntries) { + console.log(` ? [${entry.category}] ${entry.kind} ${entry.key}`); + } + } + if (remaining.length > 0) { + console.error(`\nUNEXPECTED DRIFT (${remaining.length}):`); + for (const finding of remaining) { + console.error( + ` ! [${finding.category}] ${finding.kind} ${finding.key}${finding.detail ? ` :: ${finding.detail}` : ""}`, + ); + } + console.error( + `\nLive schema diverges from supabase/schema.sql. Either codify the live state ` + + `(committed migration + schema.sql + regenerate the manifest) or fix live through ` + + `an approved migration. Raw SQL against live is how this class of incident started; ` + + `do not \"fix\" drift that way. See docs/database-drift-detection.md.`, + ); + process.exitCode = 1; + return; + } + + console.log(`\nNo unexpected schema drift between live and supabase/schema.sql.`); + if (staleEntries.length > 0) { + console.log("(Clean up the stale allowlist entries above in a follow-up commit.)"); + } +} + +const invokedDirectly = process.argv[1] && /check-drift\.(ts|mts|js)$/.test(process.argv[1]); +if (invokedDirectly) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/generate-drift-manifest.ts b/scripts/generate-drift-manifest.ts new file mode 100644 index 0000000000..65f107fc53 --- /dev/null +++ b/scripts/generate-drift-manifest.ts @@ -0,0 +1,130 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +/** + * drift:manifest — regenerates supabase/drift-manifest.json. + * + * Replays supabase/schema.sql from scratch into a disposable Supabase Postgres + * Docker container (the same image family the hosted platform runs), then + * captures public.schema_drift_snapshot() from that pristine database. The + * result is the expected-state side of `npm run check:drift`. + * + * Requires Docker. Never touches the live project. Run this whenever + * supabase/schema.sql changes — the manifest embeds schema.sql's sha256 and + * both check:drift and tests/supabase-schema.test.ts fail while it is stale. + * + * Flags: + * --keep leave the container running (for inspection / DR rehearsal) + * --port host port for the scratch Postgres (default 56599) + * --image override the Postgres image tag + */ + +const IMAGE_DEFAULT = "supabase/postgres:17.6.1.127"; +const CONTAINER = "clinical-kb-drift-manifest"; + +const repoUrl = (relative: string) => fileURLToPath(new URL(`../${relative}`, import.meta.url)); + +function arg(flag: string): string | undefined { + const index = process.argv.indexOf(flag); + return index >= 0 ? process.argv[index + 1] : undefined; +} + +function docker(args: string[], input?: string): string { + return execFileSync("docker", args, { + encoding: "utf8", + input, + maxBuffer: 64 * 1024 * 1024, + stdio: ["pipe", "pipe", "pipe"], + }); +} + +async function main() { + const image = arg("--image") ?? IMAGE_DEFAULT; + const port = arg("--port") ?? "56599"; + const keep = process.argv.includes("--keep"); + + try { + docker(["info", "--format", "{{.ServerVersion}}"]); + } catch { + throw new Error( + "Docker is required for drift:manifest (it replays schema.sql into a scratch container). Start Docker and retry.", + ); + } + + const schemaSql = readFileSync(repoUrl("supabase/schema.sql"), "utf8"); + const scaffoldSql = readFileSync(repoUrl("scripts/sql/drift-replay-scaffold.sql"), "utf8"); + const { normalizedSchemaSha256 } = await import("./check-drift"); + + console.log(`Starting scratch container ${CONTAINER} (${image}) on port ${port}…`); + try { + docker(["rm", "-f", CONTAINER]); + } catch { + // not running — fine + } + const startedAt = Date.now(); + docker(["run", "-d", "--name", CONTAINER, "-e", "POSTGRES_PASSWORD=postgres", "-p", `${port}:5432`, image]); + + try { + // Wait for Postgres readiness (image init runs its own restarts; give it time). + let ready = false; + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + docker(["exec", CONTAINER, "pg_isready", "-U", "postgres", "-q"]); + ready = true; + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 2000)); + } + } + if (!ready) throw new Error("scratch Postgres did not become ready in time"); + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const psql = (user: string, sql: string) => + docker( + ["exec", "-i", CONTAINER, "psql", "-U", user, "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-q", "-f", "-"], + sql, + ); + + console.log("Applying storage scaffold (supabase_admin)…"); + psql("supabase_admin", scaffoldSql); + console.log("Replaying supabase/schema.sql from scratch (postgres)…"); + psql("postgres", schemaSql); + const replaySeconds = ((Date.now() - startedAt) / 1000).toFixed(0); + console.log(`Replay complete in ${replaySeconds}s (container start included).`); + + const raw = docker( + ["exec", "-i", CONTAINER, "psql", "-U", "postgres", "-d", "postgres", "-tA", "-v", "ON_ERROR_STOP=1", "-f", "-"], + "select public.schema_drift_snapshot()::text;", + ).trim(); + const snapshot = JSON.parse(raw) as Record; + delete snapshot.captured_at; + + const manifest = { + generated_at: new Date().toISOString(), + generator: "scripts/generate-drift-manifest.ts", + postgres_image: image, + schema_sha256: normalizedSchemaSha256(schemaSql), + replay_seconds: Number(replaySeconds), + snapshot, + }; + writeFileSync(repoUrl("supabase/drift-manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + console.log("Wrote supabase/drift-manifest.json"); + console.log("Next: run `npm run check:drift` against live (needs service-role env)."); + } finally { + if (keep) { + console.log(`Container ${CONTAINER} kept running on port ${port} (--keep).`); + } else { + try { + docker(["rm", "-f", CONTAINER]); + } catch { + console.warn(`Could not remove container ${CONTAINER}; remove it manually.`); + } + } + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/lib/parse-differentials-export.ts b/scripts/lib/parse-differentials-export.ts index d480677112..4a5715cac8 100644 --- a/scripts/lib/parse-differentials-export.ts +++ b/scripts/lib/parse-differentials-export.ts @@ -400,8 +400,16 @@ function mergeDiagnosisRecords(existing: DifferentialRecord, incoming: Different }; } -export function parseScenarioPresets(markdown: string): DifferentialScenarioPreset[] { +// Splitting on "## " leaves any document preamble (the top-level "# Title" +// heading and intro prose) as a phantom first section; it must be dropped or +// it becomes a bogus record (e.g. a "# Scenario Presets" preset). +function markdownSections(markdown: string) { const sections = markdown.split(/(?:^|\n)##\s+/).filter(Boolean); + return markdown.trimStart().startsWith("## ") ? sections : sections.slice(1); +} + +export function parseScenarioPresets(markdown: string): DifferentialScenarioPreset[] { + const sections = markdownSections(markdown); return sections.map((section, index) => { const lines = section.split("\n"); const titleLine = lines[0]?.trim() ?? `Preset ${index + 1}`; @@ -430,7 +438,7 @@ export function parseScenarioPresets(markdown: string): DifferentialScenarioPres } export function parseRedFlagFlows(markdown: string): DifferentialRedFlagFlow[] { - const sections = markdown.split(/(?:^|\n)##\s+/).filter(Boolean); + const sections = markdownSections(markdown); return sections.map((section, index) => { const title = section.split("\n")[0]?.trim() ?? `Flow ${index + 1}`; const entryId = section.match(/Entry\s+(\d+[A-Z]?)/i)?.[1]?.toUpperCase() ?? ""; @@ -463,7 +471,9 @@ export function parseSearchAliases(markdown: string): Record { const values = match[2] .split(",") .map((item) => item.trim().toLowerCase()) - .filter(Boolean); + // The export also carries field-weight tables (e.g. "| tags | 1.1 |"); + // bare numbers are ranking weights, not clinical synonyms. + .filter((item) => Boolean(item) && !/^\d+(\.\d+)?$/.test(item)); if (token && values.length) aliases[token] = values; } return aliases; diff --git a/scripts/sql/drift-replay-scaffold.sql b/scripts/sql/drift-replay-scaffold.sql new file mode 100644 index 0000000000..efbd8bad2e --- /dev/null +++ b/scripts/sql/drift-replay-scaffold.sql @@ -0,0 +1,72 @@ +-- Minimal storage-schema scaffold so supabase/schema.sql can replay on a bare +-- supabase/postgres Docker image (used by `npm run drift:manifest` and the +-- disaster-recovery rehearsal in docs/disaster-recovery-runbook.md). +-- +-- The hosted platform provisions storage.buckets / storage.objects / +-- storage.foldername() through the Storage service's own migrations; the bare +-- image ships the empty `storage` schema only. schema.sql needs the buckets +-- table (bucket inserts), the objects table (owner-read policies), and +-- foldername() (referenced by those policy predicates). Column shapes below +-- mirror the hosted service closely enough for DDL replay; they are NOT a +-- substitute for the real Storage service. +-- +-- Idempotent; safe to re-run. Never run against the live project (it already +-- has the real storage schema, and every statement here is create-if-missing, +-- but there is no reason for it to touch live). +-- +-- Run as `supabase_admin` (the image's superuser): the bare image ships the +-- `storage` schema owned by supabase_admin, and `postgres` cannot create in +-- it. Ownership of the scaffold tables is handed to `postgres` afterwards so +-- schema.sql (applied as `postgres`, matching how live is administered) can +-- insert bucket rows and create the storage.objects policies. + +create schema if not exists storage; + +create table if not exists storage.buckets ( + id text primary key, + name text not null unique, + owner uuid, + public boolean default false, + avif_autodetection boolean default false, + file_size_limit bigint, + allowed_mime_types text[], + created_at timestamptz default now(), + updated_at timestamptz default now() +); + +create table if not exists storage.objects ( + id uuid primary key default gen_random_uuid(), + bucket_id text references storage.buckets(id), + name text, + owner uuid, + owner_id text, + version text, + metadata jsonb, + path_tokens text[] generated always as (string_to_array(name, '/')) stored, + created_at timestamptz default now(), + updated_at timestamptz default now(), + last_accessed_at timestamptz default now() +); + +alter table storage.buckets enable row level security; +alter table storage.objects enable row level security; + +create or replace function storage.foldername(name text) +returns text[] +language plpgsql +immutable +as $$ +declare + _parts text[]; +begin + select string_to_array(name, '/') into _parts; + return _parts[1 : array_length(_parts, 1) - 1]; +end; +$$; + +alter table storage.buckets owner to postgres; +alter table storage.objects owner to postgres; +alter function storage.foldername(text) owner to postgres; + +grant usage on schema storage to postgres, anon, authenticated, service_role; +grant create on schema storage to postgres; diff --git a/src/app/api/medications/route.ts b/src/app/api/medications/route.ts index 4711cafb05..cb996d8fa3 100644 --- a/src/app/api/medications/route.ts +++ b/src/app/api/medications/route.ts @@ -14,7 +14,6 @@ import { medicationValidationStatus, rowGovernance, rowToMedicationRecord, - type MedicationRecordRow, } from "@/lib/medication-records"; import { medicationToSearchResult, diff --git a/src/app/api/registry/records/route.ts b/src/app/api/registry/records/route.ts index 29b864a649..310c3c0550 100644 --- a/src/app/api/registry/records/route.ts +++ b/src/app/api/registry/records/route.ts @@ -15,7 +15,6 @@ import { rowGovernance, rowToServiceRecord, type RegistryRecordKind, - type RegistryRecordRow, } from "@/lib/registry-records"; import { fetchOwnerRegistryRowsWithSeed } from "@/lib/registry-seed"; import { rankServiceRecords, serviceRecords, type ServiceRecord, type ServiceSearchMatch } from "@/lib/services"; diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 0f2fea96db..044ab1e4ce 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -30,6 +30,7 @@ import { queryDerivedTokensForStorage, queryPrivacyMetadata, queryTextForStorage, + queryVocabularyAliasesForStorage, } from "@/lib/query-privacy"; import { safeErrorLogDetails } from "@/lib/privacy"; import { nonProductionSupabaseDemoFallbackReason } from "@/lib/supabase/errors"; @@ -414,8 +415,12 @@ function candidatePromotions(query: string, results: SearchResult[]) { document_id: label.document_id, confidence: label.confidence, })); + const rawTokens = queryDerivedTokensForStorage(Array.from(new Set(queryTerms)).slice(0, 10)); return { - aliases: queryDerivedTokensForStorage(Array.from(new Set(queryTerms)).slice(0, 10)), + // With raw retention off, fall back to curated clinical-vocabulary matches — output text + // comes from the fixed vocabulary table, never the query, so it is RET-H4 safe and keeps + // the alias-promotion pipeline fed (rag-hybrid-findings item 17). + aliases: rawTokens.length ? rawTokens : queryVocabularyAliasesForStorage(query), labels: topLabels, }; } @@ -528,6 +533,11 @@ function retrievalDecisionTelemetry(telemetry: Record) { second_stage_rerank_used: telemetryBoolean(telemetry, "second_stage_rerank_used"), second_stage_rerank_latency_ms: telemetryNumber(telemetry, "second_stage_rerank_latency_ms"), visual_direct_image_count: telemetryNumber(telemetry, "visual_direct_image_count"), + // RC9/P8b observability: these feed the synthetic-similarity gate recalibration and the + // weak-match OR-augmentation review (rag-hybrid-findings items 21 and the P8b extension) — + // without persisting them the recalibration has no data to work from. + text_or_relaxation_used: telemetryString(telemetry, "text_or_relaxation_used"), + synthetic_similarity_count: telemetryNumber(telemetry, "synthetic_similarity_count"), }; } diff --git a/src/app/applications/layout.tsx b/src/app/applications/layout.tsx index c9a0125402..641518a9fb 100644 --- a/src/app/applications/layout.tsx +++ b/src/app/applications/layout.tsx @@ -4,7 +4,7 @@ import { GlobalSearchShell } from "@/components/clinical-dashboard/global-search export default function ApplicationsLayout({ children }: { children: ReactNode }) { return ( - + {children} ); diff --git a/src/app/applications/page.tsx b/src/app/applications/page.tsx index 2f7fd01fd7..6e844a80c5 100644 --- a/src/app/applications/page.tsx +++ b/src/app/applications/page.tsx @@ -7,6 +7,19 @@ export const metadata: Metadata = { description: "Launch Clinical KB applications, workflows, and connected clinical tools.", }; -export default function ApplicationsRoute() { - return ; +type ApplicationsPageProps = { + searchParams?: Promise<{ + q?: string | string[]; + }>; +}; + +function firstSearchParam(value: string | string[] | undefined) { + return Array.isArray(value) ? value[0] : value; +} + +export default async function ApplicationsRoute({ searchParams }: ApplicationsPageProps) { + const params = searchParams ? await searchParams : {}; + const query = firstSearchParam(params.q)?.trim(); + + return query ? : ; } diff --git a/src/app/differentials/layout.tsx b/src/app/differentials/layout.tsx index 5a35655626..b37056fb20 100644 --- a/src/app/differentials/layout.tsx +++ b/src/app/differentials/layout.tsx @@ -3,5 +3,9 @@ import type { ReactNode } from "react"; import { GlobalSearchShell } from "@/components/clinical-dashboard/global-search-shell"; export default function DifferentialsLayout({ children }: { children: ReactNode }) { - return {children}; + return ( + + {children} + + ); } diff --git a/src/app/differentials/page.tsx b/src/app/differentials/page.tsx index e3d42c168c..7c3137a01b 100644 --- a/src/app/differentials/page.tsx +++ b/src/app/differentials/page.tsx @@ -1,7 +1,7 @@ import { DifferentialsHomePage } from "@/components/differentials/differentials-home-page"; type DifferentialsRouteProps = { - searchParams?: Promise<{ query?: string | string[]; q?: string | string[] }>; + searchParams?: Promise<{ query?: string | string[]; q?: string | string[]; run?: string | string[] }>; }; function firstSearchParam(value?: string | string[]) { @@ -10,11 +10,12 @@ function firstSearchParam(value?: string | string[]) { export default async function DifferentialsHomeRoute({ searchParams }: DifferentialsRouteProps) { const params = searchParams ? await searchParams : {}; - const query = firstSearchParam(params.query ?? params.q)?.trim(); + const query = (firstSearchParam(params.q) ?? firstSearchParam(params.query) ?? "").trim(); + const hasSubmittedSearch = firstSearchParam(params.run) === "1" && query.length > 0; - if (!query) { + if (!hasSubmittedSearch) { return ; } - return ; + return ; } diff --git a/src/app/favourites/layout.tsx b/src/app/favourites/layout.tsx index ab1d61dbda..abc0eb04d3 100644 --- a/src/app/favourites/layout.tsx +++ b/src/app/favourites/layout.tsx @@ -4,7 +4,7 @@ import { GlobalSearchShell } from "@/components/clinical-dashboard/global-search export default function FavouritesLayout({ children }: { children: ReactNode }) { return ( - + {children} ); diff --git a/src/app/favourites/page.tsx b/src/app/favourites/page.tsx index 38a1c38de8..6048c35e0b 100644 --- a/src/app/favourites/page.tsx +++ b/src/app/favourites/page.tsx @@ -14,5 +14,7 @@ export default async function FavouritesPage({ searchParams }: FavouritesPagePro const params = searchParams ? await searchParams : {}; const query = firstSearchParam(params.q)?.trim() ?? ""; - return ; + // No key={query} remount: query is a pure prop, and remounting on query + // change wiped the set/type/view/sort selections when clearing a search. + return ; } diff --git a/src/app/forms/page.tsx b/src/app/forms/page.tsx index 4a29418523..bd750838fd 100644 --- a/src/app/forms/page.tsx +++ b/src/app/forms/page.tsx @@ -1,5 +1,24 @@ import { FormsHomePage } from "@/components/forms/forms-home-page"; +import { FormsSearchResultsPage } from "@/components/forms/forms-search-results-page"; -export default function FormsPage() { - return ; +type FormsSearchParams = Promise<{ [key: string]: string | string[] | undefined }>; + +function readFirstSearchParam(value: string | string[] | undefined) { + return Array.isArray(value) ? value[0] : value; +} + +export default async function FormsPage({ searchParams }: { searchParams: FormsSearchParams }) { + const resolvedSearchParams = await searchParams; + const query = ( + readFirstSearchParam(resolvedSearchParams.q) ?? + readFirstSearchParam(resolvedSearchParams.query) ?? + "" + ).trim(); + const hasSubmittedSearch = readFirstSearchParam(resolvedSearchParams.run) === "1" && query.length > 0; + + if (!hasSubmittedSearch) { + return ; + } + + return ; } diff --git a/src/app/globals.css b/src/app/globals.css index 57cbbd8612..bd286e1899 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -1533,7 +1533,10 @@ summary::-webkit-details-marker { padding-bottom: max(0.45rem, var(--safe-area-bottom)); } - .answer-footer-search-dock[data-scroll-hidden="true"] { + /* Must beat the edge-to-edge dock rule above (transform: none) so scroll-hide + actually slides the bar off-screen once data-scroll-hidden is set. */ + .answer-footer-search-dock.document-mobile-search-edge.answer-footer-search-edge[data-scroll-hidden="true"], + .answer-footer-search-dock.dashboard-composer-edge.answer-footer-search-edge[data-scroll-hidden="true"] { transform: translateY(calc(100% + env(safe-area-inset-bottom))); pointer-events: none; } @@ -1852,14 +1855,29 @@ summary::-webkit-details-marker { box-shadow: 0 0 0 4px color-mix(in srgb, var(--focus) 25%, transparent) !important; } - /* Premium Hover Transitions for Source Capsules and Action row chips */ + /* Premium hover transitions for source capsules */ .source-capsule-hover { - transition: all 180ms cubic-bezier(0.34, 1.56, 0.64, 1) !important; + box-shadow: var(--glow-soft), var(--shadow-inset); + transition: + transform 180ms cubic-bezier(0.34, 1.56, 0.64, 1), + box-shadow 180ms cubic-bezier(0.22, 1, 0.36, 1), + border-color 150ms ease, + background-color 150ms ease !important; } .source-capsule-hover:hover { - transform: translateY(-1px) scale(1.015) !important; - box-shadow: 0 4px 12px color-mix(in srgb, var(--primary) 8%, transparent) !important; + transform: translateY(-1px) scale(1.01); + box-shadow: var(--shadow-tight), var(--shadow-inset); + } + + .source-capsule-hover[aria-expanded="true"] { + border-color: var(--clinical-accent); + box-shadow: var(--glow-soft), var(--shadow-inset); + } + + .source-capsule-hover[aria-expanded="true"]:hover { + transform: translateY(-1px) scale(1.01); + box-shadow: var(--shadow-tight), var(--shadow-inset); } .polished-scroll { @@ -1902,6 +1920,11 @@ summary::-webkit-details-marker { scroll-behavior: auto !important; transition-duration: 0.01ms !important; } + + .source-capsule-hover:hover, + .source-capsule-hover[aria-expanded="true"]:hover { + transform: none !important; + } } @media (forced-colors: active) { diff --git a/src/app/medications/layout.tsx b/src/app/medications/layout.tsx index 31812964b3..b891de2329 100644 --- a/src/app/medications/layout.tsx +++ b/src/app/medications/layout.tsx @@ -3,5 +3,9 @@ import type { ReactNode } from "react"; import { GlobalSearchShell } from "@/components/clinical-dashboard/global-search-shell"; export default function MedicationsLayout({ children }: { children: ReactNode }) { - return {children}; + return ( + + {children} + + ); } diff --git a/src/app/page.tsx b/src/app/page.tsx index adcf4a4668..c083f5ee9e 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,3 +1,5 @@ +import { redirect } from "next/navigation"; + import { HomePageClient } from "@/app/home-page-client"; import { isAppModeId, isAppModeVisible, type AppModeId } from "@/lib/app-modes"; @@ -20,5 +22,27 @@ export default async function Home({ searchParams }: HomeProps) { const initialSearchMode: AppModeId = isAppModeId(requestedMode) && isAppModeVisible(requestedMode) ? requestedMode : "answer"; + // /favourites is the canonical favourites surface; deep links via the + // dashboard mode param would otherwise open a divergent hub view. + if (initialSearchMode === "favourites") { + const favouriteParams = new URLSearchParams(); + const query = firstSearchParam(params.q)?.trim(); + if (query) favouriteParams.set("q", query); + if (firstSearchParam(params.focus) === "1") favouriteParams.set("focus", "1"); + if (firstSearchParam(params.run) === "1") favouriteParams.set("run", "1"); + const suffix = favouriteParams.toString(); + redirect(suffix ? `/favourites?${suffix}` : "/favourites"); + } + + if (initialSearchMode === "differentials") { + const differentialParams = new URLSearchParams(); + const query = firstSearchParam(params.q)?.trim(); + if (query) differentialParams.set("q", query); + if (firstSearchParam(params.focus) === "1") differentialParams.set("focus", "1"); + if (firstSearchParam(params.run) === "1") differentialParams.set("run", "1"); + const suffix = differentialParams.toString(); + redirect(suffix ? `/differentials?${suffix}` : "/differentials"); + } + return ; } diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 868b63df5e..81bad9e0fe 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -4,72 +4,43 @@ import { useRouter, useSearchParams } from "next/navigation"; import dynamic from "next/dynamic"; import { AlertCircle, - Bell, BookOpen, ChevronDown, - ChevronRight, - CircleUserRound, Clock3, ExternalLink, FileImage, FileText, FolderOpen, - Globe2, - HelpCircle, Heart, - Keyboard, ListChecks, Loader2, - LogOut, - Mail, - LockKeyhole, - Palette, - PanelTop, Quote, RefreshCw, Search, - Settings as SettingsIcon, ShieldAlert, - ShieldCheck, - SlidersHorizontal, - Sparkles, - Stethoscope, UploadCloud, - UserRound, WifiOff, Wrench, X, } from "lucide-react"; -import { type CSSProperties, type FormEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { type CSSProperties, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { type DocumentDeleteResult } from "@/components/DocumentManagementActions"; -import { useDismissableLayer } from "@/components/use-dismissable-layer"; import { extractSafetyFindings } from "@/lib/clinical-safety"; import { readLocalProjectIdentity, unsafeLocalProjectMessage } from "@/lib/local-project-identity"; import { isDeployedClinicalKb } from "@/lib/deployed-app"; import { isLocalNoAuthMode, publicUploadsEnabled } from "@/lib/env"; -import { - appBackdrop, - answerSurface, - cn, - fieldControlWithIcon, - fieldIcon, - floatingControl, - primaryControl, - textMuted, - toneSuccess, - toneWarning, -} from "@/components/ui-primitives"; +import { appBackdrop, answerSurface, cn, textMuted, toneSuccess, toneWarning } from "@/components/ui-primitives"; import { useAuthSession } from "@/lib/supabase/client"; -import { Sheet } from "@/components/ui/sheet"; import { AccountSetupDialog } from "@/components/clinical-dashboard/account-setup-dialog"; import { StagedAnswerResultSurface } from "@/components/clinical-dashboard/answer-result-surface"; import { CrossModeLinksSection } from "@/components/clinical-dashboard/cross-mode-links"; import { RelatedDocumentsPanel } from "@/components/clinical-dashboard/document-results"; import { AuthPanel } from "@/components/clinical-dashboard/auth-panel"; +import { buildMobileSectionFabState, MobileSectionFab, ToolsHub } from "@/components/clinical-dashboard/dashboard-nav"; +import { SettingsDialog } from "@/components/clinical-dashboard/settings-dialog"; import { useSidebarCollapsed } from "@/components/clinical-dashboard/use-sidebar-collapsed"; import { useTheme } from "@/components/clinical-dashboard/use-theme"; import { - type SidebarIdentity, deriveSidebarIdentity, ClinicalDesktopSidebar, ClinicalMobileSidebar, @@ -516,923 +487,6 @@ type LibraryHealthTarget = "documents" | "setup" | "indexing" | "failures"; type IndexingMonitorFilter = "all" | "active" | "failed"; type UploadIndexingTab = "setup" | "upload" | "jobs" | "quality"; -export function SettingsDialog({ - open, - onClose, - identity, - theme, - onToggleTheme, - onSignOut, - onOpenGuide, -}: { - open: boolean; - onClose: () => void; - identity: SidebarIdentity; - theme: "light" | "dark"; - onToggleTheme: () => void; - onSignOut: () => void; - onOpenGuide: () => void; -}) { - const closeButtonRef = useRef(null); - const settingsEmailInputRef = useRef(null); - const currentThemeLabel = theme === "dark" ? "Dark" : "Light"; - const auth = useAuthSession(); - const [settingsEmail, setSettingsEmail] = useState(""); - const [emailEntryOpen, setEmailEntryOpen] = useState(false); - const [settingsEmailAttempted, setSettingsEmailAttempted] = useState(false); - const [accountNotice, setAccountNotice] = useState(null); - const settingsAuthBusy = auth.status === "loading"; - const signedOutAccount = !identity.signedIn; - - async function submitSettingsEmail(event: FormEvent) { - event.preventDefault(); - if (!settingsEmail.trim()) return; - setAccountNotice(null); - setSettingsEmailAttempted(true); - await auth.signInWithEmail(settingsEmail.trim()); - } - - function openSettingsEmailEntry() { - setEmailEntryOpen(true); - setAccountNotice(null); - } - - function chooseSettingsProvider(provider: string) { - setAccountNotice(`${provider} sign-in is a placeholder for now. Continue with email to use this workspace.`); - } - - useEffect(() => { - if (!emailEntryOpen) return; - const focusFrame = window.requestAnimationFrame(() => { - settingsEmailInputRef.current?.focus({ preventScroll: true }); - }); - return () => window.cancelAnimationFrame(focusFrame); - }, [emailEntryOpen]); - - const settingSections = [ - { - title: "Account", - rows: [ - { icon: UserRound, label: "Profile", value: identity.displayName }, - { icon: Stethoscope, label: "Clinical role", value: "Consultant psychiatrist" }, - ], - }, - { - title: "Clinical defaults", - rows: [ - { icon: Globe2, label: "Jurisdiction", value: "Western Australia", active: true }, - { icon: CircleUserRound, label: "Default population", value: "Adults" }, - { icon: SlidersHorizontal, label: "Answer style", value: "Conservative" }, - ], - }, - { - title: "App preferences", - rows: [ - { - icon: Palette, - label: "Appearance", - value: currentThemeLabel, - onClick: onToggleTheme, - actionLabel: `Switch to ${theme === "dark" ? "light" : "dark"} mode`, - }, - { icon: SettingsIcon, label: "Interface density", value: "Comfortable" }, - ], - }, - ]; - const navItems = [ - { icon: SettingsIcon, label: "General" }, - { icon: Stethoscope, label: "Clinical defaults" }, - { icon: Sparkles, label: "Personalisation" }, - { icon: Bell, label: "Notifications" }, - { icon: LockKeyhole, label: "Security" }, - { icon: CircleUserRound, label: "Account", active: true }, - { icon: Keyboard, label: "Keyboard" }, - { - icon: HelpCircle, - label: "Help & About", - onClick: () => { - onClose(); - onOpenGuide(); - }, - }, - ]; - - const closeButton = ( - - ); - - return ( - -
- {closeButton} - - -
-
-
-

- Account & app -

-
- - Clinician account - -
- -
-

- Clinical Guide account -

-
- - {signedOutAccount ? : identity.initials} - {identity.signedIn ? ( - - ) : null} - -
-

- {identity.displayName} -

-

- {signedOutAccount ? "Sign in or create an account" : "Consultant psychiatrist, Western Australia"} -

-
- {signedOutAccount ? ( -
- - -
- ) : ( -
- - -
- )} -
- - {signedOutAccount ? ( -
-
- - -
- - {emailEntryOpen ? ( -
- - -
- ) : null} - -
- - or continue with - -
- -
- chooseSettingsProvider("Apple")} /> - chooseSettingsProvider("Google")} /> - chooseSettingsProvider("Microsoft")} /> - -
- -

- - Accounts save preferences and search history. Do not enter PHI. -

- - {(accountNotice || !auth.isConfigured || (settingsEmailAttempted && auth.error)) && ( -

- {accountNotice ?? - (settingsEmailAttempted ? auth.error : null) ?? - "Supabase browser authentication is not configured for account sign-in."} -

- )} -
- ) : ( - - )} -
- -
- - - -
- -
-
- {settingSections.map((section) => ( -
-

- {section.title} -

-
- {section.rows.map((row) => ( - - ))} - {section.title === "Account" && identity.signedIn ? ( - { - onSignOut(); - onClose(); - }} - actionLabel="Sign out" - /> - ) : null} -
-
- ))} -
- { - onClose(); - onOpenGuide(); - }} - /> -
-
-
- - ); -} - -function SettingsChip({ label }: { label: string }) { - return ( - - {label} - - ); -} - -function SettingsProviderRow({ - provider, - onClick, -}: { - provider: "Apple" | "Google" | "Microsoft" | "email"; - onClick: () => void; -}) { - const label = provider === "email" ? "Use email instead" : provider; - - return ( - - ); -} - -function SettingsProviderMark({ provider }: { provider: "Apple" | "Google" | "Microsoft" }) { - if (provider === "Microsoft") { - return ( -