From eebe938e55bbb76543228e175a0e9f0d81767733 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 28 Jul 2026 07:11:03 +0800 Subject: [PATCH 1/4] feat(rag): harden retrieval and answer reliability --- .../rag-reliability-evidence-2026-07-27.md | 70 + ...-top-local-review-manifest-2026-07-26.json | 205 ++ docs/outstanding-issues.md | 140 +- docs/rag-behaviour/refuted-approaches.md | 50 +- docs/rag-hybrid-findings-and-todo.md | 81 +- ...-governance-refresh-worklist-2026-07-22.md | 49 +- scripts/audit-source-governance.ts | 35 +- scripts/build-clinical-review-queue.ts | 19 +- scripts/eval-answer-quality.ts | 258 ++- scripts/eval-quality.ts | 97 +- scripts/eval-retrieval.ts | 67 +- scripts/eval-utils.ts | 68 + scripts/profile-retrieval-rpcs.ts | 426 +++- src/app/api/documents/[id]/reviews/route.ts | 145 +- src/app/api/documents/bulk/route.ts | 245 ++- src/lib/answer-ranking.ts | 2 +- src/lib/answer-verification.ts | 1000 ++++++++- src/lib/clinical-review-queue.ts | 193 +- src/lib/clinical-search.ts | 204 +- src/lib/clinical-value-binding.ts | 20 + src/lib/eval-document-matching.ts | 54 +- src/lib/medication-entities.ts | 525 +++++ src/lib/rag/rag-answer-text.ts | 2 +- src/lib/rag/rag-cache.ts | 2 +- src/lib/rag/rag-candidate-sources.ts | 19 +- src/lib/rag/rag-claim-support.ts | 722 ++++++- src/lib/rag/rag-comparison.ts | 52 + src/lib/rag/rag-eval-cases.ts | 51 +- src/lib/rag/rag-extractive-answer.ts | 1908 ++++++++++++++++- src/lib/rag/rag-extractive-artifacts.ts | 26 + src/lib/rag/rag-extractive-first.ts | 297 ++- src/lib/rag/rag-quote-verification.ts | 45 +- src/lib/rag/rag-routing.ts | 439 +++- src/lib/rag/rag-source-segmentation.ts | 140 ++ src/lib/rag/rag.ts | 298 ++- src/lib/retrieval-selection.ts | 23 +- src/lib/smart-rag-api.ts | 4 + src/lib/source-review.ts | 215 ++ src/lib/supabase/database.types.ts | 20 + supabase/drift-manifest.json | 32 +- ...000_bmj_third_party_source_attestation.sql | 256 +++ supabase/schema.sql | 151 +- tests/answer-ranking.test.ts | 9 + tests/answer-responsiveness-gate.test.ts | 8 + tests/answer-verification.test.ts | 243 ++- tests/clinical-review-queue.test.ts | 115 +- tests/clinical-search.test.ts | 129 ++ tests/document-mutation-routes.test.ts | 71 +- tests/eval-answer-quality.test.ts | 224 +- tests/eval-document-matching.test.ts | 82 +- tests/eval-quality.test.ts | 110 + tests/eval-retrieval.test.ts | 14 + tests/eval-utils.test.ts | 147 +- tests/extractive-answer-formatting.test.ts | 1885 ++++++++++++++++ tests/medication-entities.test.ts | 107 + tests/profile-retrieval-rpcs.test.ts | 371 ++++ tests/rag-answer-fallback.test.ts | 1619 +++++++++++++- tests/rag-answer-text.test.ts | 4 + tests/rag-claim-support.test.ts | 813 ++++++- tests/rag-document-lookup-chunk-query.test.ts | 187 ++ ...-document-lookup-escalation-rescue.test.ts | 9 +- tests/rag-eval-cases.test.ts | 56 + tests/rag-extractive-first.test.ts | 404 ++++ ...rag-extractive-procedural-artifact.test.ts | 357 +++ tests/rag-fast-path-ordering.test.ts | 117 + tests/rag-offline-answer.test.ts | 6 +- tests/rag-routing.test.ts | 741 ++++++- tests/rag-trust.test.ts | 557 ++++- tests/retrieval-selection.test.ts | 207 ++ tests/smart-rag-api.test.ts | 27 + tests/source-authority-tooling.test.ts | 2 + tests/source-review-policy.test.ts | 267 +++ tests/source-review-route.test.ts | 263 ++- tests/source-review-schema.test.ts | 129 ++ vitest.config.mts | 29 + 75 files changed, 17197 insertions(+), 767 deletions(-) create mode 100644 docs/evidence/rag-reliability-evidence-2026-07-27.md create mode 100644 docs/evidence/rag-top-local-review-manifest-2026-07-26.json create mode 100644 src/lib/clinical-value-binding.ts create mode 100644 src/lib/medication-entities.ts create mode 100644 src/lib/rag/rag-extractive-artifacts.ts create mode 100644 src/lib/rag/rag-source-segmentation.ts create mode 100644 supabase/migrations/20260727010000_bmj_third_party_source_attestation.sql create mode 100644 tests/medication-entities.test.ts create mode 100644 tests/profile-retrieval-rpcs.test.ts create mode 100644 tests/rag-document-lookup-chunk-query.test.ts create mode 100644 tests/rag-extractive-procedural-artifact.test.ts create mode 100644 tests/source-review-policy.test.ts create mode 100644 tests/source-review-schema.test.ts diff --git a/docs/evidence/rag-reliability-evidence-2026-07-27.md b/docs/evidence/rag-reliability-evidence-2026-07-27.md new file mode 100644 index 0000000000..b3348a86dc --- /dev/null +++ b/docs/evidence/rag-reliability-evidence-2026-07-27.md @@ -0,0 +1,70 @@ +# RAG reliability evidence — 2026-07-27 + +This record captures the final local/live evidence for the 2026-07-27 RAG reliability work. Raw eval +artifacts remain under the gitignored `output/` directory; this file records their paths and the metrics +needed to audit the backlog dispositions. + +## Protected retrieval canary + +- Artifact: `output/rag-retrieval-post-final.json` +- Cases: 36 +- Document recall@5: 1.0 +- Content recall@5: 1.0 +- Hit@K: 1.0 +- MRR@10: 0.8921 (unchanged) +- Content MRR@10: 0.9406 (baseline 0.9389) +- nDCG@10: 0.9308 (baseline 0.9276) +- Irrelevant-at-10 rate: 0.0917 (unchanged) +- Required-signal coverage: 1.0 +- Failed cases: 0 +- Per-case document/content reciprocal-rank regressions: 0 + +The only protected retrieval change selects document-lookup chunks using the already-ranked expanded +clinical query rather than reverting to the raw user wording. This preserves FBC/WBC/ANC and +withhold/stop aliases between document selection and chunk selection. Ranking scores, comparator order, +clamps and semantic reranking were not changed. + +## Final 44-case answer gate + +- Artifact: `output/rag-quality-final9/retrieval-quality-2026-07-27T21-19-19-573Z.json` +- Diagnostic dump: `output/rag-quality-final9-cases.json` +- Cases: 44 +- Supported: 30/30 substantive and grounded +- Unsupported: 14/14 correct +- Citation failure rate: 0 +- Numeric-grounding failure rate: 0 +- Source-backed review fallbacks: 0 +- Route-ceiling failures: 0 +- Comparison extractive fallbacks: 2, both passed with distinct attributed sources +- p95 latency: 7,494 ms +- Provider use: 1 model case, 2 request IDs +- Cost: unavailable because pricing-rate telemetry was not configured; no exact dollar value is claimed +- Blocking threshold failures: none + +The immediately preceding `final8` run had the same perfect content metrics but one transient hosted +retrieval tail: active-community ED took 15,760 ms against the 12,000 ms extractive ceiling. An isolated +repeat passed at 8,100 ms and the final 44-case run passed at 9,610 ms. No latency ceiling was weakened. + +## Targeted live confirmations + +- Community home visits: two same-document AKG citations, substantive extractive answer, zero provider + calls, 4,812 ms. +- Best Practice Prescription: two same-document AKG citations, substantive extractive answer, zero + provider calls, 1,560 ms. +- Clozapine typo threshold: one complete NMHS red-range row, substantive extractive answer, zero + provider calls, 3,160 ms. +- Clozapine ANC/FBC threshold: one complete NMHS red-range row, expected-file citation, zero provider + calls, 4,375 ms. +- Discharge summary: source-backed extractive recovery, zero review fallback, 8,831 ms on the final + targeted probe. + +## Deliberate boundaries + +- The prepared BMJ attestation migration was not applied to hosted Supabase. Qualified human review, + deliberate hosted apply/attestation and warning-rate remeasurement remain `#022`. Until that apply, + existing owner-scoped reviews keep using the legacy RPC and v2-only public/attestation requests fail + explicitly with `503 source_review_v2_unavailable`. +- Firefox/WebKit scheduled evidence and human disposition of irrelevant-at-10 labels remain `#023`. +- ADHD corpus/table accessibility and metabolic schedule evidence remain the two open parts of `#018`. +- No commit, push, deployment, ranking-score change, grounding relaxation, citation-gate relaxation or + production-data write was performed as part of this evidence run. diff --git a/docs/evidence/rag-top-local-review-manifest-2026-07-26.json b/docs/evidence/rag-top-local-review-manifest-2026-07-26.json new file mode 100644 index 0000000000..22340f6cc6 --- /dev/null +++ b/docs/evidence/rag-top-local-review-manifest-2026-07-26.json @@ -0,0 +1,205 @@ +{ + "version": "top-local-review-evidence-v1", + "source_artifact": "github-actions/run-30216191889/eval-canary-output/golden-retrieval.json", + "source_generated_at": null, + "review_status": "pending_qualified_human_review", + "attestation_applied": false, + "entry_count": 10, + "entries": [ + { + "key": "file:clozapine management by gp (nmhs).pdf", + "document_id": null, + "title": "Clozapine Management By GP(NMHS)", + "file_name": "Clozapine Management by GP (NMHS).pdf", + "publisher_code": "NMHS", + "jurisdiction": "Australia/WA", + "top_result_slots": 11, + "best_rank": 2, + "case_ids": [ + "clozapine-anc-threshold", + "clozapine-cbc-abbreviation-threshold", + "monitoring-threshold-from-chart", + "show-source-table-image" + ], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:alcohol and other drugs - addiction, toxicity and withdrawal (fsh).pdf", + "document_id": null, + "title": "Alcohol And Other Drugs - Addiction, Toxicity And Withdrawal(FSH)", + "file_name": "Alcohol and Other Drugs - Addiction, Toxicity and Withdrawal (FSH).pdf", + "publisher_code": "FSH", + "jurisdiction": "Australia/WA", + "top_result_slots": 6, + "best_rank": 1, + "case_ids": ["alcohol-ciwa-scoring", "alcohol-ciwa-threshold", "alcohol-withdrawal-management"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:mhatt assessment and treatment process (akg).pdf", + "document_id": null, + "title": "MHATT Assessment And Treatment Process(AKG)", + "file_name": "MHATT Assessment and Treatment Process (AKG).pdf", + "publisher_code": "AKG", + "jurisdiction": "Australia/WA", + "top_result_slots": 4, + "best_rank": 4, + "case_ids": ["schizophrenia-overview"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:clozapine coordinator and clozapine clinic (nmhs).pdf", + "document_id": null, + "title": "Clozapine Coordinator And Clozapine Clinic(NMHS)", + "file_name": "Clozapine Coordinator and Clozapine Clinic (NMHS).pdf", + "publisher_code": "NMHS", + "jurisdiction": "Australia/WA", + "top_result_slots": 3, + "best_rank": 1, + "case_ids": ["clozapine-anc-threshold", "clozapine-cbc-abbreviation-threshold", "show-source-table-image"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:women's and perinatal mental health referral and management guideline (kemh).pdf", + "document_id": null, + "title": "Women S And Perinatal Mental Health Referral And Management Guideline(KEMH)", + "file_name": "Women's and Perinatal Mental Health Referral and Management Guideline (KEMH).pdf", + "publisher_code": "KEMH", + "jurisdiction": "Australia/WA", + "top_result_slots": 3, + "best_rank": 7, + "case_ids": ["bipolar-management-summary", "bipolar-vs-schizoaffective", "vector-bipolar"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:mother baby unit admissions (fsh).pdf", + "document_id": null, + "title": "Mother Baby Unit Admissions(FSH)", + "file_name": "Mother Baby Unit Admissions (FSH).pdf", + "publisher_code": "FSH", + "jurisdiction": "Australia/WA", + "top_result_slots": 2, + "best_rank": 6, + "case_ids": ["bipolar-vs-schizoaffective"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:pressure injuries (akg).pdf", + "document_id": null, + "title": "Pressure Injuries(AKG)", + "file_name": "Pressure Injuries (AKG).pdf", + "publisher_code": "AKG", + "jurisdiction": "Australia/WA", + "top_result_slots": 2, + "best_rank": 6, + "case_ids": ["flowchart-next-step"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:referral to the mother baby unit mr202.09 (kemh).pdf", + "document_id": null, + "title": "Referral To The Mother Baby Unit MR202.09(KEMH)", + "file_name": "Referral to the Mother Baby Unit MR202.09 (KEMH).pdf", + "publisher_code": "KEMH", + "jurisdiction": "Australia/WA", + "top_result_slots": 2, + "best_rank": 8, + "case_ids": ["vector-postnatal"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["unknown"], + "clinical_validation_statuses": ["unverified"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:hampton house referral (fsh).pdf", + "document_id": null, + "title": "Hampton House Referral(FSH)", + "file_name": "Hampton House Referral (FSH).pdf", + "publisher_code": "FSH", + "jurisdiction": "Australia/WA", + "top_result_slots": 2, + "best_rank": 9, + "case_ids": ["bipolar-vs-schizoaffective"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + }, + { + "key": "file:arousal and agitation drug management (camhs).pdf", + "document_id": null, + "title": "Arousal And Agitation Drug Management(CAMHS)", + "file_name": "Arousal and Agitation Drug Management (CAMHS).pdf", + "publisher_code": "CAMHS", + "jurisdiction": "Australia/WA", + "top_result_slots": 1, + "best_rank": 2, + "case_ids": ["medication-chart-dose-route"], + "review_class": "local_wa", + "observed": { + "document_statuses": ["review_due"], + "clinical_validation_statuses": ["locally_reviewed"], + "extraction_qualities": ["good"] + }, + "review_status": "pending_qualified_human_review", + "attestation_applied": false + } + ] +} diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index 8c9d955b8d..e3c4b9eb5a 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -48,79 +48,74 @@ removed after current-main verification; it is not missing recommended work. database/RAG/clinical/privacy expertise; Operator = named provider/product/legal authority. - **Estimate:** focused active time, excluding approval, hosted runtime, soak, and review waits. -| Order | ID(s) | Acuity | Capability | When | Estimate | Outcome, gate, verification, and stopping condition | -| ----: | ---------------------- | -------- | ------------------------------------------- | ---------------------------------------------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | `#059` | A1 | Operator security + independent reviewer | Immediate approved security window | 1–3 hours plus verification | Verify every reported exposed credential (GitHub, OpenAI, Supabase service role/database, E2E) is retired; rotate anything still valid and update only intended secret stores. Never record values; stop before provider action without approval. | -| 2 | `#053` | A1 | Operator — legal/privacy | Start now; finish before real patient use/privacy-approved release | 4–8 hours internal; 1–6 weeks elapsed | Execute DPAs; decide ZDR/residency; obtain cache behavior in writing; review subprocessors; obtain APP 8 and APP 5/1 counsel sign-off. Do not change public copy before approval. | -| 3 | `#069` | A3 | Specialist — retrieval latency | After hosted apply of PR #1133 migrations; approval-gated live profile | 30–60 min | Operator applies `20260724120000`/`20260724130000`/`20260724130100`, then re-profiles `match_document_table_facts_text` (~70ms-class plans). Stop without mutating ranking or unpaid evals. Cloud agent blocked: no DB URL / MCP auth; live profile hit Unregistered API key. | -| 4 | `#019` | A2 | Specialist — RAG answer pipeline | Local reproducer now; behavior change after `#023` disposition | 0.5–1 day reproducer | Reproduce admission-source loss in the fallback layer using PR #1096’s source shape. Any behavior change needs protected review and an approved baseline/post canary; stop if independently non-reproducible. | -| 6 | `#022` | A2 | Operator — clinical governance + Specialist | Decision-ready | 1–2 hours policy; 0.5–1 day first ten | Decide BMJ attestation policy and review the ten highest-impact local documents. Record reviewer/evidence/time; stop after ten and remeasure warning debt. | -| 7 | `#023` | A2 | Specialist — RAG/browser diagnostics | After hosted dependency audit is green | 1–2 hours | Capture the skipped Firefox/WebKit scheduled datapoint and disposition the human irrelevant-at-10 labels. Retrieval and answer artifacts are already compared under resolved #051; do not spend on another RAG run. | -| 8 | `#018` | A2 | Specialist — clinical RAG/retrieval | After `#023`, one mechanism at a time | 1–2 days diagnosis | Give lithium, ADHD, and metabolic residuals separate current-main reproducers and candidates. Behavior canaries require approval; stop any item without a deterministic reproducer or on regression. | -| 9 | `#029` | A2 | Specialist — answer quality/clinical safety | After `#023` and `#018` | 0.5–1 day inventory; 1–3 days per fix | Re-enumerate current fallback stubs and fix one causal cluster at a time without weakening grounding/citation gates. Stop if a change merely makes the metric easier to pass. | -| 10 | `#001` | A2 | Specialist — retrieval/ranking | After `#023` and rollout approval | 0.5–1 day plus canary | Keep semantic reranking off unless an approved ambiguity comparison preserves 36/36, recall 1.0, zero per-case regressions, and shows measured gain; otherwise record keep-off and stop. | -| 11 | `#025` | A2 | Operator — Railway/GitHub/chat/Supabase | Next approved observability window | 1–3 hours/channel | Choose owned deployment, CI, ingestion, and SLO alerts; mock first, then one approved controlled provider event/channel. The merged Supabase trigger remains inert until its verified inputs are configured. Stop without an accountable responder. | -| 12 | `#055` | A2 | Specialist release owner + Operator | Before next full-confidence release/handoff | 2–4 hours plus runtime | On one exact SHA, run local/provider gates, Firefox/WebKit, required hosted CI, and close actionable GitHub threads. Stop at first failure and rerun only the repaired smallest gate. | -| 13 | `#056` | A2 | Operator — Supabase/Railway + Specialist | Next approved staging schema window | 2–4 hours | Reconcile the existing healthy, empty staging tier's 23-migration history gap using the exact repository migration chain, then re-run indexing, health, identity and data-boundary proof. Never recreate it or copy production clinical documents. | -| 14 | `#057` | A2 | High — release/SRE + Operator | After `#056` | 2–4 hours plus soak | Run documented staging soak and rollback against an exact candidate. Retain latency/error/rollback evidence; stop on unsafe data, identity mismatch, or unowned rollback. | -| 16 | `#011` | A3 | Operator — Supabase capacity | Immediately before first compute scale-up | 30–60 min plus observation | Switch Auth to percentage allocation, record before/after, and run approved advisor/health checks. Stop if no scale-up is planned. | -| 17 | `#017` | A3 | High — performance/browser | Before `#012`/`#013`/`#016`; approved live-site window | 1–2 hours | Capture reproducible mobile/desktop Lighthouse/Web-Vitals evidence and decide whether payload work is justified. Stop if metrics are acceptable or evidence is too noisy. | -| 18 | `#033` | A3 | Specialist — prompt/source governance | After `#022` and `#023` | 1–2 days plus approved eval | Design unknown-vs-adverse metadata wording and prompt tests. Require no supported-grounding drop and zero citation failures; stop on broad over-caveating or degradation. | -| 19 | `#037` | A3 | Operator — clinical/product + Standard | Next trust-policy review | 30–60 min; up to 0.5 day | Decide whether routine claims cap at medium trust. Record policy; if accepted, change only the flag/expectations and run focused tests. | -| 20 | `#012`, `#013`, `#016` | A3 | High — bundling/runtime performance | After `#017` or equivalent evidence | 0.5–2 days/route | Optimize only a production route with measured payload/render/motion harm. Require material gain plus focused, `verify:cheap`, and browser evidence; stop on small gain. | -| 21 | `#035` | A3 | Specialist — evidence rules | After a demonstrated missed conflict | 0.5–1 day design; code separate | Define a clinically reviewed conflict class with positive and negative fixtures. Stop if no bounded class can be shown; behavior change requires protected review. | -| 22 | `#027` | Optional | Operator — SRE/provider | When an owned external alert path is wanted | 1–2 hours | Decide vendor/cost/privacy/owner; if accepted, prove one non-PHI outage and recovery alert. Stop when no responder owns it. | -| 23 | `#028` | Optional | Specialist privacy/observability + Operator | After privacy/ownership/cost approval | 1–3 days | Define vendor/region/retention/redaction/sampling/source-map envelope before SDK work. Prove no clinical text, identifiers, or secrets leave; stop if unacceptable. | -| 24 | `#038` | Optional | High — product/design architecture | When a new comparison surface is approved | 0.5–1 day | Define a shared interaction contract without flattening mode-specific content. Stop when no concrete new surface exists. | -| 25 | `#040` | Optional | High — visual QA/accessibility | When baseline owner/update workflow exists | 1–2 days | Establish a small stable desktop/mobile/accessibility baseline set. Do not make it blocking if flake or maintenance cost outweighs detection value. | -| 26 | `#039` | Optional | High — frontend architecture | During a concrete catalogue-toolbar project | 0.5–1 day inventory; 1–3 days code | Converge only repeated toolbar behavior without flattening search semantics. Stop when there is no bounded implementation target. | -| 27 | `#065` | A2 | High — document-viewer UI | Only when the user explicitly resumes the paused task | 0.5–1.5 days | Finish the compact source-text accordion, citation/search auto-open, print restoration, and 320/390/1280 px coverage. Keep the preserved branch untouched until explicit resume; no provider calls. | -| 28 | `#079` | Optional | High — repository hygiene | In explicitly scheduled batches | 30–60 minutes per batch | Disposition at most ten retained worktrees per pass using owner, PR, review-ledger, ancestry, and patch evidence. Preserve every dirty, active, secret-bearing, post-freeze, or ambiguous worktree and stop rather than broad-cleaning. | - - +| Order | ID(s) | Acuity | Capability | When | Estimate | Outcome, gate, verification, and stopping condition | +| ----: | ---------------------- | -------- | ------------------------------------------- | ------------------------------------------------------------------ | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 | `#059` | A1 | Operator security + independent reviewer | Immediate approved security window | 1–3 hours plus verification | Verify every reported exposed credential (GitHub, OpenAI, Supabase service role/database, E2E) is retired; rotate anything still valid and update only intended secret stores. Never record values; stop before provider action without approval. | +| 2 | `#053` | A1 | Operator — legal/privacy | Start now; finish before real patient use/privacy-approved release | 4–8 hours internal; 1–6 weeks elapsed | Execute DPAs; decide ZDR/residency; obtain cache behavior in writing; review subprocessors; obtain APP 8 and APP 5/1 counsel sign-off. Do not change public copy before approval. | +| 6 | `#022` | A2 | Operator — clinical governance + Specialist | Policy implemented locally; hosted apply and human review pending | 1–2 hours apply; 0.5–1 day first ten | The auditable BMJ `third_party_reference_attested` policy, migration and top-ten evidence manifest are prepared without changing `clinical_validation_status=unverified`. A qualified operator must review evidence, apply the migration deliberately, attest eligible records, review the ten visible local documents, then remeasure warnings. | +| 7 | `#023` | A2 | Specialist — RAG/browser diagnostics | After hosted dependency audit is green | 1–2 hours | Capture the skipped Firefox/WebKit scheduled datapoint and disposition the human irrelevant-at-10 labels. Retrieval and answer artifacts are already compared under resolved #051; do not spend on another RAG run. | +| 8 | `#018` | A2 | Specialist — clinical RAG/retrieval | Lithium closed; ADHD/metabolic evidence debt remains | Corpus/operator follow-up | Lithium's bounded subject/row-aware fix passed its targeted answer plus the full 36-case retrieval and 44-case answer canaries. ADHD's expected CAMHS document remains absent and the surfaced chart has no accessible table; metabolic schedule evidence remains unavailable and its standalone classifier candidate was reverted. | +| 10 | `#001` | A2 | Specialist — retrieval/ranking | After rollout approval | 0.5–1 day plus canary | Keep semantic reranking off unless an approved ambiguity comparison preserves 36/36, recall 1.0, zero per-case regressions, and shows measured gain; otherwise record keep-off and stop. | +| 11 | `#025` | A2 | Operator — Railway/GitHub/chat/Supabase | Next approved observability window | 1–3 hours/channel | Choose owned deployment, CI, ingestion, and SLO alerts; mock first, then one approved controlled provider event/channel. The merged Supabase trigger remains inert until its verified inputs are configured. Stop without an accountable responder. | +| 12 | `#055` | A2 | Specialist release owner + Operator | Before next full-confidence release/handoff | 2–4 hours plus runtime | On one exact SHA, run local/provider gates, Firefox/WebKit, required hosted CI, and close actionable GitHub threads. Stop at first failure and rerun only the repaired smallest gate. | +| 13 | `#056` | A2 | Operator — Supabase/Railway + Specialist | Next approved staging schema window | 2–4 hours | Reconcile the existing healthy, empty staging tier's 23-migration history gap using the exact repository migration chain, then re-run indexing, health, identity and data-boundary proof. Never recreate it or copy production clinical documents. | +| 14 | `#057` | A2 | High — release/SRE + Operator | After `#056` | 2–4 hours plus soak | Run documented staging soak and rollback against an exact candidate. Retain latency/error/rollback evidence; stop on unsafe data, identity mismatch, or unowned rollback. | +| 16 | `#011` | A3 | Operator — Supabase capacity | Immediately before first compute scale-up | 30–60 min plus observation | Switch Auth to percentage allocation, record before/after, and run approved advisor/health checks. Stop if no scale-up is planned. | +| 17 | `#017` | A3 | High — performance/browser | Before `#012`/`#013`/`#016`; approved live-site window | 1–2 hours | Capture reproducible mobile/desktop Lighthouse/Web-Vitals evidence and decide whether payload work is justified. Stop if metrics are acceptable or evidence is too noisy. | +| 18 | `#033` | A3 | Specialist — prompt/source governance | After `#022` and explicit evaluation approval | 1–2 days plus approved eval | Design unknown-vs-adverse metadata wording and prompt tests. Require no supported-grounding drop and zero citation failures; stop on broad over-caveating or degradation. | +| 19 | `#037` | A3 | Operator — clinical/product + Standard | Next trust-policy review | 30–60 min; up to 0.5 day | Decide whether routine claims cap at medium trust. Record policy; if accepted, change only the flag/expectations and run focused tests. | +| 20 | `#012`, `#013`, `#016` | A3 | High — bundling/runtime performance | After `#017` or equivalent evidence | 0.5–2 days/route | Optimize only a production route with measured payload/render/motion harm. Require material gain plus focused, `verify:cheap`, and browser evidence; stop on small gain. | +| 21 | `#035` | A3 | Specialist — evidence rules | After a demonstrated missed conflict | 0.5–1 day design; code separate | Define a clinically reviewed conflict class with positive and negative fixtures. Stop if no bounded class can be shown; behavior change requires protected review. | +| 22 | `#027` | Optional | Operator — SRE/provider | When an owned external alert path is wanted | 1–2 hours | Decide vendor/cost/privacy/owner; if accepted, prove one non-PHI outage and recovery alert. Stop when no responder owns it. | +| 23 | `#028` | Optional | Specialist privacy/observability + Operator | After privacy/ownership/cost approval | 1–3 days | Define vendor/region/retention/redaction/sampling/source-map envelope before SDK work. Prove no clinical text, identifiers, or secrets leave; stop if unacceptable. | +| 24 | `#038` | Optional | High — product/design architecture | When a new comparison surface is approved | 0.5–1 day | Define a shared interaction contract without flattening mode-specific content. Stop when no concrete new surface exists. | +| 25 | `#040` | Optional | High — visual QA/accessibility | When baseline owner/update workflow exists | 1–2 days | Establish a small stable desktop/mobile/accessibility baseline set. Do not make it blocking if flake or maintenance cost outweighs detection value. | +| 26 | `#039` | Optional | High — frontend architecture | During a concrete catalogue-toolbar project | 0.5–1 day inventory; 1–3 days code | Converge only repeated toolbar behavior without flattening search semantics. Stop when there is no bounded implementation target. | +| 27 | `#065` | A2 | High — document-viewer UI | Only when the user explicitly resumes the paused task | 0.5–1.5 days | Finish the compact source-text accordion, citation/search auto-open, print restoration, and 320/390/1280 px coverage. Keep the preserved branch untouched until explicit resume; no provider calls. | +| 28 | `#079` | Optional | High — repository hygiene | In explicitly scheduled batches | 30–60 minutes per batch | Disposition at most ten retained worktrees per pass using owner, PR, review-ledger, ancestry, and patch evidence. Preserve every dirty, active, secret-bearing, post-freeze, or ambiguous worktree and stop rather than broad-cleaning. | + + ## Open items > **Merged-main canary update (2026-07-23, run `30018289898`):** the new structured report correctly recorded evaluated tree `c24f2e8f2d30d0c59fc1eba025d3dcd63478137e`, run/attempt identity and `cross-region-runner` latency context. Golden retrieval remained 36/36 with document/content recall 1.0 and no failed cases. The 44-case answer gate had grounded-supported and unsupported-correct rates of 1.0, but failed because `neuroleptic-side-effect-escalation` again returned one citation where two are required (citation-failure rate 0.0227). `admission-discharge-comparison` again omitted the specific AKG admission document after `comparison_source_extractive_fallback`; `admission-discharge-coverage-paraphrase` was advisory-only at 24,870 ms. Answer cost was reported as `$0.234736`. Do not retry immediately: retain this as the first structured datapoint, compare it with the scheduled 2026-07-26 report, and keep retrieval/ranking unchanged. > -> **Scheduled comparison outcome (2026-07-27, read-only):** scheduled canary run `30216191889` was compared with baseline `30018289898` without a new dispatch or spend. Both retained 36/36 retrieval, document/content recall 1.0, MRR@10 0.8921, and irrelevant-source-rate@10 0.0917. The 44-case answer report retained grounded-supported and unsupported-correct rates of 1.0, cleared the prior citation failure (0.0227 to 0), improved p95 from 18,591 ms to 17,003 ms, and cost `$0.235936` versus `$0.234736`. Both comparison cases still omitted the admission document, strengthening #019. The companion scheduled CI run `30216361999` stopped at the existing production dependency audit, so Firefox/WebKit did not run; #023 remains only for that browser datapoint and the human irrelevant-at-10 disposition. Do not dispatch another RAG run. +> **Scheduled comparison outcome (2026-07-27, runs `30018289898` → `30216191889`):** the approved read-only artifact comparison covered the same 36 retrieval and 44 answer cases. Retrieval retained document/content recall 1.0, hit rate 1.0, MRR@10 0.8921 and irrelevant@10 0.0917, with no failed cases or per-case document-RR regressions; content MRR moved 0.9344 → 0.9333 only on `clozapine-anc-threshold`. End-to-end retrieval latency materially worsened (median 12,558 → 21,021 ms; p90 32,062 → 49,682 ms), so ranking remains unchanged. The subsequent #069 profile found acceptable hosted table-facts database plans, separating this broad tail from that narrow RPC defect. Answer quality kept grounded-supported and unsupported-correct rates at 1.0, cleared the prior citation failure, and improved p95 18,591 → 17,003 ms, but #019 repeated with the identical `19baf837c890b5d3` diagnostic signature and `comparison_source_extractive_fallback`. Expected-hit moved 0.6591 → 0.6364 and cost $0.234736 → $0.235936. The scheduled CI run failed its blocking dependency audit before the Firefox/WebKit matrix, so it supplied no second browser datapoint; #024 is independently resolved and release-matrix debt remains #055. No scheduled rerun was dispatched. > -> **RAG reconciliation correction (2026-07-23):** fresh current-main live evidence supersedes the broad diagnosis in #018. The three named misses are not one composer defect. Lithium reproduced an unrelated-table retrieval fast-path defect; ADHD still retrieves a relevant chart-heavy CAMHS source but exhausts the extractive route budget; metabolic retrieves the correct AKG source but selects schedule-free prose; #019 remains post-retrieval comparison source selection. A narrow lithium subject-evidence guard improved its targeting result from 0 to 1 with golden recall 1.0 and no reciprocal-rank regressions, but was reverted and rejected because the required full canary failed. Keep #029 open for the remaining fallback-stub cases. Do not combine these residuals or change ranking scores, comparator ordering, aliases, clamps, or semantic reranking without a separate reproducer and passing canary pair. - -| ID | Pri | Type | Summary | Detail / next action | Source | Added | -| ---- | --- | ----- | --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| #059 | P1 | task | Verify containment of every credential reported exposed in chat | **Outcome:** every reported exposed credential is rejected or retired. **Next:** in approved security windows, verify and revoke or rotate the GitHub token, OpenAI key, Supabase service-role JWT, database password, and E2E credential; create replacements only when required and update only intended secret stores. **Success:** provider evidence confirms the old credentials cannot authenticate, replacements are distinct and minimally scoped, presence/readiness checks pass, and secret scans remain clean. **Stop:** no provider or secret-store action without approval; never print or paste values into Git, logs, issues, or chat. | session 2026-07-24 security reconciliation; AI Agent Target Manifest | 2026-07-24 | -| #065 | P2 | task | Complete the paused compact document source-text accordion | **Outcome:** the document viewer uses compact nested disclosures while retaining complete text, citation/search navigation, print behavior, and composer clearance. **Next:** only when the user explicitly resumes, reconcile `codex/chat-document-text-accordion-7cb4` with current `main` and complete the focused 320/390/1280 px tests. **Success:** default disclosures are closed; deep links and search open only the active passage; printing expands/restores state; no overflow. **Verify:** focused document-viewer Playwright, `verify:cheap`, `verify:ui`, and static production-readiness. **Stop:** remain paused until explicit user return; no provider calls. | paused document-viewer task; `codex/chat-document-text-accordion-7cb4` | 2026-07-24 | -| #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | -| #069 | P3 | task | Live-profile table-facts plpgsql+EXECUTE latency | Migration `20260724120000_table_facts_plpgsql_execute.sql` plus P3 follow-ups (`20260724130000_*`, `20260724130100_*`) are in PR #1133. Hosted apply is blocked in this environment (no `SUPABASE_DB_URL`; Supabase MCP `needsAuth`). A live `profile:retrieval --rpc match_document_table_facts_text --analyze` attempt returned `Unregistered API key` against the injected service-role secret. **Next:** operator applies the three pending migrations on the live target project, then re-runs approval-gated `profile:retrieval` / `explain_retrieval_rpc` and confirms ~70ms-class plans with no ranking change. **Stop** without mutating ranking. | session 2026-07-24 Database interface audit; PR #1133 | 2026-07-24 | -| #053 | P1 | task | Execute cross-border privacy/legal package | Execute OpenAI and Railway DPAs; decide ZDR and Australian data residency; obtain prompt-cache behavior in writing; review subprocessors; obtain APP 8 and APP 5/1 counsel sign-off. Do not represent the release as privacy-approved or alter final public privacy wording before sign-off. | `docs/openai-cross-border-basis.md`; `docs/privacy-impact-assessment.md` | 2026-07-24 | -| #055 | P2 | task | Run one exact-SHA full release and PR gate | Scheduled CI run `30216361999` stopped at `npm audit --omit=dev --audit-level=high` on the existing ExcelJS/brace-expansion chain, so its dependent Firefox/WebKit matrix was skipped. Let the already-open dependency PRs own that upgrade; then run one exact-SHA local/provider release gate, browser matrix, hosted CI and review-thread closure. Stop at the first actionable failure and rerun only the repaired smallest gate. | `docs/launch-operator-runbook.md`; run `30216361999`; Dependabot PRs #1268/#1269 | 2026-07-24 | -| #056 | P2 | task | Reconcile the existing isolated staging environment | `Clinical KB Staging` Supabase (`ikoiolksxqxfxgiyqpnu`) and Railway staging are active, healthy, offline-provider mode, empty of documents/registry content, and pass `search_schema_health()`. Staging migration history is 23 versions behind production (last applied `20260719055623`). Apply the exact repository chain with a credentialed Supabase CLI/database connection—do not use a helper that invents replacement timestamps—then repeat indexing, identity, health and empty-data-boundary proof. | `docs/staging-setup.md`; `docs/operator-backlog.md`; read-only provider proof 2026-07-27 | 2026-07-24 | -| #057 | P2 | task | Complete staging soak and rollback rehearsal | After #056, run the documented soak and rollback against an exact candidate; retain latency/error/rollback evidence. Stop on unsafe data, identity mismatch, or an unowned rollback decision. | `docs/launch-operator-runbook.md`; `docs/capacity-review.md` | 2026-07-24 | -| #005 | P3 | rec | `finalScore` saturates at clamp ceiling | Base + ~40 stacked boosts routinely exceed 1.0, so strong matches tie at 1.0 and order by an arbitrary `document_id` tiebreak. If ranking is ever revisited, break ties by the **pre-clamp** score rather than raising the `[0,1]` ceiling (downstream gates assume `[0,1]`). Ordering already sorts by the unbounded pre-clamp `rankScore` (`clinical-search.ts:1735,1927,1950-1955`), so the clamp confines only the reported confidence value, not result order. Not a defect on the current golden set; any change here is a protected RAG surface (canary required). | `docs/rag-hybrid-findings-and-todo.md` P1 item 4; `src/lib/clinical-search.ts:1735` | 2026-07-21 | -| #011 | P3 | task | Auth DB-connection allocation is operator-only | Supabase Auth (GoTrue) is capped at ~10 absolute DB connections (Supabase perf advisor). Switch to **percentage-based** allocation in the Supabase **dashboard** before the first compute scale-up — **not settable via SQL/MCP** (operator-owned). Verify via a staging soak + an approval-gated read-only advisor re-check. | `docs/auth-connection-cap-runbook.md`; `docs/process-hardening.md` (Known follow-up debts) | 2026-07-21 | -| #012 | P3 | rec | Slim the lazy cross-mode differentials chunk | `cross-mode-differentials.ts` is dynamically imported (correctly code-split **out** of the initial/dashboard bundle — verified), but it pulls the full ~860 KB differentials snapshot (~125 KB gzip lazy chunk) just to build a tiny `{slug,title,clinicalHinge}` + presentations + aliases catalog. A precomputed lightweight index (generator + drift check, like the `specifiers-content` split / medications `fields=index`) would cut that lazy chunk ~5–10├ù. Not a bundle leak — an M-effort slim. | `src/lib/cross-mode-differentials.ts`; `src/components/clinical-dashboard/cross-mode-links.tsx:150`; session 2026-07-21 (build:analyze) | 2026-07-21 | -| #013 | P3 | rec | Route-chunk + mockup catalogue JSON weight | `build:analyze`: `/specifiers` ships `specifiers-search-index.json` (~180 KB parsed), `/forms` ships `forms-catalog.json` (~132 KB), `/formulation` ships `formulation-content.json` (~52 KB, client-side local search — needs index/full split or a search endpoint, architectural). All route-scoped (not initial bundle). Also `*-mockups.tsx` (~100 KB across chunks) build though `/mockups` 404s in prod — exclude from the prod artifact. | session 2026-07-21 (build:analyze) | 2026-07-21 | -| #016 | P3 | rec | "Big but not easy" structural + motion perf | Deferred larger levers: (a) nonce-CSP forces every product route to `╞Æ Dynamic` (zero static generation) — evaluate Partial Prerendering / static shells for the static clinical catalogues (DSM/differentials/therapy/specifiers/formulation); (b) sidebar expand/collapse animates `grid-template-columns` (biggest smoothness cost, motion-gated — needs a transform-overlay rethink); (c) Therapy Compass fetches 692 KB / 2.5 MB JSON client-side (defer until interaction + confirm brotli); (d) settings/setup/admin dialogs static-imported into the home chunk (`next/dynamic` them). | session 2026-07-21 (build route table + design audit) | 2026-07-21 | -| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. | session 2026-07-21 (measurement pass) | 2026-07-21 | -| #018 | P2 | task | Split the lithium, ADHD and metabolic residuals by mechanism | Revalidated on current main 2026-07-23: these are not one composer defect. Lithium reproduced an unrelated-table retrieval fast-path defect; ADHD retrieves a relevant chart-heavy CAMHS source but exhausts the extractive route budget; metabolic retrieves the correct AKG source but selects schedule-free prose. The narrow lithium subject-evidence guard improved targeting from 0 to 1 with golden recall 1.0 and no reciprocal-rank regressions, but it was reverted because the full canary failed. After #051 stabilises the canary, add independent current-main reproducers and assess each mechanism separately. Do not widen the matcher or combine these into a broad ranking/composer change. | runs `30007833352` and `30009207429`; PR #1093; session 2026-07-23 | 2026-07-21 | -| #019 | P2 | task | Admission doc dropped after deterministic comparison packing | Reconfirmed in both baseline `30018289898` and scheduled run `30216191889`: retrieval remained 36/36 and retrieved the admission source, while both admission/discharge answer cases omitted it after comparison fallback. PR #1096 already proves deterministic answer ranking and cross-document packing retain both sources, so retrieval scores, aliases and comparator ordering are not the fix. Next: create a red fallback-layer unit reproducer using the live source shape; any behavior change needs a passing baseline/post canary. | runs `30018289898`/`30216191889`; PR #1096; session 2026-07-27 | 2026-07-21 | -| #021 | P3 | rec | E-3d H2 residual: strong/comparison generation discards | approx. 6 generation attempts per full 44-case run still fail the final quality gate and fall to extractive on strong-route comparison/complex shapes (the designed-conservative outcome). PARKED: weakest cost/benefit on the queue — a wave (approx. $2-4 pair + reviewer cycle) to shave seconds off a few hard cases. Revisit only if latency/waste complaints or a cheaper lever appears. | E-3c design record; runs #59-#61 diagnostics | 2026-07-21 | -| #022 | P2 | task | Source-governance metadata refresh (operator) | **Worklist generated 2026-07-22 ($0, read-only): `docs/source-governance-refresh-worklist-2026-07-22.md`.** Reframed - this is NOT 59 clinical reviews. Of the 124 documents surfacing in canary top results, 59 are review-required, and **38 (64 pct) are the BMJ published-reference tier all sitting at `clinical_validation_status: unverified`** - one attestation-policy decision, not 38 reviews. The remaining 21 are genuine local WA health-service reviews (FSH 7, NMHS 4, CAMHS 3, AKG 2, KEMH 2, RPBG 2, RKPG 1), mostly `document_status: review_due`. Burn-down: top-10 documents clear 44 pct of flagged slots, top-20 clear 66 pct. Next: decide the BMJ attestation policy, then attest local docs by visibility (start `Clozapine Management by GP (NMHS)`, 22 slots at rank 1). | runs #61/#57 Source Governance data; `docs/source-governance-refresh-worklist-2026-07-22.md` | 2026-07-21 | -| #023 | P2 | task | Complete scheduled browser and labeling disposition | The 2026-07-26 canary artifacts are read and disposed under resolved #051. Scheduled CI run `30216361999` failed its existing production dependency audit before the dependent Firefox/WebKit job, while production Chromium passed. After the already-open dependency PRs clear the audit, capture one scheduled/manual browser-matrix datapoint; separately record the human decision for the stable irrelevant-at-10 set. Do not rerun or spend on RAG for this item. | runs `30216191889`/`30216361999`; Dependabot PRs #1268/#1269; session 2026-07-27 | 2026-07-21 | -| #025 | P2 | task | Activate the three webhooks (operator secrets) | Merged (#968/#1100) + deployed but inert — verified live: `POST /api/webhooks/railway` returns `503 webhook_not_configured`; the Supabase document-change trigger exists but lacks both activation inputs. To turn on: (1) Railway ΓåÆ set `RAILWAY_WEBHOOK_SECRET` + add the `?token=…` webhook URL; (2) set `SLACK_WEBHOOK_URL`/`DISCORD_WEBHOOK_URL` in BOTH the Railway **app/server env** and **GitHub repo secrets**; (3) set one matching document-change secret in the Railway app env as `SUPABASE_INGESTION_WEBHOOK_SECRET` and in Supabase Vault as `ingestion_webhook_secret`, then set the per-environment database GUC `app.ingestion_webhook_base_url` to the deployed app origin. Each path fails closed until fully configured, so this is pure ops. See `docs/webhooks.md` for verification and rotation. | sessions 2026-07-22/24; PRs #968/#1100; docs/webhooks.md | 2026-07-22 | -| #027 | P3 | rec | External uptime monitor independent of GitHub/Railway | `live-domain-monitor.yml` runs on GitHub's cron, so it won't run in exactly the outage it should catch (Actions or the deploy itself down). Add an off-platform synthetic monitor (UptimeRobot / Better Stack / Checkly) hitting `/api/health` with a webhook alert. Provider setup, not code. | session 2026-07-22 webhook review | 2026-07-22 | -| #028 | P3 | rec | Runtime error tracking (Sentry or similar) | No error tracking in the repo — production exceptions on `psychiatry.tools`, including how often `RAG_PROVIDER_MODE=auto` silently degrades to source-only, are invisible. Weigh adding `@sentry/nextjs` (dependency + DSN secret + instrumentation) vs cost; alert ΓåÆ chat/issue. Provider-backed; needs explicit sign-off before adding the dependency. | session 2026-07-22 webhook review | 2026-07-22 | -| #029 | P2 | issue | 12 of 30 answer-quality cases return the fallback stub | run #61 --dump-answers: 12/30 quality cases emit the source_backed_review_fallback boilerplate with answer_sections: [], all grounded with 4-6 citations. Some still PASS targeting because the stub echoes query keywords (the contraindication/document_lookup matchers need only a keyword), so the targeting metric MASKS the problem for those intents. Superset of #018 — fix in the extractive composer, validate with the provider-backed answer eval. | run #61 dump artifact; session 2026-07-22 | 2026-07-22 | -| #033 | P3 | rec | Source governance metadata absent from the LLM prompt | `buildRagSourceBlock` omits `document_status`, `clinical_validation_status`, and `extraction_quality`, so the model cannot self-caveat during generation and governance is enforced only post-hoc. Generation-surface change: needs `eval:rag` plus `eval:quality --rag-only` (grounded-supported must not drop, citation-failure 0) and explicit approval. Carries the same "unknown Γëá bad" hazard as #032 — on a partially-enriched corpus the model would likely over-caveat correct sources, so design the prompt wording before spending an eval. | `src/lib/rag/rag-source-block.ts:126-198`; PR #1051 audit item 8 | 2026-07-22 | -| #035 | P3 | rec | Threshold-conflict detection covers only 3 params | `detectThresholdDisagreements` checks only ANC, WBC, and platelets paired with withholding verbs, so cross-source conflicts on medication doses, lithium/thyroid levels, or vital signs go undetected. Deliberately narrow (see the comment at `:469-474`). Broadening changes when an answer is classified `conflicting` and adds warnings — real false-positive risk. Needs new fixtures plus a behaviour review before any change. | `src/lib/evidence.ts:469-574`; PR #1051 audit item 7 | 2026-07-22 | -| #036 | P3 | rec | No explicit `is_public` visibility flag on documents | Public-corpus visibility is implicit: `owner_id IS NULL` on an `indexed` document (`resolveSearchScope`). The `metadata.public_corpus` marker is written by the promotion migrations but never used as a retrieval filter. Promotion is unconditional on `clinical_validation_status`, so unverified documents are publicly searchable — compensated by keeping `unverified_source` in the frontend-visible warning set. A hard schema flag touches RLS and the clinical-risk-gated retrieval RPCs; weigh against the existing compensating control before acting. | `supabase/schema.sql:61-108`; `src/lib/search-scope.ts:181-236`; PR #1051 audit item 3 | 2026-07-22 | -| #037 | P3 | rec | D5 trust-cap-all-claims flag parked OFF | `NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS` extends authority gating from high-risk claims to **all** supported claims (`deriveTrust`). Ships OFF by design; flipping it caps trust to `medium` for routine claims across the board — a product/clinical-UX decision, not a defect. Both states are test-pinned. Next action: product decision, then flip and re-baseline the UI expectations. | `src/lib/answer-render-policy.ts:159-177`; PR #1051 audit item 11 | 2026-07-22 | -| #038 | P3 | rec | Consolidate shared comparison behavior | Several clinical modes expose comparison workflows with similar selection, empty-state and mobile-dock needs. Define one shared behavioral contract before another comparison surface is added; keep mode-specific clinical content separate. This is a design-system recommendation, not a current defect. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | -| #039 | P3 | rec | Consolidate catalogue toolbar patterns | Catalogue/search pages have independently evolved filter, sort, result-count and mobile toolbar behavior. Inventory the existing implementations and converge only the repeated interaction contract; do not flatten mode-specific search semantics. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | -| #040 | P3 | rec | Add targeted visual-regression baselines | Keep a small approved baseline set for high-value desktop/mobile surfaces and accessibility modes instead of screenshotting every route. Start with account/settings, document viewer, mode homes and bottom-composer interactions; define an intentional-update workflow before enabling blocking comparisons. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | -| #079 | P3 | task | Disposition retained worktrees in bounded cleanup batches | **Outcome:** the retained reconciliation tail is gradually classified without another disruptive all-worktree sweep. **Next:** process no more than ten worktrees per explicitly scheduled pass using current owner/process metadata, open-PR state, exact review-ledger coverage, ancestry, and cherry-pick-aware content proof. **Success:** remove only clean, inactive, bundled worktrees whose content is merged or explicitly rejected; record every disposition and retain recovery evidence. **Stop:** preserve dirty, active, secret-bearing, post-freeze, paused, or ambiguous work and never use reset, force deletion, broad clean, or process killing. | final reconciliation inventory retained 104 independent worktrees; session 2026-07-24 | 2026-07-24 | -| #080 | P3 | rec | Re-test the removed admission-to-discharge alias widening | **Outcome:** decide with measured evidence whether the user-approved 2026-07-21 widening (NMHS admission-to-discharge titles satisfying the admission expectation) belongs back on the wide-tier `AdmissionCommunityPts` list. #030 removed it so one document could not fill both comparison slots, but coverage now enforces distinct document identities by maximum matching, so the widening is no longer a false-pass route and its removal may instead cost legitimate admission credit. **Next:** with approval, run an eval-canary baseline/post pair over the admission-discharge comparison cases with and without those two titles. **Success:** restore only on measured non-regression (recall pinned 1.0, zero per-case reciprocal-rank regressions); otherwise record keep-removed and close. **Stop:** never restore on offline reasoning alone. | `src/lib/eval-document-matching.ts`; #030 archive row; session 2026-07-25 | 2026-07-25 | +> **RAG reconciliation correction (2026-07-23, superseded for fallback status on 2026-07-27):** fresh current-main live evidence superseded the broad diagnosis in #018. The three named misses were not one composer defect. Lithium reproduced an unrelated-table retrieval fast-path defect; ADHD still retrieved a relevant chart-heavy CAMHS source but exhausted the extractive route budget; metabolic retrieved the correct AKG source but selected schedule-free prose; #019 was post-retrieval comparison source selection. The independent-fix and canary constraints remain authoritative, while #019 and #029 are now resolved by the final update below. +> +> **Final RAG verification update (2026-07-27):** the protected retrieval canary passed 36/36 with document/content recall 1.0 and zero per-case reciprocal-rank regressions; content MRR improved from 0.9389 to 0.9406 and no case regressed. The final 44-case answer gate then passed with 30/30 supported answers substantive and grounded, 14/14 unsupported answers correct, citation and numeric failure rates 0, source-backed review fallback rate 0, no route-ceiling failures, and p95 7,494 ms. The final report used two provider requests; pricing telemetry was not configured, so it does not claim an exact dollar value. #019 and #029 are resolved without weakening grounding, citation, numeric, or route gates. Detailed evidence is in `docs/evidence/rag-reliability-evidence-2026-07-27.md`. + +| ID | Pri | Type | Summary | Detail / next action | Source | Added | +| ---- | --- | ---- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| #059 | P1 | task | Verify containment of every credential reported exposed in chat | **Outcome:** every reported exposed credential is rejected or retired. **Next:** in approved security windows, verify and revoke or rotate the GitHub token, OpenAI key, Supabase service-role JWT, database password, and E2E credential; create replacements only when required and update only intended secret stores. **Success:** provider evidence confirms the old credentials cannot authenticate, replacements are distinct and minimally scoped, presence/readiness checks pass, and secret scans remain clean. **Stop:** no provider or secret-store action without approval; never print or paste values into Git, logs, issues, or chat. | session 2026-07-24 security reconciliation; AI Agent Target Manifest | 2026-07-24 | +| #065 | P2 | task | Complete the paused compact document source-text accordion | **Outcome:** the document viewer uses compact nested disclosures while retaining complete text, citation/search navigation, print behavior, and composer clearance. **Next:** only when the user explicitly resumes, reconcile `codex/chat-document-text-accordion-7cb4` with current `main` and complete the focused 320/390/1280 px tests. **Success:** default disclosures are closed; deep links and search open only the active passage; printing expands/restores state; no overflow. **Verify:** focused document-viewer Playwright, `verify:cheap`, `verify:ui`, and static production-readiness. **Stop:** remain paused until explicit user return; no provider calls. | paused document-viewer task; `codex/chat-document-text-accordion-7cb4` | 2026-07-24 | +| #001 | P2 | task | Semantic reranking still gated off | `RAG_SEMANTIC_RERANK_ENABLED=false` from PR #901. Do not enable until the provider-backed 36/36 retrieval-quality gate **and** an ambiguity-focused canary are explicitly approved and recorded. | `docs/process-hardening.md` (Semantic reranking rollout debt); PR #901 | 2026-07-21 | +| #053 | P1 | task | Execute cross-border privacy/legal package | Execute OpenAI and Railway DPAs; decide ZDR and Australian data residency; obtain prompt-cache behavior in writing; review subprocessors; obtain APP 8 and APP 5/1 counsel sign-off. Do not represent the release as privacy-approved or alter final public privacy wording before sign-off. | `docs/openai-cross-border-basis.md`; `docs/privacy-impact-assessment.md` | 2026-07-24 | +| #055 | P2 | task | Run one exact-SHA full release and PR gate | Before the next full-confidence release/handoff, record the candidate/PR SHA and run the local/provider release gates, Firefox/WebKit, required hosted CI, and actionable GitHub review-thread closure once. Stop at the first actionable failure and rerun only the repaired smallest gate. | `docs/launch-operator-runbook.md`; `docs/codex-review-protocol.md` | 2026-07-24 | +| #056 | P2 | task | Reconcile the existing staging migration history | `Clinical KB Staging` already exists as a healthy, empty Supabase/Railway tier with distinct secrets and no production clinical data, but it is 23 repository migrations behind. In the next approved staging schema window, apply the exact missing migration chain, then re-run indexing, health, identity and data-boundary proof. Do not recreate the environment or copy production clinical documents. | current-main staging verification; `docs/staging-setup.md`; `docs/operator-backlog.md` | 2026-07-27 | +| #057 | P2 | task | Complete staging soak and rollback rehearsal | After #056, run the documented soak and rollback against an exact candidate; retain latency/error/rollback evidence. Stop on unsafe data, identity mismatch, or an unowned rollback decision. | `docs/launch-operator-runbook.md`; `docs/capacity-review.md` | 2026-07-24 | +| #005 | P3 | rec | `finalScore` saturates at clamp ceiling | Base + ~40 stacked boosts routinely exceed 1.0, so strong matches tie at 1.0 and order by an arbitrary `document_id` tiebreak. If ranking is ever revisited, break ties by the **pre-clamp** score rather than raising the `[0,1]` ceiling (downstream gates assume `[0,1]`). Ordering already sorts by the unbounded pre-clamp `rankScore` (`clinical-search.ts:1735,1927,1950-1955`), so the clamp confines only the reported confidence value, not result order. Not a defect on the current golden set; any change here is a protected RAG surface (canary required). | `docs/rag-hybrid-findings-and-todo.md` P1 item 4; `src/lib/clinical-search.ts:1735` | 2026-07-21 | +| #011 | P3 | task | Auth DB-connection allocation is operator-only | Supabase Auth (GoTrue) is capped at ~10 absolute DB connections (Supabase perf advisor). Switch to **percentage-based** allocation in the Supabase **dashboard** before the first compute scale-up — **not settable via SQL/MCP** (operator-owned). Verify via a staging soak + an approval-gated read-only advisor re-check. | `docs/auth-connection-cap-runbook.md`; `docs/process-hardening.md` (Known follow-up debts) | 2026-07-21 | +| #012 | P3 | rec | Slim the lazy cross-mode differentials chunk | `cross-mode-differentials.ts` is dynamically imported (correctly code-split **out** of the initial/dashboard bundle — verified), but it pulls the full ~860 KB differentials snapshot (~125 KB gzip lazy chunk) just to build a tiny `{slug,title,clinicalHinge}` + presentations + aliases catalog. A precomputed lightweight index (generator + drift check, like the `specifiers-content` split / medications `fields=index`) would cut that lazy chunk ~5–10├ù. Not a bundle leak — an M-effort slim. | `src/lib/cross-mode-differentials.ts`; `src/components/clinical-dashboard/cross-mode-links.tsx:150`; session 2026-07-21 (build:analyze) | 2026-07-21 | +| #013 | P3 | rec | Route-chunk + mockup catalogue JSON weight | `build:analyze`: `/specifiers` ships `specifiers-search-index.json` (~180 KB parsed), `/forms` ships `forms-catalog.json` (~132 KB), `/formulation` ships `formulation-content.json` (~52 KB, client-side local search — needs index/full split or a search endpoint, architectural). All route-scoped (not initial bundle). Also `*-mockups.tsx` (~100 KB across chunks) build though `/mockups` 404s in prod — exclude from the prod artifact. | session 2026-07-21 (build:analyze) | 2026-07-21 | +| #016 | P3 | rec | "Big but not easy" structural + motion perf | Deferred larger levers: (a) nonce-CSP forces every product route to `╞Æ Dynamic` (zero static generation) — evaluate Partial Prerendering / static shells for the static clinical catalogues (DSM/differentials/therapy/specifiers/formulation); (b) sidebar expand/collapse animates `grid-template-columns` (biggest smoothness cost, motion-gated — needs a transform-overlay rethink); (c) Therapy Compass fetches 692 KB / 2.5 MB JSON client-side (defer until interaction + confirm brotli); (d) settings/setup/admin dialogs static-imported into the home chunk (`next/dynamic` them). | session 2026-07-21 (build route table + design audit) | 2026-07-21 | +| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. | session 2026-07-21 (measurement pass) | 2026-07-21 | +| #018 | P2 | task | Split the lithium, ADHD and metabolic residuals by mechanism | Current evidence keeps the mechanisms separate. **Lithium — closed within this item:** the row/atom-aware subject guard, foreign-parameter rejection and query-specific range promotion returned `0.5–1.0 mmol/L` with correct targeting/citation; the full retrieval canary remained 36/36 with recall 1.0 and zero per-case RR regressions, and the full answer canary passed every blocking gate. **ADHD — open corpus debt:** `CG.MHSP.ADHD.pdf` is absent from the hosted corpus and the retrieved chart exposes `accessible_table_count=0`; repair corpus/fixture or ingestion evidence rather than weakening extractive budgets. **Metabolic — open structured-evidence debt:** the standalone plural classifier worsened the live answer and was reverted; obtain auditable schedule text/table evidence before another candidate. | targeted live lithium/ADHD/metabolic evidence 2026-07-27; `docs/evidence/rag-reliability-evidence-2026-07-27.md`; refuted approaches | 2026-07-21 | +| #021 | P3 | rec | E-3d H2 residual: strong/comparison generation discards | approx. 6 generation attempts per full 44-case run still fail the final quality gate and fall to extractive on strong-route comparison/complex shapes (the designed-conservative outcome). PARKED: weakest cost/benefit on the queue — a wave (approx. $2-4 pair + reviewer cycle) to shave seconds off a few hard cases. Revisit only if latency/waste complaints or a cheaper lever appears. | E-3c design record; runs #59-#61 diagnostics | 2026-07-21 | +| #022 | P2 | task | Source-governance metadata refresh (operator) | The selected policy is now encoded locally as auditable `third_party_reference_attested` evidence with policy version, reviewer qualification, evidence references and append-only review history. It deliberately preserves `clinical_validation_status=unverified`; malformed, stale or non-BMJ evidence remains review debt. Migration `20260727010000_bmj_third_party_source_attestation.sql` is prepared but was **not applied**. The ten most visible local-document candidates are captured in `docs/evidence/rag-top-local-review-manifest-2026-07-26.json` with `attestation_applied=false`; qualified human review, deliberate hosted apply/attestation, and warning-rate remeasurement remain operator work. | governance worklist; local policy/migration tests; top-ten evidence manifest | 2026-07-21 | +| #023 | P2 | task | Complete scheduled browser and labeling disposition | The 2026-07-26 retrieval and answer artifacts are read and compared under resolved #051. Scheduled CI run `30216361999` failed its existing production dependency audit before Firefox/WebKit, while production Chromium passed. After that audit is green, capture one scheduled/manual browser-matrix datapoint; separately record the human decision for the stable irrelevant-at-10 set. #084 now makes each top-10 grade and matched signal reproducible, but it does not substitute for the human disposition. Do not rerun or spend on RAG for this item. | runs `30216191889`/`30216361999`; per-rank diagnostics #084; session 2026-07-27 | 2026-07-21 | +| #025 | P2 | task | Activate the three webhooks (operator secrets) | Merged (#968/#1100) + deployed but inert — verified live: `POST /api/webhooks/railway` returns `503 webhook_not_configured`; the Supabase document-change trigger exists but lacks both activation inputs. To turn on: (1) Railway ΓåÆ set `RAILWAY_WEBHOOK_SECRET` + add the `?token=…` webhook URL; (2) set `SLACK_WEBHOOK_URL`/`DISCORD_WEBHOOK_URL` in BOTH the Railway **app/server env** and **GitHub repo secrets**; (3) set one matching document-change secret in the Railway app env as `SUPABASE_INGESTION_WEBHOOK_SECRET` and in Supabase Vault as `ingestion_webhook_secret`, then set the per-environment database GUC `app.ingestion_webhook_base_url` to the deployed app origin. Each path fails closed until fully configured, so this is pure ops. See `docs/webhooks.md` for verification and rotation. | sessions 2026-07-22/24; PRs #968/#1100; docs/webhooks.md | 2026-07-22 | +| #027 | P3 | rec | External uptime monitor independent of GitHub/Railway | `live-domain-monitor.yml` runs on GitHub's cron, so it won't run in exactly the outage it should catch (Actions or the deploy itself down). Add an off-platform synthetic monitor (UptimeRobot / Better Stack / Checkly) hitting `/api/health` with a webhook alert. Provider setup, not code. | session 2026-07-22 webhook review | 2026-07-22 | +| #028 | P3 | rec | Runtime error tracking (Sentry or similar) | No error tracking in the repo — production exceptions on `psychiatry.tools`, including how often `RAG_PROVIDER_MODE=auto` silently degrades to source-only, are invisible. Weigh adding `@sentry/nextjs` (dependency + DSN secret + instrumentation) vs cost; alert ΓåÆ chat/issue. Provider-backed; needs explicit sign-off before adding the dependency. | session 2026-07-22 webhook review | 2026-07-22 | +| #033 | P3 | rec | Source governance metadata absent from the LLM prompt | `buildRagSourceBlock` omits `document_status`, `clinical_validation_status`, and `extraction_quality`, so the model cannot self-caveat during generation and governance is enforced only post-hoc. Generation-surface change: needs `eval:rag` plus `eval:quality --rag-only` (grounded-supported must not drop, citation-failure 0) and explicit approval. Carries the same "unknown Γëá bad" hazard as #032 — on a partially-enriched corpus the model would likely over-caveat correct sources, so design the prompt wording before spending an eval. | `src/lib/rag/rag-source-block.ts:126-198`; PR #1051 audit item 8 | 2026-07-22 | +| #035 | P3 | rec | Threshold-conflict detection covers only 3 params | `detectThresholdDisagreements` checks only ANC, WBC, and platelets paired with withholding verbs, so cross-source conflicts on medication doses, lithium/thyroid levels, or vital signs go undetected. Deliberately narrow (see the comment at `:469-474`). Broadening changes when an answer is classified `conflicting` and adds warnings — real false-positive risk. Needs new fixtures plus a behaviour review before any change. | `src/lib/evidence.ts:469-574`; PR #1051 audit item 7 | 2026-07-22 | +| #036 | P3 | rec | No explicit `is_public` visibility flag on documents | Public-corpus visibility is implicit: `owner_id IS NULL` on an `indexed` document (`resolveSearchScope`). The `metadata.public_corpus` marker is written by the promotion migrations but never used as a retrieval filter. Promotion is unconditional on `clinical_validation_status`, so unverified documents are publicly searchable — compensated by keeping `unverified_source` in the frontend-visible warning set. A hard schema flag touches RLS and the clinical-risk-gated retrieval RPCs; weigh against the existing compensating control before acting. | `supabase/schema.sql:61-108`; `src/lib/search-scope.ts:181-236`; PR #1051 audit item 3 | 2026-07-22 | +| #037 | P3 | rec | D5 trust-cap-all-claims flag parked OFF | `NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS` extends authority gating from high-risk claims to **all** supported claims (`deriveTrust`). Ships OFF by design; flipping it caps trust to `medium` for routine claims across the board — a product/clinical-UX decision, not a defect. Both states are test-pinned. Next action: product decision, then flip and re-baseline the UI expectations. | `src/lib/answer-render-policy.ts:159-177`; PR #1051 audit item 11 | 2026-07-22 | +| #038 | P3 | rec | Consolidate shared comparison behavior | Several clinical modes expose comparison workflows with similar selection, empty-state and mobile-dock needs. Define one shared behavioral contract before another comparison surface is added; keep mode-specific clinical content separate. This is a design-system recommendation, not a current defect. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +| #039 | P3 | rec | Consolidate catalogue toolbar patterns | Catalogue/search pages have independently evolved filter, sort, result-count and mobile toolbar behavior. Inventory the existing implementations and converge only the repeated interaction contract; do not flatten mode-specific search semantics. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +| #040 | P3 | rec | Add targeted visual-regression baselines | Keep a small approved baseline set for high-value desktop/mobile surfaces and accessibility modes instead of screenshotting every route. Start with account/settings, document viewer, mode homes and bottom-composer interactions; define an intentional-update workflow before enabling blocking comparisons. | design audit reconciliation; session 2026-07-22 | 2026-07-22 | +| #079 | P3 | task | Disposition retained worktrees in bounded cleanup batches | **Outcome:** the retained reconciliation tail is gradually classified without another disruptive all-worktree sweep. **Next:** process no more than ten worktrees per explicitly scheduled pass using current owner/process metadata, open-PR state, exact review-ledger coverage, ancestry, and cherry-pick-aware content proof. **Success:** remove only clean, inactive, bundled worktrees whose content is merged or explicitly rejected; record every disposition and retain recovery evidence. **Stop:** preserve dirty, active, secret-bearing, post-freeze, paused, or ambiguous work and never use reset, force deletion, broad clean, or process killing. | final reconciliation inventory retained 104 independent worktrees; session 2026-07-24 | 2026-07-24 | ## Resolved / archive @@ -128,10 +123,15 @@ Move resolved rows here with the resolution date and a one-line outcome. Keep th | ID | Type | Summary | Outcome | Resolved | | ---- | ----- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| #082 | issue | Bot branch-sync heads leave required checks unapproved | Retired the automatic `GITHUB_TOKEN` PR branch-update workflow instead of weakening required-check approvals or introducing a privileged automation token. The existing helper remains dry-run by default, verifies its apply identity, and refuses missing or bot identities. The fast GitHub Actions policy check rejects both direct workflow `update-branch` calls and indirect apply-helper invocation, preventing recurrence without adding runtime or hosted-wait overhead. | 2026-07-27 | -| #083 | issue | Documents-only universal search timed out on staging tenancy | A current staging nightly reproducer showed the documents-only search losing its synthetic fixture after the federated typeahead timeout was reduced to 750 ms. The search now retains 750 ms for multi-domain requests and uses the established 6,000 ms budget only when documents are the sole requested domain; fake-timer coverage proves both paths. RAG impact: no retrieval, ranking, ordering, alias, score, or result-selection change—only availability of the explicitly focused request. | 2026-07-27 | +| #029 | issue | Residual answer-quality fallback stubs | Closed after fixing each causal cluster independently. Active-community ED, community-home-visit, clozapine blood-threshold/typo, discharge source-gap recovery, and Best Practice Prescription now use narrowly validated, source-bound answers or auditable recovery; cited provider refusal prose can no longer masquerade as grounded, and terminal gaps retain no claim citations. The final 44-case gate reported 30/30 substantive grounded supported answers, 14/14 unsupported correct, zero review fallbacks, zero citation/numeric failures, and zero route-ceiling failures. Measurement still reports review fallback separately and denies targeting credit for echoed boilerplate. | 2026-07-27 | +| #019 | issue | Preserve admission/discharge sources through comparison fallback | The actual fallback path now selects source-bound facts, preserves one admission and one discharge citation from distinct documents, and terminates at an evidence gap for qualified, negated, unrelated, title-only, single-sided, or same-document traps. Both exact live cases complete in about one second with zero provider calls; the final 44-case canary passed them with two citations each, while the 36-case retrieval canary held recall 1.0 and zero RR regressions. Retrieval scores, aliases, clamps, and comparator ordering were unchanged. | 2026-07-27 | +| #084 | task | Persist per-result irrelevant-at-10 grading evidence | `eval-retrieval` now persists each top result's `relevanceGrade` and `matchedDeclaredSignals`; focused fixtures cover ideal and zero-grade rows. The final golden artifact contains 338 graded top rows, including 33 grade-zero rows. This closes the reproducibility gap only: fixture labels, ranking, thresholds, and provider behavior were not changed, and human disposition remains #023. | 2026-07-27 | +| #080 | rec | Re-test the removed admission-to-discharge alias widening | Restored the two approved NMHS Admission-to-Discharge titles only on the eval-expectation surface. Canonical document-identity dedupe plus maximum bipartite matching prevents one dual-listed physical document from satisfying both comparison slots. Focused matching tests, both targeted admission cases, the final 36-case golden retrieval run, and the 44-case answer run passed; runtime retrieval/ranking behavior was not changed. | 2026-07-27 | +| #083 | issue | Documents-only universal search timed out on staging tenancy | A current staging nightly reproducer showed the documents-only search losing its synthetic fixture after the federated typeahead timeout was reduced to 750 ms. Current main retains 750 ms for multi-domain requests and uses the established 6,000 ms budget only when documents are the sole requested domain; fake-timer coverage proves both paths. RAG impact: no retrieval, ranking, ordering, alias, score, or result-selection change—only availability of the explicitly focused request. | 2026-07-27 | +| #082 | issue | Bot branch-sync heads leave required checks unapproved | Retired the automatic `GITHUB_TOKEN` PR branch-update workflow instead of weakening required-check approvals or introducing a privileged automation token. The existing helper remains dry-run by default, verifies its apply identity, and refuses missing or bot identities. The fast GitHub Actions policy check rejects both direct workflow `update-branch` calls and indirect apply-helper invocation. | 2026-07-27 | | #058 | task | Verify production content before any seed write | Read-only production counts on project `sjrfecxgysukkwxsowpy` found 276 clinical registry, 328 medication, and 232 differential records. The required tables are non-empty, so no seed or production write was needed. | 2026-07-27 | -| #051 | task | Stabilise the live answer-quality canary before more RAG tuning | Compared scheduled run `30216191889` with baseline `30018289898` without a dispatch or new spend: both retained 36/36 retrieval, recall 1.0, MRR@10 0.8921 and irrelevant@10 0.0917; answer grounded/unsupported rates stayed 1.0, citation failure improved to zero, and p95 improved to 17,003 ms. The repeated admission-source loss remains isolated as #019; no RAG tuning is justified by this monitoring result. | 2026-07-27 | +| #069 | task | Validate hosted table-facts RPC latency | Read-only profiling on the correct hosted project separated sample 1 (`first_unprimed`) from five `warm_repeat` samples; managed Supabase buffers were not flushed, so no true-cold claim is made. First-unprimed client/DB execution was 662.916/141.537 ms (clozapine), 277.661/96.229 ms (lithium), and 322.598/148.378 ms (metabolic). Warm client median/p90 was 187.029/198.907, 167.803/174.391, and 189.065/243.324 ms; warm DB execution median/p90 was 88.292/89.355, 64.342/65.566, and 107.448/148.417 ms. Earlier exact clinical probes were lower again. Plans are not the multi-second tail; no hosted migration, ranking, or provider configuration changed. | 2026-07-27 | +| #051 | task | Stabilise the live answer-quality canary before more RAG tuning | Closed after the scheduled structured report supplied a comparable second 36-retrieval/44-answer datapoint. Content gates stayed stable, the prior citation failure cleared, and #019 repeated with an identical diagnostic signature. Retrieval latency was investigated separately: #069 subsequently found acceptable table-facts database plans, so the broad scheduled tail was not treated as ranking debt. The report/trend tooling is now sufficient to compare future approved runs; no scheduled rerun or tuning was dispatched. | 2026-07-27 | | #054 | task | Reconcile local and hosted secrets/config | Completed production names-only reconciliation on 2026-07-27. The correctly identified primary checkout received distinct gitignored local safety/query-hash/deep-probe values. A hardened checker now pins GitHub to `BigSimmo/Database` and Railway to the live production project/environment plus `Database`/`worker`, catches multiline schema and `.env.example` drift, and verifies GitHub secrets/variables and per-service Railway contracts without emitting provider values. All required names passed; the Ops Digest workflow is active with a successful scheduled run; both Railway services have later successful deployments; Supabase names-only proof found the expected cron/Vault configuration. Value equality remains deliberately unobservable, staging stays #056, webhook activation stays #025, and legal/ZDR work stays #053. | 2026-07-27 | | #064 | task | Reconcile the preserved browser and contrast patch | Landed via PR #1250 squash `b91b4600171be08198e92bcf19b7d67e8207cb2f`. Opacity-free disabled Previous/Continue styling plus native-disabled/focus/axe Playwright coverage is on `main`. Historical `agent/formulation-disabled-contrast` remained unrecovered; conflicted PRs #1219/#1223/#1226/#1231/#1249 were closed without merge. | 2026-07-26 | | #024 | issue | WebKit e2e `_rsc` prefetch access-control errors | Fixed in PR #1205 after current-main reproducers showed catch-all interception plus duplicate viewport navigation racing same-origin WebKit RSC prefetches. The harness now intercepts external HTTP(S) only, directly proves blocking/recording, navigates each route once, and leaves `pageerror` unfiltered. On `64e46036`, the local Chromium/WebKit matrix passed 24/24 and `verify:pr-local` passed 3,431 tests, build/bundle checks, and 36 offline RAG fixtures. | 2026-07-25 | diff --git a/docs/rag-behaviour/refuted-approaches.md b/docs/rag-behaviour/refuted-approaches.md index 309f3007d9..71a150ea5d 100644 --- a/docs/rag-behaviour/refuted-approaches.md +++ b/docs/rag-behaviour/refuted-approaches.md @@ -73,6 +73,38 @@ Ledger `#032` / source-governance audit (PR #1051) items that look like “gaps - **Standing guard:** `tests/retrieval-selection.test.ts` keeps relevance ordering and asserts a higher-relevance `review_due`/`unverified` source outranks a lower-relevance `current`/`reviewed` one (`docs/rag-hybrid-findings-and-todo.md` item 20). - **If ever revisited:** only via **RC8 — source-strength as a filter, not a penalty/boost in selection ordering**, gated on `eval:retrieval:quality` 36/36 plus an approved live canary pair. Prompt-side governance caveats are a separate generation-surface item (`#033`), not a ranking change. +## Refutation 4 — plural antipsychotic classifier correction alone: wrong-subject answer + +- **Shape (2026-07-27):** the exact metabolic-monitoring case was classified as + `document_lookup` because `medicationDoseRiskPattern` recognized singular `antipsychotic` + but not plural `antipsychotics`. A one-token `antipsychotics?` correction was RED-proven + locally and changed the live case to `medication_dose_risk`. +- **Live result:** targeting remained `0` with `no schedule/interval`, and the extractive + recovery changed from schedule-free metabolic prose to an unrelated clozapine-clinic + sentence. The expected metabolic document still ranked first, but its retrieved text and + table facts did not contain an auditable monitoring schedule; its table image had no + accessible table text. No model request or OpenAI cost was incurred. +- **Disposition:** the candidate was reverted immediately. Do not reintroduce the classifier + correction as a standalone answer-quality fix. The remaining work is structured + table/ingestion evidence for the schedule, followed by a fresh fixture and live canary; do + not infer a clinical schedule from an image caption or loosen answer gates. + +## Refutation 5 — generic comparator rejection or broad-citation retry for the ECT flow edge + +- **Observed artifact (2026-07-27):** flattened OCR represented a procedural edge as + `Referral for ECT > 6`. Treating every `> number` expression as structural is unsafe: real + clinical thresholds, decimal comparators such as `> 6.5`, and ECT treatment counts also use + comparator syntax. +- **Rejected recovery:** forcing the case through broader multi-source/model generation retained + irrelevant citations and ended in generic review-fallback wording rather than a substantive + ECT process answer. +- **Adopted bounded handling:** reject only the exact ECT referral-edge shape (with an optional + table-normalization colon) and unambiguous arrow glyphs, rebuild every derived evidence surface + from surviving sources, and reflow the known wrapped BASE directive. Negative fixtures preserve + ordinary measurements, `> 6.5`, `> 7`, `> 60`, and treatment-count prose. The final live ECT case + returned the complete Booking Assistant Scheduling Engine (BASE) step, grounded and substantive, + with zero provider calls in about 1.3 seconds. + ## Related follow-up plans - **Word-boundary content matcher — ✅ IMPLEMENTED (2026-07-20, same-day follow-up).** @@ -80,10 +112,14 @@ Ledger `#032` / source-governance audit (PR #1051) items that look like “gaps superset by artifact replay (canary #53: 1,126 comparisons, 0 lost matches, 7 gained — the exact known punctuation-joined occurrences). The weekly scheduled canary provides the free live confirmation; a more-tolerant matcher cannot fail a previously-passing case. -- **irrelevant@10 labeling audit — unblocked, not yet run.** The canary artifact now carries - the TOP 10 result rows per case (was 5), so the audit of broad/vector cases' extra top-10 - documents for under-labeled relevant siblings can run offline against the next artifact. - Until it runs, treat the 0.108 rate as a labeling question, not ranking debt - (`docs/observability-slos.md` §3.1). -- **Phase E (answer-side quality):** untouched by this cycle; requires its own approval and - spend (~$2–5/run) per the master plan. +- **irrelevant@10 labeling audit — diagnostics completed 2026-07-27; no ranking action.** Scheduled run + `30216191889` kept irrelevant@10 exactly 0.0917 versus baseline `30018289898`. The 12 cases with + non-zero rates and their top-10 titles/previews showed no broad under-labelled-sibling pattern; + the obvious tail rows were off-topic service records, unrelated conditions, or generic policy + fragments. The current artifact now persists each row's `relevanceGrade` and + `matchedDeclaredSignals`; focused tests cover ideal and zero-grade rows, closing #084. Human + disposition remains #023. Keep treating this as an evaluation-label audit surface, not permission + to change ranking (`docs/observability-slos.md` §3.1). +- **Answer-side quality:** the final 44-case run passed every blocking gate with 30/30 substantive + grounded supported answers and zero source-backed review stubs. #029 is resolved, and the quality + gate now blocks any recurrence of that fallback rather than merely reporting it. diff --git a/docs/rag-hybrid-findings-and-todo.md b/docs/rag-hybrid-findings-and-todo.md index 1a780aaf01..5eec32af5a 100644 --- a/docs/rag-hybrid-findings-and-todo.md +++ b/docs/rag-hybrid-findings-and-todo.md @@ -108,9 +108,9 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows the golden retrieval eval. If differentials-mode retrieval quality needs a lift, re-propose the action-signal part (without the metadata condition) through the golden eval; the original code is in PR #120 history (`git show 635485998^1:src/lib/retrieval-selection.ts`). - - Higher-value redirect than mrr-chasing: **item 9 (enrichment/reindex — the OCR extraction drops - letters, e.g. "score"→"core", "psychosis"→"p ycho i ", which hurts both lexical matching and the - readability of quoted answer text)** and **item 10 (DB-backed synonyms/typos)**. + - The former higher-value redirect than mrr-chasing was **item 9 (OCR enrichment/reindex)** plus + **item 10 (DB-backed synonyms/typos)**. The bounded reference-backed OCR repair has since landed + (item 24); only live backfill/measurement remains, not another broad heuristic. ## P2 — latency, eval coverage, data @@ -148,8 +148,8 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows `alcohol-ciwa-threshold` golden case guard it. **This is a general recall win, not just CIWA** — any long multi-term query previously risked silent 0-match FTS. -9. ⚠️ **OCR "dropped-s" defect — real but NOT reliably heuristically-detectable; guard attempted then - REVERTED (2026-07-01). Honest post-mortem below.** +9. ⚠️ **OCR "dropped-s" heuristic — attempted then REVERTED (2026-07-01); bounded reference repair + later landed (item 24). Historical post-mortem below.** - **What's true:** real dropped-'s' corruption exists in some table-derived index units ("psychosocial"→"p ycho ocial", "1st mood stabiliser"→"1 t mood tabili er"). The **raw `document_chunks` (answer context) are clean** — 0 docs below 0.025 s-ratio — so **generated @@ -172,11 +172,12 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows exposed the detector's false positives. Those images were legitimately refreshed (they were version-stale anyway); a few units carry harmless appended source-chunk context from the since-reverted guard — will normalize on the next reindex. No further docs were processed. - - **If ever pursued (low priority, modest impact):** reliable detection needs a **dictionary/ - spellcheck approach** ("fraction of tokens that aren't valid English/clinical words") or fixing - the **upstream table-OCR** step — not a token heuristic. Neither is warranted by the impact. - Remaining true enrichment items: confirm `20260627000000_retrieval_hnsw_ef_search.sql` on live; run - `enrich:backfill` / `tags:backfill` for any genuinely missing synopsis/labels. + - **Current disposition (2026-07-27):** the broad low-`s` and token-fragment heuristics remain + rejected. Item 24's dictionary/reference-backed repair now fixes only whitespace-fragmented OCR + words aligned to clean same-page source text and directly covers `p ycho ocial`. The implementation + is complete; a provider-backed backfill and live warning/repair-rate measurement were not run. + Other enrichment work remains separate: confirm `20260627000000_retrieval_hnsw_ef_search.sql` on + live; run `enrich:backfill` / `tags:backfill` only for genuinely missing synopsis/labels. 10. 🔧 **Query understanding (RC6/E) — pg_trgm typo correction started (2026-07-01).** - **Data-driven promotion is blocked:** `rag_query_misses` (71 rows) are privacy-redacted hashes with empty `candidate_aliases`, so the plan's "promote real misses to aliases" path can't run. @@ -247,8 +248,9 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows ## P2 — naturalness residual -15. ⏳ One flattened-table run-on still slips through (TPR / postural-BP line). Mostly handled by v15 + - `separateSettingRunOns`; extend the deterministic separator or the prompt if it recurs. +15. ✅ **Flattened TPR / postural-BP run-on — FIXED.** `polishClinicalAnswerProse` now separates the + inpatient/community clauses deterministically with and without the comma form. Focused coverage + in `tests/answer-prose-runons.test.ts` pins the original line and guards normal prose. ## Security (do outside this repo) @@ -331,18 +333,21 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows that non-blocking performance debt remains tracked in item 25. Current productization boundary: there are no mutating registry edit routes today, so the re-embed-on-edit helper is present but not wired to a route; any future registry `POST`/`PATCH`/`PUT` path must call `bestEffortReembedRegistryRecordAfterEdit` after the - write commits. Remaining non-blocking UX follow-up: an answer-mode check that registry-backed - citations render as curated registry records rather than primary source documents. + write commits. The former citation UX follow-up is complete: `documentCitationHref` routes + registry-backed citations to their native service/differential detail pages, with focused tests + in `tests/citations.test.ts`. 23. ✅ **Finding #11 full fix — CLOSED.** The corpus-grounded relevance implementation described in item 10 (2026-07-07) superseded the earlier classifier-memo-only state. In-corpus bare topics and corpus-absent invented terms now follow deterministic corpus evidence; no duplicate Phase-2 task remains. Item 17's broader alias-promotion privacy/design work remains separate. -24. ⏳ **OCR dropped-letter corruption in table index units** — no reliable detector exists (82% - false positives; guard reverted). Next viable angle: dictionary-based repair at INGESTION - (compare table-cell tokens against the document's own clean chunk text — "p ycho ocial" - aligns to "psychosocial" within the same page's raw text) rather than heuristic detection at - query time. Scope to `worker/` table extraction; requires the Python OCR stack to test. -25. 🔶 **Retrieval/RAG latency — serial-depth fixes landed locally; live tail isolated to DB RPCs.** +24. ✅ **OCR dropped-letter corruption in table index units — bounded repair landed.** The broad + low-`s` heuristic remains rejected, but `repairOcrDropoutAgainstReference` now repairs only + whitespace-fragmented OCR words that align to the same page's clean source-chunk text and stamps + `ocr_repair_version=clean-chunk-fragment-v1`. `tests/document-index-units.test.ts` covers + `p ycho ocial` → `psychosocial` plus the single-letter clinical-token false-positive guard. + No broad provider-backed backfill was performed during this reconciliation. +25. 🔶 **Retrieval/RAG latency — serial-depth fixes landed; targeted table-facts DB plans are now + acceptable, while the broader end-to-end tail remains.** Post-registry retrieval stayed quality-clean (`top_k_hit_rate=1`, `document_recall_at_5=1`, `content_recall_at_5=1`, `force_embedding_failure_count=0`) with local `p90_latency_ms=13145`. Post-routing RAG-only passed with `p95_latency_ms=20385` and no blocking threshold failures, but @@ -359,9 +364,33 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows material temporary I/O. Forced-embedding/model-routing evaluation stopped after the provider returned quota exhaustion; no model or production configuration was changed. - **Next smallest performance work:** capture `EXPLAIN (ANALYZE, BUFFERS)` for the slow RPC shapes - with non-sensitive fixtures, optimise those plans, and then compare Micro versus Small primary - compute if the plans remain memory-bound. Do not create a Singapore read replica yet: replicas - require at least Small compute and would duplicate the same slow plans while adding asynchronous - freshness/read-routing work. Reconsider only after database execution is fast enough that network - RTT is again a material share. + **Superseded next step:** do not optimise the table-facts plan, alter ranking, or scale compute from + the earlier broad tail alone. The phase-aware profile below shows bounded database execution and a + visible client/edge component. If the broad end-to-end tail persists, profile first-unprimed versus + warm orchestration phases and remaining serial RPC depth separately; only reconsider compute after + a repeatable database-plan bottleneck is demonstrated. + + **Targeted hosted profile (2026-07-27):** the first exact clinical-query pass recorded warm + PostgreSQL median/p90 execution of 43.045/43.757 ms for clozapine, 75.489/81.738 ms for lithium, + and 59.243/60.309 ms for metabolic monitoring; corresponding client median/p90 was + 210.193/217.388, 226.029/253.273, and 204.702/213.108 ms. A subsequent six-sample phase-aware pass + labelled sample 1 `first_unprimed` and samples 2–6 `warm_repeat`: first-unprimed client/DB execution + was 662.916/141.537, 277.661/96.229, and 322.598/148.378 ms respectively; warm DB median/p90 was + 88.292/89.355, 64.342/65.566, and 107.448/148.417 ms, while warm client median/p90 was + 187.029/198.907, 167.803/174.391, and 189.065/243.324 ms. Managed Supabase buffers were not flushed, + so this deliberately does not claim a true cold-start measurement. The table-facts RPC is not the + multi-second tail and is closed without hosted migration or ranking changes; broader latency remains + an orchestration/RTT observation to diagnose independently. + +26. ✅ **Fallback answer quality and measurement — CLOSED (2026-07-27).** The earlier 12/30 and + 3/44 review-fallback inventories are superseded. Evaluation now separates + `source_backed_review_fallback` from substantive grounded answers and does not award targeting + credit to echoed query boilerplate. Narrow source-bound paths cover the reproduced + admission/discharge, active-community ED, community-home-visit, clozapine threshold/typo and Best + Practice Prescription shapes. A cited provider source-gap is no longer promoted to grounded merely + because its citation IDs are valid; it retries or enters source-backed recovery, while terminal gaps + clear claim citations and retain an auditable reason. The final live 44-case gate recorded 30/30 + substantive grounded supported answers, 14/14 unsupported correct, zero review fallbacks, zero + citation or numeric failures, zero route-ceiling failures, and p95 7,494 ms. The protected 36-case + retrieval canary stayed at document/content recall 1.0 with no per-case reciprocal-rank regression. + See `docs/evidence/rag-reliability-evidence-2026-07-27.md`. diff --git a/docs/source-governance-refresh-worklist-2026-07-22.md b/docs/source-governance-refresh-worklist-2026-07-22.md index 811c803216..d86c1af79c 100644 --- a/docs/source-governance-refresh-worklist-2026-07-22.md +++ b/docs/source-governance-refresh-worklist-2026-07-22.md @@ -1,8 +1,31 @@ # Source-governance refresh worklist — 2026-07-22 -Successor to [`source-review-priority-2026-07-02.md`](source-review-priority-2026-07-02.md), regenerated -from live canary artifacts. Ledger item: **#022**. Produced read-only at **$0** — no provider calls, no -live queries; everything below is derived from the Eval Canary artifacts for runs **#61** and **#57**. +Successor to [`source-review-priority-2026-07-02.md`](source-review-priority-2026-07-02.md). Ledger item: +**#022**. The historical worklist below was derived read-only from Eval Canary artifacts for runs **#61** and +**#57**. The 2026-07-27 implementation-status section and tracked ten-document manifest use the retained +`golden-retrieval.json` artifact from GitHub run **30216191889**. No new provider run or live application query was +dispatched for either analysis. + +## 2026-07-27 policy implementation status + +The selected BMJ policy is now represented locally as +`third_party_reference_attested` with exact policy version +`bmj-third-party-reference-attestation-v1`. It is deliberately not a local clinical validation: + +- only an administrator may submit it through the source-review route; +- the source must be indexed and carry compatible, recognized BMJ publisher metadata; +- evidence references, a non-future attestation date, the exact policy version, and reviewer qualification are + required in both TypeScript and SQL; +- the append-only event records the evidence while `clinical_validation_status=unverified`, the document currentness, + the document review date, and the user-facing **Not locally validated** caveat remain unchanged; +- raw warning/unverified telemetry remains unchanged. A separately named operational measure excludes only complete, + current, structured BMJ attestations; malformed, old-policy, non-BMJ, and review-due records remain debt. + +The forward migration is prepared but has **not** been applied to hosted Supabase, and no source has been attested. +The current single-run local review evidence pack is +[`rag-top-local-review-manifest-2026-07-26.json`](evidence/rag-top-local-review-manifest-2026-07-26.json). Its ten +entries remain `pending_qualified_human_review` with `attestation_applied=false`; it is evidence preparation, not a +clinical decision. ## What the two governance numbers actually mean @@ -17,10 +40,11 @@ Policy (verbatim from the canary log): _"unknown, unverified, review_due, outdat and poor extraction metadata are treated as review-required; do not silently default them to current or approved."_ -> **Reporting gap worth noting:** run #61's own `## Source Governance` table reports `Top results | 0` and -> all-zero rates, even though the underlying `topResults` records carry full governance metadata. The -> operator-facing table in the log is therefore **not** populated — this worklist had to be derived from -> the raw JSON. Worth fixing so the canary log surfaces this directly. +> **Reporting gap resolved (2026-07-27):** the answer-quality step now consumes the preceding +> `golden-retrieval.json` through `--source-governance-results`, so its operator-facing +> `## Source Governance` table is populated from the same raw `topResults` metadata. Offline replay +> populated 338 top results, including 202 review-required entries, without adding retrieval cases or +> changing ranking. ## The reframing: this is not 59 document reviews @@ -85,15 +109,20 @@ of these slots in one action. ## Suggested order of work -1. **Decide the BMJ attestation policy** (clears ~64% of review-required documents in one action) — any chosen option must be an explicit, auditable metadata update that never silently defaults to `current`/`approved` and preserves the third-party/unverified distinction; this remains open debt (ledger #022) until the approved policy is implemented, not merely decided. +1. **Apply and use the decided BMJ attestation policy deliberately** — the local + `third_party_reference_attested` policy, schema, API validation, migration, and evidence manifest now + exist. A qualified reviewer must assess the evidence, an operator must deliberately apply the hosted + migration, and only eligible sources may then be attested. Never silently default a source to + `current`/`approved`; the third-party/unverified distinction remains visible. 2. **Attest the local documents by visibility** — start with `Clozapine Management by GP (NMHS)` (22 slots, rank 1), then the FSH addiction/withdrawal document, then the remaining NMHS/AKG/CAMHS/KEMH items. 3. **Re-read the warning rate** on the next canary to confirm the burn-down. ## Scope and limits -- This is **operator work** (document review-status attestation in the app), not a code defect. No code - change is proposed here. +- The policy/tooling code and forward migration are implemented locally. Hosted application, qualified + evidence review, source attestation, local-document clinical review, and warning-rate remeasurement + remain **operator/clinical-governance work**; no hosted migration or source attestation was performed. - The list reflects **only documents that surfaced in golden-case top results**, so it is a visibility-weighted worklist, not a full corpus audit. The 2026-07-02 predecessor notes the live corpus is far larger (~2,065 indexed documents at that time). diff --git a/scripts/audit-source-governance.ts b/scripts/audit-source-governance.ts index 93f2a011ba..7cbb517315 100644 --- a/scripts/audit-source-governance.ts +++ b/scripts/audit-source-governance.ts @@ -3,6 +3,11 @@ import { pathToFileURL } from "node:url"; import * as nextEnv from "@next/env"; import { auditSourceAuthorityDocuments, isRegistryRecordSource } from "@/lib/source-authority-metadata"; +import { + BMJ_THIRD_PARTY_ATTESTATION_BASIS, + countOperationalUnattestedReviewDebt, + type BmjThirdPartyAttestationEvent, +} from "@/lib/source-review"; import type { DocumentLabel } from "@/lib/types"; const loadEnvConfig = @@ -128,7 +133,7 @@ function usage() { async function fetchAll( supabase: SupabaseAdmin, - table: "documents" | "document_labels", + table: "documents" | "document_labels" | "source_review_events", select: string, filter?: (query: QueryBuilder) => QueryBuilder, ) { @@ -269,6 +274,19 @@ export async function main(argv = process.argv.slice(2)) { ); const clinicalGovernanceDocuments = documents.filter((document) => !isRegistryRecordSource(document)); const registryRecordsExcludedFromClinicalMetadataGate = documents.length - clinicalGovernanceDocuments.length; + const clinicalGovernanceDocumentIds = new Set(clinicalGovernanceDocuments.map((document) => document.id)); + // source_review_events is service-role-only. This global operator audit uses + // the existing admin client, reads only the fields needed to bind an + // attestation, scopes them back to audited documents, and never emits event + // evidence in either the JSON report or console output. + const bmjAttestationEvents = ( + await fetchAll( + supabase, + "source_review_events", + "id,document_id,reviewer_id,decision,evidence_references,new_document_status,new_validation_status,review_date,policy_version,reviewer_qualification,created_at", + (query) => query.eq("decision", BMJ_THIRD_PARTY_ATTESTATION_BASIS), + ) + ).filter((event) => clinicalGovernanceDocumentIds.has(event.document_id)); const statusCounts = new Map(); const validationCounts = new Map(); @@ -309,6 +327,17 @@ export async function main(argv = process.argv.slice(2)) { partial_extraction: extractionCounts.get("partial") ?? 0, missing_smart_v2_labels: missingSmartV2LabelDocuments.length, }; + // Raw validation telemetry above is intentionally unchanged. This second, + // explicitly operational measure removes only complete, current, structured + // BMJ attestations from the work queue; review-due or malformed attestations + // remain debt and the source remains clinically unverified. + const operationalDebtCounts = countOperationalUnattestedReviewDebt( + clinicalGovernanceDocuments.map((document) => ({ + documentId: document.id, + metadata: metadataRecord(document.metadata), + })), + bmjAttestationEvents, + ); const debtPolicyFailures: string[] = []; if (debtPolicy) { @@ -389,6 +418,7 @@ export async function main(argv = process.argv.slice(2)) { indexed_without_smart_v2_labels: missingSmartV2LabelDocuments.length, }, debt_counts: debtCounts, + operational_review_debt_counts: operationalDebtCounts, sample_review_due_documents: clinicalGovernanceDocuments .filter((document) => metadataRecord(document.metadata).document_status === "review_due") .slice(0, 10) @@ -445,6 +475,9 @@ export async function main(argv = process.argv.slice(2)) { .map(([value, count]) => `${value}=${count}`) .join(", ")}`, ); + console.log( + `Operational unattested review debt: ${report.operational_review_debt_counts.unattested_review_debt} (complete current BMJ attestations: ${report.operational_review_debt_counts.complete_bmj_third_party_attestations}; raw unverified: ${report.operational_review_debt_counts.raw_unverified_validation})`, + ); console.log( `Extraction quality: ${Object.entries(report.extraction_quality_counts) .map(([value, count]) => `${value}=${count}`) diff --git a/scripts/build-clinical-review-queue.ts b/scripts/build-clinical-review-queue.ts index cfa91d52eb..a1f7dff8c8 100644 --- a/scripts/build-clinical-review-queue.ts +++ b/scripts/build-clinical-review-queue.ts @@ -1,22 +1,27 @@ import { readFile, writeFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; -import { buildClinicalReviewQueue } from "@/lib/clinical-review-queue"; +import { buildClinicalReviewQueue, buildTopLocalReviewEvidenceManifest } from "@/lib/clinical-review-queue"; -type Args = { input?: string; output?: string; help: boolean }; +type Args = { input?: string; output?: string; sourceArtifact?: string; topLocalManifest: boolean; help: boolean }; function parseArgs(argv: string[]): Args { - const args: Args = { help: false }; + const args: Args = { help: false, topLocalManifest: false }; for (let index = 0; index < argv.length; index += 1) { const token = argv[index]; if (token === "--help" || token === "-h") { args.help = true; continue; } + if (token === "--top-local-manifest") { + args.topLocalManifest = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); if (token === "--input") args.input = value; else if (token === "--output") args.output = value; + else if (token === "--source-artifact") args.sourceArtifact = value; else throw new Error(`Unknown option: ${token}`); index += 1; } @@ -25,9 +30,10 @@ function parseArgs(argv: string[]): Args { function usage() { return [ - "Usage: node scripts/run-tsx.mjs scripts/build-clinical-review-queue.ts --input [--output ]", + "Usage: node scripts/run-tsx.mjs scripts/build-clinical-review-queue.ts --input [--output ] [--top-local-manifest] [--source-artifact