From d27504a6c41039a2b7db7e6aea2096a9af98bf1f Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:22:51 +0800 Subject: [PATCH 1/3] Land codex RAG_FIX WIP: soften metadata weighting, frontend-visible governance warnings, tags/labels hardening Recovered verbatim from the main checkout stash 'codex/RAG_FIX uncommitted work before switch to main (2026-07-02)', base 3eeb9b6bd. - Reduce metadata/governance boost magnitudes in retrieval-selection resultBoost, answer-ranking sourceQualityScore, and rag secondStageScore so document status/validation/extraction metadata nudges rather than dominates ordering; drop review_due and unverified penalties. - Add frontend-visible source-governance warning filtering (isFrontendVisibleSourceGovernanceWarning / frontendSourceGovernanceWarnings) and surface it in ClinicalDashboard. - Harden document tags/organization: expanded tag derivation rules, organization badge fixes, label coverage checker improvements, and backfill/classify script updates; extend matching tests. - Add scripts/audit-source-governance.ts and the audit:source-governance npm script; update retrieval-quality runbook and metadata debt snapshot. - Bump next build heap to 16 GB. Excluded deliberately: mockups/best/ screenshot set (20 MB of PNGs; repo tracks no images) and its README reference, docs/backup-pr-111.md (machine-local restore metadata), docs/codex-prompt-playbook.md (unrelated doc). All remain preserved in the main checkout / stash. Co-Authored-By: Claude Fable 5 --- ...lease-source-metadata-debt-2026-06-30.json | 31 +- docs/retrieval-quality-runbook.md | 9 +- package.json | 3 +- scripts/audit-source-governance.ts | 285 ++++++++++++++++++ scripts/backfill-source-metadata.ts | 40 ++- scripts/check-document-label-coverage.ts | 55 +++- scripts/classify-documents.ts | 5 + scripts/eval-quality.ts | 14 +- src/components/ClinicalDashboard.tsx | 19 +- src/components/DocumentOrganizationBadges.tsx | 15 +- src/components/DocumentTagCloud.tsx | 5 +- src/lib/answer-ranking.ts | 12 +- src/lib/clinical-search.ts | 54 ++-- src/lib/document-organization.ts | 35 ++- src/lib/document-tags.ts | 164 ++++++++-- src/lib/rag.ts | 2 +- src/lib/retrieval-selection.ts | 20 +- src/lib/source-governance.ts | 14 + tests/document-organization.test.ts | 34 +++ tests/document-tags.test.ts | 23 +- tests/eval-quality.test.ts | 8 +- tests/rag-answer-fallback.test.ts | 4 +- tests/retrieval-selection.test.ts | 75 ++++- tests/source-governance.test.ts | 60 ++++ 24 files changed, 836 insertions(+), 150 deletions(-) create mode 100644 scripts/audit-source-governance.ts diff --git a/docs/release-source-metadata-debt-2026-06-30.json b/docs/release-source-metadata-debt-2026-06-30.json index 2e88122db8..01028412fe 100644 --- a/docs/release-source-metadata-debt-2026-06-30.json +++ b/docs/release-source-metadata-debt-2026-06-30.json @@ -1,20 +1,39 @@ { "accepted": true, "accepted_by": "repository owner", - "accepted_at": "2026-06-30T00:00:00.000+08:00", + "accepted_at": "2026-07-02T00:00:00.000+08:00", "expires_at": "2026-07-31T23:59:59.000+08:00", - "reason": "Temporary release metadata debt acceptance for the existing eval corpus while document-level source status and local validation metadata are backfilled. This does not mark any source current, locally reviewed, or approved.", + "reason": "Bounded release metadata debt acceptance for residual corpus-state governance values after required source metadata key coverage was completed. This accepts review_due, unknown, and unverified states as tracked backend debt only; it does not mark any source current, locally reviewed, or approved.", "scope": "eval-quality retrieval source metadata thresholds only", - "source_report": "output/evals/eval-quality-final.json", + "source_report": "output/evals/retrieval-quality-2026-07-02T04-58-43-365Z.json", + "observed_live_corpus": { + "observed_at": "2026-07-02T00:00:00.000+08:00", + "indexed_documents": 2065, + "missing_required_source_metadata": 0, + "review_due_documents": 481, + "unknown_status_documents": 132, + "unverified_documents": 130, + "poor_extraction_documents": 0, + "missing_smart_v2_labels": 0 + }, + "observed_retrieval_eval": { + "stale_top_results": 0, + "review_due_top_results": 5, + "unverified_top_results": 6, + "review_required_top_results": 7, + "stale_rate": 0, + "review_required_rate": 0.14 + }, "required_follow_up": [ - "Review high-frequency golden top-result documents first.", + "Review high-frequency review_due, unknown, and unverified documents first.", + "Keep required source metadata key coverage at zero missing fields.", "Backfill documents.metadata.document_status only after confirming source currentness.", "Backfill documents.metadata.clinical_validation_status only after local clinical review.", "Replace this debt file with stricter ceilings or remove it before expiry." ], "ceilings": { - "max_stale_rate": 1, - "max_review_required_rate": 1, + "max_stale_rate": 0, + "max_review_required_rate": 0.2, "max_outdated_top_results": 0, "max_poor_extraction_top_results": 0, "max_source_governance_danger_failure_rate": 0 diff --git a/docs/retrieval-quality-runbook.md b/docs/retrieval-quality-runbook.md index 3367a63843..a02dd0cf6c 100644 --- a/docs/retrieval-quality-runbook.md +++ b/docs/retrieval-quality-runbook.md @@ -85,14 +85,15 @@ Source governance: - unverified top-result count - unknown-extraction top-result count - poor-extraction top-result count -- combined stale/review/unknown top-result rate +- stale/outdated top-result rate +- combined stale/review/unknown top-result audit rate - review-required top-result count and rate Metadata policy: - `unknown`, `unverified`, `review_due`, `outdated`, unknown extraction, and poor extraction are treated as review-required. - Do not silently default missing corpus metadata to `current` or `approved`. -- Reduce the warning rate by backfilling source metadata through ingestion/enrichment or by explicitly accepting the review-required baseline in a versioned release metadata debt file. +- Reduce the review-required rate by backfilling source metadata through ingestion/enrichment or by explicitly accepting a bounded review-required baseline in a versioned release metadata debt file. - Danger-class source governance warnings are blocking. - Warning-class retrieval source metadata notes may be accepted only by passing `--source-metadata-debt ` to `npm run eval:quality -- --fail-on-threshold`. - Source metadata debt acceptance does not mark sources current or approved. It only removes the accepted retrieval metadata threshold failures from the blocking failure list. @@ -118,7 +119,7 @@ Answer quality: - retrieval hit@K is below `0.8` - document recall@5 is below `0.8` - content recall@5 is below `0.8` -- stale/review/unknown top-result rate is above `0.25` +- stale/outdated top-result rate is above `0.25` - review-required top-result rate is above `0.25` - grounded supported answer rate is below `0.9` - unsupported-answer correctness is below `1.0` @@ -153,4 +154,4 @@ Run full quality evals after: - clinical output changes - release or handoff confidence checks -`npm run verify:release` includes `npm run eval:quality:release` after cheaper local gates. `eval:quality:release` passes `docs/release-source-metadata-debt-2026-06-30.json` while that temporary release debt is active. Use focused variants such as `--retrieval-only`, `--rag-only`, `--limit`, `--query`, or `--question` during development to avoid unnecessary provider-backed cost. +`npm run verify:release` includes `npm run eval:quality:release` after cheaper local gates. `eval:quality:release` passes `docs/release-source-metadata-debt-2026-06-30.json` while that bounded release debt is active. Use `npm run audit:source-governance` to refresh the live corpus debt counts, and use focused variants such as `--retrieval-only`, `--rag-only`, `--limit`, `--query`, or `--question` during development to avoid unnecessary provider-backed cost. diff --git a/package.json b/package.json index b8a99eaecb..b935173156 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "dev": "node scripts/dev-free-port.mjs", "preinstall": "node scripts/check-node-engine.cjs", "ensure": "node scripts/ensure-local-server.mjs", - "build": "node scripts/guard-next-build.mjs && node --max-old-space-size=8192 ./node_modules/next/dist/bin/next build --webpack", + "build": "node scripts/guard-next-build.mjs && node --max-old-space-size=16384 ./node_modules/next/dist/bin/next build --webpack", "start": "node scripts/dev-free-port.mjs start", "lint": "node --max-old-space-size=8192 ./node_modules/eslint/bin/eslint.js src tests scripts worker supabase playwright eslint.config.mjs next.config.ts playwright.config.ts playwright.visual.config.ts vitest.config.mts --no-error-on-unmatched-pattern", "typecheck": "node ./node_modules/typescript/bin/tsc --noEmit", @@ -41,6 +41,7 @@ "enrich:backfill": "tsx scripts/backfill-enrichment.ts", "classify:documents": "tsx scripts/classify-documents.ts", "check:document-label-coverage": "tsx scripts/check-document-label-coverage.ts", + "audit:source-governance": "tsx scripts/audit-source-governance.ts", "tags:backfill": "tsx scripts/backfill-document-tags.ts", "index:backfill": "tsx scripts/backfill-smart-index.ts", "visual:backfill": "tsx scripts/backfill-visual-intelligence.ts", diff --git a/scripts/audit-source-governance.ts b/scripts/audit-source-governance.ts new file mode 100644 index 0000000000..65e1598bb4 --- /dev/null +++ b/scripts/audit-source-governance.ts @@ -0,0 +1,285 @@ +import * as nextEnv from "@next/env"; +import type { DocumentLabel } from "@/lib/types"; + +const loadEnvConfig = + nextEnv.loadEnvConfig ?? + (nextEnv as unknown as { default?: { loadEnvConfig?: typeof nextEnv.loadEnvConfig } }).default?.loadEnvConfig; + +if (!loadEnvConfig) throw new Error("Unable to load @next/env loadEnvConfig."); +loadEnvConfig(process.cwd()); + +type AuditArgs = { + json: boolean; + help: boolean; +}; + +type SupabaseAdmin = Awaited>; + +type DocumentRow = { + id: string; + title: string; + file_name: string; + status: string; + metadata: Record | null; +}; + +type LabelRow = { + id: string; + document_id: string; + label_type: DocumentLabel["label_type"]; + source: DocumentLabel["source"]; +}; + +type QueryResult = { + data: T[] | null; + error: { message: string } | null; +}; + +type QueryBuilder = PromiseLike> & { + eq(column: string, value: unknown): QueryBuilder; + order(column: string, options: { ascending: boolean }): QueryBuilder; + range(from: number, to: number): QueryBuilder; +}; + +const requiredMetadataKeys = [ + "document_status", + "clinical_validation_status", + "clinical_validation_evidence", + "extraction_quality", +] as const; + +const smartV2LabelTypes = new Set([ + "clinical_action", + "care_phase", + "document_intent", + "content_feature", +]); + +async function loadAdminClient() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + return createAdminClient(); +} + +function parseArgs(argv: string[]): AuditArgs { + const args: AuditArgs = { json: false, help: false }; + for (const token of argv) { + if (token === "--json") { + args.json = true; + continue; + } + if (token === "--help" || token === "-h") { + args.help = true; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + return args; +} + +function usage() { + return [ + "Usage: npm run audit:source-governance -- [options]", + "", + "Read-only audit of source governance metadata and smart-v2 label debt.", + "", + "Options:", + " --json Print machine-readable JSON.", + " --help Show this help.", + ].join("\n"); +} + +async function fetchAll( + supabase: SupabaseAdmin, + table: "documents" | "document_labels", + select: string, + filter?: (query: QueryBuilder) => QueryBuilder, +) { + const rows: T[] = []; + const pageSize = 1000; + + for (let from = 0; ; from += pageSize) { + let query = supabase + .from(table) + .select(select) + .order("id", { ascending: true }) + .range(from, from + pageSize - 1) as unknown as QueryBuilder; + if (filter) query = filter(query); + const { data, error } = await query; + if (error) throw new Error(error.message); + rows.push(...(data ?? [])); + if (!data || data.length < pageSize) break; + } + + return rows; +} + +function metadataRecord(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function stringValue(value: unknown) { + return typeof value === "string" && value.trim() ? value.trim() : "missing"; +} + +function increment(counts: Map, key: string) { + counts.set(key, (counts.get(key) ?? 0) + 1); +} + +function sortedCounts(counts: Map) { + return Object.fromEntries([...counts.entries()].sort(([left], [right]) => left.localeCompare(right))); +} + +function compactDocument(document: DocumentRow) { + const metadata = metadataRecord(document.metadata); + return { + id: document.id, + title: document.title, + file_name: document.file_name, + document_status: stringValue(metadata.document_status), + clinical_validation_status: stringValue(metadata.clinical_validation_status), + extraction_quality: stringValue(metadata.extraction_quality), + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const supabase = await loadAdminClient(); + const documents = await fetchAll(supabase, "documents", "id,title,file_name,status,metadata", (query) => + query.eq("status", "indexed"), + ); + const labels = await fetchAll(supabase, "document_labels", "id,document_id,label_type,source", (query) => + query.eq("source", "generated"), + ); + + const statusCounts = new Map(); + const validationCounts = new Map(); + const extractionCounts = new Map(); + const requiredMissingCounts = new Map(); + const missingRequiredDocuments: Array & { missing_keys: string[] }> = []; + + for (const document of documents) { + const metadata = metadataRecord(document.metadata); + increment(statusCounts, stringValue(metadata.document_status)); + increment(validationCounts, stringValue(metadata.clinical_validation_status)); + increment(extractionCounts, stringValue(metadata.extraction_quality)); + + const missingKeys = requiredMetadataKeys.filter((key) => { + const value = metadata[key]; + return value === undefined || value === null || value === ""; + }); + for (const key of missingKeys) increment(requiredMissingCounts, key); + if (missingKeys.length > 0) { + missingRequiredDocuments.push({ ...compactDocument(document), missing_keys: missingKeys }); + } + } + + const indexedDocumentIds = new Set(documents.map((document) => document.id)); + const generatedLabelDocumentIds = new Set(labels.map((label) => label.document_id)); + const smartV2DocumentIds = new Set( + labels.filter((label) => smartV2LabelTypes.has(label.label_type)).map((label) => label.document_id), + ); + const missingGeneratedLabelDocuments = documents.filter((document) => !generatedLabelDocumentIds.has(document.id)); + const missingSmartV2LabelDocuments = documents.filter((document) => !smartV2DocumentIds.has(document.id)); + const requiredMetadataMissingTotal = [...requiredMissingCounts.values()].reduce((total, count) => total + count, 0); + + const report = { + mode: "read-only", + indexed_documents: documents.length, + required_metadata_missing_total: requiredMetadataMissingTotal, + required_metadata_missing_counts: Object.fromEntries( + requiredMetadataKeys.map((key) => [key, requiredMissingCounts.get(key) ?? 0]), + ), + document_status_counts: sortedCounts(statusCounts), + clinical_validation_status_counts: sortedCounts(validationCounts), + extraction_quality_counts: sortedCounts(extractionCounts), + generated_label_coverage: { + documents_with_generated_labels: generatedLabelDocumentIds.size, + indexed_without_generated_labels: missingGeneratedLabelDocuments.length, + }, + smart_v2_label_coverage: { + documents_with_smart_v2_labels: smartV2DocumentIds.size, + indexed_without_smart_v2_labels: missingSmartV2LabelDocuments.length, + }, + debt_counts: { + review_due: statusCounts.get("review_due") ?? 0, + unknown_status: statusCounts.get("unknown") ?? 0, + unverified_validation: validationCounts.get("unverified") ?? 0, + poor_extraction: extractionCounts.get("poor") ?? 0, + partial_extraction: extractionCounts.get("partial") ?? 0, + missing_smart_v2_labels: missingSmartV2LabelDocuments.length, + }, + sample_review_due_documents: documents + .filter((document) => metadataRecord(document.metadata).document_status === "review_due") + .slice(0, 10) + .map(compactDocument), + sample_unknown_status_documents: documents + .filter((document) => metadataRecord(document.metadata).document_status === "unknown") + .slice(0, 10) + .map(compactDocument), + sample_unverified_documents: documents + .filter((document) => metadataRecord(document.metadata).clinical_validation_status === "unverified") + .slice(0, 10) + .map(compactDocument), + missing_required_metadata_documents: missingRequiredDocuments.slice(0, 25), + missing_smart_v2_label_documents: missingSmartV2LabelDocuments.map((document) => ({ + id: document.id, + title: document.title, + file_name: document.file_name, + })), + indexed_document_id_count: indexedDocumentIds.size, + passed_required_metadata_gate: requiredMetadataMissingTotal === 0, + }; + + if (args.json) { + console.log(JSON.stringify(report, null, 2)); + } else { + console.log("[Source Governance Audit]"); + console.log(`Mode: ${report.mode}`); + console.log(`Indexed documents: ${report.indexed_documents}`); + console.log(`Required metadata missing: ${report.required_metadata_missing_total}`); + console.log( + `Document status: ${Object.entries(report.document_status_counts) + .map(([value, count]) => `${value}=${count}`) + .join(", ")}`, + ); + console.log( + `Clinical validation: ${Object.entries(report.clinical_validation_status_counts) + .map(([value, count]) => `${value}=${count}`) + .join(", ")}`, + ); + console.log( + `Extraction quality: ${Object.entries(report.extraction_quality_counts) + .map(([value, count]) => `${value}=${count}`) + .join(", ")}`, + ); + console.log( + `Generated labels: missing=${report.generated_label_coverage.indexed_without_generated_labels}, covered=${report.generated_label_coverage.documents_with_generated_labels}`, + ); + console.log( + `Smart-v2 labels: missing=${report.smart_v2_label_coverage.indexed_without_smart_v2_labels}, covered=${report.smart_v2_label_coverage.documents_with_smart_v2_labels}`, + ); + if (report.missing_smart_v2_label_documents.length) { + console.log("Documents missing smart-v2 labels:"); + for (const document of report.missing_smart_v2_label_documents) { + console.log(`- ${document.title} (${document.file_name})`); + } + } + console.log( + report.passed_required_metadata_gate + ? "PASS: required source governance metadata is complete." + : "FAIL: required source governance metadata has gaps.", + ); + } + + if (!report.passed_required_metadata_gate) process.exitCode = 1; +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/scripts/backfill-source-metadata.ts b/scripts/backfill-source-metadata.ts index 8f6a13c31b..171438b2b5 100644 --- a/scripts/backfill-source-metadata.ts +++ b/scripts/backfill-source-metadata.ts @@ -38,9 +38,20 @@ type ClinicalValidationEvidence = { const APPLY = process.argv.includes("--apply"); const EVAL_ONLY = process.argv.includes("--eval-only"); -const NOW = new Date("2026-06-30T00:00:00+08:00"); const BACKFILL_VERSION = "source_metadata_backfill_2026_06_30_v1"; +function backfillAsOfDate() { + const index = process.argv.indexOf("--as-of"); + if (index < 0) return new Date(); + const raw = process.argv[index + 1]; + if (!raw || raw.startsWith("--")) throw new Error("Missing value for --as-of"); + const date = /^\d{4}-\d{2}-\d{2}$/.test(raw) ? new Date(`${raw}T00:00:00+08:00`) : new Date(raw); + if (Number.isNaN(date.getTime())) throw new Error(`Invalid --as-of date: ${raw}`); + return date; +} + +const NOW = backfillAsOfDate(); + const publisherByCode: Record = { AKG: { publisher: "Armadale Kalamunda Group", jurisdiction: "Australia/WA" }, BMJ: { publisher: "BMJ Best Practice", jurisdiction: "International" }, @@ -87,7 +98,8 @@ function publisherCodeFor(document: DocumentRow, text = "") { function sourceTypeFor(document: DocumentRow, text: string) { const haystack = `${document.title} ${document.file_name} ${text.slice(0, 1500)}`.toLowerCase(); - if (haystack.includes("standard operational procedure") || /\bsop\b/.test(haystack)) return "standard_operating_procedure"; + if (haystack.includes("standard operational procedure") || /\bsop\b/.test(haystack)) + return "standard_operating_procedure"; if (haystack.includes("policy and procedure")) return "policy_procedure"; if (/\bprocedure\b/.test(haystack)) return "procedure"; if (/\bpolicy\b/.test(haystack)) return "policy"; @@ -265,7 +277,9 @@ function firstMatchDate(text: string, labels: string[], endOfMonth: boolean) { function standaloneReviewDate(text: string) { const datePattern = "([0-3]?\\d[/-][01]?\\d[/-]20\\d{2}|[01]?\\d[/-]20\\d{2}|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sept?(?:ember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\s+[0-3]?\\d,?\\s+20\\d{2}|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sept?(?:ember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\s+20\\d{2}|[0-3]?\\d\\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sept?(?:ember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\s+20\\d{2})"; - const reviewedThenReview = text.match(new RegExp(`\\bReviewed\\s+${datePattern}[\\s\\S]{0,100}?\\bReview\\s+${datePattern}`, "i")); + const reviewedThenReview = text.match( + new RegExp(`\\bReviewed\\s+${datePattern}[\\s\\S]{0,100}?\\bReview\\s+${datePattern}`, "i"), + ); if (reviewedThenReview?.[2]) { const parsed = parseClinicalDate(reviewedThenReview[2], { endOfMonth: true }); if (parsed) return { date: parsed, raw: normalizeWhitespace(reviewedThenReview[0]) }; @@ -281,11 +295,9 @@ function standaloneReviewDate(text: string) { } function extractDates(text: string) { - const review = firstMatchDate( - text, - ["Review Due", "Revision Due", "Revision Date", "Review Date", "Next Review"], - true, - ) ?? standaloneReviewDate(text); + const review = + firstMatchDate(text, ["Review Due", "Revision Due", "Revision Date", "Review Date", "Next Review"], true) ?? + standaloneReviewDate(text); const publication = firstMatchDate( text, @@ -305,8 +317,7 @@ function extractDates(text: string) { "Endorsed", ], false, - ) ?? - null; + ) ?? null; const lastUpdated = firstMatchDate(text, ["Last updated", "Updated"], false); const reviewCycle = /\b(?:reviewed|evaluated)[^.]{0,120}\bat least every three\s*(?:\(\s*3\s*\)|3)?\s*years?\b/i.test(text) || @@ -423,7 +434,11 @@ function extractionQualityFor(quality: QualityRow | undefined, existing: string) const score = typeof quality?.quality_score === "number" ? quality.quality_score : null; const issues = Array.isArray(quality?.issues) ? quality.issues.map(String).join(" ") : String(quality?.issues ?? ""); if (qualityValue === "poor" || (score !== null && score < 0.52)) return "poor"; - if (qualityValue === "good" && (score === null || score >= 0.72) && !/\b(?:failed|ocr|missing text)\b/i.test(issues)) { + if ( + qualityValue === "good" && + (score === null || score >= 0.72) && + !/\b(?:failed|ocr|missing text)\b/i.test(issues) + ) { return "good"; } if (qualityValue === "partial" || qualityValue === "good" || (score !== null && score >= 0.52)) return "partial"; @@ -504,8 +519,7 @@ function deriveMetadata(document: DocumentRow, text: string, quality: QualityRow publisher: publisherCode ? "filename/source_path code" : "not inferred", document_status: dates.review?.raw ?? dates.lastUpdated?.raw ?? "not inferred", publication_date: dates.publication?.raw ?? "not inferred", - clinical_validation_status: - clinicalValidation.basis, + clinical_validation_status: clinicalValidation.basis, extraction_quality: quality ? `document_index_quality:${quality.extraction_quality ?? "unknown"} score:${quality.quality_score ?? "unknown"}` : "existing metadata", diff --git a/scripts/check-document-label-coverage.ts b/scripts/check-document-label-coverage.ts index c5d49ab082..d49df13467 100644 --- a/scripts/check-document-label-coverage.ts +++ b/scripts/check-document-label-coverage.ts @@ -1,6 +1,8 @@ import * as nextEnv from "@next/env"; import { promises as fs } from "node:fs"; import { resolve } from "node:path"; +import { reviewDocumentTagQuality } from "@/lib/document-tags"; +import type { DocumentLabel } from "@/lib/types"; const loadEnvConfig = nextEnv.loadEnvConfig ?? @@ -20,12 +22,17 @@ type SupabaseAdmin = Awaited>; type DocumentRow = { id: string; + title: string; + file_name: string; }; type LabelRow = { id: string; document_id: string; - label_type: string; + label: string; + label_type: DocumentLabel["label_type"]; + source: DocumentLabel["source"]; + confidence: number; }; type QueryResult = { @@ -164,6 +171,12 @@ function countByLabelType(labels: LabelRow[]) { const smartV2LabelTypes = new Set(["clinical_action", "care_phase", "document_intent", "content_feature"]); +function countQualityIssues(issues: ReturnType) { + const counts = new Map(); + for (const issue of issues) counts.set(issue.kind, (counts.get(issue.kind) ?? 0) + 1); + return Object.fromEntries([...counts.entries()].sort()); +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (args.help) { @@ -174,9 +187,14 @@ async function main() { const supabase = await loadAdminClient(); const allowedSiteMissing = await loadAllowlist(args.allowedSiteMissingPath); const allowedDocumentTypeMissing = await loadAllowlist(args.allowedDocumentTypeMissingPath); - const documents = await fetchAll(supabase, "documents", "id", (query) => query.eq("status", "indexed")); - const labels = await fetchAll(supabase, "document_labels", "id,document_id,label_type", (query) => - query.eq("source", "generated"), + const documents = await fetchAll(supabase, "documents", "id,title,file_name", (query) => + query.eq("status", "indexed"), + ); + const labels = await fetchAll( + supabase, + "document_labels", + "id,document_id,label,label_type,source,confidence", + (query) => query.eq("source", "generated"), ); const documentIds = new Set(documents.map((document) => document.id)); @@ -202,6 +220,17 @@ async function main() { const missingDocumentType = [...documentIds].filter( (id) => !documentTypeDocumentIds.has(id) && !allowedDocumentTypeMissing.has(id), ); + const labelsByDocument = new Map(); + for (const label of labels) { + labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); + } + const qualityIssues = reviewDocumentTagQuality( + documents.map((document) => ({ + ...document, + labels: labelsByDocument.get(document.id) ?? [], + })), + { overusedThreshold: Math.max(100, Math.ceil(documents.length * 0.35)) }, + ); const passed = missingGenerated.length === 0 && missingSite.length === 0 && missingDocumentType.length === 0; @@ -217,6 +246,18 @@ async function main() { indexed_without_smart_v2: missingSmartV2.length, labels_by_type: countByLabelType(labels), smart_v2_labels_by_type: countByLabelType(smartV2Labels), + label_quality_issue_count: qualityIssues.length, + label_quality_issue_counts: countQualityIssues(qualityIssues), + sample_label_quality_issues: qualityIssues.slice(0, 10).map((issue) => ({ + kind: issue.kind, + label: issue.label, + canonical_label: issue.canonicalLabel, + label_type: issue.label_type, + count: issue.count, + reason: issue.reason, + examples: issue.examples, + document_titles: issue.documentTitles, + })), sample_missing_generated: missingGenerated.slice(0, 10), sample_missing_site: missingSite.slice(0, 10), sample_missing_document_type: missingDocumentType.slice(0, 10), @@ -251,6 +292,12 @@ async function main() { .map(([type, count]) => `${type}=${count}`) .join(", ")}`, ); + console.log(`Label quality issues: ${report.label_quality_issue_count}`); + console.log( + `Label quality issue counts: ${Object.entries(report.label_quality_issue_counts) + .map(([type, count]) => `${type}=${count}`) + .join(", ")}`, + ); if (allowedSiteMissingDocs.length) { console.log(`Allowed indexed docs without site labels (from allowlist): ${allowedSiteMissingDocs.length}`); } diff --git a/scripts/classify-documents.ts b/scripts/classify-documents.ts index cf46f4e92f..aefec264a1 100644 --- a/scripts/classify-documents.ts +++ b/scripts/classify-documents.ts @@ -1,4 +1,5 @@ import * as nextEnv from "@next/env"; +import { documentLabelTier } from "@/lib/document-tags"; const loadEnvConfig = nextEnv.loadEnvConfig ?? @@ -42,6 +43,8 @@ type GeneratedLabelRow = { generated_by: "document-organization-classifier"; organization_profile_version: "document-organization-v1"; classified_at: string; + label_tier: ReturnType; + review_status: Classification["profile"]["review_status"]; }; }; @@ -264,6 +267,8 @@ function generatedLabelsForPlan(plan: ClassificationPlan, stampedAt: string): Ge generated_by: "document-organization-classifier", organization_profile_version: "document-organization-v1", classified_at: stampedAt, + label_tier: documentLabelTier(label.label, label.label_type), + review_status: plan.classification.profile.review_status, }, })); } diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 5136bac03a..a56727b12f 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -226,9 +226,7 @@ function failureCategoryCounts(results: Array<{ failures: string[] }>) { } function isSourceMetadataDebtThresholdFailure(failure: string) { - return ( - failure.startsWith("top-result stale/review/unknown rate") || failure.startsWith("top-result review_required_rate") - ); + return failure.startsWith("top-result stale_rate") || failure.startsWith("top-result review_required_rate"); } function isIsoDateString(value: string) { @@ -265,7 +263,7 @@ function evaluateSourceMetadataDebtAcceptance(args: { } if (args.governance.stale_rate > acceptance.max_stale_rate) { rejectionReasons.push( - `stale/review/unknown rate ${args.governance.stale_rate} exceeds accepted ceiling ${acceptance.max_stale_rate}`, + `stale rate ${args.governance.stale_rate} exceeds accepted ceiling ${acceptance.max_stale_rate}`, ); } if (args.governance.review_required_rate > acceptance.max_review_required_rate) { @@ -339,7 +337,8 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { unverified_top_results: unverified, unknown_extraction_top_results: unknownExtraction, poor_extraction_top_results: poorExtraction, - stale_rate: rate(stale + reviewDue + unknown, total), + stale_rate: rate(stale, total), + stale_review_unknown_rate: rate(stale + reviewDue + unknown, total), review_required_top_results: reviewRequired, review_required_rate: rate(reviewRequired, total), metadata_policy: @@ -417,7 +416,7 @@ export function buildEvalQualityReport(args: { } if (governance.stale_rate > qualityThresholds.staleTopResultRate) { thresholdFailures.push( - `top-result stale/review/unknown rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`, + `top-result stale_rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`, ); } if (governance.review_required_rate > qualityThresholds.reviewRequiredTopResultRate) { @@ -607,7 +606,8 @@ ${markdownTable([ ["Unverified top results", governance.unverified_top_results], ["Unknown-extraction top results", governance.unknown_extraction_top_results], ["Poor-extraction top results", governance.poor_extraction_top_results], - ["Stale/review/unknown rate", governance.stale_rate], + ["Stale rate", governance.stale_rate], + ["Stale/review/unknown rate", governance.stale_review_unknown_rate], ["Review-required top results", governance.review_required_top_results], ["Review-required rate", governance.review_required_rate], ])} diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 2353de34f4..ab5962fbf3 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -198,7 +198,11 @@ import { searchServiceRecords } from "@/lib/services"; import { buildAnswerRenderModel, type AnswerRenderModel, type SourceLink } from "@/lib/answer-render-policy"; import { SourceActionRow, sourceResultHref } from "@/components/clinical-dashboard/source-actions"; import { clinicalProseUsefulness, sourceTextForCompactDisplay } from "@/lib/source-text-sanitizer"; -import { groupSourceGovernanceWarnings, type SourceGovernanceWarning } from "@/lib/source-governance"; +import { + frontendSourceGovernanceWarnings, + groupSourceGovernanceWarnings, + type SourceGovernanceWarning, +} from "@/lib/source-governance"; import { smartEvidenceTags } from "@/lib/evidence-tags"; import { reviewDocumentTagQuality, @@ -606,7 +610,7 @@ function ScopeAndGovernanceNotice({ scope: SearchScopeSummary | null; warnings: SourceGovernanceWarning[]; }) { - const groupedWarnings = groupSourceGovernanceWarnings(warnings).slice(0, 4); + const groupedWarnings = groupSourceGovernanceWarnings(frontendSourceGovernanceWarnings(warnings)).slice(0, 4); const showScope = Boolean(scope && scope.activeFilterCount > 0) || Boolean(scope?.warnings?.length) || @@ -2196,6 +2200,7 @@ function AnswerInsightBar({ queryMode: ClinicalQueryMode; sourceGovernanceWarnings: SourceGovernanceWarning[]; }) { + const frontendGovernanceWarnings = frontendSourceGovernanceWarnings(sourceGovernanceWarnings); const metadata = normalizeSourceMetadata( bestSource?.source_metadata ?? answer.sources?.[0]?.source_metadata ?? answer.citations?.[0]?.source_metadata, ); @@ -2205,8 +2210,8 @@ function AnswerInsightBar({ queryModeLabel(queryMode); const sourceCount = answer.evidenceSummary?.total_sources ?? answer.sources?.length ?? answer.citations.length; const support = relevanceChipLabel(relevance ?? answer.relevance, answer.grounded); - const sourceStatus = sourceGovernanceWarnings.length - ? `${sourceGovernanceWarnings.length} source status note${sourceGovernanceWarnings.length === 1 ? "" : "s"}` + const sourceStatus = frontendGovernanceWarnings.length + ? `${frontendGovernanceWarnings.length} source status note${frontendGovernanceWarnings.length === 1 ? "" : "s"}` : sourceStatusLabel(metadata); const retrievalGate = answer.retrievalDiagnostics?.gateStatus; const items = [ @@ -7137,7 +7142,11 @@ export function ClinicalDashboard({ const showSystemNotice = Boolean(setupWarning && !demoMode); const groupedGovernanceWarningCount = useMemo( - () => groupSourceGovernanceWarnings(sourceGovernanceWarnings).reduce((total, warning) => total + warning.count, 0), + () => + groupSourceGovernanceWarnings(frontendSourceGovernanceWarnings(sourceGovernanceWarnings)).reduce( + (total, warning) => total + warning.count, + 0, + ), [sourceGovernanceWarnings], ); const mobileFabState = useMemo( diff --git a/src/components/DocumentOrganizationBadges.tsx b/src/components/DocumentOrganizationBadges.tsx index 1980ba09fa..5f8549ecb8 100644 --- a/src/components/DocumentOrganizationBadges.tsx +++ b/src/components/DocumentOrganizationBadges.tsx @@ -2,6 +2,7 @@ import { AlertTriangle, Building2, FileText, Tag } from "lucide-react"; import { cn, metadataPill, toneInfo, toneNeutral, toneWarning } from "@/components/ui-primitives"; import { canonicalDocumentDisplayTitle } from "@/lib/document-organization"; +import { formatDocumentLabelDisplay } from "@/lib/document-tags"; import type { DocumentLabel, DocumentOrganizationProfile } from "@/lib/types"; type OrganizationDocument = { @@ -29,16 +30,6 @@ export function documentDisplayTitle(document: Pick - word.length <= 4 && word === word.toUpperCase() ? word : `${word[0]?.toUpperCase()}${word.slice(1)}`, - ) - .join(" "); -} - function labelFromLabels( labels: OrganizationDocument["labels"], type: DocumentLabel["label_type"], @@ -79,7 +70,7 @@ export function DocumentOrganizationBadges({ {siteLabel ? ( - {compact && siteShortLabel ? siteShortLabel : displayLabel(siteLabel)} + {compact && siteShortLabel ? siteShortLabel : formatDocumentLabelDisplay(siteLabel, "site")} ) : needsReview && candidateCount > 0 ? ( @@ -90,7 +81,7 @@ export function DocumentOrganizationBadges({ {typeLabel ? ( - {displayLabel(typeLabel)} + {formatDocumentLabelDisplay(typeLabel, "document_type")} ) : null} {needsReview ? ( diff --git a/src/components/DocumentTagCloud.tsx b/src/components/DocumentTagCloud.tsx index e6a5390c4b..362e337263 100644 --- a/src/components/DocumentTagCloud.tsx +++ b/src/components/DocumentTagCloud.tsx @@ -118,7 +118,10 @@ export function DocumentTagCloud({ selectedTagKeys, grouped = false, }: DocumentTagCloudProps) { - const tags = useMemo(() => buildSmartDocumentTags(labels, { query, includeManualGroup: true }), [labels, query]); + const tags = useMemo( + () => buildSmartDocumentTags(labels, { query, includeManualGroup: true }).filter((tag) => tag.tier !== "ranking"), + [labels, query], + ); const groupedTags = useMemo( () => groupSmartDocumentTags(labels, { query, includeManualGroup: true }), [labels, query], diff --git a/src/lib/answer-ranking.ts b/src/lib/answer-ranking.ts index c3a0023a40..9a5523a469 100644 --- a/src/lib/answer-ranking.ts +++ b/src/lib/answer-ranking.ts @@ -163,12 +163,12 @@ function classSignalScore(queryClass: RagQueryClass, result: SearchResult, combi function sourceQualityScore(result: SearchResult) { const metadata = result.source_metadata; let score = 0; - if (metadata?.document_status === "current") score += 0.035; - if (metadata?.document_status === "outdated") score -= 0.08; - if (metadata?.clinical_validation_status === "approved") score += 0.035; - if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.02; - if (metadata?.extraction_quality === "good") score += 0.035; - if (metadata?.extraction_quality === "poor") score -= 0.06; + if (metadata?.document_status === "current") score += 0.004; + if (metadata?.document_status === "outdated") score -= 0.04; + if (metadata?.clinical_validation_status === "approved") score += 0.004; + if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.003; + if (metadata?.extraction_quality === "good") score += 0.002; + if (metadata?.extraction_quality === "poor") score -= 0.05; return score; } diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index e443239497..9e0de4f1c0 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -576,9 +576,8 @@ function hasImageEvidenceNeed(query: string) { function extractionQualityScore(result: SearchResult) { const quality = result.source_metadata?.extraction_quality; - if (quality === "good") return 0.03; - if (quality === "partial") return -0.01; - if (quality === "poor") return -0.04; + if (quality === "good") return 0.002; + if (quality === "poor") return -0.05; return 0; } @@ -614,17 +613,14 @@ function sourceQualityRankSignal(result: SearchResult, queryClass: RagQueryClass if (result.source_strength === "moderate") score += 0.02; if (result.source_strength === "limited") score -= 0.015; - if (metadata?.document_status === "current") score += 0.035; - if (metadata?.document_status === "review_due") score -= 0.025; - if (metadata?.document_status === "outdated") score -= 0.09; + if (metadata?.document_status === "current") score += 0.004; + if (metadata?.document_status === "outdated") score -= 0.04; - if (metadata?.clinical_validation_status === "approved") score += 0.035; - if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.025; - if (metadata?.clinical_validation_status === "unverified") score -= 0.02; + if (metadata?.clinical_validation_status === "approved") score += 0.004; + if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.003; - if (metadata?.extraction_quality === "good") score += 0.025; - if (metadata?.extraction_quality === "partial") score -= 0.015; - if (metadata?.extraction_quality === "poor") score -= 0.075; + if (metadata?.extraction_quality === "good") score += 0.002; + if (metadata?.extraction_quality === "poor") score -= 0.06; const tableFocusedQuery = queryClass === "table_threshold" || queryClass === "medication_dose_risk"; if (tableFocusedQuery && (result.table_facts?.length ?? 0) > 0) score += 0.055; @@ -1162,8 +1158,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se : 0; const status = result.source_metadata?.document_status; const validation = result.source_metadata?.clinical_validation_status; - const statusBoost = status === "current" ? 0.05 : status === "review_due" ? -0.04 : status === "outdated" ? -0.18 : 0; - const validationBoost = validation === "approved" ? 0.04 : validation === "locally_reviewed" ? 0.025 : 0; + const statusBoost = status === "current" ? 0.004 : status === "outdated" ? -0.04 : 0; + const validationBoost = validation === "approved" ? 0.004 : validation === "locally_reviewed" ? 0.003 : 0; const publicationYearsAgo = parseDateAsYearsAgo(result.source_metadata?.publication_date); const reviewYearsAgo = parseDateAsYearsAgo(result.source_metadata?.review_date); const freshnessBoost = @@ -1290,7 +1286,11 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se agitationArousalQuery && /\bagitation\b/.test(titleTokenText) && (/\barousal\b/.test(titleTokenText) || /\bpharma mgt\b/.test(titleTokenText)); - const agitationArousalCanonicalBoost = agitationArousalCanonicalTitle ? 0.34 : agitationArousalQuery && agitationArousalSource ? 0.18 : 0; + const agitationArousalCanonicalBoost = agitationArousalCanonicalTitle + ? 0.34 + : agitationArousalQuery && agitationArousalSource + ? 0.18 + : 0; const agitationArousalGenericPenalty = agitationArousalQuery && !agitationArousalSource ? -0.32 : 0; const activeCommunityEdQuery = queryClass === "document_lookup" && @@ -1298,27 +1298,34 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se /\bcommunity\b/i.test(query) && /\b(?:ed|emergency department)\b/i.test(query); const activeCommunityEdSource = - /\bactive\b/.test(haystack) && - /\bcommunity\b/.test(haystack) && - /\b(?:ed|emergency department)\b/.test(haystack); + /\bactive\b/.test(haystack) && /\bcommunity\b/.test(haystack) && /\b(?:ed|emergency department)\b/.test(haystack); const activeCommunityCanonicalTitle = activeCommunityEdQuery && /\bactive\b/.test(titleTokenText) && /\bcommunity\b/.test(titleTokenText) && /\b(?:ed|emergency department)\b/.test(titleTokenText); - const activeCommunityCanonicalBoost = activeCommunityCanonicalTitle ? 0.38 : activeCommunityEdQuery && activeCommunityEdSource ? 0.18 : 0; + const activeCommunityCanonicalBoost = activeCommunityCanonicalTitle + ? 0.38 + : activeCommunityEdQuery && activeCommunityEdSource + ? 0.18 + : 0; const activeCommunityGenericPenalty = activeCommunityEdQuery && !activeCommunityEdSource ? -0.22 : 0; const riskFlowchartQuery = queryClass === "document_lookup" && /\b(?:flow\s*chart|flowchart|algorithm|pathway)\b/i.test(query) && /\b(?:risk|red\s*zone|red|next step|step after)\b/i.test(query); const riskFlowchartSource = - /\b(?:flowchart|flow chart|flow|algorithm|pathway|matrix)\b/.test(haystack) && /\b(?:risk|red zone|red)\b/.test(haystack); + /\b(?:flowchart|flow chart|flow|algorithm|pathway|matrix)\b/.test(haystack) && + /\b(?:risk|red zone|red)\b/.test(haystack); const riskFlowchartCanonicalTitle = riskFlowchartQuery && /\b(?:flow|flowchart|flow chart|algorithm|pathway|matrix)\b/.test(titleTokenText) && /\brisk\b/.test(titleTokenText); - const riskFlowchartCanonicalBoost = riskFlowchartCanonicalTitle ? 0.32 : riskFlowchartQuery && riskFlowchartSource ? 0.16 : 0; + const riskFlowchartCanonicalBoost = riskFlowchartCanonicalTitle + ? 0.32 + : riskFlowchartQuery && riskFlowchartSource + ? 0.16 + : 0; const riskFlowchartGenericPenalty = riskFlowchartQuery && !riskFlowchartSource ? -0.18 : 0; const structuredTableBoost = (queryClass === "table_threshold" || queryClass === "medication_dose_risk") && (result.table_facts?.length ?? 0) > 0 @@ -1357,9 +1364,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se const comparisonCoverageBoost = queryClass === "comparison" && titleCoverageBoost > 0 && evidenceBoost > 0.02 ? 0.025 : 0; const routeSignal = (() => { - if (result.source_metadata?.document_status === "review_due") return -0.03; - if (result.source_metadata?.document_status === "outdated") return -0.1; - if (result.source_metadata?.extraction_quality === "poor") return -0.05; + if (result.source_metadata?.document_status === "outdated") return -0.04; + if (result.source_metadata?.extraction_quality === "poor") return -0.04; return 0; })(); const lowLexicalCoverage = normalizedTokens.length > 0 && evidenceBoost < 0.035 && titleCoverageBoost < 0.045; diff --git a/src/lib/document-organization.ts b/src/lib/document-organization.ts index fa9bc18758..c30cbb73b1 100644 --- a/src/lib/document-organization.ts +++ b/src/lib/document-organization.ts @@ -384,7 +384,11 @@ const documentTypePatterns: Array<{ confidence: 0.82, patterns: [/\bprescrib\b/i, /\baid\b/i, /\bcalculator\b/i, /\bdosing\b/i, /\bnomogram\b/i], }, - { label: "reference", confidence: 0.72, patterns: [/\breference\b/i, /\binformation sheet\b/i, /\bplacecard\b/i] }, + { + label: "reference", + confidence: 0.72, + patterns: [/\breference\b/i, /\bquick reference\b/i, /\bquick guide\b/i, /\bqrg\b/i, /\binformation sheet\b/i, /\bplacecard\b/i], + }, ]; const secondaryFacetLimits: Record = { @@ -853,8 +857,8 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "assess", label_type: "clinical_action", - strong: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination)\b/i], - body: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination)\b/i], + strong: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b|bleeding event)\b/i], + body: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b)\b/i], minBodyMatches: 1, }, { @@ -865,7 +869,7 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "administer", label_type: "clinical_action", - strong: [/\b(?:administer|administration|infusions?|eye drops?|medication administration)\b/i], + strong: [/\b(?:administer|administration|infusions?|eye drops?|medication administration|vaccination|prophylaxis|chloramphenicol|gentamicin|oxybuprocaine|phenylephrine|naloxone|prenoxad|niacin|polystyrene sulphonate|resonium)\b/i], }, { label: "monitor", @@ -891,8 +895,8 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "document", label_type: "clinical_action", - strong: [/\b(?:document|documentation|record in)\b/i], - body: [/\b(?:documentation|record in)\b/i], + strong: [/\b(?:document|documentation|record in|medical records?|my health record|uploading|amending|removing documents)\b/i], + body: [/\b(?:documentation|record in|medical records?|my health record)\b/i], minBodyMatches: 3, }, { label: "notify", label_type: "clinical_action", strong: [/\b(?:notify|notification|report to)\b/i] }, @@ -944,22 +948,22 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "clinical-instruction", label_type: "document_intent", - strong: [/\b(?:guideline|procedure|protocol|qrg\b|quick reference guide|sop\b|standard operating procedure|clinical instruction|clinical management|clinical summary|insertion summary|clinical poster|sdg\b|standing drug guideline)\b/i], + strong: [/\b(?:guideline|procedure|protocol|qrg\b|quick reference|quick guide|sop\b|standard operating procedure|clinical instruction|clinical management|clinical summary|insertion summary|clinical poster|sdg\b|standing drug guideline|apheresis|biopsy|perfusion|ventilation|cardiac arrest|stemi activation|therapy recipients|micro alerts)\b/i], }, { label: "decision-support", label_type: "document_intent", - strong: [/\b(?:algorithm|flowchart|decision tree|criteria|threshold|diagnos|diagnosis)\b/i], + strong: [/\b(?:algorithm|flowchart|decision tree|criteria|threshold|diagnos|diagnosis|scale\b|aims\b|bleeding event|hypothermia|cardiac arrest|stemi activation|pathway)\b/i], }, { label: "patient-information", label_type: "document_intent", - strong: [/\b(?:patient information|consumer information|factsheet|leaflet|flyer|brochure|booklet|poster|info sheet|information sheet|print ready pi|food and nutrition|huffers and puffers|common discomforts in pregnancy|for patients)\b/i], + strong: [/\b(?:patient information|consumer information|factsheet|leaflet|flyer|brochure|booklet|poster|info sheet|information sheet|print ready pi|food and nutrition|huffers and puffers|common discomforts in pregnancy|caring for your|living with|before surgery|tips for reducing|genetic cholesterol|for patients)\b/i], }, { label: "staff-guidance", label_type: "document_intent", - strong: [/\b(?:staff guidance|staff access|staff only|registrar role|staff role|roles? and responsibilities|orientation|training|education|competenc|whs\b|work health and safety)\b/i], + strong: [/\b(?:staff guidance|staff access|staff only|registrar role|staff role|roles? and responsibilities|reflective practice|peer workforce|research nurse|orientation|training|education|competenc|whs\b|work health and safety)\b/i], }, { label: "legal-governance", @@ -969,17 +973,17 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "operational-process", label_type: "document_intent", - strong: [/\b(?:workflow|process|administration|operational|procedure|sop\b|standard operating procedure|access to|access model|capacity model|management sop|roles? and responsibilities)\b/i], + strong: [/\b(?:workflow|process|administration|operational|procedure|sop\b|standard operating procedure|access to|access model|capacity model|remote network access|my health record|medical records?|contingency plan|workloads|staffing|management sop|roles? and responsibilities)\b/i], }, { label: "documentation-requirement", label_type: "document_intent", - strong: [/\b(?:documentation|record in|form required|documented in)\b/i], + strong: [/\b(?:documentation|record in|form required|documented in|medical records?|my health record|uploading|amending|removing documents)\b/i], }, { label: "medication-instruction", label_type: "document_intent", - strong: [/\b(?:prescrib|dose|dosing|medication instruction|medication management|medications?\b|drug guideline|standing drug guideline|sdg\b|drug infusions?|infusions?|eye drops?|permethrin)\b/i], + strong: [/\b(?:prescrib|dose|dosing|medication instruction|medication management|medications?\b|drug guideline|standing drug guideline|sdg\b|drug infusions?|infusions?|eye drops?|over the counter|complementary medicines|vaccination|prophylaxis|chloramphenicol|gentamicin|oxybuprocaine|phenylephrine|naloxone|prenoxad|niacin|polystyrene sulphonate|resonium|permethrin)\b/i], }, { label: "contains-table", label_type: "content_feature", strong: [/\b(?:table|matrix|schedule)\b/i] }, @@ -994,6 +998,11 @@ const smartFacetRules: SmartFacetRule[] = [ label_type: "content_feature", strong: [/\b(?:dose|dosing|dosage|nomogram|drug guideline|standing drug guideline|sdg\b|drug infusions?|infusions?|eye drops?)\b/i], }, + { + label: "contains-quick-reference", + label_type: "content_feature", + strong: [/\b(?:quick reference|quick guide|qrg\b)\b/i], + }, { label: "contains-monitoring-schedule", label_type: "content_feature", diff --git a/src/lib/document-tags.ts b/src/lib/document-tags.ts index 2d1b60d115..44ba2785db 100644 --- a/src/lib/document-tags.ts +++ b/src/lib/document-tags.ts @@ -16,12 +16,15 @@ export type SmartDocumentTagGroup = | "Content feature" | "Manual"; +export type SmartDocumentTagTier = "primary" | "secondary" | "ranking"; + export type SmartDocumentTag = { key: string; label: string; searchText: string; label_type: DocumentLabelType; group: SmartDocumentTagGroup; + tier: SmartDocumentTagTier; source: DocumentLabel["source"]; confidence: number; score: number; @@ -97,33 +100,34 @@ const groupLabels: Record = { }; const groupRank: Record = { - Site: 0, - Medication: 1, - Risk: 2, - Workflow: 3, - Topic: 4, - Population: 5, - Setting: 6, - Service: 7, - "Document type": 8, - "Clinical action": 9, - "Care phase": 10, - "Document intent": 11, + Site: 1, + Medication: 2, + "Clinical action": 3, + "Care phase": 4, + Risk: 5, + "Document intent": 6, + Topic: 7, + Service: 8, + Setting: 9, + Population: 10, + Workflow: 11, "Content feature": 12, - Manual: 13, + "Document type": 13, + Manual: 14, }; export const smartDocumentFacetGroups: SmartDocumentTagGroup[] = [ "Site", "Medication", - "Risk", - "Workflow", "Clinical action", "Care phase", "Document intent", + "Risk", + "Workflow", + "Topic", + "Service", "Setting", "Population", - "Service", "Document type", ]; @@ -139,6 +143,7 @@ const acronymDisplay = new Map([ ["ect", "ECT"], ["fbc", "FBC"], ["gp", "GP"], + ["hr", "HR"], ["honos", "HoNOS"], ["honosca", "HoNOSCA"], ["im", "IM"], @@ -177,6 +182,98 @@ const siteShortLabels = new Map([ ["wa health", "WA Health"], ]); +const displayLabelOverrides = new Map([ + ["assessment_tool", "Assessment tool"], + ["prescribing_aid", "Prescribing aid"], + ["blood test monitoring", "Blood test monitoring"], + ["community program for opioid pharmacotherapy", "CPOP"], + ["electroconvulsive-therapy", "Electroconvulsive therapy"], + ["substance use alcohol and drugs", "Substance use, alcohol and drugs"], + ["aggression violence code black", "Aggression, violence and Code Black"], + ["admission waitlist bed access", "Admission, waitlist and bed access"], + ["transport transfer escort", "Transport, transfer and escort"], + ["rights carers advocates", "Rights, carers and advocates"], + ["consent capacity confidentiality", "Consent, capacity and confidentiality"], + ["incident notification open disclosure", "Incident, notification and open disclosure"], + ["psychosis schizophrenia", "Psychosis and schizophrenia"], + ["depression mood disorders", "Depression and mood disorders"], + ["bipolar mood episode", "Bipolar and mood episode"], + ["anxiety trauma", "Anxiety and trauma"], + ["cognitive impairment learning disability", "Cognitive impairment and learning disability"], + ["shared care gp liaison", "Shared care and GP liaison"], + ["care coordination case management", "Care coordination and case management"], + ["mental state examination", "Mental state examination"], + ["community treatment order", "Community treatment order"], + ["substance withdrawal", "Substance withdrawal"], + ["acute psychosis", "Acute psychosis"], + ["mood episode", "Mood episode"], + ["initial assessment", "Initial assessment"], + ["acute management", "Acute management"], + ["crisis response", "Crisis response"], + ["ongoing management", "Ongoing management"], + ["maintenance treatment", "Maintenance treatment"], + ["discharge planning", "Discharge planning"], + ["post discharge follow up", "Post-discharge follow-up"], + ["clinical instruction", "Clinical instruction"], + ["decision support", "Decision support"], + ["patient information", "Patient information"], + ["staff guidance", "Staff guidance"], + ["legal governance", "Legal and governance"], + ["operational process", "Operational process"], + ["documentation requirement", "Documentation requirement"], + ["medication instruction", "Medication instruction"], + ["contains table", "Contains table"], + ["contains flowchart", "Contains flowchart"], + ["contains form", "Contains form"], + ["contains dosage guidance", "Contains dosage guidance"], + ["contains monitoring schedule", "Contains monitoring schedule"], + ["contains referral criteria", "Contains referral criteria"], + ["contains escalation criteria", "Contains escalation criteria"], + ["contains legal criteria", "Contains legal criteria"], + ["contains quick reference", "Contains quick reference"], + ["de escalate", "De-escalate"], + ["qtc monitoring", "QTc monitoring"], + ["nocc outcome measures", "NOCC outcome measures"], +]); + +const displayLowercaseWords = new Set(["and", "or", "of", "to", "for", "in", "on", "with", "from", "by"]); + +const rankingOnlyLabels = new Set([ + "clinical risk", + "mental health", + "inpatient", + "assessment", + "monitoring", + "physical health care", + "admission waitlist bed access", + "clinical", + "non clinical", + "admin", + "assess", + "refer", + "review", + "document", + "clinical instruction", + "documentation requirements", +]); + +const validClinicalActionLabels = new Set([ + "assess", + "prescribe", + "administer", + "monitor", + "escalate", + "refer", + "admit", + "discharge", + "transfer", + "observe", + "document", + "notify", + "review", + "de escalate", +]); + export const clinicalDocumentTagAliases = [ { from: "a n c", to: "absolute neutrophil count" }, { from: "absolute neutrophils", to: "absolute neutrophil count" }, @@ -327,6 +424,7 @@ function isNoisyLabel(label: string, labelType: DocumentLabelType) { if (!label || label.length < 2 || label.length > 64) return true; if (lowValuePattern.test(label)) return true; if (/\b(?:docx?|xlsx?|pptx?|pdf)\b/.test(label)) return true; + if (labelType === "clinical_action" && validClinicalActionLabels.has(label)) return false; if ( labelType === "document_type" && /^(?:policy|guideline|procedure|protocol|form|checklist|pathway|reference|algorithm|factsheet|manual|assessment_tool|prescribing_aid)$/.test( @@ -361,19 +459,40 @@ function sourceValue(value: unknown): DocumentLabel["source"] { return value === "manual" ? "manual" : "generated"; } -function displayLabel(value: string, labelType?: DocumentLabelType) { +export function formatDocumentLabelDisplay(value: string, labelType?: DocumentLabelType) { const displayValue = value.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim(); if (labelType === "site") { const siteShortLabel = siteShortLabels.get(displayValue); if (siteShortLabel) return siteShortLabel; } + const override = displayLabelOverrides.get(value) ?? displayLabelOverrides.get(displayValue); + if (override) return override; return displayValue .split(" ") .filter(Boolean) - .map((word) => acronymDisplay.get(word) ?? `${word[0]?.toUpperCase() ?? ""}${word.slice(1)}`) + .map((word, index) => { + const acronym = acronymDisplay.get(word); + if (acronym) return acronym; + if (index > 0 && displayLowercaseWords.has(word)) return word; + return `${word[0]?.toUpperCase() ?? ""}${word.slice(1)}`; + }) .join(" "); } +function displayLabel(value: string, labelType?: DocumentLabelType) { + return formatDocumentLabelDisplay(value, labelType); +} + +export function documentLabelTier(label: string, labelType: DocumentLabelType): SmartDocumentTagTier { + const normalized = normalizedText(label); + if (rankingOnlyLabels.has(normalized)) return "ranking"; + if (labelType === "site" || labelType === "medication" || labelType === "risk") return "primary"; + if (labelType === "clinical_action" || labelType === "care_phase" || labelType === "document_intent") + return "primary"; + if (labelType === "topic" && !rankingOnlyLabels.has(normalized)) return "primary"; + return "secondary"; +} + function queryTerms(query?: string) { return new Set( normalizedText(query ?? "") @@ -450,6 +569,7 @@ export function buildSmartDocumentTags( searchText: normalized.label, label_type: normalized.label_type, group, + tier: documentLabelTier(normalized.label, normalized.label_type), source, confidence: normalized.confidence, score, @@ -472,9 +592,11 @@ export function buildSmartDocumentTags( export function groupSmartDocumentTags( labels: Array> | null | undefined, - options: { limit?: number; query?: string; includeManualGroup?: boolean } = {}, + options: { limit?: number; query?: string; includeManualGroup?: boolean; includeRankingOnly?: boolean } = {}, ) { - const tags = buildSmartDocumentTags(labels, options); + const tags = buildSmartDocumentTags(labels, options).filter( + (tag) => options.includeRankingOnly || tag.tier !== "ranking", + ); return [...new Set(tags.map((tag) => tag.group))] .sort((a, b) => groupRank[a] - groupRank[b]) .map((group) => ({ @@ -592,6 +714,7 @@ export function reviewDocumentTagQuality< for (const global of globalUsage.values()) { if (global.documents.size < overusedThreshold) continue; + if (documentLabelTier(global.canonicalLabel, global.label_type) === "ranking") continue; addQualityIssue(issues, { kind: "overused", label: global.label, @@ -625,6 +748,7 @@ export function buildSmartDocumentTagFacets( const documentTagKeys = new Set(); const tags = buildSmartDocumentTags(document.labels, { query: options.query, includeManualGroup: false }); for (const tag of tags) { + if (tag.tier === "ranking") continue; if (!allowedGroups.has(tag.group) || documentTagKeys.has(tag.key)) continue; documentTagKeys.add(tag.key); const existing = facets.get(tag.key); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index a5822606f9..56b8428fc7 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -536,7 +536,7 @@ function secondStageScore(result: SearchResult, queryClass: RagQueryClass | unde if (tableVisualEvidenceUnitTypes.has(unitType)) score += 0.08; else if (visualEvidenceUnitTypes.has(unitType)) score += 0.04; if (source === "visual_intelligence") score += Math.min(0.035, Math.max(0, sourceQuality - 0.55) * 0.08); - if (result.source_metadata?.document_status === "outdated") score -= 0.05; + if (result.source_metadata?.document_status === "outdated") score -= 0.04; if (result.source_metadata?.extraction_quality === "poor") score -= 0.05; if (result.indexing_quality?.quality_score !== undefined && result.indexing_quality.quality_score < 0.55) score -= 0.035; diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index b7ffa25eb4..0dbd1d8301 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -323,10 +323,9 @@ function resultBoost(args: { intent: RetrievalIntent; candidate: RetrievalCandid if ( hasActionSignal && metadata?.document_status === "current" && - (metadata.clinical_validation_status === "approved" || - metadata.clinical_validation_status === "locally_reviewed") + (metadata.clinical_validation_status === "approved" || metadata.clinical_validation_status === "locally_reviewed") ) { - boost += 0.05; + boost += 0.003; } if (args.candidate.chunkType === "flowchart" && !hasActionSignal) boost -= 0.14; } @@ -347,17 +346,14 @@ function resultBoost(args: { intent: RetrievalIntent; candidate: RetrievalCandid if (args.intent.requiredTermSignals.length > 0 && args.candidate.lexicalScore === 1) boost += 0.1; if (args.intent.requiredTermSignals.length > 0 && (args.candidate.lexicalScore ?? 0) === 0) boost -= 0.08; - if (metadata?.document_status === "current") boost += 0.06; - if (metadata?.document_status === "review_due") boost -= 0.12; - if (metadata?.document_status === "outdated") boost -= 0.24; + if (metadata?.document_status === "current") boost += 0.004; + if (metadata?.document_status === "outdated") boost -= 0.04; - if (metadata?.clinical_validation_status === "approved") boost += 0.06; - if (metadata?.clinical_validation_status === "locally_reviewed") boost += 0.05; - if (metadata?.clinical_validation_status === "unverified") boost -= 0.02; + if (metadata?.clinical_validation_status === "approved") boost += 0.004; + if (metadata?.clinical_validation_status === "locally_reviewed") boost += 0.003; - if (metadata?.extraction_quality === "good") boost += 0.02; - if (metadata?.extraction_quality === "partial") boost -= 0.04; - if (metadata?.extraction_quality === "poor") boost -= 0.12; + if (metadata?.extraction_quality === "good") boost += 0.002; + if (metadata?.extraction_quality === "poor") boost -= 0.06; return boost; } diff --git a/src/lib/source-governance.ts b/src/lib/source-governance.ts index 4d03ef4773..df9ffab7e1 100644 --- a/src/lib/source-governance.ts +++ b/src/lib/source-governance.ts @@ -30,6 +30,12 @@ export type GroupedSourceGovernanceWarning = { export const sourceGovernanceRefusalAnswer = "I cannot provide a clinical answer because one or more matched documents are not suitable for clinical use yet. Try a narrower clinical term or scope the search to a current approved document."; +const frontendVisibleWarningCodes = new Set([ + "outdated_source", + "poor_extraction", + "weak_evidence", +]); + function isLocalMetadataText(value: string) { return /\b(?:wa|western australia|perth|north metropolitan|east metropolitan|south metropolitan|health service)\b/i.test( value, @@ -197,6 +203,14 @@ export function groupSourceGovernanceWarnings(warnings: SourceGovernanceWarning[ return Array.from(grouped.values()).sort((a, b) => severityRank[a.severity] - severityRank[b.severity]); } +export function isFrontendVisibleSourceGovernanceWarning(warning: SourceGovernanceWarning) { + return frontendVisibleWarningCodes.has(warning.code); +} + +export function frontendSourceGovernanceWarnings(warnings: SourceGovernanceWarning[]) { + return warnings.filter(isFrontendVisibleSourceGovernanceWarning); +} + export function hasDangerSourceGovernanceWarning(warnings: SourceGovernanceWarning[]) { return warnings.some((warning) => warning.severity === "danger"); } diff --git a/tests/document-organization.test.ts b/tests/document-organization.test.ts index 3a1a071199..ea6abee6c4 100644 --- a/tests/document-organization.test.ts +++ b/tests/document-organization.test.ts @@ -432,5 +432,39 @@ describe("document organization classifier", () => { expect(roles.profile.secondary_facets.document_intent).toEqual( expect.arrayContaining(["staff-guidance", "operational-process"]), ); + + const quickReference = classifyDocumentOrganization({ + title: "Cleaning Of The Ultrasound Probe Quick Reference(RPBG)", + file_name: "Cleaning of the Ultrasound Probe Quick Reference (RPBG).pdf", + contentText: "Royal Perth Bentley Group quick reference for cleaning.", + }); + expect(quickReference.profile.document_type.label).toBe("reference"); + expect(quickReference.profile.secondary_facets.document_intent).toContain("clinical-instruction"); + expect(quickReference.profile.secondary_facets.content_feature).toContain("contains-quick-reference"); + + const medicalRecords = classifyDocumentOrganization({ + title: "After-Hours Medical Records - Mental Health(AKG)", + file_name: "After-Hours Medical Records - Mental Health (AKG).pdf", + contentText: "Armadale Kalamunda Group medical records process.", + }); + expect(medicalRecords.profile.secondary_facets.clinical_action).toContain("document"); + expect(medicalRecords.profile.secondary_facets.document_intent).toEqual( + expect.arrayContaining(["operational-process", "documentation-requirement"]), + ); + + const medicineOnly = classifyDocumentOrganization({ + title: "Chloramphenicol(RPBG)", + file_name: "Chloramphenicol (RPBG).pdf", + contentText: "Royal Perth Bentley Group chloramphenicol medication instruction.", + }); + expect(medicineOnly.profile.secondary_facets.clinical_action).toContain("administer"); + expect(medicineOnly.profile.secondary_facets.document_intent).toContain("medication-instruction"); + + const patientLeaflet = classifyDocumentOrganization({ + title: "Living With A Defibrillator(RPBG)", + file_name: "Living with a Defibrillator (RPBG).pdf", + contentText: "Royal Perth Bentley Group patient information.", + }); + expect(patientLeaflet.profile.secondary_facets.document_intent).toContain("patient-information"); }); }); diff --git a/tests/document-tags.test.ts b/tests/document-tags.test.ts index fec37951ff..91bde5299a 100644 --- a/tests/document-tags.test.ts +++ b/tests/document-tags.test.ts @@ -3,6 +3,7 @@ import { buildSmartDocumentTagFacets, buildSmartDocumentTags, filterDocumentsBySmartTagFacets, + formatDocumentLabelDisplay, normalizeDocumentLabelForStorage, reviewDocumentTagQuality, tagSearchText, @@ -118,11 +119,29 @@ describe("smart document tags", () => { buildSmartDocumentTags([label({ label: "electroconvulsive therapy", label_type: "topic" })])[0], ).toMatchObject({ searchText: "electroconvulsive-therapy", - label: "Electroconvulsive Therapy", + label: "Electroconvulsive therapy", }); expect( buildSmartDocumentTags([label({ label: "substance-use-alcohol-and-drugs", label_type: "topic" })])[0].label, - ).toBe("Substance Use Alcohol And Drugs"); + ).toBe("Substance use, alcohol and drugs"); + }); + + it("separates stable machine labels from polished display labels and visibility tiers", () => { + expect(formatDocumentLabelDisplay("contains_quick-reference", "content_feature")).toBe( + "Contains quick reference", + ); + expect(formatDocumentLabelDisplay("post-discharge-follow-up", "care_phase")).toBe("Post-discharge follow-up"); + expect(formatDocumentLabelDisplay("fiona stanley hospital", "site")).toBe("FSH"); + + const tags = buildSmartDocumentTags([ + label({ label: "clinical-risk", label_type: "risk", confidence: 0.9 }), + label({ label: "lithium", label_type: "medication", confidence: 0.9 }), + ]); + expect(tags.find((tag) => tag.searchText === "clinical risk")).toMatchObject({ tier: "ranking" }); + expect(tags.find((tag) => tag.searchText === "lithium")).toMatchObject({ tier: "primary" }); + + const facets = buildSmartDocumentTagFacets([{ labels: tags }]); + expect(facets.flatMap((group) => group.facets).map((facet) => facet.searchText)).not.toContain("clinical risk"); }); it("builds grouped tag facets with document counts", () => { diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index cfcf53a02c..915b822281 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -241,10 +241,10 @@ describe("eval quality reporting", () => { }); expect(report.accepted_threshold_failures).toEqual( - expect.arrayContaining([ - expect.stringContaining("top-result stale/review/unknown rate"), - expect.stringContaining("top-result review_required_rate"), - ]), + expect.arrayContaining([expect.stringContaining("top-result review_required_rate")]), + ); + expect(report.accepted_threshold_failures).not.toEqual( + expect.arrayContaining([expect.stringContaining("top-result stale_rate")]), ); expect(report.blocking_threshold_failures).toEqual( expect.arrayContaining([expect.stringContaining("RAG source_governance_danger_failure_rate")]), diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 212f253aa4..d69991d70f 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1433,8 +1433,8 @@ describe("RAG structured-output fallback", () => { candidateCount: 2, retrievalDepth: 2, distinctDocumentCount: 2, - topScore: 0.6, - secondScore: 0.58, + topScore: 0.63, + secondScore: 0.61, scoreSpread: 0.02, routeMode: "unsupported", }); diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index 6b4334f441..70c4ad21d6 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -327,7 +327,7 @@ describe("retrieval source selection", () => { expect(selection.summary.matchedSignals).toEqual(expect.arrayContaining(["risk", "red_zone"])); }); - it("prefers current locally reviewed clozapine threshold evidence over close review-required sources", () => { + it("keeps stronger clozapine threshold evidence visible even when routine governance review is required", () => { const selection = selectRetrievalEvidence({ query: "What ANC or FBC threshold should withhold clozapine?", queryClass: "table_threshold", @@ -377,18 +377,67 @@ describe("retrieval source selection", () => { }); expect(selection.results).toHaveLength(5); - expect(selection.results.map((result) => result.id)).toEqual([ - "current-local-fsh", - "current-local-nmhs", - "current-local-akg", - "current-local-camhs", - "current-local-smhs", - ]); + expect(selection.results.map((result) => result.id)).toContain("review-due-shared-care"); expect( - selection.results.every((result) => result.source_metadata?.clinical_validation_status === "locally_reviewed"), + selection.results.some((result) => result.source_metadata?.clinical_validation_status === "locally_reviewed"), ).toBe(true); }); + it("does not let unverified validation displace directly relevant evidence", () => { + const selection = selectRetrievalEvidence({ + query: "What dose and route are shown in the agitation medication chart?", + queryClass: "medication_dose_risk", + topK: 2, + maxResultsPerDocument: 2, + results: [ + source({ + id: "unverified-dose-route", + document_id: "unverified-dose-doc", + title: "Agitation Medication Chart", + file_name: "agitation-chart.pdf", + section_heading: "Medication chart", + content: "Agitation medication chart lists lorazepam 1 mg IM or PO.", + hybrid_score: 0.56, + source_metadata: sourceMetadata({ + document_status: "current", + clinical_validation_status: "unverified", + extraction_quality: "good", + }), + index_unit: { + id: "unit-unverified-dose-route", + unit_type: "medication_chart_row", + title: "Lorazepam route row", + content: "Lorazepam 1 mg IM or PO.", + source_chunk_id: "unverified-dose-route", + source_image_id: null, + page_start: 5, + page_end: 5, + heading_path: ["Medication chart"], + normalized_terms: ["agitation", "lorazepam", "1 mg", "im", "po"], + quality_score: 0.9, + extraction_mode: "hybrid", + }, + }), + source({ + id: "current-generic-policy", + document_id: "current-generic-policy-doc", + title: "Current Medication Policy", + file_name: "current-medication-policy.pdf", + content: "General current medication governance policy without agitation dose or route detail.", + hybrid_score: 0.64, + source_metadata: sourceMetadata({ + document_status: "current", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }), + }), + ], + }); + + expect(selection.results[0].id).toBe("unverified-dose-route"); + expect(selection.results[0].source_metadata?.clinical_validation_status).toBe("unverified"); + }); + it("prefers risk/red-zone flowchart evidence over generic flowchart evidence", () => { const selection = selectRetrievalEvidence({ query: "In the clinical flowchart, what is the next step after red-zone risk?", @@ -460,7 +509,7 @@ describe("retrieval source selection", () => { expect(selection.summary.matchedSignals).toEqual(expect.arrayContaining(["risk", "red_zone"])); }); - it("prefers current locally reviewed risk-flowchart action evidence over review-required action evidence", () => { + it("does not let routine governance metadata displace stronger risk-flowchart action evidence", () => { const selection = selectRetrievalEvidence({ query: "In the clinical flowchart, what is the next step after red-zone risk?", queryClass: "document_lookup", @@ -496,8 +545,8 @@ describe("retrieval source selection", () => { ], }); - expect(selection.results[0].id).toBe("current-action"); - expect(selection.results[0].source_metadata?.document_status).toBe("current"); - expect(selection.results[0].source_metadata?.clinical_validation_status).toBe("locally_reviewed"); + expect(selection.results[0].id).toBe("review-due-action"); + expect(selection.results[0].source_metadata?.document_status).toBe("review_due"); + expect(selection.results[0].source_metadata?.clinical_validation_status).toBe("unverified"); }); }); diff --git a/tests/source-governance.test.ts b/tests/source-governance.test.ts index 8558d36430..ba23e5d121 100644 --- a/tests/source-governance.test.ts +++ b/tests/source-governance.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + frontendSourceGovernanceWarnings, groupSourceGovernanceWarnings, hasDangerSourceGovernanceWarning, sourceGovernanceRefusalAnswer, @@ -149,6 +150,65 @@ describe("source governance warnings", () => { expect(hasDangerSourceGovernanceWarning(warnings)).toBe(false); }); + it("keeps routine review metadata notes out of frontend-visible governance warnings", () => { + const warnings = sourceGovernanceWarnings({ + results: [ + result({ + source_metadata: { + source_title: "Review due local source", + publisher: "WA Health", + jurisdiction: "Australia/WA", + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "review_due", + clinical_validation_status: "unverified", + extraction_quality: "good", + }, + indexing_quality: { + document_id: "doc-1", + quality_score: 0.9, + extraction_quality: "good", + metrics: {}, + issues: [], + }, + table_facts: [], + }), + ], + }); + + expect(warnings.map((warning) => warning.code)).toEqual( + expect.arrayContaining(["review_due_source", "unverified_source"]), + ); + expect(frontendSourceGovernanceWarnings(warnings)).toEqual([]); + }); + + it("surfaces only clinically material warnings to the frontend", () => { + const warnings = sourceGovernanceWarnings({ + results: [result()], + relevance: { + verdict: "none", + label: "No evidence", + matchedTerms: [], + missingTerms: [], + directSourceCount: 0, + weakSourceCount: 0, + score: 0, + supportReason: "No source-backed evidence.", + isSourceBacked: false, + }, + }); + const visibleCodes = frontendSourceGovernanceWarnings(warnings).map((warning) => warning.code); + + expect(visibleCodes).toEqual(expect.arrayContaining(["weak_evidence", "outdated_source", "poor_extraction"])); + expect(visibleCodes).not.toContain("review_due_source"); + expect(visibleCodes).not.toContain("unverified_source"); + expect(visibleCodes).not.toContain("non_local_source"); + }); + it("keeps the refusal message free of backend and source-backed wording", () => { expect(sourceGovernanceRefusalAnswer).not.toMatch(/source-backed|source-governance|admin|need review/i); expect(sourceGovernanceRefusalAnswer).toContain("matched documents are not suitable for clinical use yet"); From a91acefd781d067a4645b26b1e528ab04c5d58cd Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:35:57 +0800 Subject: [PATCH 2/3] fix(test): restore main's weak-retrieval gate fixture scores Codex bumped topScore/secondScore to 0.63/0.61 to suit its pre-merge weights; under the merged (main) confidence-gate thresholds 0.63 is no longer 'weak', so the gate correctly declines to block and the test failed. Main's 0.6/0.58 fixture keeps the scenario inside the weak band the test is about. Co-Authored-By: Claude Fable 5 --- tests/rag-answer-fallback.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 7f500c47c0..36d3caf349 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1434,8 +1434,8 @@ describe("RAG structured-output fallback", () => { candidateCount: 2, retrievalDepth: 2, distinctDocumentCount: 2, - topScore: 0.63, - secondScore: 0.61, + topScore: 0.6, + secondScore: 0.58, scoreSpread: 0.02, routeMode: "unsupported", }); From 763932079d6dab67bcc1e51b5ac314070896bd0c Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:40:03 +0800 Subject: [PATCH 3/3] style: prettier formatting for merged files Co-Authored-By: Claude Fable 5 --- src/lib/clinical-search.ts | 3 ++- src/lib/document-organization.ts | 33 +++++++++++++++++++++++++------- src/lib/rag.ts | 5 ++++- tests/document-tags.test.ts | 4 +--- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 15b9f92bfd..a7fab396a9 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -1406,7 +1406,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se // the same result. const nextStepActionQuery = /\b(?:next step|step after|action)\b/i.test(query); const riskFlowchartSource = nextStepActionQuery - ? riskFlowchartZoneActionSource || (riskFlowchartLexicalSource && riskZoneActionPattern.test(riskFlowchartEvidenceText)) + ? riskFlowchartZoneActionSource || + (riskFlowchartLexicalSource && riskZoneActionPattern.test(riskFlowchartEvidenceText)) : riskFlowchartLexicalSource || riskFlowchartZoneActionSource; const riskFlowchartCanonicalTitle = riskFlowchartQuery && diff --git a/src/lib/document-organization.ts b/src/lib/document-organization.ts index 344801e2c4..67c19f2f27 100644 --- a/src/lib/document-organization.ts +++ b/src/lib/document-organization.ts @@ -390,7 +390,14 @@ const documentTypePatterns: Array<{ { label: "reference", confidence: 0.72, - patterns: [/\breference\b/i, /\bquick reference\b/i, /\bquick guide\b/i, /\bqrg\b/i, /\binformation sheet\b/i, /\bplacecard\b/i], + patterns: [ + /\breference\b/i, + /\bquick reference\b/i, + /\bquick guide\b/i, + /\bqrg\b/i, + /\binformation sheet\b/i, + /\bplacecard\b/i, + ], }, ]; @@ -860,8 +867,12 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "assess", label_type: "clinical_action", - strong: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b|bleeding event)\b/i], - body: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b)\b/i], + strong: [ + /\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b|bleeding event)\b/i, + ], + body: [ + /\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b)\b/i, + ], minBodyMatches: 1, }, { @@ -872,7 +883,9 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "administer", label_type: "clinical_action", - strong: [/\b(?:administer|administration|infusions?|eye drops?|medication administration|vaccination|prophylaxis|chloramphenicol|gentamicin|oxybuprocaine|phenylephrine|naloxone|prenoxad|niacin|polystyrene sulphonate|resonium)\b/i], + strong: [ + /\b(?:administer|administration|infusions?|eye drops?|medication administration|vaccination|prophylaxis|chloramphenicol|gentamicin|oxybuprocaine|phenylephrine|naloxone|prenoxad|niacin|polystyrene sulphonate|resonium)\b/i, + ], }, { label: "monitor", @@ -898,7 +911,9 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "document", label_type: "clinical_action", - strong: [/\b(?:document|documentation|record in|medical records?|my health record|uploading|amending|removing documents)\b/i], + strong: [ + /\b(?:document|documentation|record in|medical records?|my health record|uploading|amending|removing documents)\b/i, + ], body: [/\b(?:documentation|record in|medical records?|my health record)\b/i], minBodyMatches: 3, }, @@ -958,7 +973,9 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "decision-support", label_type: "document_intent", - strong: [/\b(?:algorithm|flowchart|decision tree|criteria|threshold|diagnos|diagnosis|scale\b|aims\b|bleeding event|hypothermia|cardiac arrest|stemi activation|pathway)\b/i], + strong: [ + /\b(?:algorithm|flowchart|decision tree|criteria|threshold|diagnos|diagnosis|scale\b|aims\b|bleeding event|hypothermia|cardiac arrest|stemi activation|pathway)\b/i, + ], }, { label: "patient-information", @@ -989,7 +1006,9 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "documentation-requirement", label_type: "document_intent", - strong: [/\b(?:documentation|record in|form required|documented in|medical records?|my health record|uploading|amending|removing documents)\b/i], + strong: [ + /\b(?:documentation|record in|form required|documented in|medical records?|my health record|uploading|amending|removing documents)\b/i, + ], }, { label: "medication-instruction", diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 763f721936..4f43ee18ca 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3308,7 +3308,10 @@ function hasRiskFlowchartActionEvidence(query: string, results: SearchResult[], const evidenceText = evidenceTextForGate(result); if (!riskZoneActionPattern.test(evidenceText)) return false; if (zonePhrasePattern.test(evidenceText)) return true; - return ["risk_matrix_cell", "flowchart_step", "diagram_decision"].includes(result.index_unit?.unit_type ?? "") && bareColourPattern.test(evidenceText); + return ( + ["risk_matrix_cell", "flowchart_step", "diagram_decision"].includes(result.index_unit?.unit_type ?? "") && + bareColourPattern.test(evidenceText) + ); }); } diff --git a/tests/document-tags.test.ts b/tests/document-tags.test.ts index 91bde5299a..51cbfdbb17 100644 --- a/tests/document-tags.test.ts +++ b/tests/document-tags.test.ts @@ -127,9 +127,7 @@ describe("smart document tags", () => { }); it("separates stable machine labels from polished display labels and visibility tiers", () => { - expect(formatDocumentLabelDisplay("contains_quick-reference", "content_feature")).toBe( - "Contains quick reference", - ); + expect(formatDocumentLabelDisplay("contains_quick-reference", "content_feature")).toBe("Contains quick reference"); expect(formatDocumentLabelDisplay("post-discharge-follow-up", "care_phase")).toBe("Post-discharge follow-up"); expect(formatDocumentLabelDisplay("fiona stanley hospital", "site")).toBe("FSH");