diff --git a/docs/release-source-metadata-debt-2026-06-30.json b/docs/release-source-metadata-debt-2026-06-30.json index 2e88122db8..01028412fe 100644 --- a/docs/release-source-metadata-debt-2026-06-30.json +++ b/docs/release-source-metadata-debt-2026-06-30.json @@ -1,20 +1,39 @@ { "accepted": true, "accepted_by": "repository owner", - "accepted_at": "2026-06-30T00:00:00.000+08:00", + "accepted_at": "2026-07-02T00:00:00.000+08:00", "expires_at": "2026-07-31T23:59:59.000+08:00", - "reason": "Temporary release metadata debt acceptance for the existing eval corpus while document-level source status and local validation metadata are backfilled. This does not mark any source current, locally reviewed, or approved.", + "reason": "Bounded release metadata debt acceptance for residual corpus-state governance values after required source metadata key coverage was completed. This accepts review_due, unknown, and unverified states as tracked backend debt only; it does not mark any source current, locally reviewed, or approved.", "scope": "eval-quality retrieval source metadata thresholds only", - "source_report": "output/evals/eval-quality-final.json", + "source_report": "output/evals/retrieval-quality-2026-07-02T04-58-43-365Z.json", + "observed_live_corpus": { + "observed_at": "2026-07-02T00:00:00.000+08:00", + "indexed_documents": 2065, + "missing_required_source_metadata": 0, + "review_due_documents": 481, + "unknown_status_documents": 132, + "unverified_documents": 130, + "poor_extraction_documents": 0, + "missing_smart_v2_labels": 0 + }, + "observed_retrieval_eval": { + "stale_top_results": 0, + "review_due_top_results": 5, + "unverified_top_results": 6, + "review_required_top_results": 7, + "stale_rate": 0, + "review_required_rate": 0.14 + }, "required_follow_up": [ - "Review high-frequency golden top-result documents first.", + "Review high-frequency review_due, unknown, and unverified documents first.", + "Keep required source metadata key coverage at zero missing fields.", "Backfill documents.metadata.document_status only after confirming source currentness.", "Backfill documents.metadata.clinical_validation_status only after local clinical review.", "Replace this debt file with stricter ceilings or remove it before expiry." ], "ceilings": { - "max_stale_rate": 1, - "max_review_required_rate": 1, + "max_stale_rate": 0, + "max_review_required_rate": 0.2, "max_outdated_top_results": 0, "max_poor_extraction_top_results": 0, "max_source_governance_danger_failure_rate": 0 diff --git a/docs/retrieval-quality-runbook.md b/docs/retrieval-quality-runbook.md index 3367a63843..a02dd0cf6c 100644 --- a/docs/retrieval-quality-runbook.md +++ b/docs/retrieval-quality-runbook.md @@ -85,14 +85,15 @@ Source governance: - unverified top-result count - unknown-extraction top-result count - poor-extraction top-result count -- combined stale/review/unknown top-result rate +- stale/outdated top-result rate +- combined stale/review/unknown top-result audit rate - review-required top-result count and rate Metadata policy: - `unknown`, `unverified`, `review_due`, `outdated`, unknown extraction, and poor extraction are treated as review-required. - Do not silently default missing corpus metadata to `current` or `approved`. -- Reduce the warning rate by backfilling source metadata through ingestion/enrichment or by explicitly accepting the review-required baseline in a versioned release metadata debt file. +- Reduce the review-required rate by backfilling source metadata through ingestion/enrichment or by explicitly accepting a bounded review-required baseline in a versioned release metadata debt file. - Danger-class source governance warnings are blocking. - Warning-class retrieval source metadata notes may be accepted only by passing `--source-metadata-debt ` to `npm run eval:quality -- --fail-on-threshold`. - Source metadata debt acceptance does not mark sources current or approved. It only removes the accepted retrieval metadata threshold failures from the blocking failure list. @@ -118,7 +119,7 @@ Answer quality: - retrieval hit@K is below `0.8` - document recall@5 is below `0.8` - content recall@5 is below `0.8` -- stale/review/unknown top-result rate is above `0.25` +- stale/outdated top-result rate is above `0.25` - review-required top-result rate is above `0.25` - grounded supported answer rate is below `0.9` - unsupported-answer correctness is below `1.0` @@ -153,4 +154,4 @@ Run full quality evals after: - clinical output changes - release or handoff confidence checks -`npm run verify:release` includes `npm run eval:quality:release` after cheaper local gates. `eval:quality:release` passes `docs/release-source-metadata-debt-2026-06-30.json` while that temporary release debt is active. Use focused variants such as `--retrieval-only`, `--rag-only`, `--limit`, `--query`, or `--question` during development to avoid unnecessary provider-backed cost. +`npm run verify:release` includes `npm run eval:quality:release` after cheaper local gates. `eval:quality:release` passes `docs/release-source-metadata-debt-2026-06-30.json` while that bounded release debt is active. Use `npm run audit:source-governance` to refresh the live corpus debt counts, and use focused variants such as `--retrieval-only`, `--rag-only`, `--limit`, `--query`, or `--question` during development to avoid unnecessary provider-backed cost. diff --git a/package.json b/package.json index cbcf4f82f5..b6397a2318 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "dev": "node scripts/dev-free-port.mjs", "preinstall": "node scripts/check-node-engine.cjs", "ensure": "node scripts/ensure-local-server.mjs", - "build": "node scripts/guard-next-build.mjs && node --max-old-space-size=8192 ./node_modules/next/dist/bin/next build --webpack", + "build": "node scripts/guard-next-build.mjs && node --max-old-space-size=16384 ./node_modules/next/dist/bin/next build --webpack", "start": "node scripts/dev-free-port.mjs start", "lint": "node --max-old-space-size=8192 ./node_modules/eslint/bin/eslint.js src tests scripts worker supabase playwright eslint.config.mjs next.config.ts playwright.config.ts playwright.visual.config.ts vitest.config.mts --no-error-on-unmatched-pattern", "typecheck": "node ./node_modules/typescript/bin/tsc --noEmit", @@ -41,6 +41,7 @@ "enrich:backfill": "tsx scripts/backfill-enrichment.ts", "classify:documents": "tsx scripts/classify-documents.ts", "check:document-label-coverage": "tsx scripts/check-document-label-coverage.ts", + "audit:source-governance": "tsx scripts/audit-source-governance.ts", "tags:backfill": "tsx scripts/backfill-document-tags.ts", "index:backfill": "tsx scripts/backfill-smart-index.ts", "visual:backfill": "tsx scripts/backfill-visual-intelligence.ts", diff --git a/scripts/audit-source-governance.ts b/scripts/audit-source-governance.ts new file mode 100644 index 0000000000..65e1598bb4 --- /dev/null +++ b/scripts/audit-source-governance.ts @@ -0,0 +1,285 @@ +import * as nextEnv from "@next/env"; +import type { DocumentLabel } from "@/lib/types"; + +const loadEnvConfig = + nextEnv.loadEnvConfig ?? + (nextEnv as unknown as { default?: { loadEnvConfig?: typeof nextEnv.loadEnvConfig } }).default?.loadEnvConfig; + +if (!loadEnvConfig) throw new Error("Unable to load @next/env loadEnvConfig."); +loadEnvConfig(process.cwd()); + +type AuditArgs = { + json: boolean; + help: boolean; +}; + +type SupabaseAdmin = Awaited>; + +type DocumentRow = { + id: string; + title: string; + file_name: string; + status: string; + metadata: Record | null; +}; + +type LabelRow = { + id: string; + document_id: string; + label_type: DocumentLabel["label_type"]; + source: DocumentLabel["source"]; +}; + +type QueryResult = { + data: T[] | null; + error: { message: string } | null; +}; + +type QueryBuilder = PromiseLike> & { + eq(column: string, value: unknown): QueryBuilder; + order(column: string, options: { ascending: boolean }): QueryBuilder; + range(from: number, to: number): QueryBuilder; +}; + +const requiredMetadataKeys = [ + "document_status", + "clinical_validation_status", + "clinical_validation_evidence", + "extraction_quality", +] as const; + +const smartV2LabelTypes = new Set([ + "clinical_action", + "care_phase", + "document_intent", + "content_feature", +]); + +async function loadAdminClient() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + return createAdminClient(); +} + +function parseArgs(argv: string[]): AuditArgs { + const args: AuditArgs = { json: false, help: false }; + for (const token of argv) { + if (token === "--json") { + args.json = true; + continue; + } + if (token === "--help" || token === "-h") { + args.help = true; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + return args; +} + +function usage() { + return [ + "Usage: npm run audit:source-governance -- [options]", + "", + "Read-only audit of source governance metadata and smart-v2 label debt.", + "", + "Options:", + " --json Print machine-readable JSON.", + " --help Show this help.", + ].join("\n"); +} + +async function fetchAll( + supabase: SupabaseAdmin, + table: "documents" | "document_labels", + select: string, + filter?: (query: QueryBuilder) => QueryBuilder, +) { + const rows: T[] = []; + const pageSize = 1000; + + for (let from = 0; ; from += pageSize) { + let query = supabase + .from(table) + .select(select) + .order("id", { ascending: true }) + .range(from, from + pageSize - 1) as unknown as QueryBuilder; + if (filter) query = filter(query); + const { data, error } = await query; + if (error) throw new Error(error.message); + rows.push(...(data ?? [])); + if (!data || data.length < pageSize) break; + } + + return rows; +} + +function metadataRecord(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function stringValue(value: unknown) { + return typeof value === "string" && value.trim() ? value.trim() : "missing"; +} + +function increment(counts: Map, key: string) { + counts.set(key, (counts.get(key) ?? 0) + 1); +} + +function sortedCounts(counts: Map) { + return Object.fromEntries([...counts.entries()].sort(([left], [right]) => left.localeCompare(right))); +} + +function compactDocument(document: DocumentRow) { + const metadata = metadataRecord(document.metadata); + return { + id: document.id, + title: document.title, + file_name: document.file_name, + document_status: stringValue(metadata.document_status), + clinical_validation_status: stringValue(metadata.clinical_validation_status), + extraction_quality: stringValue(metadata.extraction_quality), + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const supabase = await loadAdminClient(); + const documents = await fetchAll(supabase, "documents", "id,title,file_name,status,metadata", (query) => + query.eq("status", "indexed"), + ); + const labels = await fetchAll(supabase, "document_labels", "id,document_id,label_type,source", (query) => + query.eq("source", "generated"), + ); + + const statusCounts = new Map(); + const validationCounts = new Map(); + const extractionCounts = new Map(); + const requiredMissingCounts = new Map(); + const missingRequiredDocuments: Array & { missing_keys: string[] }> = []; + + for (const document of documents) { + const metadata = metadataRecord(document.metadata); + increment(statusCounts, stringValue(metadata.document_status)); + increment(validationCounts, stringValue(metadata.clinical_validation_status)); + increment(extractionCounts, stringValue(metadata.extraction_quality)); + + const missingKeys = requiredMetadataKeys.filter((key) => { + const value = metadata[key]; + return value === undefined || value === null || value === ""; + }); + for (const key of missingKeys) increment(requiredMissingCounts, key); + if (missingKeys.length > 0) { + missingRequiredDocuments.push({ ...compactDocument(document), missing_keys: missingKeys }); + } + } + + const indexedDocumentIds = new Set(documents.map((document) => document.id)); + const generatedLabelDocumentIds = new Set(labels.map((label) => label.document_id)); + const smartV2DocumentIds = new Set( + labels.filter((label) => smartV2LabelTypes.has(label.label_type)).map((label) => label.document_id), + ); + const missingGeneratedLabelDocuments = documents.filter((document) => !generatedLabelDocumentIds.has(document.id)); + const missingSmartV2LabelDocuments = documents.filter((document) => !smartV2DocumentIds.has(document.id)); + const requiredMetadataMissingTotal = [...requiredMissingCounts.values()].reduce((total, count) => total + count, 0); + + const report = { + mode: "read-only", + indexed_documents: documents.length, + required_metadata_missing_total: requiredMetadataMissingTotal, + required_metadata_missing_counts: Object.fromEntries( + requiredMetadataKeys.map((key) => [key, requiredMissingCounts.get(key) ?? 0]), + ), + document_status_counts: sortedCounts(statusCounts), + clinical_validation_status_counts: sortedCounts(validationCounts), + extraction_quality_counts: sortedCounts(extractionCounts), + generated_label_coverage: { + documents_with_generated_labels: generatedLabelDocumentIds.size, + indexed_without_generated_labels: missingGeneratedLabelDocuments.length, + }, + smart_v2_label_coverage: { + documents_with_smart_v2_labels: smartV2DocumentIds.size, + indexed_without_smart_v2_labels: missingSmartV2LabelDocuments.length, + }, + debt_counts: { + review_due: statusCounts.get("review_due") ?? 0, + unknown_status: statusCounts.get("unknown") ?? 0, + unverified_validation: validationCounts.get("unverified") ?? 0, + poor_extraction: extractionCounts.get("poor") ?? 0, + partial_extraction: extractionCounts.get("partial") ?? 0, + missing_smart_v2_labels: missingSmartV2LabelDocuments.length, + }, + sample_review_due_documents: documents + .filter((document) => metadataRecord(document.metadata).document_status === "review_due") + .slice(0, 10) + .map(compactDocument), + sample_unknown_status_documents: documents + .filter((document) => metadataRecord(document.metadata).document_status === "unknown") + .slice(0, 10) + .map(compactDocument), + sample_unverified_documents: documents + .filter((document) => metadataRecord(document.metadata).clinical_validation_status === "unverified") + .slice(0, 10) + .map(compactDocument), + missing_required_metadata_documents: missingRequiredDocuments.slice(0, 25), + missing_smart_v2_label_documents: missingSmartV2LabelDocuments.map((document) => ({ + id: document.id, + title: document.title, + file_name: document.file_name, + })), + indexed_document_id_count: indexedDocumentIds.size, + passed_required_metadata_gate: requiredMetadataMissingTotal === 0, + }; + + if (args.json) { + console.log(JSON.stringify(report, null, 2)); + } else { + console.log("[Source Governance Audit]"); + console.log(`Mode: ${report.mode}`); + console.log(`Indexed documents: ${report.indexed_documents}`); + console.log(`Required metadata missing: ${report.required_metadata_missing_total}`); + console.log( + `Document status: ${Object.entries(report.document_status_counts) + .map(([value, count]) => `${value}=${count}`) + .join(", ")}`, + ); + console.log( + `Clinical validation: ${Object.entries(report.clinical_validation_status_counts) + .map(([value, count]) => `${value}=${count}`) + .join(", ")}`, + ); + console.log( + `Extraction quality: ${Object.entries(report.extraction_quality_counts) + .map(([value, count]) => `${value}=${count}`) + .join(", ")}`, + ); + console.log( + `Generated labels: missing=${report.generated_label_coverage.indexed_without_generated_labels}, covered=${report.generated_label_coverage.documents_with_generated_labels}`, + ); + console.log( + `Smart-v2 labels: missing=${report.smart_v2_label_coverage.indexed_without_smart_v2_labels}, covered=${report.smart_v2_label_coverage.documents_with_smart_v2_labels}`, + ); + if (report.missing_smart_v2_label_documents.length) { + console.log("Documents missing smart-v2 labels:"); + for (const document of report.missing_smart_v2_label_documents) { + console.log(`- ${document.title} (${document.file_name})`); + } + } + console.log( + report.passed_required_metadata_gate + ? "PASS: required source governance metadata is complete." + : "FAIL: required source governance metadata has gaps.", + ); + } + + if (!report.passed_required_metadata_gate) process.exitCode = 1; +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/scripts/backfill-source-metadata.ts b/scripts/backfill-source-metadata.ts index 14773887a4..171438b2b5 100644 --- a/scripts/backfill-source-metadata.ts +++ b/scripts/backfill-source-metadata.ts @@ -38,9 +38,20 @@ type ClinicalValidationEvidence = { const APPLY = process.argv.includes("--apply"); const EVAL_ONLY = process.argv.includes("--eval-only"); -const NOW = new Date("2026-06-30T00:00:00+08:00"); const BACKFILL_VERSION = "source_metadata_backfill_2026_06_30_v1"; +function backfillAsOfDate() { + const index = process.argv.indexOf("--as-of"); + if (index < 0) return new Date(); + const raw = process.argv[index + 1]; + if (!raw || raw.startsWith("--")) throw new Error("Missing value for --as-of"); + const date = /^\d{4}-\d{2}-\d{2}$/.test(raw) ? new Date(`${raw}T00:00:00+08:00`) : new Date(raw); + if (Number.isNaN(date.getTime())) throw new Error(`Invalid --as-of date: ${raw}`); + return date; +} + +const NOW = backfillAsOfDate(); + const publisherByCode: Record = { AKG: { publisher: "Armadale Kalamunda Group", jurisdiction: "Australia/WA" }, BMJ: { publisher: "BMJ Best Practice", jurisdiction: "International" }, diff --git a/scripts/check-document-label-coverage.ts b/scripts/check-document-label-coverage.ts index c5d49ab082..d49df13467 100644 --- a/scripts/check-document-label-coverage.ts +++ b/scripts/check-document-label-coverage.ts @@ -1,6 +1,8 @@ import * as nextEnv from "@next/env"; import { promises as fs } from "node:fs"; import { resolve } from "node:path"; +import { reviewDocumentTagQuality } from "@/lib/document-tags"; +import type { DocumentLabel } from "@/lib/types"; const loadEnvConfig = nextEnv.loadEnvConfig ?? @@ -20,12 +22,17 @@ type SupabaseAdmin = Awaited>; type DocumentRow = { id: string; + title: string; + file_name: string; }; type LabelRow = { id: string; document_id: string; - label_type: string; + label: string; + label_type: DocumentLabel["label_type"]; + source: DocumentLabel["source"]; + confidence: number; }; type QueryResult = { @@ -164,6 +171,12 @@ function countByLabelType(labels: LabelRow[]) { const smartV2LabelTypes = new Set(["clinical_action", "care_phase", "document_intent", "content_feature"]); +function countQualityIssues(issues: ReturnType) { + const counts = new Map(); + for (const issue of issues) counts.set(issue.kind, (counts.get(issue.kind) ?? 0) + 1); + return Object.fromEntries([...counts.entries()].sort()); +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (args.help) { @@ -174,9 +187,14 @@ async function main() { const supabase = await loadAdminClient(); const allowedSiteMissing = await loadAllowlist(args.allowedSiteMissingPath); const allowedDocumentTypeMissing = await loadAllowlist(args.allowedDocumentTypeMissingPath); - const documents = await fetchAll(supabase, "documents", "id", (query) => query.eq("status", "indexed")); - const labels = await fetchAll(supabase, "document_labels", "id,document_id,label_type", (query) => - query.eq("source", "generated"), + const documents = await fetchAll(supabase, "documents", "id,title,file_name", (query) => + query.eq("status", "indexed"), + ); + const labels = await fetchAll( + supabase, + "document_labels", + "id,document_id,label,label_type,source,confidence", + (query) => query.eq("source", "generated"), ); const documentIds = new Set(documents.map((document) => document.id)); @@ -202,6 +220,17 @@ async function main() { const missingDocumentType = [...documentIds].filter( (id) => !documentTypeDocumentIds.has(id) && !allowedDocumentTypeMissing.has(id), ); + const labelsByDocument = new Map(); + for (const label of labels) { + labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); + } + const qualityIssues = reviewDocumentTagQuality( + documents.map((document) => ({ + ...document, + labels: labelsByDocument.get(document.id) ?? [], + })), + { overusedThreshold: Math.max(100, Math.ceil(documents.length * 0.35)) }, + ); const passed = missingGenerated.length === 0 && missingSite.length === 0 && missingDocumentType.length === 0; @@ -217,6 +246,18 @@ async function main() { indexed_without_smart_v2: missingSmartV2.length, labels_by_type: countByLabelType(labels), smart_v2_labels_by_type: countByLabelType(smartV2Labels), + label_quality_issue_count: qualityIssues.length, + label_quality_issue_counts: countQualityIssues(qualityIssues), + sample_label_quality_issues: qualityIssues.slice(0, 10).map((issue) => ({ + kind: issue.kind, + label: issue.label, + canonical_label: issue.canonicalLabel, + label_type: issue.label_type, + count: issue.count, + reason: issue.reason, + examples: issue.examples, + document_titles: issue.documentTitles, + })), sample_missing_generated: missingGenerated.slice(0, 10), sample_missing_site: missingSite.slice(0, 10), sample_missing_document_type: missingDocumentType.slice(0, 10), @@ -251,6 +292,12 @@ async function main() { .map(([type, count]) => `${type}=${count}`) .join(", ")}`, ); + console.log(`Label quality issues: ${report.label_quality_issue_count}`); + console.log( + `Label quality issue counts: ${Object.entries(report.label_quality_issue_counts) + .map(([type, count]) => `${type}=${count}`) + .join(", ")}`, + ); if (allowedSiteMissingDocs.length) { console.log(`Allowed indexed docs without site labels (from allowlist): ${allowedSiteMissingDocs.length}`); } diff --git a/scripts/classify-documents.ts b/scripts/classify-documents.ts index c399dfc2cb..f102bc7220 100644 --- a/scripts/classify-documents.ts +++ b/scripts/classify-documents.ts @@ -1,4 +1,5 @@ import * as nextEnv from "@next/env"; +import { documentLabelTier } from "@/lib/document-tags"; const loadEnvConfig = nextEnv.loadEnvConfig ?? @@ -42,6 +43,8 @@ type GeneratedLabelRow = { generated_by: "document-organization-classifier"; organization_profile_version: "document-organization-v1"; classified_at: string; + label_tier: ReturnType; + review_status: Classification["profile"]["review_status"]; }; }; @@ -263,6 +266,8 @@ function generatedLabelsForPlan(plan: ClassificationPlan, stampedAt: string): Ge generated_by: "document-organization-classifier", organization_profile_version: "document-organization-v1", classified_at: stampedAt, + label_tier: documentLabelTier(label.label, label.label_type), + review_status: plan.classification.profile.review_status, }, })); } diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 5136bac03a..a56727b12f 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -226,9 +226,7 @@ function failureCategoryCounts(results: Array<{ failures: string[] }>) { } function isSourceMetadataDebtThresholdFailure(failure: string) { - return ( - failure.startsWith("top-result stale/review/unknown rate") || failure.startsWith("top-result review_required_rate") - ); + return failure.startsWith("top-result stale_rate") || failure.startsWith("top-result review_required_rate"); } function isIsoDateString(value: string) { @@ -265,7 +263,7 @@ function evaluateSourceMetadataDebtAcceptance(args: { } if (args.governance.stale_rate > acceptance.max_stale_rate) { rejectionReasons.push( - `stale/review/unknown rate ${args.governance.stale_rate} exceeds accepted ceiling ${acceptance.max_stale_rate}`, + `stale rate ${args.governance.stale_rate} exceeds accepted ceiling ${acceptance.max_stale_rate}`, ); } if (args.governance.review_required_rate > acceptance.max_review_required_rate) { @@ -339,7 +337,8 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { unverified_top_results: unverified, unknown_extraction_top_results: unknownExtraction, poor_extraction_top_results: poorExtraction, - stale_rate: rate(stale + reviewDue + unknown, total), + stale_rate: rate(stale, total), + stale_review_unknown_rate: rate(stale + reviewDue + unknown, total), review_required_top_results: reviewRequired, review_required_rate: rate(reviewRequired, total), metadata_policy: @@ -417,7 +416,7 @@ export function buildEvalQualityReport(args: { } if (governance.stale_rate > qualityThresholds.staleTopResultRate) { thresholdFailures.push( - `top-result stale/review/unknown rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`, + `top-result stale_rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`, ); } if (governance.review_required_rate > qualityThresholds.reviewRequiredTopResultRate) { @@ -607,7 +606,8 @@ ${markdownTable([ ["Unverified top results", governance.unverified_top_results], ["Unknown-extraction top results", governance.unknown_extraction_top_results], ["Poor-extraction top results", governance.poor_extraction_top_results], - ["Stale/review/unknown rate", governance.stale_rate], + ["Stale rate", governance.stale_rate], + ["Stale/review/unknown rate", governance.stale_review_unknown_rate], ["Review-required top results", governance.review_required_top_results], ["Review-required rate", governance.review_required_rate], ])} diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 6211a879e1..eaee4b8d3a 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -206,7 +206,11 @@ import { sourceTextForCompactDisplay, sourceTextForVerbatimQuote, } from "@/lib/source-text-sanitizer"; -import { groupSourceGovernanceWarnings, type SourceGovernanceWarning } from "@/lib/source-governance"; +import { + frontendSourceGovernanceWarnings, + groupSourceGovernanceWarnings, + type SourceGovernanceWarning, +} from "@/lib/source-governance"; import { smartEvidenceTags } from "@/lib/evidence-tags"; import { reviewDocumentTagQuality, @@ -614,7 +618,7 @@ function ScopeAndGovernanceNotice({ scope: SearchScopeSummary | null; warnings: SourceGovernanceWarning[]; }) { - const groupedWarnings = groupSourceGovernanceWarnings(warnings).slice(0, 4); + const groupedWarnings = groupSourceGovernanceWarnings(frontendSourceGovernanceWarnings(warnings)).slice(0, 4); const showScope = Boolean(scope && scope.activeFilterCount > 0) || Boolean(scope?.warnings?.length) || @@ -2216,6 +2220,7 @@ function AnswerInsightBar({ queryMode: ClinicalQueryMode; sourceGovernanceWarnings: SourceGovernanceWarning[]; }) { + const frontendGovernanceWarnings = frontendSourceGovernanceWarnings(sourceGovernanceWarnings); const metadata = normalizeSourceMetadata( bestSource?.source_metadata ?? answer.sources?.[0]?.source_metadata ?? answer.citations?.[0]?.source_metadata, ); @@ -2225,8 +2230,8 @@ function AnswerInsightBar({ queryModeLabel(queryMode); const sourceCount = answer.evidenceSummary?.total_sources ?? answer.sources?.length ?? answer.citations.length; const support = relevanceChipLabel(relevance ?? answer.relevance, answer.grounded); - const sourceStatus = sourceGovernanceWarnings.length - ? `${sourceGovernanceWarnings.length} source status note${sourceGovernanceWarnings.length === 1 ? "" : "s"}` + const sourceStatus = frontendGovernanceWarnings.length + ? `${frontendGovernanceWarnings.length} source status note${frontendGovernanceWarnings.length === 1 ? "" : "s"}` : sourceStatusLabel(metadata); const retrievalGate = answer.retrievalDiagnostics?.gateStatus; const items = [ @@ -7182,7 +7187,11 @@ export function ClinicalDashboard({ const showSystemNotice = Boolean(setupWarning && !demoMode); const groupedGovernanceWarningCount = useMemo( - () => groupSourceGovernanceWarnings(sourceGovernanceWarnings).reduce((total, warning) => total + warning.count, 0), + () => + groupSourceGovernanceWarnings(frontendSourceGovernanceWarnings(sourceGovernanceWarnings)).reduce( + (total, warning) => total + warning.count, + 0, + ), [sourceGovernanceWarnings], ); const mobileFabState = useMemo( diff --git a/src/components/DocumentOrganizationBadges.tsx b/src/components/DocumentOrganizationBadges.tsx index 1980ba09fa..5f8549ecb8 100644 --- a/src/components/DocumentOrganizationBadges.tsx +++ b/src/components/DocumentOrganizationBadges.tsx @@ -2,6 +2,7 @@ import { AlertTriangle, Building2, FileText, Tag } from "lucide-react"; import { cn, metadataPill, toneInfo, toneNeutral, toneWarning } from "@/components/ui-primitives"; import { canonicalDocumentDisplayTitle } from "@/lib/document-organization"; +import { formatDocumentLabelDisplay } from "@/lib/document-tags"; import type { DocumentLabel, DocumentOrganizationProfile } from "@/lib/types"; type OrganizationDocument = { @@ -29,16 +30,6 @@ export function documentDisplayTitle(document: Pick - word.length <= 4 && word === word.toUpperCase() ? word : `${word[0]?.toUpperCase()}${word.slice(1)}`, - ) - .join(" "); -} - function labelFromLabels( labels: OrganizationDocument["labels"], type: DocumentLabel["label_type"], @@ -79,7 +70,7 @@ export function DocumentOrganizationBadges({ {siteLabel ? ( - {compact && siteShortLabel ? siteShortLabel : displayLabel(siteLabel)} + {compact && siteShortLabel ? siteShortLabel : formatDocumentLabelDisplay(siteLabel, "site")} ) : needsReview && candidateCount > 0 ? ( @@ -90,7 +81,7 @@ export function DocumentOrganizationBadges({ {typeLabel ? ( - {displayLabel(typeLabel)} + {formatDocumentLabelDisplay(typeLabel, "document_type")} ) : null} {needsReview ? ( diff --git a/src/components/DocumentTagCloud.tsx b/src/components/DocumentTagCloud.tsx index e6a5390c4b..362e337263 100644 --- a/src/components/DocumentTagCloud.tsx +++ b/src/components/DocumentTagCloud.tsx @@ -118,7 +118,10 @@ export function DocumentTagCloud({ selectedTagKeys, grouped = false, }: DocumentTagCloudProps) { - const tags = useMemo(() => buildSmartDocumentTags(labels, { query, includeManualGroup: true }), [labels, query]); + const tags = useMemo( + () => buildSmartDocumentTags(labels, { query, includeManualGroup: true }).filter((tag) => tag.tier !== "ranking"), + [labels, query], + ); const groupedTags = useMemo( () => groupSmartDocumentTags(labels, { query, includeManualGroup: true }), [labels, query], diff --git a/src/lib/answer-ranking.ts b/src/lib/answer-ranking.ts index 1414ed8715..254f5cd29f 100644 --- a/src/lib/answer-ranking.ts +++ b/src/lib/answer-ranking.ts @@ -153,12 +153,12 @@ function classSignalScore(queryClass: RagQueryClass, result: SearchResult, combi function sourceQualityScore(result: SearchResult) { const metadata = result.source_metadata; let score = 0; - if (metadata?.document_status === "current") score += 0.035; - if (metadata?.document_status === "outdated") score -= 0.08; - if (metadata?.clinical_validation_status === "approved") score += 0.035; - if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.02; - if (metadata?.extraction_quality === "good") score += 0.035; - if (metadata?.extraction_quality === "poor") score -= 0.06; + if (metadata?.document_status === "current") score += 0.004; + if (metadata?.document_status === "outdated") score -= 0.04; + if (metadata?.clinical_validation_status === "approved") score += 0.004; + if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.003; + if (metadata?.extraction_quality === "good") score += 0.002; + if (metadata?.extraction_quality === "poor") score -= 0.05; return score; } diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index d144e4f9aa..a7fab396a9 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -630,9 +630,8 @@ function hasImageEvidenceNeed(query: string) { function extractionQualityScore(result: SearchResult) { const quality = result.source_metadata?.extraction_quality; - if (quality === "good") return 0.03; - if (quality === "partial") return -0.01; - if (quality === "poor") return -0.04; + if (quality === "good") return 0.002; + if (quality === "poor") return -0.05; return 0; } @@ -668,17 +667,14 @@ function sourceQualityRankSignal(result: SearchResult, queryClass: RagQueryClass if (result.source_strength === "moderate") score += 0.02; if (result.source_strength === "limited") score -= 0.015; - if (metadata?.document_status === "current") score += 0.035; - if (metadata?.document_status === "review_due") score -= 0.025; - if (metadata?.document_status === "outdated") score -= 0.09; + if (metadata?.document_status === "current") score += 0.004; + if (metadata?.document_status === "outdated") score -= 0.04; - if (metadata?.clinical_validation_status === "approved") score += 0.035; - if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.025; - if (metadata?.clinical_validation_status === "unverified") score -= 0.02; + if (metadata?.clinical_validation_status === "approved") score += 0.004; + if (metadata?.clinical_validation_status === "locally_reviewed") score += 0.003; - if (metadata?.extraction_quality === "good") score += 0.025; - if (metadata?.extraction_quality === "partial") score -= 0.015; - if (metadata?.extraction_quality === "poor") score -= 0.075; + if (metadata?.extraction_quality === "good") score += 0.002; + if (metadata?.extraction_quality === "poor") score -= 0.06; const tableFocusedQuery = queryClass === "table_threshold" || queryClass === "medication_dose_risk"; if (tableFocusedQuery && (result.table_facts?.length ?? 0) > 0) score += 0.055; @@ -1222,8 +1218,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se : 0; const status = result.source_metadata?.document_status; const validation = result.source_metadata?.clinical_validation_status; - const statusBoost = status === "current" ? 0.05 : status === "review_due" ? -0.04 : status === "outdated" ? -0.18 : 0; - const validationBoost = validation === "approved" ? 0.04 : validation === "locally_reviewed" ? 0.025 : 0; + const statusBoost = status === "current" ? 0.004 : status === "outdated" ? -0.04 : 0; + const validationBoost = validation === "approved" ? 0.004 : validation === "locally_reviewed" ? 0.003 : 0; const publicationYearsAgo = parseDateAsYearsAgo(result.source_metadata?.publication_date); const reviewYearsAgo = parseDateAsYearsAgo(result.source_metadata?.review_date); const freshnessBoost = @@ -1410,7 +1406,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se // the same result. const nextStepActionQuery = /\b(?:next step|step after|action)\b/i.test(query); const riskFlowchartSource = nextStepActionQuery - ? riskFlowchartZoneActionSource || (riskFlowchartLexicalSource && riskZoneActionPattern.test(riskFlowchartEvidenceText)) + ? riskFlowchartZoneActionSource || + (riskFlowchartLexicalSource && riskZoneActionPattern.test(riskFlowchartEvidenceText)) : riskFlowchartLexicalSource || riskFlowchartZoneActionSource; const riskFlowchartCanonicalTitle = riskFlowchartQuery && @@ -1459,9 +1456,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se const comparisonCoverageBoost = queryClass === "comparison" && titleCoverageBoost > 0 && evidenceBoost > 0.02 ? 0.025 : 0; const routeSignal = (() => { - if (result.source_metadata?.document_status === "review_due") return -0.03; - if (result.source_metadata?.document_status === "outdated") return -0.1; - if (result.source_metadata?.extraction_quality === "poor") return -0.05; + if (result.source_metadata?.document_status === "outdated") return -0.04; + if (result.source_metadata?.extraction_quality === "poor") return -0.04; return 0; })(); const lowLexicalCoverage = normalizedTokens.length > 0 && evidenceBoost < 0.035 && titleCoverageBoost < 0.045; diff --git a/src/lib/document-organization.ts b/src/lib/document-organization.ts index 40d1a11b28..67c19f2f27 100644 --- a/src/lib/document-organization.ts +++ b/src/lib/document-organization.ts @@ -387,7 +387,18 @@ const documentTypePatterns: Array<{ confidence: 0.82, patterns: [/\bprescrib\b/i, /\baid\b/i, /\bcalculator\b/i, /\bdosing\b/i, /\bnomogram\b/i], }, - { label: "reference", confidence: 0.72, patterns: [/\breference\b/i, /\binformation sheet\b/i, /\bplacecard\b/i] }, + { + label: "reference", + confidence: 0.72, + patterns: [ + /\breference\b/i, + /\bquick reference\b/i, + /\bquick guide\b/i, + /\bqrg\b/i, + /\binformation sheet\b/i, + /\bplacecard\b/i, + ], + }, ]; const secondaryFacetLimits: Record = { @@ -856,8 +867,12 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "assess", label_type: "clinical_action", - strong: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination)\b/i], - body: [/\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination)\b/i], + strong: [ + /\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b|bleeding event)\b/i, + ], + body: [ + /\b(?:assess|assessment|screening|screen for|diagnos|diagnosis|evaluate|evaluation|examination|scale\b|aims\b)\b/i, + ], minBodyMatches: 1, }, { @@ -868,7 +883,9 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "administer", label_type: "clinical_action", - strong: [/\b(?:administer|administration|infusions?|eye drops?|medication administration)\b/i], + strong: [ + /\b(?:administer|administration|infusions?|eye drops?|medication administration|vaccination|prophylaxis|chloramphenicol|gentamicin|oxybuprocaine|phenylephrine|naloxone|prenoxad|niacin|polystyrene sulphonate|resonium)\b/i, + ], }, { label: "monitor", @@ -894,8 +911,10 @@ const smartFacetRules: SmartFacetRule[] = [ { label: "document", label_type: "clinical_action", - strong: [/\b(?:document|documentation|record in)\b/i], - body: [/\b(?:documentation|record in)\b/i], + strong: [ + /\b(?:document|documentation|record in|medical records?|my health record|uploading|amending|removing documents)\b/i, + ], + body: [/\b(?:documentation|record in|medical records?|my health record)\b/i], minBodyMatches: 3, }, { label: "notify", label_type: "clinical_action", strong: [/\b(?:notify|notification|report to)\b/i] }, @@ -948,26 +967,28 @@ const smartFacetRules: SmartFacetRule[] = [ label: "clinical-instruction", label_type: "document_intent", strong: [ - /\b(?:guideline|procedure|protocol|qrg\b|quick reference guide|sop\b|standard operating procedure|clinical instruction|clinical management|clinical summary|insertion summary|clinical poster|sdg\b|standing drug guideline)\b/i, + /\b(?:guideline|procedure|protocol|qrg\b|quick reference|quick guide|sop\b|standard operating procedure|clinical instruction|clinical management|clinical summary|insertion summary|clinical poster|sdg\b|standing drug guideline|apheresis|biopsy|perfusion|ventilation|cardiac arrest|stemi activation|therapy recipients|micro alerts)\b/i, ], }, { label: "decision-support", label_type: "document_intent", - strong: [/\b(?:algorithm|flowchart|decision tree|criteria|threshold|diagnos|diagnosis)\b/i], + strong: [ + /\b(?:algorithm|flowchart|decision tree|criteria|threshold|diagnos|diagnosis|scale\b|aims\b|bleeding event|hypothermia|cardiac arrest|stemi activation|pathway)\b/i, + ], }, { label: "patient-information", label_type: "document_intent", strong: [ - /\b(?:patient information|consumer information|factsheet|leaflet|flyer|brochure|booklet|poster|info sheet|information sheet|print ready pi|food and nutrition|huffers and puffers|common discomforts in pregnancy|for patients)\b/i, + /\b(?:patient information|consumer information|factsheet|leaflet|flyer|brochure|booklet|poster|info sheet|information sheet|print ready pi|food and nutrition|huffers and puffers|common discomforts in pregnancy|caring for your|living with|before surgery|tips for reducing|genetic cholesterol|for patients)\b/i, ], }, { label: "staff-guidance", label_type: "document_intent", strong: [ - /\b(?:staff guidance|staff access|staff only|registrar role|staff role|roles? and responsibilities|orientation|training|education|competenc|whs\b|work health and safety)\b/i, + /\b(?:staff guidance|staff access|staff only|registrar role|staff role|roles? and responsibilities|reflective practice|peer workforce|research nurse|orientation|training|education|competenc|whs\b|work health and safety)\b/i, ], }, { @@ -979,19 +1000,21 @@ const smartFacetRules: SmartFacetRule[] = [ label: "operational-process", label_type: "document_intent", strong: [ - /\b(?:workflow|process|administration|operational|procedure|sop\b|standard operating procedure|access to|access model|capacity model|management sop|roles? and responsibilities)\b/i, + /\b(?:workflow|process|administration|operational|procedure|sop\b|standard operating procedure|access to|access model|capacity model|remote network access|my health record|medical records?|contingency plan|workloads|staffing|management sop|roles? and responsibilities)\b/i, ], }, { label: "documentation-requirement", label_type: "document_intent", - strong: [/\b(?:documentation|record in|form required|documented in)\b/i], + strong: [ + /\b(?:documentation|record in|form required|documented in|medical records?|my health record|uploading|amending|removing documents)\b/i, + ], }, { label: "medication-instruction", label_type: "document_intent", strong: [ - /\b(?:prescrib|dose|dosing|medication instruction|medication management|medications?\b|drug guideline|standing drug guideline|sdg\b|drug infusions?|infusions?|eye drops?|permethrin)\b/i, + /\b(?:prescrib|dose|dosing|medication instruction|medication management|medications?\b|drug guideline|standing drug guideline|sdg\b|drug infusions?|infusions?|eye drops?|over the counter|complementary medicines|vaccination|prophylaxis|chloramphenicol|gentamicin|oxybuprocaine|phenylephrine|naloxone|prenoxad|niacin|polystyrene sulphonate|resonium|permethrin)\b/i, ], }, @@ -1009,6 +1032,11 @@ const smartFacetRules: SmartFacetRule[] = [ /\b(?:dose|dosing|dosage|nomogram|drug guideline|standing drug guideline|sdg\b|drug infusions?|infusions?|eye drops?)\b/i, ], }, + { + label: "contains-quick-reference", + label_type: "content_feature", + strong: [/\b(?:quick reference|quick guide|qrg\b)\b/i], + }, { label: "contains-monitoring-schedule", label_type: "content_feature", diff --git a/src/lib/document-tags.ts b/src/lib/document-tags.ts index f8ec9b10dd..d205ca1b70 100644 --- a/src/lib/document-tags.ts +++ b/src/lib/document-tags.ts @@ -16,12 +16,15 @@ export type SmartDocumentTagGroup = | "Content feature" | "Manual"; +export type SmartDocumentTagTier = "primary" | "secondary" | "ranking"; + export type SmartDocumentTag = { key: string; label: string; searchText: string; label_type: DocumentLabelType; group: SmartDocumentTagGroup; + tier: SmartDocumentTagTier; source: DocumentLabel["source"]; confidence: number; score: number; @@ -97,33 +100,34 @@ const groupLabels: Record = { }; const groupRank: Record = { - Site: 0, - Medication: 1, - Risk: 2, - Workflow: 3, - Topic: 4, - Population: 5, - Setting: 6, - Service: 7, - "Document type": 8, - "Clinical action": 9, - "Care phase": 10, - "Document intent": 11, + Site: 1, + Medication: 2, + "Clinical action": 3, + "Care phase": 4, + Risk: 5, + "Document intent": 6, + Topic: 7, + Service: 8, + Setting: 9, + Population: 10, + Workflow: 11, "Content feature": 12, - Manual: 13, + "Document type": 13, + Manual: 14, }; export const smartDocumentFacetGroups: SmartDocumentTagGroup[] = [ "Site", "Medication", - "Risk", - "Workflow", "Clinical action", "Care phase", "Document intent", + "Risk", + "Workflow", + "Topic", + "Service", "Setting", "Population", - "Service", "Document type", ]; @@ -139,6 +143,7 @@ const acronymDisplay = new Map([ ["ect", "ECT"], ["fbc", "FBC"], ["gp", "GP"], + ["hr", "HR"], ["honos", "HoNOS"], ["honosca", "HoNOSCA"], ["im", "IM"], @@ -177,6 +182,98 @@ const siteShortLabels = new Map([ ["wa health", "WA Health"], ]); +const displayLabelOverrides = new Map([ + ["assessment_tool", "Assessment tool"], + ["prescribing_aid", "Prescribing aid"], + ["blood test monitoring", "Blood test monitoring"], + ["community program for opioid pharmacotherapy", "CPOP"], + ["electroconvulsive-therapy", "Electroconvulsive therapy"], + ["substance use alcohol and drugs", "Substance use, alcohol and drugs"], + ["aggression violence code black", "Aggression, violence and Code Black"], + ["admission waitlist bed access", "Admission, waitlist and bed access"], + ["transport transfer escort", "Transport, transfer and escort"], + ["rights carers advocates", "Rights, carers and advocates"], + ["consent capacity confidentiality", "Consent, capacity and confidentiality"], + ["incident notification open disclosure", "Incident, notification and open disclosure"], + ["psychosis schizophrenia", "Psychosis and schizophrenia"], + ["depression mood disorders", "Depression and mood disorders"], + ["bipolar mood episode", "Bipolar and mood episode"], + ["anxiety trauma", "Anxiety and trauma"], + ["cognitive impairment learning disability", "Cognitive impairment and learning disability"], + ["shared care gp liaison", "Shared care and GP liaison"], + ["care coordination case management", "Care coordination and case management"], + ["mental state examination", "Mental state examination"], + ["community treatment order", "Community treatment order"], + ["substance withdrawal", "Substance withdrawal"], + ["acute psychosis", "Acute psychosis"], + ["mood episode", "Mood episode"], + ["initial assessment", "Initial assessment"], + ["acute management", "Acute management"], + ["crisis response", "Crisis response"], + ["ongoing management", "Ongoing management"], + ["maintenance treatment", "Maintenance treatment"], + ["discharge planning", "Discharge planning"], + ["post discharge follow up", "Post-discharge follow-up"], + ["clinical instruction", "Clinical instruction"], + ["decision support", "Decision support"], + ["patient information", "Patient information"], + ["staff guidance", "Staff guidance"], + ["legal governance", "Legal and governance"], + ["operational process", "Operational process"], + ["documentation requirement", "Documentation requirement"], + ["medication instruction", "Medication instruction"], + ["contains table", "Contains table"], + ["contains flowchart", "Contains flowchart"], + ["contains form", "Contains form"], + ["contains dosage guidance", "Contains dosage guidance"], + ["contains monitoring schedule", "Contains monitoring schedule"], + ["contains referral criteria", "Contains referral criteria"], + ["contains escalation criteria", "Contains escalation criteria"], + ["contains legal criteria", "Contains legal criteria"], + ["contains quick reference", "Contains quick reference"], + ["de escalate", "De-escalate"], + ["qtc monitoring", "QTc monitoring"], + ["nocc outcome measures", "NOCC outcome measures"], +]); + +const displayLowercaseWords = new Set(["and", "or", "of", "to", "for", "in", "on", "with", "from", "by"]); + +const rankingOnlyLabels = new Set([ + "clinical risk", + "mental health", + "inpatient", + "assessment", + "monitoring", + "physical health care", + "admission waitlist bed access", + "clinical", + "non clinical", + "admin", + "assess", + "refer", + "review", + "document", + "clinical instruction", + "documentation requirements", +]); + +const validClinicalActionLabels = new Set([ + "assess", + "prescribe", + "administer", + "monitor", + "escalate", + "refer", + "admit", + "discharge", + "transfer", + "observe", + "document", + "notify", + "review", + "de escalate", +]); + export const clinicalDocumentTagAliases = [ { from: "a n c", to: "absolute neutrophil count" }, { from: "absolute neutrophils", to: "absolute neutrophil count" }, @@ -334,6 +431,7 @@ function isNoisyLabel(label: string, labelType: DocumentLabelType) { if (!label || label.length < 2 || label.length > 64) return true; if (lowValuePattern.test(label)) return true; if (/\b(?:docx?|xlsx?|pptx?|pdf)\b/.test(label)) return true; + if (labelType === "clinical_action" && validClinicalActionLabels.has(label)) return false; if ( labelType === "document_type" && /^(?:policy|guideline|procedure|protocol|form|checklist|pathway|reference|algorithm|factsheet|manual|assessment_tool|prescribing_aid)$/.test( @@ -361,19 +459,40 @@ function sourceValue(value: unknown): DocumentLabel["source"] { return value === "manual" ? "manual" : "generated"; } -function displayLabel(value: string, labelType?: DocumentLabelType) { +export function formatDocumentLabelDisplay(value: string, labelType?: DocumentLabelType) { const displayValue = value.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim(); if (labelType === "site") { const siteShortLabel = siteShortLabels.get(displayValue); if (siteShortLabel) return siteShortLabel; } + const override = displayLabelOverrides.get(value) ?? displayLabelOverrides.get(displayValue); + if (override) return override; return displayValue .split(" ") .filter(Boolean) - .map((word) => acronymDisplay.get(word) ?? `${word[0]?.toUpperCase() ?? ""}${word.slice(1)}`) + .map((word, index) => { + const acronym = acronymDisplay.get(word); + if (acronym) return acronym; + if (index > 0 && displayLowercaseWords.has(word)) return word; + return `${word[0]?.toUpperCase() ?? ""}${word.slice(1)}`; + }) .join(" "); } +function displayLabel(value: string, labelType?: DocumentLabelType) { + return formatDocumentLabelDisplay(value, labelType); +} + +export function documentLabelTier(label: string, labelType: DocumentLabelType): SmartDocumentTagTier { + const normalized = normalizedText(label); + if (rankingOnlyLabels.has(normalized)) return "ranking"; + if (labelType === "site" || labelType === "medication" || labelType === "risk") return "primary"; + if (labelType === "clinical_action" || labelType === "care_phase" || labelType === "document_intent") + return "primary"; + if (labelType === "topic" && !rankingOnlyLabels.has(normalized)) return "primary"; + return "secondary"; +} + function queryTerms(query?: string) { return new Set( normalizedText(query ?? "") @@ -450,6 +569,7 @@ export function buildSmartDocumentTags( searchText: normalized.label, label_type: normalized.label_type, group, + tier: documentLabelTier(normalized.label, normalized.label_type), source, confidence: normalized.confidence, score, @@ -472,9 +592,11 @@ export function buildSmartDocumentTags( export function groupSmartDocumentTags( labels: Array> | null | undefined, - options: { limit?: number; query?: string; includeManualGroup?: boolean } = {}, + options: { limit?: number; query?: string; includeManualGroup?: boolean; includeRankingOnly?: boolean } = {}, ) { - const tags = buildSmartDocumentTags(labels, options); + const tags = buildSmartDocumentTags(labels, options).filter( + (tag) => options.includeRankingOnly || tag.tier !== "ranking", + ); return [...new Set(tags.map((tag) => tag.group))] .sort((a, b) => groupRank[a] - groupRank[b]) .map((group) => ({ @@ -592,6 +714,7 @@ export function reviewDocumentTagQuality< for (const global of globalUsage.values()) { if (global.documents.size < overusedThreshold) continue; + if (documentLabelTier(global.canonicalLabel, global.label_type) === "ranking") continue; addQualityIssue(issues, { kind: "overused", label: global.label, @@ -625,6 +748,7 @@ export function buildSmartDocumentTagFacets( const documentTagKeys = new Set(); const tags = buildSmartDocumentTags(document.labels, { query: options.query, includeManualGroup: false }); for (const tag of tags) { + if (tag.tier === "ranking") continue; if (!allowedGroups.has(tag.group) || documentTagKeys.has(tag.key)) continue; documentTagKeys.add(tag.key); const existing = facets.get(tag.key); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index ce6fbaf59d..4f43ee18ca 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -579,7 +579,7 @@ function secondStageScore(result: SearchResult, queryClass: RagQueryClass | unde if (tableVisualEvidenceUnitTypes.has(unitType)) score += 0.08; else if (visualEvidenceUnitTypes.has(unitType)) score += 0.04; if (source === "visual_intelligence") score += Math.min(0.035, Math.max(0, sourceQuality - 0.55) * 0.08); - if (result.source_metadata?.document_status === "outdated") score -= 0.05; + if (result.source_metadata?.document_status === "outdated") score -= 0.04; if (result.source_metadata?.extraction_quality === "poor") score -= 0.05; if (result.indexing_quality?.quality_score !== undefined && result.indexing_quality.quality_score < 0.55) score -= 0.035; @@ -3308,7 +3308,10 @@ function hasRiskFlowchartActionEvidence(query: string, results: SearchResult[], const evidenceText = evidenceTextForGate(result); if (!riskZoneActionPattern.test(evidenceText)) return false; if (zonePhrasePattern.test(evidenceText)) return true; - return ["risk_matrix_cell", "flowchart_step", "diagram_decision"].includes(result.index_unit?.unit_type ?? "") && bareColourPattern.test(evidenceText); + return ( + ["risk_matrix_cell", "flowchart_step", "diagram_decision"].includes(result.index_unit?.unit_type ?? "") && + bareColourPattern.test(evidenceText) + ); }); } diff --git a/src/lib/source-governance.ts b/src/lib/source-governance.ts index 4d03ef4773..df9ffab7e1 100644 --- a/src/lib/source-governance.ts +++ b/src/lib/source-governance.ts @@ -30,6 +30,12 @@ export type GroupedSourceGovernanceWarning = { export const sourceGovernanceRefusalAnswer = "I cannot provide a clinical answer because one or more matched documents are not suitable for clinical use yet. Try a narrower clinical term or scope the search to a current approved document."; +const frontendVisibleWarningCodes = new Set([ + "outdated_source", + "poor_extraction", + "weak_evidence", +]); + function isLocalMetadataText(value: string) { return /\b(?:wa|western australia|perth|north metropolitan|east metropolitan|south metropolitan|health service)\b/i.test( value, @@ -197,6 +203,14 @@ export function groupSourceGovernanceWarnings(warnings: SourceGovernanceWarning[ return Array.from(grouped.values()).sort((a, b) => severityRank[a.severity] - severityRank[b.severity]); } +export function isFrontendVisibleSourceGovernanceWarning(warning: SourceGovernanceWarning) { + return frontendVisibleWarningCodes.has(warning.code); +} + +export function frontendSourceGovernanceWarnings(warnings: SourceGovernanceWarning[]) { + return warnings.filter(isFrontendVisibleSourceGovernanceWarning); +} + export function hasDangerSourceGovernanceWarning(warnings: SourceGovernanceWarning[]) { return warnings.some((warning) => warning.severity === "danger"); } diff --git a/tests/document-organization.test.ts b/tests/document-organization.test.ts index d3bdbc4a1b..e061648113 100644 --- a/tests/document-organization.test.ts +++ b/tests/document-organization.test.ts @@ -456,5 +456,39 @@ describe("document organization classifier", () => { expect(roles.profile.secondary_facets.document_intent).toEqual( expect.arrayContaining(["staff-guidance", "operational-process"]), ); + + const quickReference = classifyDocumentOrganization({ + title: "Cleaning Of The Ultrasound Probe Quick Reference(RPBG)", + file_name: "Cleaning of the Ultrasound Probe Quick Reference (RPBG).pdf", + contentText: "Royal Perth Bentley Group quick reference for cleaning.", + }); + expect(quickReference.profile.document_type.label).toBe("reference"); + expect(quickReference.profile.secondary_facets.document_intent).toContain("clinical-instruction"); + expect(quickReference.profile.secondary_facets.content_feature).toContain("contains-quick-reference"); + + const medicalRecords = classifyDocumentOrganization({ + title: "After-Hours Medical Records - Mental Health(AKG)", + file_name: "After-Hours Medical Records - Mental Health (AKG).pdf", + contentText: "Armadale Kalamunda Group medical records process.", + }); + expect(medicalRecords.profile.secondary_facets.clinical_action).toContain("document"); + expect(medicalRecords.profile.secondary_facets.document_intent).toEqual( + expect.arrayContaining(["operational-process", "documentation-requirement"]), + ); + + const medicineOnly = classifyDocumentOrganization({ + title: "Chloramphenicol(RPBG)", + file_name: "Chloramphenicol (RPBG).pdf", + contentText: "Royal Perth Bentley Group chloramphenicol medication instruction.", + }); + expect(medicineOnly.profile.secondary_facets.clinical_action).toContain("administer"); + expect(medicineOnly.profile.secondary_facets.document_intent).toContain("medication-instruction"); + + const patientLeaflet = classifyDocumentOrganization({ + title: "Living With A Defibrillator(RPBG)", + file_name: "Living with a Defibrillator (RPBG).pdf", + contentText: "Royal Perth Bentley Group patient information.", + }); + expect(patientLeaflet.profile.secondary_facets.document_intent).toContain("patient-information"); }); }); diff --git a/tests/document-tags.test.ts b/tests/document-tags.test.ts index fec37951ff..51cbfdbb17 100644 --- a/tests/document-tags.test.ts +++ b/tests/document-tags.test.ts @@ -3,6 +3,7 @@ import { buildSmartDocumentTagFacets, buildSmartDocumentTags, filterDocumentsBySmartTagFacets, + formatDocumentLabelDisplay, normalizeDocumentLabelForStorage, reviewDocumentTagQuality, tagSearchText, @@ -118,11 +119,27 @@ describe("smart document tags", () => { buildSmartDocumentTags([label({ label: "electroconvulsive therapy", label_type: "topic" })])[0], ).toMatchObject({ searchText: "electroconvulsive-therapy", - label: "Electroconvulsive Therapy", + label: "Electroconvulsive therapy", }); expect( buildSmartDocumentTags([label({ label: "substance-use-alcohol-and-drugs", label_type: "topic" })])[0].label, - ).toBe("Substance Use Alcohol And Drugs"); + ).toBe("Substance use, alcohol and drugs"); + }); + + it("separates stable machine labels from polished display labels and visibility tiers", () => { + expect(formatDocumentLabelDisplay("contains_quick-reference", "content_feature")).toBe("Contains quick reference"); + expect(formatDocumentLabelDisplay("post-discharge-follow-up", "care_phase")).toBe("Post-discharge follow-up"); + expect(formatDocumentLabelDisplay("fiona stanley hospital", "site")).toBe("FSH"); + + const tags = buildSmartDocumentTags([ + label({ label: "clinical-risk", label_type: "risk", confidence: 0.9 }), + label({ label: "lithium", label_type: "medication", confidence: 0.9 }), + ]); + expect(tags.find((tag) => tag.searchText === "clinical risk")).toMatchObject({ tier: "ranking" }); + expect(tags.find((tag) => tag.searchText === "lithium")).toMatchObject({ tier: "primary" }); + + const facets = buildSmartDocumentTagFacets([{ labels: tags }]); + expect(facets.flatMap((group) => group.facets).map((facet) => facet.searchText)).not.toContain("clinical risk"); }); it("builds grouped tag facets with document counts", () => { diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index cfcf53a02c..915b822281 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -241,10 +241,10 @@ describe("eval quality reporting", () => { }); expect(report.accepted_threshold_failures).toEqual( - expect.arrayContaining([ - expect.stringContaining("top-result stale/review/unknown rate"), - expect.stringContaining("top-result review_required_rate"), - ]), + expect.arrayContaining([expect.stringContaining("top-result review_required_rate")]), + ); + expect(report.accepted_threshold_failures).not.toEqual( + expect.arrayContaining([expect.stringContaining("top-result stale_rate")]), ); expect(report.blocking_threshold_failures).toEqual( expect.arrayContaining([expect.stringContaining("RAG source_governance_danger_failure_rate")]), diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index 1d730fcb09..bac8cb8b33 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -310,6 +310,55 @@ describe("retrieval source selection", () => { ); }); + it("keeps directly relevant content selectable when review metadata is absent", () => { + const selection = selectRetrievalEvidence({ + query: "What is the next step after red-zone risk?", + queryClass: "document_lookup", + topK: 2, + maxResultsPerDocument: 2, + results: [ + source({ + id: "metadata-missing-flowchart", + document_id: "metadata-missing-doc", + title: "Risk Matrix Flowchart", + file_name: "risk-matrix.pdf", + section_heading: "Red zone", + content: "Risk matrix flowchart red zone: next step is urgent review and escalation.", + hybrid_score: 0.52, + source_metadata: undefined, + index_unit: { + id: "unit-red-risk", + unit_type: "flowchart_step", + title: "Red-zone risk next step", + content: "Red-zone risk requires urgent review and escalation.", + source_chunk_id: "metadata-missing-flowchart", + source_image_id: "image-red-risk", + page_start: 3, + page_end: 3, + heading_path: ["Risk matrix", "Red zone"], + normalized_terms: ["risk", "red zone", "next step", "urgent review"], + quality_score: 0.9, + extraction_mode: "hybrid", + }, + }), + source({ + id: "current-generic", + document_id: "current-generic-doc", + title: "Generic Current Source", + file_name: "generic-current.pdf", + content: "Generic process overview and routine administrative guidance.", + hybrid_score: 0.66, + source_metadata: sourceMetadata(), + }), + ], + }); + + expect(selection.results[0].id).toBe("metadata-missing-flowchart"); + expect(selection.results[0].source_metadata).toBeUndefined(); + expect(selection.summary.requiredSignalsSatisfied).toBe(true); + expect(selection.summary.matchedSignals).toEqual(expect.arrayContaining(["risk", "red_zone"])); + }); + // Contract changed 2026-07-02 (measured): source-governance metadata must NOT reorder retrieval // selection. The corpus is only partially enriched — unenriched documents normalize to // unknown/unverified — so metadata weighting in selection buried correct documents on the golden @@ -377,6 +426,61 @@ describe("retrieval source selection", () => { ]); }); + it("does not let unverified validation displace directly relevant evidence", () => { + const selection = selectRetrievalEvidence({ + query: "What dose and route are shown in the agitation medication chart?", + queryClass: "medication_dose_risk", + topK: 2, + maxResultsPerDocument: 2, + results: [ + source({ + id: "unverified-dose-route", + document_id: "unverified-dose-doc", + title: "Agitation Medication Chart", + file_name: "agitation-chart.pdf", + section_heading: "Medication chart", + content: "Agitation medication chart lists lorazepam 1 mg IM or PO.", + hybrid_score: 0.56, + source_metadata: sourceMetadata({ + document_status: "current", + clinical_validation_status: "unverified", + extraction_quality: "good", + }), + index_unit: { + id: "unit-unverified-dose-route", + unit_type: "medication_chart_row", + title: "Lorazepam route row", + content: "Lorazepam 1 mg IM or PO.", + source_chunk_id: "unverified-dose-route", + source_image_id: null, + page_start: 5, + page_end: 5, + heading_path: ["Medication chart"], + normalized_terms: ["agitation", "lorazepam", "1 mg", "im", "po"], + quality_score: 0.9, + extraction_mode: "hybrid", + }, + }), + source({ + id: "current-generic-policy", + document_id: "current-generic-policy-doc", + title: "Current Medication Policy", + file_name: "current-medication-policy.pdf", + content: "General current medication governance policy without agitation dose or route detail.", + hybrid_score: 0.64, + source_metadata: sourceMetadata({ + document_status: "current", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }), + }), + ], + }); + + expect(selection.results[0].id).toBe("unverified-dose-route"); + expect(selection.results[0].source_metadata?.clinical_validation_status).toBe("unverified"); + }); + it("prefers risk/red-zone flowchart evidence over generic flowchart evidence", () => { const selection = selectRetrievalEvidence({ query: "In the clinical flowchart, what is the next step after red-zone risk?", @@ -447,4 +551,45 @@ describe("retrieval source selection", () => { expect(selection.summary.requiredSignalsSatisfied).toBe(true); expect(selection.summary.matchedSignals).toEqual(expect.arrayContaining(["risk", "red_zone"])); }); + + it("does not let routine governance metadata displace stronger risk-flowchart action evidence", () => { + const selection = selectRetrievalEvidence({ + query: "In the clinical flowchart, what is the next step after red-zone risk?", + queryClass: "document_lookup", + topK: 2, + maxResultsPerDocument: 2, + results: [ + source({ + id: "review-due-action", + document_id: "review-due-doc", + title: "Mental Health Emergency Flowchart", + file_name: "review-due-flowchart.pdf", + content: "Flowchart red-zone risk: urgent psychiatric review is required.", + hybrid_score: 0.72, + source_metadata: sourceMetadata({ + document_status: "review_due", + clinical_validation_status: "unverified", + extraction_quality: "good", + }), + }), + source({ + id: "current-action", + document_id: "current-doc", + title: "Acute Deterioration Risk Pathway", + file_name: "current-risk-pathway.pdf", + content: "Risk pathway red-zone risk: next step is urgent senior review and escalation.", + hybrid_score: 0.69, + source_metadata: sourceMetadata({ + document_status: "current", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }), + }), + ], + }); + + expect(selection.results[0].id).toBe("review-due-action"); + expect(selection.results[0].source_metadata?.document_status).toBe("review_due"); + expect(selection.results[0].source_metadata?.clinical_validation_status).toBe("unverified"); + }); }); diff --git a/tests/source-governance.test.ts b/tests/source-governance.test.ts index 8558d36430..ba23e5d121 100644 --- a/tests/source-governance.test.ts +++ b/tests/source-governance.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + frontendSourceGovernanceWarnings, groupSourceGovernanceWarnings, hasDangerSourceGovernanceWarning, sourceGovernanceRefusalAnswer, @@ -149,6 +150,65 @@ describe("source governance warnings", () => { expect(hasDangerSourceGovernanceWarning(warnings)).toBe(false); }); + it("keeps routine review metadata notes out of frontend-visible governance warnings", () => { + const warnings = sourceGovernanceWarnings({ + results: [ + result({ + source_metadata: { + source_title: "Review due local source", + publisher: "WA Health", + jurisdiction: "Australia/WA", + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "review_due", + clinical_validation_status: "unverified", + extraction_quality: "good", + }, + indexing_quality: { + document_id: "doc-1", + quality_score: 0.9, + extraction_quality: "good", + metrics: {}, + issues: [], + }, + table_facts: [], + }), + ], + }); + + expect(warnings.map((warning) => warning.code)).toEqual( + expect.arrayContaining(["review_due_source", "unverified_source"]), + ); + expect(frontendSourceGovernanceWarnings(warnings)).toEqual([]); + }); + + it("surfaces only clinically material warnings to the frontend", () => { + const warnings = sourceGovernanceWarnings({ + results: [result()], + relevance: { + verdict: "none", + label: "No evidence", + matchedTerms: [], + missingTerms: [], + directSourceCount: 0, + weakSourceCount: 0, + score: 0, + supportReason: "No source-backed evidence.", + isSourceBacked: false, + }, + }); + const visibleCodes = frontendSourceGovernanceWarnings(warnings).map((warning) => warning.code); + + expect(visibleCodes).toEqual(expect.arrayContaining(["weak_evidence", "outdated_source", "poor_extraction"])); + expect(visibleCodes).not.toContain("review_due_source"); + expect(visibleCodes).not.toContain("unverified_source"); + expect(visibleCodes).not.toContain("non_local_source"); + }); + it("keeps the refusal message free of backend and source-backed wording", () => { expect(sourceGovernanceRefusalAnswer).not.toMatch(/source-backed|source-governance|admin|need review/i); expect(sourceGovernanceRefusalAnswer).toContain("matched documents are not suitable for clinical use yet");