Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitleaksignore
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,3 +8,5 @@
# generic-api-key rule on historical commit content scanned in PR history.
27121756b10df6d93841ebf48b0f1991e0144a1f:data/medications-snapshot.json:generic-api-key:22253
27121756b10df6d93841ebf48b0f1991e0144a1f:data/medications-snapshot.json:generic-api-key:47690
b65acddaa13e69e0ad2a49783116056b3a1a3639:data/medications-snapshot.json:generic-api-key:21520
b65acddaa13e69e0ad2a49783116056b3a1a3639:data/medications-snapshot.json:generic-api-key:46330
16 changes: 15 additions & 1 deletion scripts/eval-quality.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,6 +45,7 @@ type EvalQualityArgs = {
retrievalOnly: boolean;
ragOnly: boolean;
skipPreflight: boolean;
forceEmbedding: boolean;
};

export type RagQualityResult = {
Expand DownExpand Up@@ -150,6 +151,7 @@ function parseArgs(argv: string[]): EvalQualityArgs {
retrievalOnly: false,
ragOnly: false,
skipPreflight: false,
forceEmbedding: false,
};

for (let index = 0; index < argv.length; index += 1) {
Expand All@@ -176,6 +178,10 @@ function parseArgs(argv: string[]): EvalQualityArgs {
args.skipPreflight = true;
continue;
}
if (token === "--force-embedding") {
args.forceEmbedding = true;
continue;
}

const value = argv[index + 1];
if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
Expand DownExpand Up@@ -476,6 +482,11 @@ export function buildEvalQualityReport(args: {
`retrieval content_recall_at_5 ${retrievalSummary.content_recall_at_5} below ${qualityThresholds.retrievalContentRecallAt5}`,
);
}
if (retrievalSummary.force_embedding_failure_count > 0) {
thresholdFailures.push(
`retrieval force_embedding_failure_count ${retrievalSummary.force_embedding_failure_count} above 0`,
);
}
if (governance.stale_rate > qualityThresholds.staleTopResultRate) {
thresholdFailures.push(
`top-result stale_rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`,
Expand DownExpand Up@@ -664,6 +675,8 @@ ${markdownTable([
## Retrieval Decision Metrics

${markdownTable([
["Force-embedding cases", retrieval.force_embedding_case_count],
["Force-embedding failures", retrieval.force_embedding_failure_count],
["Embedding skipped rate", retrieval.embedding_skipped_rate],
["Median text candidate budget", retrieval.median_text_candidate_budget],
["Second-stage rerank rate", retrieval.second_stage_rerank_rate],
Expand DownExpand Up@@ -728,6 +741,7 @@ async function runRetrievalQualityCases(args: {
ownerId?: string;
limit?: number;
query?: string;
forceEmbedding?: boolean;
supabase: Awaited<ReturnType<typeof loadAdminClient>>;
}) {
const [{ searchChunksWithTelemetry }, capturedCases] = await Promise.all([
Expand DownExpand Up@@ -756,7 +770,7 @@ async function runRetrievalQualityCases(args: {
topK: retrievalLimitForGoldenCase(testCase),
minSimilarity: 0.12,
skipCache: true,
forceEmbedding: testCase.forceEmbedding,
forceEmbedding: testCase.forceEmbedding || args.forceEmbedding,
}),
);
const latencyMs =
Expand Down
24 changes: 24 additions & 0 deletions scripts/eval-retrieval.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,6 +47,7 @@ type EvalArgs = {
export type GoldenRetrievalResult = {
id: string;
query: string;
forceEmbedding: boolean;
expectedQueryClass: string;
actualQueryClass: string | null;
expectedDocumentSubstrings: string[];
Expand DownExpand Up@@ -516,6 +517,11 @@ export function evaluateGoldenRetrievalCase(args: {
const tableEvidenceFoundAtK = hasTableEvidence(args.results, topK);
const actualQueryClass = args.telemetry.query_class ?? null;
const failures: string[] = [];
const vectorLayerCount = Object.entries(args.telemetry.retrieval_layer_counts ?? {}).reduce(
(sum, [layer, count]) =>
["embedding_fields", "index_units", "hybrid_vector", "vector_fallback"].includes(layer) ? sum + count : sum,
0,
);
const hitAtK =
documentHitsAtK.missing.length === 0 &&
contentHitsAtK.missing.length === 0 &&
Expand All@@ -533,10 +539,23 @@ export function evaluateGoldenRetrievalCase(args: {
if (args.testCase.expectTableEvidence && !tableEvidenceFound) {
failures.push("expected table evidence in top 5");
}
if (args.testCase.forceEmbedding) {
if (args.telemetry.embedding_skipped) failures.push("forceEmbedding expected embedding to run");
if (args.telemetry.retrieval_strategy === "search_cache") failures.push("forceEmbedding served search cache");
if (
args.telemetry.retrieval_strategy === "text_fast_path" ||
args.telemetry.retrieval_strategy === "document_lookup_fast_path"
) {
failures.push(`forceEmbedding returned lexical strategy ${args.telemetry.retrieval_strategy}`);
}
if (args.telemetry.coverage_gate_decision === "accepted") failures.push("forceEmbedding returned coverage gate");
if (vectorLayerCount <= 0) failures.push("forceEmbedding found no vector-layer candidates");
}

return {
id: args.testCase.id,
query: args.testCase.query,
forceEmbedding: args.testCase.forceEmbedding ?? false,
expectedQueryClass: args.testCase.expectedQueryClass,
actualQueryClass,
expectedDocumentSubstrings: args.testCase.expectedDocumentSubstrings,
Expand DownExpand Up@@ -612,6 +631,7 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
}
return counts;
}, {});
const forceEmbeddingResults = results.filter((result) => result.forceEmbedding);
return {
case_count: results.length,
document_recall_at_5: Number(
Expand DownExpand Up@@ -647,6 +667,8 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
embedding_skip_reason_counts: embeddingSkipReasonCounts,
text_fast_path_reason_counts: textFastPathReasonCounts,
retrieval_layer_counts: layerCounts,
force_embedding_case_count: forceEmbeddingResults.length,
force_embedding_failure_count: forceEmbeddingResults.filter((result) => result.failures.length > 0).length,
median_text_candidate_budget: percentile(textCandidateBudgets, 50),
second_stage_rerank_rate: Number(
(results.filter((result) => result.secondStageRerankUsed).length / Math.max(results.length, 1)).toFixed(4),
Expand DownExpand Up@@ -723,6 +745,8 @@ function printHumanSummary(summary: ReturnType<typeof summarizeGoldenRetrievalRe
console.log(` retrieval_strategy_counts=${JSON.stringify(summary.retrieval_strategy_counts)}`);
console.log(` retrieval_plan_counts=${JSON.stringify(summary.retrieval_plan_counts)}`);
console.log(` retrieval_layer_counts=${JSON.stringify(summary.retrieval_layer_counts)}`);
console.log(` force_embedding_case_count=${summary.force_embedding_case_count}`);
console.log(` force_embedding_failure_count=${summary.force_embedding_failure_count}`);
console.log(` embedding_skipped_rate=${summary.embedding_skipped_rate}`);
console.log(` embedding_skip_reason_counts=${JSON.stringify(summary.embedding_skip_reason_counts)}`);
console.log(` text_fast_path_reason_counts=${JSON.stringify(summary.text_fast_path_reason_counts)}`);
Expand Down
4 changes: 2 additions & 2 deletions src/app/api/answer/route.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,7 +4,7 @@ import { demoAnswer } from "@/lib/demo-data";
import { isDemoMode, isLocalNoAuthMode } from "@/lib/env";
import { answerQuestionWithScope } from "@/lib/rag";
import { jsonError, PublicApiError } from "@/lib/http";
import { consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { allowRateLimitInMemoryFallbackOnUnavailable, consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { publicAccessContext } from "@/lib/public-api-access";
import { classifyRagQuery } from "@/lib/clinical-search";
import { buildSmartRagApiPlan } from "@/lib/smart-rag-api";
Expand DownExpand Up@@ -70,7 +70,7 @@ export async function POST(request: Request) {
supabase,
subject: access.rateLimitSubject,
bucket: "answer",
allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(),
allowInMemoryFallbackOnUnavailable: allowRateLimitInMemoryFallbackOnUnavailable(),
});
if (rateLimit.limited) {
return rateLimitJsonResponse("Too many answer requests. Retry shortly.", rateLimit);
Expand Down
4 changes: 2 additions & 2 deletions src/app/api/answer/stream/route.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,7 @@ import { z } from "zod";
import { demoAnswer } from "@/lib/demo-data";
import { isDemoMode, isLocalNoAuthMode } from "@/lib/env";
import { PublicApiError, jsonError } from "@/lib/http";
import { consumeSubjectApiRateLimit, type ApiRateLimitResult } from "@/lib/api-rate-limit";
import { allowRateLimitInMemoryFallbackOnUnavailable, consumeSubjectApiRateLimit, type ApiRateLimitResult } from "@/lib/api-rate-limit";
import { publicAccessContext } from "@/lib/public-api-access";
import { answerQuestionWithScope, type AnswerProgressEvent } from "@/lib/rag";
import { classifyRagQuery } from "@/lib/clinical-search";
Expand DownExpand Up@@ -232,7 +232,7 @@ export async function POST(request: Request) {
supabase,
subject: access.rateLimitSubject,
bucket: "answer",
allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(),
allowInMemoryFallbackOnUnavailable: allowRateLimitInMemoryFallbackOnUnavailable(),
});
if (rateLimit.limited) return rateLimitStream(rateLimit);

Expand Down
8 changes: 4 additions & 4 deletions src/app/api/differentials/[slug]/route.ts
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
import { NextResponse } from "next/server";
import { z } from "zod";

import { consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { allowRateLimitInMemoryFallbackOnUnavailable, consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import {
deriveGovernanceFromSnapshot,
normalizeDifferentialSlug,
Expand All@@ -14,7 +14,7 @@ import { ensureDifferentialsSeeded, loadDifferentialSnapshot } from "@/lib/diffe
import { getDifferentialRecord, getPresentationWorkflow } from "@/lib/differentials";
import { isDemoMode, isLocalNoAuthMode } from "@/lib/env";
import { jsonError } from "@/lib/http";
import { hasPublicApiAuthSignal, publicAccessContext } from "@/lib/public-api-access";
import { publicAccessContext, shouldResolvePublicCatalogAccess } from "@/lib/public-api-access";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, unauthorizedResponse } from "@/lib/supabase/auth";
import { parseRequestQuery } from "@/lib/validation/query";
Expand DownExpand Up@@ -63,7 +63,7 @@ export async function GET(request: Request, context: { params: Promise<{ slug: s
});
}

if (!hasPublicApiAuthSignal(request)) {
if (!shouldResolvePublicCatalogAccess(request)) {
const snapshot = loadDifferentialSnapshot();
const governance = deriveGovernanceFromSnapshot(snapshot);
if (kind === "presentation") {
Expand DownExpand Up@@ -91,7 +91,7 @@ export async function GET(request: Request, context: { params: Promise<{ slug: s
supabase,
subject: access.rateLimitSubject,
bucket: "registry",
allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(),
allowInMemoryFallbackOnUnavailable: allowRateLimitInMemoryFallbackOnUnavailable(),
});
if (rateLimit.limited) {
return rateLimitJsonResponse("Differential requests are rate limited. Try again shortly.", rateLimit);
Expand Down
8 changes: 4 additions & 4 deletions src/app/api/differentials/presentations/[slug]/route.ts
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
import { NextResponse } from "next/server";

import { consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { allowRateLimitInMemoryFallbackOnUnavailable, consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import {
deriveGovernanceFromSnapshot,
normalizeDifferentialSlug,
Expand All@@ -13,7 +13,7 @@ import { ensureDifferentialsSeeded, loadDifferentialSnapshot } from "@/lib/diffe
import { getDifferentialRecord, getPresentationWorkflow } from "@/lib/differentials";
import { isDemoMode, isLocalNoAuthMode } from "@/lib/env";
import { jsonError } from "@/lib/http";
import { hasPublicApiAuthSignal, publicAccessContext } from "@/lib/public-api-access";
import { publicAccessContext, shouldResolvePublicCatalogAccess } from "@/lib/public-api-access";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, unauthorizedResponse } from "@/lib/supabase/auth";

Expand DownExpand Up@@ -53,7 +53,7 @@ export async function GET(request: Request, context: { params: Promise<{ slug: s
});
}

if (!hasPublicApiAuthSignal(request)) {
if (!shouldResolvePublicCatalogAccess(request)) {
const snapshot = loadDifferentialSnapshot();
const workflow = getPresentationWorkflow(normalizedSlug);
if (!workflow) return notFoundResponse(normalizedSlug);
Expand All@@ -78,7 +78,7 @@ export async function GET(request: Request, context: { params: Promise<{ slug: s
supabase,
subject: access.rateLimitSubject,
bucket: "registry",
allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(),
allowInMemoryFallbackOnUnavailable: allowRateLimitInMemoryFallbackOnUnavailable(),
});
if (rateLimit.limited) {
return rateLimitJsonResponse("Differential requests are rate limited. Try again shortly.", rateLimit);
Expand Down
8 changes: 4 additions & 4 deletions src/app/api/differentials/route.ts
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
import { NextResponse } from "next/server";
import { z } from "zod";

import { consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { allowRateLimitInMemoryFallbackOnUnavailable, consumeSubjectApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit";
import {
deriveGovernanceFromSnapshot,
rowGovernance,
Expand All@@ -14,7 +14,7 @@ import { ensureDifferentialsSeeded, loadDifferentialSnapshot } from "@/lib/diffe
import { differentialRecords, searchDifferentialRecords, searchPresentationWorkflows } from "@/lib/differentials";
import { isDemoMode, isLocalNoAuthMode } from "@/lib/env";
import { jsonError } from "@/lib/http";
import { hasPublicApiAuthSignal, publicAccessContext } from "@/lib/public-api-access";
import { publicAccessContext, shouldResolvePublicCatalogAccess } from "@/lib/public-api-access";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, unauthorizedResponse } from "@/lib/supabase/auth";
import { parseRequestQuery, queryInteger } from "@/lib/validation/query";
Expand DownExpand Up@@ -68,7 +68,7 @@ export async function GET(request: Request) {
});
}

if (!hasPublicApiAuthSignal(request)) {
if (!shouldResolvePublicCatalogAccess(request)) {
return differentialResponse({
...publicDifferentialPayload(kind, q, limit),
publicAccess: true,
Expand All@@ -82,7 +82,7 @@ export async function GET(request: Request) {
supabase,
subject: access.rateLimitSubject,
bucket: "registry",
allowInMemoryFallbackOnUnavailable: isLocalNoAuthMode(),
allowInMemoryFallbackOnUnavailable: allowRateLimitInMemoryFallbackOnUnavailable(),
});
if (rateLimit.limited) {
return rateLimitJsonResponse("Differential requests are rate limited. Try again shortly.", rateLimit);
Expand Down
8 changes: 6 additions & 2 deletions src/app/api/documents/[id]/route.ts
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
import { NextResponse } from "next/server";
import type { Json } from "@/lib/supabase/database.types";
import { z } from "zod";
import { rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { getDemoDocumentPayload } from "@/lib/demo-data";
import { env, isDemoMode } from "@/lib/env";
import { jsonError, PublicApiError } from "@/lib/http";
import { invalidateRagCachesForDocumentMutation } from "@/lib/rag";
import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth";
import { publicAccessContext, withOwnerReadScope } from "@/lib/public-api-access";
import { enforceDocumentReadRateLimit, withOwnerReadScope } from "@/lib/public-api-access";
import { writeAuditLog } from "@/lib/audit";
import { parseJsonBody } from "@/lib/validation/body";
import { parseRouteParams } from "@/lib/validation/params";
Expand DownExpand Up@@ -281,7 +282,10 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:

const { id } = parseRouteParams({ id: rawId }, documentRouteParamsSchema, "Invalid document id.");
const supabase = createAdminClient();
const access = await publicAccessContext(request, supabase);
const { access, rateLimit } = await enforceDocumentReadRateLimit(request, supabase);
if (rateLimit.limited) {
return rateLimitJsonResponse("Document requests are rate limited. Try again shortly.", rateLimit);
}
const { data: document, error } = await withOwnerReadScope(
supabase.from("documents").select("*").eq("id", id),
access.ownerId,
Expand Down
8 changes: 6 additions & 2 deletions src/app/api/documents/[id]/search/route.ts
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
import { NextResponse } from "next/server";
import { z } from "zod";
import { rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { demoChunks, getDemoDocument } from "@/lib/demo-data";
import { isDemoMode } from "@/lib/env";
import { jsonError } from "@/lib/http";
import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, unauthorizedResponse } from "@/lib/supabase/auth";
import { publicAccessContext, withOwnerReadScope } from "@/lib/public-api-access";
import { enforceDocumentReadRateLimit, withOwnerReadScope } from "@/lib/public-api-access";
import { parseRouteParams } from "@/lib/validation/params";
import { parseRequestQuery, queryInteger } from "@/lib/validation/query";

Expand DownExpand Up@@ -182,7 +183,10 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:

const { id } = parseRouteParams({ id: rawId }, documentSearchParamsSchema, "Invalid document id.");
const supabase = createAdminClient();
const access = await publicAccessContext(request, supabase);
const { access, rateLimit } = await enforceDocumentReadRateLimit(request, supabase);
if (rateLimit.limited) {
return rateLimitJsonResponse("Document requests are rate limited. Try again shortly.", rateLimit);
}
const { data: document, error: documentError } = await withOwnerReadScope(
supabase.from("documents").select("id,metadata").eq("id", id),
access.ownerId,
Expand Down
8 changes: 6 additions & 2 deletions src/app/api/documents/[id]/signed-url/route.ts
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
import { NextResponse } from "next/server";
import { z } from "zod";
import { rateLimitJsonResponse } from "@/lib/api-rate-limit";
import { getDemoDocument } from "@/lib/demo-data";
import { env } from "@/lib/env";
import { isDemoMode } from "@/lib/env";
import { jsonError, PublicApiError } from "@/lib/http";
import { createAdminClient } from "@/lib/supabase/admin";
import { AuthenticationError, unauthorizedResponse } from "@/lib/supabase/auth";
import { publicAccessContext, withOwnerReadScope } from "@/lib/public-api-access";
import { enforceDocumentReadRateLimit, withOwnerReadScope } from "@/lib/public-api-access";

export const runtime = "nodejs";

Expand All@@ -32,7 +33,10 @@ export async function GET(_request: Request, { params }: { params: Promise<{ id:
if (!routeIdSchema.safeParse(id).success) throw new PublicApiError("Invalid document id.");

const supabase = createAdminClient();
const access = await publicAccessContext(_request, supabase);
const { access, rateLimit } = await enforceDocumentReadRateLimit(_request, supabase);
if (rateLimit.limited) {
return rateLimitJsonResponse("Document requests are rate limited. Try again shortly.", rateLimit);
}
const { data: document, error } = await withOwnerReadScope(
supabase.from("documents").select("storage_path,file_type").eq("id", id),
access.ownerId,
Expand Down
Loading