[WIP] Optimize search and ingestion performance

src/components/ClinicalDashboard.tsx

@@ -5368,7 +5368,12 @@ export function ClinicalDashboard() {
  const clientDemoMode = explicitDemoMode || browserAuthUnavailableDemoFallback || localNoAuthMode;
  const uploadReadOnlyMode =
  demoMode || process.env.NEXT_PUBLIC_DEMO_MODE === "true" || browserAuthUnavailableDemoFallback;
- const canUsePrivateApis = localProjectReady && (localNoAuthMode || authStatus === "authenticated");
+ const storedSessionExists =
+ typeof window !== "undefined" &&
+ Object.keys(localStorage).some((k) => k.startsWith("sb-") && k.endsWith("-auth-token"));
+ const canUsePrivateApis =
+ localProjectReady &&
+ (localNoAuthMode || authStatus === "authenticated" || (supabaseEnvStatus === "ready" && storedSessionExists));
  const canRunSearch = explicitDemoMode || (hasReadyPublicSearchSetup(setupChecks) && canUsePrivateApis);
  const openGuide = useCallback(() => setGuideOpen(true), []);
  const closeGuide = useCallback(() => setGuideOpen(false), []);

src/components/DocumentViewer.tsx

@@ -1910,7 +1910,7 @@ export function DocumentViewer({
  const [mobileActionsOpen, setMobileActionsOpen] = useState(false);
  const generatedSummaryRef = useRef<HTMLElement | null>(null);
  const { status: authStatus, isConfigured, authorizationHeader, markSessionExpired } = useAuthSession();
- const [serverDemoMode, setServerDemoMode] = useState(process.env.NEXT_PUBLIC_DEMO_MODE === "true" || !isConfigured);
+ const [serverDemoMode, setServerDemoMode] = useState(process.env.NEXT_PUBLIC_DEMO_MODE === "true");
  const localNoAuthMode = isLocalNoAuthMode();
  const clientDemoMode = localNoAuthMode || serverDemoMode;
  const canUsePrivateApis = localProjectReady && (clientDemoMode || authStatus === "authenticated");

src/lib/search-scope.ts

@@ -4,6 +4,9 @@ import type { ClinicalSourceMetadata, DocumentLabelType } from "@/lib/types";
 import { normalizeSourceMetadata } from "@/lib/source-metadata";
 
 const labelTypes = ["site", "medication", "topic", "document_type"] as const satisfies readonly DocumentLabelType[];
+const sourceStatusValues = ["current", "review_due", "outdated", "unknown"] as const;
+const validationStatusValues = ["unverified", "locally_reviewed", "approved"] as const;
+const documentScopeQueryPageSize = 1000;
 
 export const searchScopeFiltersSchema = z
  .object({
@@ -64,6 +67,30 @@ function hasValues(values?: string[]) {
  return Boolean(values?.length);
 }
 
+function escapePostgrestValue(value: string) {
+ return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+}
+
+function buildStatusFallbackClauses(args: {
+ fieldName: string;
+ values?: string[];
+ fallbackValue: string;
+ knownValues: readonly string[];
+ orParts: string[];
+}) {
+ const normalizedValues = (args.values ?? []).map(normalizeFilterText);
+ for (const value of normalizedValues) {
+ args.orParts.push(`${args.fieldName}.eq.${value}`);
+ }
+
+ if (!normalizedValues.includes(args.fallbackValue)) {
+ return;
+ }
+
+ args.orParts.push(`${args.fieldName}.is.null`);
+ args.orParts.push(`${args.fieldName}.not.in.(${args.knownValues.join(",")})`);
+}
+
 export function activeScopeFilterCount(filters: SearchScopeFilters) {
  return [
  filters.medications,
@@ -92,19 +119,9 @@ function isLocalSource(metadata: ClinicalSourceMetadata) {
  );
 }
 
-function metadataMatches(row: ScopeDocumentRow, filters: SearchScopeFilters) {
+function metadataMatchesLocality(row: ScopeDocumentRow, filters: SearchScopeFilters) {
+ if (!filters.locality) return true;
  const source = normalizeSourceMetadata(row.metadata);
- const metadata = row.metadata && typeof row.metadata === "object" ? (row.metadata as Record<string, unknown>) : {};
- const collection = typeof metadata.collection === "string" ? metadata.collection.trim().toLowerCase() : "";
-
- if (filters.sourceStatuses?.length && !filters.sourceStatuses.includes(source.document_status)) return false;
- if (filters.validationStatuses?.length && !filters.validationStatuses.includes(source.clinical_validation_status))
- return false;
- if (filters.extractionQualities?.length && !filters.extractionQualities.includes(source.extraction_quality))
- return false;
- if (filters.importBatchIds?.length && (!row.import_batch_id || !filters.importBatchIds.includes(row.import_batch_id)))
- return false;
- if (filters.collections?.length && !filters.collections.map(normalizeFilterText).includes(collection)) return false;
  if (filters.locality === "local" && !isLocalSource(source)) return false;
  if (filters.locality === "non_local" && isLocalSource(source)) return false;
  return true;
@@ -136,30 +153,74 @@ export async function resolveSearchScope(args: {
  }
 
  const maxResolvedDocuments = args.maxResolvedDocuments ?? 5000;
+
  const documentRows: ScopeDocumentRow[] = [];
- const pageSize = 1000;
- for (let offset = 0; offset < maxResolvedDocuments; offset += pageSize) {
+ for (let offset = 0; offset < maxResolvedDocuments; offset += documentScopeQueryPageSize) {
  let documentQuery = args.supabase
  .from("documents")
  .select("id,metadata,import_batch_id")
  .eq("status", "indexed")
- .range(offset, Math.min(offset + pageSize - 1, maxResolvedDocuments - 1));
+ .range(offset, Math.min(offset + documentScopeQueryPageSize - 1, maxResolvedDocuments - 1));
  if (args.ownerId) documentQuery = documentQuery.eq("owner_id", args.ownerId);
  if (explicitIds.length) documentQuery = documentQuery.in("id", explicitIds);
 
+ // Push metadata enum filters into SQL using JSONB text extraction. Keep
+ // fallback semantics by matching nulls, fallback values, and malformed rows.
+ if (filters.sourceStatuses?.length) {
+ const orParts: string[] = [];
+ buildStatusFallbackClauses({
+ fieldName: "metadata->>document_status",
+ values: filters.sourceStatuses,
+ fallbackValue: "unknown",
+ knownValues: sourceStatusValues,
+ orParts,
+ });
+ documentQuery = documentQuery.or(orParts.join(","));
+ }
+ if (filters.validationStatuses?.length) {
+ const orParts: string[] = [];
+ buildStatusFallbackClauses({
+ fieldName: "metadata->>clinical_validation_status",
+ values: filters.validationStatuses,
+ fallbackValue: "unverified",
+ knownValues: validationStatusValues,
+ orParts,
+ });
+ documentQuery = documentQuery.or(orParts.join(","));
+ }
+ if (filters.extractionQualities?.length) {
+ const orParts = filters.extractionQualities.map((q) => `metadata->>extraction_quality.eq.${normalizeFilterText(q)}`);
+ if (filters.extractionQualities.includes("unknown")) orParts.push("metadata->>extraction_quality.is.null");
+ documentQuery = documentQuery.or(orParts.join(","));
+ }
+ if (filters.importBatchIds?.length) {
+ documentQuery = documentQuery.in("import_batch_id", filters.importBatchIds);
+ }
+ if (filters.collections?.length) {
+ const orParts = filters.collections.map((collection) =>
+ `metadata->>collection.ilike.${escapePostgrestValue(normalizeFilterText(collection))}`,
+ );
+ documentQuery = documentQuery.or(orParts.join(","));
+ }
+
  const { data, error: documentError } = await documentQuery;
  if (documentError) throw new Error(documentError.message);
- const page = (data ?? []) as ScopeDocumentRow[];
- documentRows.push(...page);
- if (page.length < pageSize || documentRows.length >= maxResolvedDocuments) break;
+ documentRows.push(...((data ?? []) as ScopeDocumentRow[]));
+
+ if ((data ?? []).length < documentScopeQueryPageSize) {
+ break;
+ }
  }
+
  if (documentRows.length >= maxResolvedDocuments) {
  warnings.push(
- `Scope resolution read the first ${maxResolvedDocuments} indexed documents; narrow the filters if expected documents are missing.`,
+ `Scope resolution matched more than ${maxResolvedDocuments} indexed documents; narrow the filters if expected documents are missing.`,
  );
  }
 
- const rows = documentRows.filter((row) => metadataMatches(row, filters));
+ // Apply locality filter in application code — it requires a regex match
+ // across two JSONB fields that cannot be expressed via PostgREST alone.
+ const rows = filters.locality ? documentRows.filter((row) => metadataMatchesLocality(row, filters)) : documentRows;
  const candidateIds = rows.map((row) => row.id);
  if (candidateIds.length === 0) {
  warnings.push("No indexed documents matched the selected filters.");