diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..ffd4e6bc1a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +.git +.github +.claude +node_modules +.next +coverage +test-results +playwright-report +scratch +.tmp-visual +sample-documents +dev-server.log +*.log +.env +.env.* +!.env.example +.vscode +.idea +Dockerfile +Dockerfile.worker +.dockerignore diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml new file mode 100644 index 0000000000..f756c3c17b --- /dev/null +++ b/.github/workflows/eval-canary.yml @@ -0,0 +1,121 @@ +# Nightly production eval canary. See docs/observability-slos.md §3. +# +# Runs the golden retrieval eval (which PR CI can never run — it needs live +# Supabase + OpenAI keys) plus a small answer-quality subset against the live +# project, and fails loudly on regression: red run + a GitHub issue on +# scheduled failures. +# +# The schedule only fires from the default branch. After merging, trigger one +# workflow_dispatch run and confirm it is green before trusting the nightly +# cadence. +name: Eval Canary + +on: + workflow_dispatch: + inputs: + answer_case_limit: + description: "Number of answer-quality cases to run (--limit)" + required: false + default: "8" + schedule: + # 18:00 UTC = 02:00 Australia/Perth — off-peak for clinicians. + - cron: "0 18 * * *" + +concurrency: + group: eval-canary + cancel-in-progress: false + +permissions: + contents: read + issues: write + +env: + NEXT_PUBLIC_SUPABASE_URL: https://sjrfecxgysukkwxsowpy.supabase.co + SUPABASE_PROJECT_REF: sjrfecxgysukkwxsowpy + SUPABASE_PROJECT_NAME: Clinical KB Database + # Evals use the service-role admin client; the publishable key only needs to + # satisfy env validation (same placeholder approach as ci.yml). + NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + RAG_EVAL_OWNER_EMAIL: ${{ secrets.E2E_USER_EMAIL }} + +jobs: + eval-canary: + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Preflight required secrets + run: | + missing="" + [ -z "$SUPABASE_SERVICE_ROLE_KEY" ] && missing="$missing SUPABASE_SERVICE_ROLE_KEY" + [ -z "$OPENAI_API_KEY" ] && missing="$missing OPENAI_API_KEY" + [ -z "$RAG_EVAL_OWNER_EMAIL" ] && missing="$missing E2E_USER_EMAIL" + if [ -n "$missing" ]; then + echo "::error::Eval canary cannot run — missing repo secrets:$missing" + exit 1 + fi + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Guard Supabase project identity + run: npm run check:supabase-project + + - name: Golden retrieval eval (live corpus) + run: npm run eval:retrieval:quality -- --fail-on-threshold + + - name: Answer-quality subset (live generation) + run: npm run eval:quality -- --rag-only --limit ${{ github.event.inputs.answer_case_limit || '8' }} --fail-on-threshold + + - name: Open or update regression issue + if: failure() && github.event_name == 'schedule' + uses: actions/github-script@v8 + with: + script: | + const label = "eval-canary"; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + `Nightly eval canary failed on ${new Date().toISOString()}.`, + "", + `Run: ${runUrl}`, + "", + "Triage order (docs/observability-slos.md §3): rerun via workflow_dispatch,", + "check hybrid_rpc_errors and `npm run check:indexing`, then bisect code.", + "A failure can be corpus-state-dependent — confirm before reverting anything.", + ].join("\n"); + const { data: existing } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: "open", + labels: label, + }); + if (existing.length > 0) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing[0].number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Eval canary regression: nightly golden eval failed", + labels: [label], + body, + }); + } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..0694c45608 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,61 @@ +# syntax=docker/dockerfile:1 +# Clinical KB app tier (Next.js). See docs/deployment-architecture.md. +# +# The repo is engine-strict (Node 24.x / npm 11.x via .npmrc + preinstall +# guard), so every stage pins the same Node 24 base image. The build stage +# runs the repo's own `npm run build` (guard-next-build + next build) so the +# image build fails exactly where a local build would. +# +# NEXT_PUBLIC_* values are inlined into the client bundle at build time. +# The publishable key is public by design; pass the real one for a +# production image: +# docker build \ +# --build-arg NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=sb_publishable_... \ +# -t clinical-kb-app . +# Server-side secrets (SUPABASE_SERVICE_ROLE_KEY, OPENAI_API_KEY, ...) are +# NEVER baked into the image — inject them at run time from the host's +# secret store. + +FROM node:24-bookworm-slim AS deps +WORKDIR /app +# check-node-engine.cjs runs as the npm preinstall hook, so it must be in +# place before `npm ci`. +COPY package.json package-lock.json .npmrc ./ +COPY scripts/check-node-engine.cjs scripts/check-node-engine.cjs +RUN npm ci + +FROM node:24-bookworm-slim AS build +WORKDIR /app +ENV NEXT_TELEMETRY_DISABLED=1 +COPY --from=deps /app/node_modules ./node_modules +COPY . . +ARG NEXT_PUBLIC_SUPABASE_URL=https://sjrfecxgysukkwxsowpy.supabase.co +ARG NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=placeholder-build-publishable-key +ENV NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL} +ENV NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=${NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY} +# The repo build script allocates an 8 GiB heap; give the builder >= 10 GiB. +RUN npm run build + +FROM node:24-bookworm-slim AS prod-deps +WORKDIR /app +COPY package.json package-lock.json .npmrc ./ +COPY scripts/check-node-engine.cjs scripts/check-node-engine.cjs +RUN npm ci --omit=dev + +FROM node:24-bookworm-slim AS runner +WORKDIR /app +ENV NODE_ENV=production +ENV NEXT_TELEMETRY_DISABLED=1 +ENV PORT=3000 +COPY --from=prod-deps /app/node_modules ./node_modules +COPY --from=build /app/.next ./.next +COPY public ./public +COPY package.json next.config.ts ./ +USER node +EXPOSE 3000 +# /api/health is the app's own ops health route. +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||3000)+'/api/health').then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" +# Bypass scripts/dev-free-port.mjs (a local-dev port picker): a container has +# exactly one app, so bind 0.0.0.0 on $PORT directly. +CMD ["sh", "-c", "node node_modules/next/dist/bin/next start -H 0.0.0.0 -p ${PORT:-3000}"] diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000000..26c1499d95 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,38 @@ +# syntax=docker/dockerfile:1 +# Clinical KB ingestion worker (Node pipeline + Python OCR stack). +# See docs/deployment-architecture.md for why the worker ships as a +# container instead of completing the edge-agent migration. +# +# Runtime contents: +# - Node 24 + full (dev-inclusive) node_modules: the worker runs through +# tsx, which is a devDependency. +# - Tesseract OCR (Debian package, bundles English language data). +# - Python venv with worker/python/requirements.txt (PyMuPDF, Pillow, +# pytesseract). The venv's `python` matches the PYTHON_BIN default. +# +# Build: docker build -f Dockerfile.worker -t clinical-kb-worker . +# Run: docker run --env-file clinical-kb-worker +# Secrets are injected at run time; nothing is baked into the image. + +FROM node:24-bookworm-slim AS deps +WORKDIR /app +COPY package.json package-lock.json .npmrc ./ +COPY scripts/check-node-engine.cjs scripts/check-node-engine.cjs +RUN npm ci + +FROM node:24-bookworm-slim AS runner +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-venv tesseract-ocr \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY worker/python/requirements.txt worker/python/requirements.txt +RUN python3 -m venv /opt/ocr-venv \ + && /opt/ocr-venv/bin/pip install --no-cache-dir -r worker/python/requirements.txt +ENV PATH="/opt/ocr-venv/bin:${PATH}" +ENV NODE_ENV=production +COPY --from=deps /app/node_modules ./node_modules +COPY . . +USER node +# Long-poll worker; WORKER_* env vars control claim batch size, concurrency, +# and the stale-claim window (see src/lib/env.ts). +CMD ["node", "node_modules/tsx/dist/cli.mjs", "worker/index.ts"] diff --git a/docs/capacity-review.md b/docs/capacity-review.md new file mode 100644 index 0000000000..1fff509f88 --- /dev/null +++ b/docs/capacity-review.md @@ -0,0 +1,161 @@ +# Capacity Review — Concurrent Clinician Load + +Models the reference load ("30 clinicians on a ward round") against the known +constraints of the current stack, names the first bottleneck, and defines the +soak test that validates the model against staging. Written 2026-07-06. +Topology assumptions come from `docs/deployment-architecture.md` (single warm +app container, Sydney, co-located with Supabase `sjrfecxgysukkwxsowpy`). + +## 1. Load model + +Ward-round profile, 30 concurrent users over ~60 minutes: + +| Behavior | Assumption | Steady-state rate | +| -------------- | ------------------------------------------------ | ------------------------------------ | +| Sign-in burst | all 30 authenticate within ~2 min at round start | 15 auth ops/min, once | +| Answer queries | 1 question / user / 2 min | ~15 answers/min (~0.25/s) | +| Searches | 2 searches per answer (typeahead + refine) | ~30 searches/min | +| Document opens | 1 per answer (citation click) | ~15 reads/min | +| Overlap | ward rounds ask repeated questions | ~30 % of answers are near-duplicates | + +## 2. Constraint-by-constraint analysis + +### Auth: 10 absolute DB connections (the hard cap) + +The Supabase auth server (GoTrue) is capped at **10 absolute database +connections** (advisor finding, recorded in `docs/process-hardening.md`). Auth +work is bursty and short, but a synchronized sign-in burst (round start, token +refresh storms after an app deploy) queues behind 10 connections and shows up +as login latency or timeouts — a hard, user-visible failure while the rest of +the app looks healthy. + +Mitigations, in order: + +1. **Switch auth to percentage-based connection allocation** in the Supabase + dashboard (a documented follow-up debt; not settable via SQL/MCP — operator + action, ask before touching live settings). +2. Persistent cookie sessions (`@supabase/ssr`, already shipped) mean sign-in + is amortized: a returning clinician refreshes a token rather than + re-authenticating, so the burst is mostly first-day-of-rotation shaped. +3. Keep the app tier at one warm instance: every additional cold instance + multiplies token-refresh traffic at deploy time. + +### Data path: per-answer RPC fan-out vs pooling + +Answer retrieval fans out to **~6 hybrid RPCs** (chunks, embedding fields, +index units, memory cards, table facts, documents-for-query) plus cache reads +and telemetry writes. Critically, the app reaches Postgres through +**PostgREST/Supavisor (HTTP)**, not direct Postgres connections — so 30 users +do not consume 30+ DB connections; they consume PostgREST pool slots for the +duration of each RPC. + +Volume math at steady state: 15 answers/min × ~6 RPCs ≈ 1.5 RPC/s plus ~0.5 +search RPC/s — trivial as _throughput_. The pressure point is **per-RPC cost**: +each hybrid RPC does vector (HNSW) + trigram/tsvector work over ~69k chunks on +a small shared compute tier. Under concurrency the failure shape is CPU +saturation → every RPC slows together → answer p95 inflates → users retry → +amplification. This degrades before anything errors, which is why the p95 +latency SLOs by route mode exist (`docs/observability-slos.md`). + +Existing dampers: the 5-minute answer/search caches, the shared +`rag_response_cache`, and in-flight answer coalescing +(`answer_inflight_coalesced`) — with ~30 % duplicate questions on a ward +round, coalescing + cache absorb roughly a third of the fan-out at exactly the +moment load is highest. This only works while the app is a **single process** +(see deployment doc §2). + +### OpenAI: rate limits and generation concurrency + +Per answer: 1 embedding call (unless the lexical fast path skips it) + 1–2 +generations (fast route, escalation to strong). 15 answers/min with grounded +prompts (~5–15k tokens each) lands in the low hundreds of thousands of +tokens/min at worst — within Tier-2+ gpt-5.5 TPM limits, but _bursts_ of +simultaneous strong-route generations can trip request-per-minute limits. +Existing dampers: coalescing (duplicate questions never reach OpenAI), the +answer cache, `OPENAI_MAX_RETRIES`, and graceful degradation to source-only +answers (which must stay _visible_ — see the degraded-rate SLO). + +### App-layer rate limits (protective, not a bottleneck) + +Per-owner buckets (`src/lib/api-rate-limit.ts`): answer 30/min, search +240/min, document_read 180/min. A single clinician cannot realistically hit +these; they exist to stop runaway clients. Anonymous buckets are far tighter +(answer 6/min) — load testing unauthenticated will measure the limiter, not +the system (the soak script reports 429s separately for this reason). + +### App tier: Node process + +One Node process handles ~0.75 req/s of API traffic with almost all wall time +spent waiting on Supabase/OpenAI. Not a factor at 30 users; becomes one only +if replicas are added carelessly (cache/coalescing dilution) or the box is +undersized for the 8 GiB build-time heap (build happens in CI, not on the +serving instance). + +## 3. Verdict: first bottleneck and what to change + +1. **First hard failure: the auth 10-connection cap** during synchronized + sign-in/token-refresh bursts. Fix: percentage-based allocation in the + dashboard (operator action; requires explicit approval before touching live + settings), and keep single-instance deploys so refresh storms stay small. +2. **First soft failure: Postgres CPU under hybrid-RPC concurrency** — answer + p95 inflates well before errors appear. Watch the latency SLOs; the + remedies in order are: confirm cache/coalescing hit rates, then compute + upgrade, then (measured, eval-gated) retrieval fan-out reduction. Do not + touch retrieval code for capacity reasons without the golden eval. +3. **Bounded third: OpenAI RPM/TPM bursts** — mitigated by coalescing and by + spreading strong-route escalation; if the degraded-answer SLO trips in + correlation with 429s, raise the OpenAI tier before changing code. + +Explicit non-actions: no read replicas (retrieval is CPU-bound, not +read-connection-bound); no horizontal app scaling at this load; no retrieval +concurrency semaphore until soak data shows queueing. + +## 4. Soak test + +`scripts/soak-test.ts` — a dependency-free load driver for the ward-round +profile. **Staging only. Never point it at production.** + +```bash +# 30 virtual clinicians, 10 min, ward-round mix (75% search / 25% answer) +npx tsx scripts/soak-test.ts \ + --target https:// \ + --confirm-staging \ + --users 30 --duration-s 600 --ramp-s 120 + +# Authenticated run (bypasses anonymous rate limits): +npx tsx scripts/soak-test.ts --target https:// --confirm-staging \ + --bearer "$STAGING_ACCESS_TOKEN" +``` + +What it does: + +- Ramps up `--users` virtual users over `--ramp-s`, each looping: pick a query + (from `scripts/fixtures/rag-retrieval-golden.json` when present, otherwise a + built-in clinical list), issue a search request (75 %) or an answer request + (25 %), then think-time pause (`--think-ms`, default 15 s mean, jittered). +- Records per-endpoint latency percentiles (p50/p90/p95/max), HTTP error + counts, and 429s (reported separately — a 429 is the limiter working, not a + system failure). +- Exits non-zero if the non-429 error rate exceeds 5 % — so it can gate a + staging deploy. + +Safety rails (enforced in the script): + +- Requires an explicit `--target` **and** `--confirm-staging`; refuses to run + otherwise. +- Refuses any target whose host matches production markers (the production + Supabase ref, or hosts passed via `--forbid-host`, repeatable). +- Read-only traffic: only search and answer endpoints — no uploads, no + mutations, no admin routes. + +Success criteria for a 30-user staging soak (maps to the SLO table): + +| Metric | Target | +| ------------------------- | --------------------- | +| `/api/search` p95 | ≤ 3 s | +| `/api/answer` p95 | ≤ 25 s (mixed routes) | +| Non-429 error rate | < 1 % | +| Auth failures during ramp | 0 | + +Follow-ups after the first staging soak: record results here, compare against +the model in §2, and revisit the verdict in §3 if the ordering was wrong. diff --git a/docs/deployment-architecture.md b/docs/deployment-architecture.md new file mode 100644 index 0000000000..579f9366ab --- /dev/null +++ b/docs/deployment-architecture.md @@ -0,0 +1,234 @@ +# Deployment Architecture + +Decision record for the production topology of Clinical KB. Written 2026-07-06. +Companion documents: `docs/observability-slos.md` (SLOs + eval canary) and +`docs/capacity-review.md` (load model, first bottleneck, soak test). + +Status of this document: **decided and partially implemented**. The app-tier and +worker container images ship in this repo (`Dockerfile`, `Dockerfile.worker`). +Host provisioning, staging setup, and secret placement are operator actions and +are specified here but not executed by this change. + +## 1. Current state (what exists today) + +- **No production deployment target.** There is no hosting config; before this + change there was no Dockerfile. `npm run check:deployment-readiness` boots + `next start` locally and verifies project identity — it proves the build can + serve, not that anything is deployed. +- **Database/auth/storage:** live Supabase project `Clinical KB Database` + (`sjrfecxgysukkwxsowpy`), region **ap-southeast-2 (Sydney)**, Postgres 17, + ~2,000 indexed documents / ~69k chunks. RLS is service-role-only; the app + layer is the ownership boundary. +- **Ingestion:** a local worker (`npm run worker`) that needs a Python OCR + stack (PyMuPDF, Pillow, pytesseract + the Tesseract binary), plus the + `indexing-v3-agent` Supabase Edge Function acting as a cron-triggered + completion/repair gate — not a full extraction pipeline. +- **Known failure mode:** silent degradation. Hybrid retrieval RPCs once died + quietly while the app kept serving from fallbacks. Every topology decision + below biases toward _loud_ failure and standing guards. + +## 2. App tier + +### Decision + +Run the Next.js app as a **single long-lived container** (Node 24, image built +from `Dockerfile`) on a managed container host **in Sydney, co-located with +the Supabase project's ap-southeast-2 region**. + +Recommended host: **Fly.io (`syd` region)**, because it runs plain OCI images +with per-app secrets, health checks, and rollback to previous releases, and it +has a Sydney region. Google Cloud Run (`australia-southeast2`) is an equivalent +alternative if a GCP account is preferred; the image is host-agnostic either +way. Railway is _not_ suitable today (no Sydney region — cross-region RPC +latency would multiply across the per-answer fan-out). + +### Why a long-lived container and not serverless (Vercel et al.) + +- **In-memory coalescing and caches are load-bearing.** The answer pipeline + coalesces identical in-flight questions (`answer_inflight_coalesced` in + `src/lib/rag.ts`) and holds LRU answer/search caches + (`RAG_ANSWER_CACHE_TTL_MS`/`RAG_ANSWER_CACHE_SIZE`). Serverless isolates get + one request each, so coalescing never fires and every duplicate ward-round + question pays the full ~6-RPC fan-out plus an OpenAI generation. +- **Fire-and-forget background work.** Cache invalidation and telemetry writes + run as `void (async () => ...)` after the response; serverless platforms may + freeze the isolate at response end. +- **Long requests.** The strong answer route runs up to + `OPENAI_ANSWER_TIMEOUT_MS` (30 s) plus retrieval; streaming responses run + longer. That is hostile to per-request serverless billing/limits. +- **Connection amplification.** Many cold instances multiply concurrent + PostgREST/auth traffic against a database whose auth server is capped at 10 + absolute connections (see `docs/capacity-review.md`). + +Scale-out plan: stay at 1 instance (vertical scaling first) until sustained +load demands more; replicas are safe but dilute in-memory coalescing, so add +them only after the shared `rag_response_cache` hit rate is confirmed healthy. + +### Image contract (`Dockerfile`) + +- `node:24-bookworm-slim` in all stages — respects `engines`/`engine-strict` + and the `preinstall` engine guard. +- The build stage runs the repo's own `npm run build` + (`guard-next-build.mjs` + `next build --webpack`) — **the image build fails + exactly where a local build would**. The build allocates an 8 GiB heap; give + the Docker builder ≥ 10 GiB memory. +- `NEXT_PUBLIC_SUPABASE_URL` and `NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY` are + build args (they inline into the client bundle). The publishable key is + public by design; the placeholder default exists so CI can build without + secrets. **Production images must be built with the real publishable key.** +- Runtime is a non-root `node` user, prod-only `node_modules`, direct + `next start -H 0.0.0.0 -p $PORT` (the local port-picker script is + deliberately bypassed), and a `HEALTHCHECK` against `/api/health`. +- No secret is ever baked into a layer. `SUPABASE_SERVICE_ROLE_KEY`, + `OPENAI_API_KEY`, etc. are injected at run time by the host's secret store. + +Minimal Fly config (create at deploy time; not committed until an app is +provisioned): + +```toml +# fly.toml (sketch — values fixed at provisioning time) +app = "clinical-kb" +primary_region = "syd" +[build] +[env] + PORT = "3000" +[http_service] + internal_port = 3000 + force_https = true + min_machines_running = 1 # keep the warm instance: caches + coalescing + auto_stop_machines = false # no scale-to-zero: cold starts defeat the SLOs +[[http_service.checks]] + path = "/api/health" + interval = "30s" + timeout = "5s" +``` + +## 3. Ingestion tier + +### Decision: containerized worker (recommended) over completing the edge-agent migration + +Ship the existing worker as a container (`Dockerfile.worker`: Node 24 + tsx + +Tesseract + a Python venv with `worker/python/requirements.txt`) and run **one +always-on worker instance** co-located in Sydney. The `indexing-v3-agent` Edge +Function **stays** in its current role as the cron-triggered completion/repair +gate — the two are complementary, not alternatives. + +Reasoning: + +1. **The OCR stack cannot run at the edge.** PyMuPDF and Tesseract are native + binaries driven from Python. Supabase Edge Functions are Deno isolates with + no native-binary support and hard wall-clock/memory ceilings. "Completing + the migration" would mean reimplementing PDF parsing, OCR fallback, image + captioning, and table extraction inside those ceilings — a rewrite with a + strictly worse capability ceiling, not a migration. +2. **Job shape mismatch.** Large guideline PDFs take multi-minute processing + (the queue's stale-claim window is 45 minutes); edge functions are built for + sub-minute invocations. +3. **The worker is already multi-instance safe.** `claim_ingestion_jobs` uses + `FOR UPDATE SKIP LOCKED` with per-document exclusivity, so containerizing it + verbatim gives horizontal scaling for free (see queue semantics below). +4. **Smallest delta.** `worker/main.ts` runs unchanged in the container; the + only new artifact is the image. The edge path would fork the pipeline into + two implementations that drift — this repo's defining failure mode. + +Scaling: raise `WORKER_BATCH_SIZE` / `WORKER_CONCURRENCY` on the single +instance first; add replicas only for sustained backlog (safe by construction). + +### Queue durability when a worker dies mid-job + +Semantics of `claim_ingestion_jobs` (migration +`20260615114506_claim_ingestion_jobs_document_lock.sql`): + +- **Claim:** `status → processing`, `locked_at = now()`, `locked_by = worker`, + and — important — **`attempt_count` is incremented at claim time**, not at + failure time. Claims take `FOR UPDATE SKIP LOCKED` over the job _and_ its + document row, rank one job per document, and exclude any document that + already has a _fresh_ processing job. +- **There is no heartbeat.** The worker never refreshes `locked_at` mid-job. + If the worker dies, the job sits in `processing` until `locked_at` is older + than the stale window (`p_stale_after_minutes`, default 45, worker-side + `WORKER_STALE_AFTER_MINUTES`), after which any worker reclaims it + (`stage = 'reclaimed stale job'`). +- **Dead-lettering is implicit.** Because attempts are consumed at claim, a + crash-looping job exhausts `max_attempts` (default 3) after ~3 stale windows + and becomes terminally `failed` — the de-facto dead-letter state. Recovery is + operator-driven: `npm run recover:ingestion` or the retry API, both protected + by the ingestion rollback fence (`updated_at` fence) against retry/reindex + overlap races. + +Operational rules that follow: + +- **The stale window must exceed the worst-case job runtime.** If a live + worker runs a job longer than 45 minutes, a second worker can reclaim and + double-process the same document (the per-document exclusion only respects + _fresh_ locks). The rollback fence bounds the damage but does not prevent the + wasted work. When adding worker replicas, first confirm p100 job duration + against the window. +- **Worker death costs at most one stale window of latency** for the in-flight + job and zero data loss: all artifact writes are idempotent per + generation/chunk-key, and completion is gated by the strict completion RPCs + plus the edge agent. +- **Backlog improvement (not in this change):** a heartbeat that refreshes + `locked_at` could ride the existing throttled progress updates + (`WORKER_PROGRESS_UPDATE_MIN_INTERVAL_MS`, 60 s), which would let the stale + window shrink from 45 min to ~5 min without double-claim risk. Touches + worker + RPC; needs its own migration and review. + +## 4. Secrets management + +| Variable | Sensitivity | Build-time or runtime | Where it lives | +| ------------------------------------------------ | ---------------- | --------------------- | ------------------------------------------------------------------- | +| `NEXT_PUBLIC_SUPABASE_URL` | public | build (inlined) | Dockerfile ARG / repo | +| `NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY` | public-by-design | build (inlined) | Dockerfile ARG; real value passed by the release pipeline | +| `SUPABASE_SERVICE_ROLE_KEY` | **critical** | runtime | host secret store; GitHub repo secret (CI boot smoke + eval canary) | +| `OPENAI_API_KEY` | **critical** | runtime | host secret store; GitHub repo secret | +| `SUPABASE_PROJECT_REF` / `SUPABASE_PROJECT_NAME` | low | runtime | plain env (pins `check:supabase-project`) | +| `INDEXING_V3_AGENT_SECRET` | high | runtime | Supabase Edge Function secrets | +| `RAG_QUERY_HASH_SECRET` | high | runtime | host secret store | +| `E2E_USER_EMAIL` / `E2E_USER_PASSWORD` | medium | CI only | GitHub repo secrets | + +Rules: + +- Secrets never enter images, the repo, or `NEXT_PUBLIC_*` names. `.env.local` + is a local-dev convenience only. +- Each environment (production, staging, CI) gets **separate** service-role and + OpenAI keys so rotation and blast radius stay per-environment. +- Rotation: publishable-key rotation is already an operator runbook item + (`docs/archive/operator-decisions-2026-07-04.md`); service-role rotation is a + Supabase dashboard action + host secret update + redeploy. +- `npm run check:supabase-project` runs after any Supabase env change (repo + rule), and the eval canary runs it before every scheduled eval. + +## 5. Staging environment + +- **A second, dedicated Supabase project** (same org, ap-southeast-2) — not a + branch of production. Rationale: staging must absorb soak tests, destructive + ingestion experiments, and migration rehearsal without any shared compute, + pooling, or the production auth 10-connection cap; per-environment keys fall + out naturally. +- Seeded via the existing pipeline (`npm run import:docs`, `registry:seed`, + `differentials:seed`, `medications:seed`) with a small (~50-document) + synthetic/public corpus. `public/demo-documents/` plus generated samples + (`npm run samples`) are sufficient for load-shape realism; do not copy + clinical production documents into staging. +- One staging app container + one staging worker container from the _same_ + images, different env. `RAG_PROVIDER_MODE=auto` with staging OpenAI key. +- **Known change required:** `src/lib/supabase/project.ts` pins the expected + project to production, so `check:supabase-project` will (correctly) fail on + staging until the expected-project table is made environment-aware. Do this + when the staging project is provisioned — a deliberate speed bump so staging + cannot silently be pointed at production. +- The soak test (`scripts/soak-test.ts`) targets staging **only** — see + `docs/capacity-review.md`. + +## 6. Rollout and rollback + +- Images are built from `main` (CI job to be added once a host account + exists), tagged with the git SHA, and deployed after the standard gates + (`verify` + `ui-smoke` + the clinical governance preflight where relevant). +- Rollback = redeploy the previous image tag. Database migrations follow the + existing rule: committed migrations + `schema.sql` reconciliation only, never + raw SQL against live. +- The nightly eval canary (`.github/workflows/eval-canary.yml`) is the standing + guard that retrieval/answer quality did not silently regress after any + deploy — see `docs/observability-slos.md`. diff --git a/docs/observability-slos.md b/docs/observability-slos.md new file mode 100644 index 0000000000..67648b28fc --- /dev/null +++ b/docs/observability-slos.md @@ -0,0 +1,176 @@ +# Observability & SLOs + +Service-level objectives for the Clinical KB answer pipeline, the alert +thresholds attached to them, and the nightly production eval canary that turns +the golden eval into a standing guard. Written 2026-07-06. + +Context: this repo's defining failure mode is **silent degradation** — hybrid +retrieval RPCs once died quietly while the app kept serving from fallbacks. +Every SLO below is chosen so that the degraded state is _visible_ even when the +app keeps returning 200s. + +## 1. Telemetry sources (what exists today) + +| Source | What it carries | +| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rag_queries.metadata` (jsonb, one row per answered query) | `routing_mode` (`fast` / `strong` / `extractive` / `unsupported`), `confidence` (`high` / `medium` / `low` / `unsupported`), `query_class`, `fallback_reason`, `grounded`, per-stage latencies (`total_latency_ms`, `supabase_rpc_latency_ms`, `embedding_latency_ms`, …), cache/coalescing flags, and `hybrid_rpc_errors` (map of RPC → error) when any hybrid RPC failed | +| `rag_query_misses` | weak-search/miss review queue with `miss_reason` | +| Server logs | `logger.error("hybrid_rpc_failed", …)` per failing RPC (also emitted by deep-memory) | +| `search_schema_health()` | execution smoke over all hybrid RPCs; surfaced by `npm run check:indexing` | +| `/api/health` | app liveness + Supabase reachability (container HEALTHCHECK target) | +| Answer API response | `degradedMode` / `answerQualityTier: "source_only"` signal per response | + +Query-text privacy: `rag_queries.query` is centrally redacted +(`queryTextForStorage`); SLO queries below only aggregate metadata, never raw +text. + +## 2. SLOs and alert thresholds + +Measurement window is trailing 24 h unless stated; "page" means the loudest +channel available (today: GitHub issue from the canary + host alert; later: +host-native alerting per `docs/deployment-architecture.md`). + +### Latency — answer p95 by route mode + +| Route mode (`metadata->>'routing_mode'`) | SLO (p95) | Warn | Page | +| ---------------------------------------- | --------- | ------------------ | ------------------ | +| `fast` | ≤ 10 s | p95 > 10 s for 1 h | p95 > 20 s for 1 h | +| `strong` | ≤ 25 s | p95 > 25 s for 1 h | p95 > 35 s for 1 h | +| `extractive` / source-only | ≤ 6 s | p95 > 6 s for 1 h | p95 > 12 s for 1 h | +| `/api/search` (route timing) | ≤ 3 s | p95 > 3 s for 1 h | p95 > 8 s for 1 h | + +Anchors: `OPENAI_ANSWER_TIMEOUT_MS` is 30 s; the retrieval latency eval budget +is p90 ≤ 20 s with a 25 s case timeout. A `strong` p95 near 35 s means answers +are riding the timeout and silently falling back. + +```sql +select + metadata->>'routing_mode' as route_mode, + percentile_cont(0.95) within group (order by (metadata->>'total_latency_ms')::numeric) as p95_ms, + count(*) as n +from rag_queries +where created_at > now() - interval '24 hours' +group by 1; +``` + +### Quality — source-gap rate + +Share of answered queries whose confidence collapsed to a gap +(`fallback_reason` or support modules report `source_gap`). + +- **SLO:** ≤ 15 % of answers over 7 days. +- **Warn:** > 20 % over 24 h. **Page:** > 30 % over 6 h (step change — + suggests retrieval, enrichment, or corpus regression, not user behavior). + +### Quality — unsupported rate + +Share of queries with `routing_mode = 'unsupported'` or +`confidence = 'unsupported'`. A base rate is legitimate (out-of-corpus +questions), so alert on deviation, not existence. + +- **SLO:** ≤ 10 % of queries over 7 days. +- **Warn:** > 15 % over 24 h. **Page:** > 25 % over 6 h, or a doubling versus + the trailing-7-day rate. Known confounder: the nondeterministic + unsupported short-circuit (finding #11) — check its memoization before + declaring a regression. + +```sql +select + count(*) filter (where metadata->>'routing_mode' = 'unsupported' + or metadata->>'confidence' = 'unsupported')::float + / greatest(count(*), 1) as unsupported_rate +from rag_queries +where created_at > now() - interval '24 hours'; +``` + +### Reliability — hybrid_rpc_errors rate + +Share of queries whose metadata contains a non-empty `hybrid_rpc_errors` map. +This is the direct guard against the historical silent-RPC-death incident, so +tolerance is near zero. + +- **SLO:** 0 sustained errors. Isolated blips (< 0.1 % over 24 h) tolerated. +- **Warn:** > 0.5 % of queries in any 1 h window. +- **Page:** any _sustained_ nonzero rate across 3 consecutive hours, or the + same RPC name failing repeatedly — the app will look healthy while serving + fallback-quality answers, which is exactly the failure mode to catch. + +```sql +select + metadata->'hybrid_rpc_errors' as errors, count(*) +from rag_queries +where created_at > now() - interval '6 hours' + and metadata ? 'hybrid_rpc_errors' +group by 1 order by 2 desc; +``` + +Complementary standing checks: `search_schema_health()` via +`npm run check:indexing` (fails closed on RPC regression) and the nightly eval +canary below (fails closed on quality regression). + +### Reliability — degraded/source-only answer rate + +`RAG_PROVIDER_MODE=auto` silently degrades to deterministic "Source-only" +answers when generation fails quality gates. Expected occasionally; a spike +means the OpenAI path is broken while users still get 200s. + +- **SLO:** ≤ 10 % of grounded answers over 24 h. +- **Warn:** > 20 % over 1 h. **Page:** > 50 % over 1 h (generation is + effectively down). + +Measure via `metadata->>'fallback_reason'` / `answer_model_demoted`. + +## 3. Nightly production eval canary + +`.github/workflows/eval-canary.yml` — scheduled nightly (18:00 UTC = 02:00 +Australia/Perth) plus `workflow_dispatch` for on-demand runs. + +What it does, in order: + +1. `npm run check:supabase-project` — hard guard that the configured env + points at `sjrfecxgysukkwxsowpy` and nothing else. +2. `npm run eval:retrieval:quality -- --fail-on-threshold` — the golden + retrieval eval (34 cases incl. forced-vector probes) against the live + corpus. This is the eval CI never runs on PRs (it needs live Supabase + + OpenAI keys); the canary makes it a standing nightly guard instead of a + manual pre-merge step that can be skipped. +3. `npm run eval:quality -- --rag-only --limit 8 --fail-on-threshold` — a + small answer-quality subset (grounding, citations, unsupported-correctness) + to bound OpenAI spend while still catching generation-side regressions. + +Failing loudly: + +- Any threshold failure fails the workflow run (red nightly badge, email per + GitHub notification settings). +- On scheduled failures the workflow **opens a GitHub issue** labeled + `eval-canary` (or comments on the existing open one), so a regression + creates a durable, assignable artifact rather than a missed notification. + +Required repo secrets (same ones CI's deployment boot smoke already uses, +plus the eval owner): `SUPABASE_SERVICE_ROLE_KEY`, `OPENAI_API_KEY`, +`E2E_USER_EMAIL` (resolved to the eval owner via `RAG_EVAL_OWNER_EMAIL`). +The workflow preflights these and fails with an explicit message when absent. + +Operational notes: + +- The canary reads live shared corpus state; a pass is a snapshot, and a + failure can be corpus-state-dependent (see the clozapine-wcc history). + Triage order: rerun via `workflow_dispatch` → check `hybrid_rpc_errors` and + `check:indexing` → only then bisect code. +- Evals write telemetry rows (`rag_queries`) but mutate no content. +- Cost bound: ~34 retrieval cases (embedding calls only on forced-vector + probes) + 8 generated answers per night. +- **The schedule only runs from `main`.** After merging, trigger one + `workflow_dispatch` run and confirm it goes green before trusting the + nightly cadence (repo gate for this workflow). + +## 4. Gaps / next steps (not in this change) + +- Host-level metrics (CPU, memory, restart count) and log drains once the + container host exists (`docs/deployment-architecture.md` §2). +- A lightweight `/api/health` extension exposing cache hit-rate and + `hybrid_rpc_errors`-in-last-hour counters for host-native alerting, so the + SQL above can become a scrape instead of a manual query. (Touches app code — + deliberately excluded from this change.) +- Wire the warn/page thresholds into an actual alerting channel (Supabase log + drain → host alerts, or a scheduled workflow evaluating the SQL above). diff --git a/scripts/soak-test.ts b/scripts/soak-test.ts new file mode 100644 index 0000000000..37df6589d9 --- /dev/null +++ b/scripts/soak-test.ts @@ -0,0 +1,276 @@ +/** + * Ward-round soak test for the Clinical KB app tier. + * + * STAGING ONLY. This script drives sustained answer/search load and must never + * point at production. See docs/capacity-review.md §4 for the load model, + * usage examples, and success criteria. + * + * Safety rails: + * - requires an explicit --target and --confirm-staging; + * - refuses targets that look like production (the live Supabase project ref + * in the host, or any host passed via --forbid-host); + * - issues read-only traffic only (POST /api/search and POST /api/answer). + */ + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +const PRODUCTION_MARKERS = ["sjrfecxgysukkwxsowpy"]; + +type SoakArgs = { + target: string; + confirmStaging: boolean; + users: number; + durationS: number; + rampS: number; + thinkMs: number; + answerShare: number; + timeoutMs: number; + bearer?: string; + forbidHosts: string[]; +}; + +type RequestSample = { + endpoint: "search" | "answer"; + status: number; + latencyMs: number; + timedOut: boolean; +}; + +function usage(): never { + console.log( + [ + "Usage: npx tsx scripts/soak-test.ts --target --confirm-staging [options]", + "", + "Options:", + " --target Base URL of the STAGING app (required)", + " --confirm-staging Acknowledge the target is staging (required)", + " --users Virtual users (default 30)", + " --duration-s Steady-state duration in seconds (default 300)", + " --ramp-s Ramp-up window in seconds (default 60)", + " --think-ms Mean think time between requests (default 15000)", + " --answer-share <0..1> Fraction of requests that are answers (default 0.25)", + " --timeout-ms Per-request timeout (default 60000)", + " --bearer Authorization bearer token (bypasses anonymous limits)", + " --forbid-host Extra host substring to refuse (repeatable)", + ].join("\n"), + ); + process.exit(1); +} + +function parseArgs(argv: string[]): SoakArgs { + const args: SoakArgs = { + target: "", + confirmStaging: false, + users: 30, + durationS: 300, + rampS: 60, + thinkMs: 15_000, + answerShare: 0.25, + timeoutMs: 60_000, + forbidHosts: [], + }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + const value = argv[index + 1]; + if (token === "--help" || token === "-h") usage(); + if (token === "--confirm-staging") { + args.confirmStaging = true; + continue; + } + if (!value) continue; + if (token === "--target") args.target = value; + if (token === "--users") args.users = Number.parseInt(value, 10); + if (token === "--duration-s") args.durationS = Number.parseInt(value, 10); + if (token === "--ramp-s") args.rampS = Number.parseInt(value, 10); + if (token === "--think-ms") args.thinkMs = Number.parseInt(value, 10); + if (token === "--answer-share") args.answerShare = Number.parseFloat(value); + if (token === "--timeout-ms") args.timeoutMs = Number.parseInt(value, 10); + if (token === "--bearer") args.bearer = value; + if (token === "--forbid-host") args.forbidHosts.push(value.toLowerCase()); + } + + if (!args.target) { + console.error("Missing --target. This script never assumes a default target."); + usage(); + } + if (!args.confirmStaging) { + console.error( + "Refusing to run without --confirm-staging. This script is for STAGING only; do not point it at production.", + ); + process.exit(1); + } + if (!Number.isInteger(args.users) || args.users < 1 || args.users > 500) { + throw new Error("--users must be an integer between 1 and 500."); + } + if (!Number.isFinite(args.answerShare) || args.answerShare < 0 || args.answerShare > 1) { + throw new Error("--answer-share must be between 0 and 1."); + } + return args; +} + +function assertTargetIsNotProduction(args: SoakArgs) { + const url = new URL(args.target); + const host = url.host.toLowerCase(); + const markers = [...PRODUCTION_MARKERS.map((marker) => marker.toLowerCase()), ...args.forbidHosts]; + for (const marker of markers) { + if (host.includes(marker)) { + console.error(`Refusing target ${host}: matches forbidden production marker "${marker}".`); + process.exit(1); + } + } +} + +const fallbackQueries = [ + "clozapine monitoring requirements", + "lithium toxicity management", + "acute dystonia treatment", + "venlafaxine discontinuation symptoms", + "sodium valproate in pregnancy", + "serotonin syndrome recognition", + "rapid tranquillisation protocol", + "metformin renal dosing", + "warfarin reversal steps", + "delirium screening tools", +]; + +function loadQueries(): string[] { + const fixturePath = join(process.cwd(), "scripts", "fixtures", "rag-retrieval-golden.json"); + if (!existsSync(fixturePath)) return fallbackQueries; + try { + const parsed: unknown = JSON.parse(readFileSync(fixturePath, "utf8")); + if (!Array.isArray(parsed)) return fallbackQueries; + const queries = parsed + .map((entry) => (entry && typeof entry === "object" ? (entry as { query?: unknown }).query : null)) + .filter((query): query is string => typeof query === "string" && query.length > 0); + return queries.length > 0 ? queries : fallbackQueries; + } catch { + return fallbackQueries; + } +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function jitteredThink(meanMs: number) { + // 0.5x..1.5x uniform jitter around the mean keeps users out of lockstep. + return meanMs * (0.5 + Math.random()); +} + +function percentile(sorted: number[], fraction: number) { + if (sorted.length === 0) return 0; + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1)); + return sorted[index]; +} + +async function issueRequest(args: SoakArgs, endpoint: "search" | "answer", query: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), args.timeoutMs); + const startedAt = Date.now(); + try { + const response = await fetch(new URL(`/api/${endpoint}`, args.target), { + method: "POST", + headers: { + "content-type": "application/json", + ...(args.bearer ? { authorization: `Bearer ${args.bearer}` } : {}), + }, + body: JSON.stringify({ query }), + signal: controller.signal, + }); + // Drain the body so keep-alive sockets are reusable. + await response.arrayBuffer().catch(() => undefined); + return { endpoint, status: response.status, latencyMs: Date.now() - startedAt, timedOut: false }; + } catch (error) { + const timedOut = error instanceof Error && error.name === "AbortError"; + return { endpoint, status: 0, latencyMs: Date.now() - startedAt, timedOut }; + } finally { + clearTimeout(timer); + } +} + +async function runVirtualUser( + args: SoakArgs, + userIndex: number, + queries: string[], + endAtMs: number, + samples: RequestSample[], +) { + // Stagger starts across the ramp window. + await sleep((args.rampS * 1000 * userIndex) / Math.max(args.users, 1)); + while (Date.now() < endAtMs) { + const query = queries[Math.floor(Math.random() * queries.length)]; + const endpoint = Math.random() < args.answerShare ? "answer" : "search"; + samples.push(await issueRequest(args, endpoint, query)); + const remaining = endAtMs - Date.now(); + if (remaining <= 0) break; + await sleep(Math.min(jitteredThink(args.thinkMs), remaining)); + } +} + +function summarizeEndpoint(samples: RequestSample[], endpoint: "search" | "answer") { + const scoped = samples.filter((sample) => sample.endpoint === endpoint); + const ok = scoped.filter((sample) => sample.status >= 200 && sample.status < 400); + const rateLimited = scoped.filter((sample) => sample.status === 429); + const failed = scoped.filter((sample) => sample.status === 0 || (sample.status >= 400 && sample.status !== 429)); + const latencies = ok.map((sample) => sample.latencyMs).sort((a, b) => a - b); + return { + endpoint, + total: scoped.length, + ok: ok.length, + rateLimited: rateLimited.length, + failed: failed.length, + timedOut: scoped.filter((sample) => sample.timedOut).length, + p50: percentile(latencies, 0.5), + p90: percentile(latencies, 0.9), + p95: percentile(latencies, 0.95), + max: latencies.at(-1) ?? 0, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + assertTargetIsNotProduction(args); + const queries = loadQueries(); + + console.log(`Soak target: ${args.target}`); + console.log( + `Profile: ${args.users} users, ramp ${args.rampS}s, steady ${args.durationS}s, ` + + `answer share ${Math.round(args.answerShare * 100)}%, think ~${args.thinkMs}ms, ${queries.length} queries.`, + ); + console.log(args.bearer ? "Auth: bearer token supplied." : "Auth: anonymous (expect tight 429 limits)."); + + const endAtMs = Date.now() + (args.rampS + args.durationS) * 1000; + const samples: RequestSample[] = []; + await Promise.all( + Array.from({ length: args.users }, (_, userIndex) => runVirtualUser(args, userIndex, queries, endAtMs, samples)), + ); + + const summaries = [summarizeEndpoint(samples, "search"), summarizeEndpoint(samples, "answer")]; + console.log("\nResults:"); + for (const summary of summaries) { + console.log( + ` /api/${summary.endpoint}: n=${summary.total} ok=${summary.ok} 429=${summary.rateLimited} ` + + `failed=${summary.failed} timeouts=${summary.timedOut}`, + ); + console.log( + ` latency ms (ok only): p50=${summary.p50} p90=${summary.p90} p95=${summary.p95} max=${summary.max}`, + ); + } + + const total = samples.length; + const hardFailures = summaries.reduce((sum, summary) => sum + summary.failed, 0); + const failureRate = total > 0 ? hardFailures / total : 0; + console.log(`\nTotal requests: ${total}; non-429 failure rate: ${(failureRate * 100).toFixed(2)}% (gate: 5%).`); + if (failureRate > 0.05) { + console.error("FAIL: non-429 failure rate exceeded 5%."); + process.exit(1); + } + console.log("PASS: failure rate within budget. Compare percentiles against docs/capacity-review.md §4."); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +});