diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index e57bad6745..d3f68a05b6 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -568,3 +568,4 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-07-31 | 1489 | 67d5cb91083f9b0e9d3017816cbf68abab102688 | PR 1489 review — Therapy startup/sidebar perf, catalogue split, bundle-budget, phone-chrome | approved with follow-ups; merged 945148251. No P0/P1. Findings fixed on claude/pr-1489-review-786e01: inferred modality mislabelled ECT/rTMS as ACT and Psychoanalysis as CBT (pre-existing on main); hashed catalogue assets never pruned (2 stranded in-PR); classifyPullRequestFiles returned clinicalRisk:false for 205 clinical records; viewportHeightChanged guard outranked topRevealOffset; guard keyed innerHeight not visualViewport; sk-proj- keys unescaped; bundle-budget step timeout 3m too tight. Bundling note: operationalRisk+clinicalRisk in one squash, so no per-item revert. | verify:cheap static gates pass; lint pass; typecheck exit 0; vitest 449 files/4700 pass; verify:phone-chrome contracts 116 pass + focused browser 13 pass; verify:ui 342 pass/2 fail, both pass isolated (composer hero-vs-dock hydration race, no position: assignment in use-hide-on-scroll) | | 2026-07-31 | claude/fable-implementation-fc937c | 2147572428271278462db2aac169a5c3c0a2bcc7 | pr-1531 unblock | reverted CodeRabbit autofix (next.config env import broke build; prettier docs); merged origin/main behind-but-clean; merge-tree-clean | prettier design-system+next.config; eslint next.config --max-warnings 0; merge-tree clean vs origin/main | | 2026-07-31 | cursor/sentry-db-query-tracing-0546 | 0dfb7a35a3a2623794a2d3e4c657268018604d18 | pr-1540-unblock | fixed static-pr knip unlisted @sentry/core; merge-tree clean; no blocking threads | check:knip:pass; vitest:error-tracking:5/5; tsc:pass; merge-tree:clean | +| 2026-07-31 | claude/sentry-agent-monitoring-eri94v | 38378ac78c0b288b6e93b638019653001f375042 | Sentry AI agent monitoring (OpenAI wrap, gen_ai scrubber allowlist, conversation id) | pass — metadata-only instrumentation; privacy boundary preserved | verify:pr-local,verify:cheap,typecheck,vitest 458 files green | diff --git a/docs/codebase-index.md b/docs/codebase-index.md index 428cc71207..c55f14bdfb 100644 --- a/docs/codebase-index.md +++ b/docs/codebase-index.md @@ -171,12 +171,12 @@ domain-extracted directory; imported as `@/lib/rag/rag*`). Other modules below r ### Infra helpers -| Module | Role | -| ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ | -| `openai.ts`, `embedding-dimensions.ts`, `api-rate-limit.ts` | External APIs and rate limits | -| `observability/` — `answer-slo.ts`, `cache-metrics.ts`, `spend-metrics.ts`, `error-tracking.ts` | Deep-health SLO / cache-hit / answer-spend snapshots; privacy-safe Sentry error + DB-span scrubbers (`docs/error-tracking.md`) | -| `validation/` | `body.ts`, `query.ts`, `params.ts`, `http.ts`, `form-data.ts` | -| `app-modes.ts`, `document-flow-routes.ts`, `local-project-identity.ts`, `local-server-utils.mjs` | Routing and project identity | +| Module | Role | +| ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `openai.ts`, `embedding-dimensions.ts`, `api-rate-limit.ts` | External APIs and rate limits | +| `observability/` — `answer-slo.ts`, `cache-metrics.ts`, `spend-metrics.ts`, `error-tracking.ts`, `agent-monitoring.ts` | Deep-health SLO / cache-hit / answer-spend snapshots; privacy-safe Sentry error + DB-span scrubbers and metadata-only OpenAI agent monitoring (`docs/error-tracking.md`) | +| `validation/` | `body.ts`, `query.ts`, `params.ts`, `http.ts`, `form-data.ts` | +| `app-modes.ts`, `document-flow-routes.ts`, `local-project-identity.ts`, `local-server-utils.mjs` | Routing and project identity | --- diff --git a/docs/error-tracking.md b/docs/error-tracking.md index 4ff6735455..a4972603b4 100644 --- a/docs/error-tracking.md +++ b/docs/error-tracking.md @@ -23,6 +23,20 @@ Privacy constraints for traces: View samples in Sentry under **Explore → Traces**, and aggregated DB performance under **Dashboards → Sentry Built → Queries**. +### AI agent monitoring (OpenAI spans) + +The OpenAI client is wrapped by `src/lib/observability/agent-monitoring.ts` (`Sentry.instrumentOpenAiClient`, SDK ≥ 10.67) so Sentry's AI/agents views show per-call operation, model, latency, and token usage. The wrap is inert unless `SENTRY_DSN` is set, the resolved traces sample rate is greater than zero, **and** the runtime actually initialized Sentry — so it is always a no-op in the ingestion worker and in tests, which import the module but never run `Sentry.init()`. + +Privacy constraints for agent monitoring: + +- `recordInputs` / `recordOutputs` are **false** on the wrap and `dataCollection.genAI` is `{ inputs: false, outputs: false }` in both runtime configs — prompts, clinical queries, source evidence, generated answers, and embedding inputs are never recorded. +- `privacySafeTransactionEvent` allowlists only gen_ai metadata attributes (system, operation name, request/response model, response id, finish reasons, token usage, conversation id) and rebuilds gen_ai span descriptions as ` ` from those attributes. Message, prompt, tool-payload, and embedding-input attributes are stripped on export even if a future SDK version records them. +- Each answer request (`/api/answer` and `/api/answer/stream`, summaries included) calls `Sentry.setConversationId()` — the request's synthetic UUID — so the embedding/generation calls of one request group into one conversation without carrying any query text. +- User identification (`Sentry.setUser`) is deliberately **not** wired: the committed privacy boundary strips `user` from every outgoing event (see the tests), and linking clinical-query telemetry to an identity would need its own governance review first. +- `responses.parse` (schema-parsed generation) is not in the SDK's instrumentation registry and emits no gen_ai span; `responses.create` and `embeddings.create` are covered. + +Rollback matches tracing: set `SENTRY_TRACES_SAMPLE_RATE=0` (agent spans stop; error capture stays) or remove `SENTRY_DSN`. Raising the sample rate above the 0.1 default captures a larger share of answer requests in the agents view and is an operator decision. + ## Operator approval and rollout Before setting `SENTRY_DSN`, the operator must approve the vendor/project, data region, retention period, access roles, sampling rate, cost budget, and alert destination. Configure a server-side DSN only; never use a `NEXT_PUBLIC_*` DSN. Keep provider-side IP/user enrichment disabled and restrict project access. Start with a non-production synthetic exception and inspect the received event before enabling production alerts. diff --git a/package-lock.json b/package-lock.json index 6736581994..ed60042b22 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "license": "UNLICENSED", "dependencies": { "@next/env": "16.2.12", - "@sentry/nextjs": "^10.17.0", + "@sentry/nextjs": "^10.67.0", "@supabase/ssr": "^0.12.3", "@supabase/supabase-js": "^2.110.8", "exceljs": "^4.4.0", diff --git a/package.json b/package.json index 397c94c72d..ff2dac014a 100644 --- a/package.json +++ b/package.json @@ -224,7 +224,7 @@ }, "dependencies": { "@next/env": "16.2.12", - "@sentry/nextjs": "^10.17.0", + "@sentry/nextjs": "^10.67.0", "@supabase/ssr": "^0.12.3", "@supabase/supabase-js": "^2.110.8", "exceljs": "^4.4.0", diff --git a/src/app/api/answer/route.ts b/src/app/api/answer/route.ts index 02f1a83102..2d576b36bd 100644 --- a/src/app/api/answer/route.ts +++ b/src/app/api/answer/route.ts @@ -11,6 +11,7 @@ import { rateLimitJsonResponse, } from "@/lib/api-rate-limit"; import { publicAccessContext } from "@/lib/public-api-access"; +import { setAgentConversationId } from "@/lib/observability/agent-monitoring"; import { classifyRagQuery } from "@/lib/clinical-search"; import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { queryClassForClinicalMode, queryForClinicalMode } from "@/lib/clinical-query-mode"; @@ -58,6 +59,10 @@ function buildDemoAnswerPayload(body: AnswerRequestBody, fallbackReason?: string export async function POST(request: Request) { const interactionId = randomUUID(); + // Group this request's LLM calls (embedding, generation, verification) into + // one Sentry agent-monitoring conversation keyed by the synthetic interaction + // UUID — never by query text. + setAgentConversationId(interactionId); const routeStartedAt = Date.now(); let body: AnswerRequestBody | null = null; try { diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index 46f440651c..e2923f6464 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -11,6 +11,7 @@ import { type ApiRateLimitResult, } from "@/lib/api-rate-limit"; import { publicAccessContext } from "@/lib/public-api-access"; +import { setAgentConversationId } from "@/lib/observability/agent-monitoring"; import { answerDegradedModeSignal, buildGovernedAnswerClientResponse, @@ -158,6 +159,9 @@ function streamAnswer( const ownerId = accessScope.ownerId; const encoder = new TextEncoder(); const interactionId = randomUUID(); + // Group this request's LLM calls into one Sentry agent-monitoring + // conversation keyed by the synthetic interaction UUID — never by query text. + setAgentConversationId(interactionId); return new Response( new ReadableStream({ diff --git a/src/lib/observability/agent-monitoring.ts b/src/lib/observability/agent-monitoring.ts new file mode 100644 index 0000000000..dd8fb8852b --- /dev/null +++ b/src/lib/observability/agent-monitoring.ts @@ -0,0 +1,65 @@ +import "server-only"; + +import * as Sentry from "@sentry/nextjs"; + +import { resolveTracesSampleRate } from "@/lib/observability/error-tracking"; + +/** + * Node-only OpenAI client instrumentation for privacy-safe AI agent monitoring. + * Kept out of `error-tracking.ts` for the same reason as `supabase-tracing.ts`: + * Edge instrumentation imports the scrubbers without pulling the Node Sentry entry. + */ + +/** + * Agent monitoring is span-based, so it needs a DSN, a positive traces sample + * rate, AND a runtime where `Sentry.init()` actually ran (`src/sentry.*.config.ts` + * via Next instrumentation). The client gate keeps the wrap inert in the + * ingestion worker and in tests, which import this module but never init Sentry. + */ +export function isSentryAgentMonitoringEnabled(): boolean { + try { + return Boolean(process.env.SENTRY_DSN?.trim()) && resolveTracesSampleRate() > 0 && Boolean(Sentry.getClient()); + } catch { + return false; + } +} + +/** + * Wrap an OpenAI client so `responses.create` / `embeddings.create` emit gen_ai + * spans (model, operation, latency, token usage). Inputs and outputs are never + * recorded — clinical queries, source evidence, and generated answers must not + * leave the server. `privacySafeTransactionEvent` enforces the same boundary + * again on export (docs/error-tracking.md). + */ +export function instrumentOpenAIClientForAgentMonitoring(client: T): T { + if (!isSentryAgentMonitoringEnabled()) { + return client; + } + + try { + return Sentry.instrumentOpenAiClient(client, { + recordInputs: false, + recordOutputs: false, + }); + } catch { + // Optional observability must never take down answer generation. + return client; + } +} + +/** + * Group every LLM call made while serving one answer request into a single + * Sentry agent-monitoring conversation. The id must be the request's synthetic + * interaction UUID — never query text or any derived clinical identifier. + */ +export function setAgentConversationId(conversationId: string): void { + if (!isSentryAgentMonitoringEnabled()) { + return; + } + + try { + Sentry.setConversationId(conversationId); + } catch { + // Optional observability must never take down answer generation. + } +} diff --git a/src/lib/observability/error-tracking.ts b/src/lib/observability/error-tracking.ts index a344c09fb1..f49498b47d 100644 --- a/src/lib/observability/error-tracking.ts +++ b/src/lib/observability/error-tracking.ts @@ -28,6 +28,40 @@ const SAFE_SPAN_DATA_KEYS = [ "sentry.sample_rate", ] as const; +/** + * gen_ai span attributes safe to export for AI agent monitoring: operation and + * model metadata, token usage, and the synthetic per-request conversation id. + * Prompt, message, tool, embedding-input, and response-text attributes are + * deliberately absent — clinical queries, source evidence, and generated + * answers never leave the server (docs/error-tracking.md). + */ +const SAFE_GEN_AI_SPAN_DATA_KEYS = [ + "gen_ai.system", + "gen_ai.operation.name", + "gen_ai.agent.name", + "gen_ai.pipeline.name", + "gen_ai.conversation.id", + "gen_ai.request.model", + "gen_ai.request.stream", + "gen_ai.request.temperature", + "gen_ai.request.max_tokens", + "gen_ai.request.top_p", + "gen_ai.request.frequency_penalty", + "gen_ai.request.presence_penalty", + "gen_ai.request.encoding_format", + "gen_ai.request.dimensions", + "gen_ai.response.model", + "gen_ai.response.id", + "gen_ai.response.finish_reasons", + "gen_ai.response.stop_reason", + "gen_ai.response.streaming", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.usage.input_tokens.cached", + "gen_ai.usage.input_tokens.cache_write", +] as const; + const DEFAULT_TRACES_SAMPLE_RATE = 0.1; /** @@ -123,6 +157,19 @@ function privacySafeSpanDescription(data: Record, fallback: str return `from(${table})`; } + // gen_ai (agent monitoring) spans: rebuild " " from the + // allowlisted attributes; the SDK's own span name is treated as free-form text. + const genAiOperation = typeof data["gen_ai.operation.name"] === "string" ? data["gen_ai.operation.name"] : undefined; + if (genAiOperation) { + const genAiModel = + typeof data["gen_ai.request.model"] === "string" + ? data["gen_ai.request.model"] + : typeof data["gen_ai.response.model"] === "string" + ? data["gen_ai.response.model"] + : undefined; + return genAiModel ? `${genAiOperation} ${genAiModel}` : genAiOperation; + } + // Keep parameterized framework/route span names; drop free-form or query-bearing text. if ( typeof fallback === "string" && @@ -141,7 +188,9 @@ function privacySafeSpanDescription(data: Record, fallback: str function privacySafeSpan(span: ScrubbedSpan): ScrubbedSpan { const rawData = (span.data ?? {}) as Record; const data = Object.fromEntries( - SAFE_SPAN_DATA_KEYS.flatMap((key) => (rawData[key] === undefined ? [] : [[key, rawData[key]]])), + [...SAFE_SPAN_DATA_KEYS, ...SAFE_GEN_AI_SPAN_DATA_KEYS].flatMap((key) => + rawData[key] === undefined ? [] : [[key, rawData[key]]], + ), ); return { diff --git a/src/lib/openai.ts b/src/lib/openai.ts index 91c7cbec52..1033ab31cd 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -3,6 +3,7 @@ import OpenAI from "openai"; import { zodTextFormat } from "openai/helpers/zod"; import type { ZodType } from "zod"; import { env, requireOpenAIEnv } from "@/lib/env"; +import { instrumentOpenAIClientForAgentMonitoring } from "@/lib/observability/agent-monitoring"; import { assessClinicalImageUse } from "@/lib/image-filtering"; import { PublicApiError } from "@/lib/http"; import { @@ -78,11 +79,15 @@ export function createOpenAIClient() { }); } - openAIClient ??= new OpenAI({ - apiKey: env.OPENAI_API_KEY, - timeout: env.OPENAI_REQUEST_TIMEOUT_MS, - maxRetries: env.OPENAI_MAX_RETRIES, - }); + // Metadata-only Sentry agent monitoring (model, latency, token usage — never + // inputs/outputs); inert unless the runtime initialized Sentry with tracing on. + openAIClient ??= instrumentOpenAIClientForAgentMonitoring( + new OpenAI({ + apiKey: env.OPENAI_API_KEY, + timeout: env.OPENAI_REQUEST_TIMEOUT_MS, + maxRetries: env.OPENAI_MAX_RETRIES, + }), + ); return openAIClient; } diff --git a/src/sentry.edge.config.ts b/src/sentry.edge.config.ts index a7edbe6ea1..ba19fa8677 100644 --- a/src/sentry.edge.config.ts +++ b/src/sentry.edge.config.ts @@ -20,6 +20,8 @@ try { sendDefaultPii: false, dataCollection: { databaseQueryData: false, + // Mirror the server runtime: AI inputs/outputs are never collected. + genAI: { inputs: false, outputs: false }, }, enableLogs: false, maxBreadcrumbs: 0, diff --git a/src/sentry.server.config.ts b/src/sentry.server.config.ts index 94fa9fac75..02b95a6b68 100644 --- a/src/sentry.server.config.ts +++ b/src/sentry.server.config.ts @@ -79,6 +79,10 @@ try { sendDefaultPii: false, dataCollection: { databaseQueryData: false, + // Defense in depth for AI agent monitoring: the OpenAI wrap in + // agent-monitoring.ts already sets recordInputs/recordOutputs false, and + // privacySafeTransactionEvent strips prompt/output attributes on export. + genAI: { inputs: false, outputs: false }, }, includeLocalVariables: false, enableLogs: false, diff --git a/tests/agent-monitoring.test.ts b/tests/agent-monitoring.test.ts new file mode 100644 index 0000000000..8dfe47c0e7 --- /dev/null +++ b/tests/agent-monitoring.test.ts @@ -0,0 +1,39 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +describe("Sentry agent monitoring gating", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + it("stays disabled without a DSN, with tracing off, or without an initialized Sentry client", async () => { + vi.stubEnv("SENTRY_DSN", ""); + vi.stubEnv("SENTRY_TRACES_SAMPLE_RATE", "0.1"); + const noDsn = await import("@/lib/observability/agent-monitoring"); + expect(noDsn.isSentryAgentMonitoringEnabled()).toBe(false); + + vi.resetModules(); + vi.stubEnv("SENTRY_DSN", "https://public@o0.ingest.sentry.io/1"); + vi.stubEnv("SENTRY_TRACES_SAMPLE_RATE", "0"); + const rateZero = await import("@/lib/observability/agent-monitoring"); + expect(rateZero.isSentryAgentMonitoringEnabled()).toBe(false); + + vi.resetModules(); + vi.stubEnv("SENTRY_DSN", "https://public@o0.ingest.sentry.io/1"); + vi.stubEnv("SENTRY_TRACES_SAMPLE_RATE", "0.1"); + const noClient = await import("@/lib/observability/agent-monitoring"); + // Sentry.init never runs in Vitest (or in the ingestion worker), so the + // client gate keeps agent monitoring off even with DSN + tracing set. + expect(noClient.isSentryAgentMonitoringEnabled()).toBe(false); + }); + + it("returns the client unwrapped and keeps scope calls inert when disabled", async () => { + vi.stubEnv("SENTRY_DSN", ""); + const { instrumentOpenAIClientForAgentMonitoring, setAgentConversationId } = + await import("@/lib/observability/agent-monitoring"); + + const client = { responses: { create: async () => ({}) } }; + expect(instrumentOpenAIClientForAgentMonitoring(client)).toBe(client); + expect(() => setAgentConversationId("11111111-2222-4333-8444-555555555555")).not.toThrow(); + }); +}); diff --git a/tests/error-tracking.test.ts b/tests/error-tracking.test.ts index 5c3ec714ea..95118c250e 100644 --- a/tests/error-tracking.test.ts +++ b/tests/error-tracking.test.ts @@ -139,6 +139,83 @@ describe("privacySafeTransactionEvent", () => { expect(event.spans?.[0]?.data).not.toHaveProperty("db.url"); expect(event.contexts?.trace?.data).toBeUndefined(); }); + + it("keeps gen_ai agent-monitoring metadata and strips prompts, messages, outputs, and tool payloads", () => { + const event = privacySafeTransactionEvent({ + type: "transaction", + event_id: "txn-2", + transaction: "/api/answer", + contexts: { trace: { trace_id: "trace-1", span_id: "span-root", op: "http.server" } }, + spans: [ + { + span_id: "span-ai", + trace_id: "trace-1", + op: "gen_ai.chat", + origin: "auto.ai.openai", + // Free-form SDK span names are never trusted; the description is + // rebuilt from allowlisted attributes only. + description: "chat gpt-5.2 for Jane Doe", + start_timestamp: 1, + timestamp: 2.5, + data: { + "gen_ai.system": "openai", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "gpt-5.2", + "gen_ai.response.model": "gpt-5.2", + "gen_ai.response.id": "resp_abc", + "gen_ai.conversation.id": "3e4f6a52-interaction-uuid", + "gen_ai.request.stream": false, + "gen_ai.usage.input_tokens": 1200, + "gen_ai.usage.output_tokens": 300, + "gen_ai.usage.total_tokens": 1500, + "gen_ai.request.messages": '[{"role":"user","content":"Jane Doe MRN 123456 lithium level"}]', + "gen_ai.input.messages": '[{"role":"user","content":"Jane Doe suicidal ideation"}]', + "gen_ai.output.messages": '[{"role":"assistant","content":"clinical output about Jane"}]', + "gen_ai.response.text": "clinical output about Jane Doe", + "gen_ai.system_instructions": "clinical synthesis prompt", + "gen_ai.prompt": "Jane Doe MRN 123456", + "gen_ai.embeddings.input": "Jane Doe query text", + "gen_ai.request.available_tools": '[{"name":"tool","description":"..."}]', + "gen_ai.response.tool_calls": '[{"name":"tool","arguments":"Jane"}]', + "gen_ai.tool.input": "Jane Doe", + "gen_ai.tool.output": "clinical output", + }, + }, + ], + }); + + expect(JSON.stringify(event)).not.toMatch(/Jane|123456|suicidal|clinical output|synthesis prompt|query text/); + expect(event.spans?.[0]).toMatchObject({ + op: "gen_ai.chat", + origin: "auto.ai.openai", + description: "chat gpt-5.2", + data: { + "gen_ai.system": "openai", + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "gpt-5.2", + "gen_ai.response.id": "resp_abc", + "gen_ai.conversation.id": "3e4f6a52-interaction-uuid", + "gen_ai.usage.input_tokens": 1200, + "gen_ai.usage.output_tokens": 300, + "gen_ai.usage.total_tokens": 1500, + }, + }); + for (const payloadKey of [ + "gen_ai.request.messages", + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.response.text", + "gen_ai.system_instructions", + "gen_ai.prompt", + "gen_ai.embeddings.input", + "gen_ai.request.available_tools", + "gen_ai.response.tool_calls", + "gen_ai.tool.input", + "gen_ai.tool.output", + ]) { + expect(event.spans?.[0]?.data).not.toHaveProperty(payloadKey); + } + }); }); describe("isSentryDbTracingEnabled", () => {