diff --git a/docs/error-tracking.md b/docs/error-tracking.md index 41349bccba..1bd417d768 100644 --- a/docs/error-tracking.md +++ b/docs/error-tracking.md @@ -31,12 +31,27 @@ Privacy constraints for agent monitoring: - `recordInputs` / `recordOutputs` are **false** on the wrap and `dataCollection.genAI` is `{ inputs: false, outputs: false }` in both runtime configs — prompts, clinical queries, source evidence, generated answers, and embedding inputs are never recorded. - `privacySafeTransactionEvent` allowlists only gen_ai metadata attributes (system, operation name, request/response model, response id, finish reasons, token usage, conversation id) and rebuilds gen_ai span descriptions as ` ` from those attributes. Message, prompt, tool-payload, and embedding-input attributes are stripped on export even if a future SDK version records them. -- Each answer request (`/api/answer` and `/api/answer/stream`, summaries included) calls `Sentry.setConversationId()` — the request's synthetic UUID — so the embedding/generation calls of one request group into one conversation without carrying any query text. +- Each answer / stream / document-summarize request calls `Sentry.setConversationId()` **before** OpenAI work — the request's synthetic UUID — so the embedding/generation calls of one request group into one conversation without carrying any query text. - User identification (`Sentry.setUser`) is deliberately **not** wired: the committed privacy boundary strips `user` from every outgoing event (see the tests), and linking clinical-query telemetry to an identity would need its own governance review first. - `responses.parse` (schema-parsed generation) is not in the SDK's instrumentation registry and emits no gen_ai span; `responses.create` and `embeddings.create` are covered. Rollback matches tracing: set `SENTRY_TRACES_SAMPLE_RATE=0` (agent spans stop; error capture stays) or remove `SENTRY_DSN`. Raising the sample rate above the 0.1 default captures a larger share of answer requests in the agents view and is an operator decision. +### Sentry “Agent Monitoring” wizard mapping + +The product wizard’s copy is generic. Map it to this repo as follows — do **not** paste the wizard’s sample code: + +| Wizard step | This repo | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Install `@sentry/nextjs` ≥ 10.67 | repository dependency `@sentry/nextjs@^10.69.0` (lockfile resolves `10.69.0`); `10.67` is the minimum supported version; do not switch to pnpm | +| `Sentry.init` + tracing | `src/sentry.server.config.ts` / `src/sentry.edge.config.ts`; DSN from `SENTRY_DSN` only (never hardcode); default `tracesSampleRate` 0.1 via `SENTRY_TRACES_SAMPLE_RATE` | +| `dataCollection.genAI` | Explicitly `{ inputs: false, outputs: false }` | +| `instrumentOpenAiClient` | `instrumentOpenAIClientForAgentMonitoring` in `createOpenAIClient()` — `recordInputs`/`recordOutputs` **false** | +| `setConversationId` | `setAgentConversationId(interactionId)` on `/api/answer`, `/api/answer/stream`, and `/api/documents/[id]/summarize` | +| `setUser` (optional) | **Rejected** — scrubbers strip `user`; do not wire | + +Leaving wizard checkboxes for “record inputs/outputs” or “identify users” unchecked is expected and correct for this clinical app. + ### Structured logs (Sentry Logs) Server/edge `enableLogs` turns on when `SENTRY_DSN` is set (disable with `SENTRY_ENABLE_LOGS=false`). There is still no browser logging path and no `consoleLoggingIntegration` — console output is too easy to leak clinical text. diff --git a/src/app/api/documents/[id]/summarize/route.ts b/src/app/api/documents/[id]/summarize/route.ts index e2e0fc7564..25e3f90434 100644 --- a/src/app/api/documents/[id]/summarize/route.ts +++ b/src/app/api/documents/[id]/summarize/route.ts @@ -10,6 +10,7 @@ import { logAnswerDiagnostics } from "@/lib/answer-telemetry"; import { answerFeedbackMetadata } from "@/lib/answer-feedback-token"; import { jsonError } from "@/lib/http"; import { consumeApiRateLimit, rateLimitJsonResponse } from "@/lib/api-rate-limit"; +import { setAgentConversationId } from "@/lib/observability/agent-monitoring"; import { createAdminClient } from "@/lib/supabase/admin"; import { AuthenticationError, requireAuthenticatedUser, unauthorizedResponse } from "@/lib/supabase/auth"; import { parseRouteParams } from "@/lib/validation/params"; @@ -39,6 +40,10 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: const rateLimit = await consumeApiRateLimit({ supabase, ownerId: user.id, bucket: "document_summarize" }); if (rateLimit.limited) return rateLimitJsonResponse("Too many document summary requests. Retry shortly.", rateLimit); + // Group this request's LLM calls into one Sentry agent-monitoring conversation + // before any OpenAI work starts. Synthetic UUID only — never document/query text. + const interactionId = randomUUID(); + setAgentConversationId(interactionId); const answer = await summarizeDocument(id, user.id, { signal: request.signal }); const governedResponse = buildGovernedAnswerClientResponse(answer); logAnswerDiagnostics({ @@ -47,7 +52,6 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: ownerId: user.id, answer: governedResponse.telemetryAnswer, }); - const interactionId = randomUUID(); return NextResponse.json({ ...governedResponse.payload, ...answerFeedbackMetadata(interactionId, governedResponse.payload.answer), diff --git a/tests/summarize-agent-conversation.test.ts b/tests/summarize-agent-conversation.test.ts new file mode 100644 index 0000000000..98ff029716 --- /dev/null +++ b/tests/summarize-agent-conversation.test.ts @@ -0,0 +1,95 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const setAgentConversationId = vi.fn(); +const summarizeDocument = vi.fn(async () => ({ + answer: "Summary body.", + grounded: true, + confidence: "high", + citations: [], + smartPanel: { query: "summary" }, + smartApiPlan: { displayMode: "direct" }, +})); + +vi.mock("@/lib/observability/agent-monitoring", () => ({ + setAgentConversationId, +})); + +vi.mock("@/lib/demo-data", () => ({ + isDemoMode: () => false, + getDemoDocument: () => null, + demoSummary: () => ({}), +})); + +vi.mock("@/lib/env", () => ({ + isDemoMode: () => false, +})); + +vi.mock("@/lib/rag/rag", () => ({ + summarizeDocument, +})); + +vi.mock("@/lib/answer-response", () => ({ + buildGovernedAnswerClientResponse: (answer: { answer: string }) => ({ + payload: { answer: answer.answer, grounded: true }, + telemetryAnswer: answer, + }), + buildGovernedDemoAnswerClientResponse: (payload: unknown) => payload, +})); + +vi.mock("@/lib/answer-telemetry", () => ({ + logAnswerDiagnostics: vi.fn(), +})); + +vi.mock("@/lib/answer-feedback-token", () => ({ + answerFeedbackMetadata: (interactionId: string) => ({ interactionId }), +})); + +vi.mock("@/lib/api-rate-limit", () => ({ + consumeApiRateLimit: vi.fn(async () => ({ + limited: false, + limit: 12, + remaining: 11, + retryAfterSeconds: 60, + resetAt: new Date().toISOString(), + })), + rateLimitJsonResponse: vi.fn(), +})); + +vi.mock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({}), +})); + +vi.mock("@/lib/supabase/auth", () => ({ + AuthenticationError: class AuthenticationError extends Error {}, + requireAuthenticatedUser: vi.fn(async () => ({ id: "11111111-1111-4111-8111-111111111111" })), + unauthorizedResponse: () => new Response(null, { status: 401 }), +})); + +describe("document summarize Sentry conversation id", () => { + afterEach(() => { + setAgentConversationId.mockClear(); + summarizeDocument.mockClear(); + vi.resetModules(); + }); + + it("sets the agent conversation id before any LLM summarize work", async () => { + const { POST } = await import("../src/app/api/documents/[id]/summarize/route"); + const documentId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + + const response = await POST( + new Request(`http://localhost/api/documents/${documentId}/summarize`, { method: "POST" }), + { + params: Promise.resolve({ id: documentId }), + }, + ); + + expect(response.status).toBe(200); + expect(setAgentConversationId).toHaveBeenCalledTimes(1); + expect(setAgentConversationId.mock.invocationCallOrder[0]).toBeLessThan( + summarizeDocument.mock.invocationCallOrder[0]!, + ); + const body = (await response.json()) as { interactionId?: string }; + expect(body.interactionId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); + expect(setAgentConversationId).toHaveBeenCalledWith(body.interactionId); + }); +});