Skip to content

test(cloudflare): Proof that AI providers work OOTB on Cloudflare#22313

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JPeer264 merged 1 commit into from
Jul 16, 2026
Merged

test(cloudflare): Proof that AI providers work OOTB on Cloudflare #22313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion dev-packages/cloudflare-integration-tests/package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,12 +15,17 @@
},
"dependencies": {
"ai": "^6.0.0",
"@anthropic-ai/sdk": "0.63.0",
"@google/genai": "^1.20.0",
"@langchain/core": "^0.3.80",
"@langchain/langgraph": "^1.0.1",
"@langchain/openai": "^0.5.0",
"@prisma/adapter-d1": "6.15.0",
"@prisma/client": "6.15.0",
"@sentry/cloudflare": "10.66.0",
"@sentry/hono": "10.66.0",
"hono": "^4.12.25"
"hono": "^4.12.25",
"openai": "5.18.1"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20260426.0",
Expand Down
25 changes: 18 additions & 7 deletions dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,27 @@
import Anthropic from '@anthropic-ai/sdk';
import * as Sentry from '@sentry/cloudflare';
import type { AnthropicAiClient } from '@sentry/core';
import { MockAnthropic } from './mocks';

interface Env {
SENTRY_DSN: string;
}

const mockClient = new MockAnthropic({
apiKey: 'mock-api-key',
});
// Return a canned response so the `@anthropic-ai/sdk` runs on workerd without hitting the network.
const mockFetch: typeof fetch = async () =>
new Response(
JSON.stringify({
id: 'msg_mock123',
type: 'message',
role: 'assistant',
model: 'claude-3-haiku-20240307',
content: [{ type: 'text', text: 'Hello from Anthropic!' }],
stop_reason: 'end_turn',
stop_sequence: null,
usage: { input_tokens: 10, output_tokens: 15 },
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);

const client: AnthropicAiClient = Sentry.instrumentAnthropicAiClient(mockClient);
const client= Sentry.instrumentAnthropicAiClient(new Anthropic({ apiKey: 'mock-api-key', fetch: mockFetch }));

export default Sentry.withSentry(
(env: Env) => ({
Expand All@@ -20,7 +31,7 @@ export default Sentry.withSentry(
}),
{
async fetch(_request, _env, _ctx) {
const response = await client.messages?.create({
const response = await client.messages.create({
model: 'claude-3-haiku-20240307',
messages: [{ role: 'user', content: 'What is the capital of France?' }],
temperature: 0.7,
Expand Down
68 changes: 0 additions & 68 deletions dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/mocks.ts
View file
Open in desktop

This file was deleted.

56 changes: 29 additions & 27 deletions dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,57 @@
import { expect, it } from 'vitest';
import {
GEN_AI_OPERATION_NAME_ATTRIBUTE,
GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE,
GEN_AI_REQUEST_MODEL_ATTRIBUTE,
GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE,
GEN_AI_RESPONSE_ID_ATTRIBUTE,
GEN_AI_RESPONSE_MODEL_ATTRIBUTE,
GEN_AI_SYSTEM_ATTRIBUTE,
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
} from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
import { createRunner } from '../../../runner';

// These tests are not exhaustive because the instrumentation is
// already tested in the node integration tests and we merely
// want to test that the instrumentation does not break in our
// cloudflare SDK.
// This test runs the `@anthropic-ai/sdk` on the Workers runtime (with a
// canned fetch) to verify the instrumentation works end-to-end on Cloudflare,
// not just against a hand-written mock client.

it('traces a basic message creation request', async ({ signal }) => {
it('traces a basic message creation request with the anthropic SDK', async ({ signal }) => {
const runner = createRunner(__dirname)
.ignore('event')
.expect(envelope => {
// Transaction item (first item in envelope)
const transactionEvent = envelope[1]?.[0]?.[1] as any;
expect(transactionEvent.transaction).toBe('GET /');

// Span container item (second item in same envelope)
const container = envelope[1]?.[1]?.[1] as any;
expect(container).toBeDefined();

expect(container.items).toHaveLength(1);
const [firstSpan] = container.items;

// [0] chat claude-3-haiku-20240307
expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307');
expect(firstSpan!.status).toBe('ok');
expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' });
expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' });
expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.anthropic' });
expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'anthropic' });
expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({
type: 'string',
value: 'claude-3-haiku-20240307',
});
expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 });
expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({
type: 'string',
value: 'claude-3-haiku-20240307',
expect(container.items[0]).toEqual({
trace_id: expect.any(String),
span_id: expect.any(String),
parent_span_id: expect.any(String),
name: 'chat claude-3-haiku-20240307',
start_timestamp: expect.any(Number),
end_timestamp: expect.any(Number),
status: 'ok',
is_segment: false,
attributes: {
'sentry.origin': { value: 'auto.ai.anthropic', type: 'string' },
'sentry.op': { value: 'gen_ai.chat', type: 'string' },
[GEN_AI_SYSTEM_ATTRIBUTE]: { value: 'anthropic', type: 'string' },
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: { value: 'chat', type: 'string' },
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: { value: 'claude-3-haiku-20240307', type: 'string' },
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: { value: 0.7, type: 'double' },
[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: { value: 100, type: 'integer' },
[GEN_AI_RESPONSE_ID_ATTRIBUTE]: { value: 'msg_mock123', type: 'string' },
[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: { value: 'claude-3-haiku-20240307', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: { value: 10, type: 'integer' },
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: { value: 15, type: 'integer' },
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: { value: 25, type: 'integer' },
},

@cursorcursorBotJul 16, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Broken exact span assertions

High Severity

The new exact toEqual checks on streamed AI spans only list gen-ai attributes. Streamed spans also get common fields from applyCommonSpanAttributes (sentry.trace.lifecycle, sentry.segment.*, sentry.sdk.*, sentry.release, sentry.environment), so these assertions fail. The langchain suite has the same issue.

Additional Locations (2)
Fix in CursorFix in Web

Triggered by project rule: PR Review Guidelines for Cursor Bot

Reviewed by Cursor Bugbot for commit 10d6264. Configure here.

});
expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'msg_mock123' });
expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 });
expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 });
})
.start(signal);
await runner.makeRequest('get', '/');
Expand Down
2 changes: 1 addition & 1 deletion dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/wrangler.jsonc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
{
"name": "worker-name",
"name": "anthropic-ai-worker",
"compatibility_date": "2025-06-17",
"main": "index.ts",
"compatibility_flags": ["nodejs_als"],
Expand Down
51 changes: 39 additions & 12 deletions dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/index.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,45 @@
import { GoogleGenAI } from '@google/genai';
import * as Sentry from '@sentry/cloudflare';
import type { GoogleGenAIClient } from '@sentry/core';
import { MockGoogleGenAI } from './mocks';

interface Env {
SENTRY_DSN: string;
}

const mockClient = new MockGoogleGenAI({
apiKey: 'mock-api-key',
});
// `@google/genai` has no per-client fetch option — it calls the global `fetch`
// directly. Override it (gated to the Gemini host) so the SDK runs on
// workerd without hitting the network, while Sentry envelope delivery still
// uses the original fetch.
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;

const client: GoogleGenAIClient = Sentry.instrumentGoogleGenAIClient(mockClient);
if (!url.includes('generativelanguage.googleapis.com')) {
return originalFetch(input, init);
}

if (url.includes(':embedContent') || url.includes(':batchEmbedContents')) {
return new Response(JSON.stringify({ embeddings: [{ values: [0.1, 0.2, 0.3, 0.4, 0.5] }] }), {
status: 200,
headers: { 'content-type': 'application/json' },
});
}

return new Response(
JSON.stringify({
candidates: [
{
content: { parts: [{ text: 'Hello from Google GenAI!' }], role: 'model' },
finishReason: 'stop',
index: 0,
},
],
usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 12, totalTokenCount: 20 },
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}) as typeof fetch;

const client = Sentry.instrumentGoogleGenAIClient(new GoogleGenAI({ apiKey: 'mock-api-key' }));

export default Sentry.withSentry(
(env: Env) => ({
Expand All@@ -36,12 +65,10 @@
],
});

const chatResponse = await chat.sendMessage({
message: 'Tell me a joke',
});
await chat.sendMessage({ message: 'Tell me a joke' });

// Test 2: models.generateContent
const modelResponse = await client.models.generateContent({
await client.models.generateContent({
model: 'gemini-1.5-flash',
config: {
temperature: 0.7,
Expand All@@ -57,12 +84,12 @@
});

// Test 3: models.embedContent
const embedResponse = await client.models.embedContent({
await client.models.embedContent({
model: 'text-embedding-004',
contents: 'Hello world',
});

return new Response(JSON.stringify({ chatResponse, modelResponse, embedResponse }));
return new Response(JSON.stringify({ success: true }));
},
},
);
Loading
Loading