fix(anthropic): capture time_to_first_token when using .text_stream

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_async_text_stream.yaml

@@ -0,0 +1,140 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":1024,"messages":[{"role":"user","content":"what is 1+1?,
+ just return the number"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "145"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - AsyncAnthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - async:asyncio
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01Hwq7nXVJreMkdBttJ9H46B","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"2"}
+ }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c89e6a908c63-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:49 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1499000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:49Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:48Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9499000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:48Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5r8E8kny9TZdxz4Q6v
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_streaming_sync_text_stream.yaml

@@ -0,0 +1,139 @@
+interactions:
+ - request:
+ body:
+ '{"max_tokens":300,"messages":[{"role":"user","content":"what is 2+2? (just
+ the number)"}],"model":"claude-3-haiku-20240307","stream":true}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ anthropic-version:
+ - "2023-06-01"
+ connection:
+ - keep-alive
+ content-length:
+ - "138"
+ content-type:
+ - application/json
+ host:
+ - api.anthropic.com
+ user-agent:
+ - Anthropic/Python 0.52.1
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - "false"
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 0.52.1
+ x-stainless-read-timeout:
+ - "600"
+ x-stainless-retry-count:
+ - "0"
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.13.3
+ x-stainless-stream-helper:
+ - messages
+ x-stainless-timeout:
+ - NOT_GIVEN
+ method: POST
+ uri: https://api.anthropic.com/v1/messages
+ response:
+ body:
+ string: 'event: message_start
+
+ data: {"type":"message_start","message":{"id":"msg_01NR8LiXqHhf3PCEc3k3m3hJ","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":4,"service_tier":"standard"}} }
+
+
+ event: content_block_start
+
+ data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
+
+
+ event: ping
+
+ data: {"type": "ping"}
+
+
+ event: content_block_delta
+
+ data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4"} }
+
+
+ event: content_block_stop
+
+ data: {"type":"content_block_stop","index":0 }
+
+
+ event: message_delta
+
+ data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5} }
+
+
+ event: message_stop
+
+ data: {"type":"message_stop" }
+
+
+ '
+ headers:
+ CF-RAY:
+ - 9472c8a8e81323dd-EWR
+ Cache-Control:
+ - no-cache
+ Connection:
+ - keep-alive
+ Content-Type:
+ - text/event-stream; charset=utf-8
+ Date:
+ - Thu, 29 May 2025 03:07:50 GMT
+ Server:
+ - cloudflare
+ Transfer-Encoding:
+ - chunked
+ X-Robots-Tag:
+ - none
+ anthropic-organization-id:
+ - 27796668-7351-40ac-acc4-024aee8995a5
+ anthropic-ratelimit-input-tokens-limit:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-remaining:
+ - "8000000"
+ anthropic-ratelimit-input-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-output-tokens-limit:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-remaining:
+ - "1500000"
+ anthropic-ratelimit-output-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-requests-limit:
+ - "10000"
+ anthropic-ratelimit-requests-remaining:
+ - "9999"
+ anthropic-ratelimit-requests-reset:
+ - "2025-05-29T03:07:50Z"
+ anthropic-ratelimit-tokens-limit:
+ - "9500000"
+ anthropic-ratelimit-tokens-remaining:
+ - "9500000"
+ anthropic-ratelimit-tokens-reset:
+ - "2025-05-29T03:07:50Z"
+ cf-cache-status:
+ - DYNAMIC
+ request-id:
+ - req_011CPb5rFK3186WDZPzKJEuC
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ via:
+ - 1.1 google
+ status:
+ code: 200
+ message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -582,6 +582,87 @@ def test_anthropic_messages_streaming_sync(memory_logger):
  assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
 
 
+@pytest.mark.vcr
+def test_anthropic_messages_streaming_sync_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_client())
+ msg_in = {"role": "user", "content": "what is 2+2? (just the number)"}
+
+ start = time.time()
+ with client.messages.stream(model=MODEL, max_tokens=300, messages=[msg_in]) as stream:
+ texts = list(stream.text_stream)
+ end = time.time()
+ msg_out = stream.get_final_message()
+ usage = msg_out.usage
+
+ text = "".join(texts)
+ assert "4" in text
+ assert "4" in msg_out.content[0].text
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "2+2" in str(log["input"])
+ assert "4" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 300
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
+@pytest.mark.vcr
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_async_text_stream(memory_logger):
+ """time_to_first_token is captured when iterating via .text_stream on async streams (BT-4702)."""
+ assert not memory_logger.pop()
+
+ client = wrap_anthropic(_get_async_client())
+ msgs_in = [{"role": "user", "content": "what is 1+1?, just return the number"}]
+
+ start = time.time()
+ async with client.messages.stream(max_tokens=1024, messages=msgs_in, model=MODEL) as stream:
+ texts = [t async for t in stream.text_stream]
+ msg_out = await stream.get_final_message()
+ usage = msg_out.usage
+ end = time.time()
+
+ text = "".join(texts)
+ assert "2" in text
+ assert msg_out.content[0].text == "2"
+
+ logs = memory_logger.pop()
+ assert len(logs) == 1
+ log = logs[0]
+ assert "user" in str(log["input"])
+ assert "1+1" in str(log["input"])
+ assert "2" in str(log["output"])
+ assert log["project_id"] == PROJECT_NAME
+ assert log["span_attributes"]["type"] == "llm"
+ assert log["metadata"]["model"] == MODEL
+ assert log["metadata"]["max_tokens"] == 1024
+ assert log["output"]["role"] == "assistant"
+ assert log["output"]["model"] == msg_out.model
+ assert log["output"]["stop_reason"] == msg_out.stop_reason
+ _assert_metrics_are_valid(log["metrics"], start, end)
+ assert log["metrics"]["prompt_tokens"] == usage.input_tokens
+ assert log["metrics"]["completion_tokens"] == usage.output_tokens
+ assert log["metrics"]["tokens"] == usage.input_tokens + usage.output_tokens
+ assert log["metrics"]["prompt_cached_tokens"] == usage.cache_read_input_tokens
+ assert log["metrics"]["prompt_cache_creation_tokens"] == usage.cache_creation_input_tokens
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_sync(memory_logger):
  assert not memory_logger.pop()

py/src/braintrust/integrations/anthropic/tracing.py

@@ -331,6 +331,26 @@ def __next__(self):
  self.__process_message(m)
  return m
 
+ @property
+ def text_stream(self):
+ if hasattr(self.__msg_stream, "__aiter__"):
+ return self.__async_text_stream()
+ return self.__sync_text_stream()
+
+ def __sync_text_stream(self):
+ for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
+ async def __async_text_stream(self):
+ async for event in self:
+ if getattr(event, "type", None) == "content_block_delta":
+ delta = getattr(event, "delta", None)
+ if getattr(delta, "type", None) == "text_delta":
+ yield getattr(delta, "text", "")
+
  def __process_message(self, m):
  if self.__time_to_first_token is None:
  self.__time_to_first_token = time.time() - self.__request_start_time