fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Praneeth Yenugutala (praneethy91) wants to merge 1 commit into from
Closed

fix: surface experiment-comparison2 fetch failures and retry transient 5xx #640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 38 additions & 5 deletions py/src/braintrust/logger.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
_state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
"""An HTTP adapter that automatically retries requests on connection exceptions.

This adapter extends requests' HTTPAdapter to add retry logic for common network-related
exceptions including connection errors, timeouts, and other HTTP errors. It implements
an exponential backoff strategy between retries to avoid overwhelming servers during
intermittent connectivity issues.
exceptions including connection errors, timeouts, and other HTTP errors, as well as
responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
It implements an exponential backoff strategy between retries to avoid overwhelming
servers during intermittent connectivity issues.

Attributes:
base_num_retries: Maximum number of retries before giving up and re-raising the exception.
base_num_retries: Maximum number of retries before giving up and re-raising the exception
(or returning the last error response).
backoff_factor: A multiplier used to determine the time to wait between retries.
The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
# downloading.
if not response.is_redirect and response.content:
pass
if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
# Unlike connection-level failures, a completed response with an error
# status doesn't raise -- retry it here so transient 5xx/429 responses
# get the same backoff treatment as network exceptions.
sleep_s = self.backoff_factor * (2**num_prev_retries)
print(
"Retrying request after HTTP",
response.status_code,
"response",
file=sys.stderr,
)
print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
time.sleep(sleep_s)
num_prev_retries += 1
continue
return response
except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
score_summary = {}
metric_summary = {}
comparison_experiment_name = None
scores_fetch_error = None
if summarize_scores:
# Get the comparison experiment
if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
},
)
except Exception as e:
scores_fetch_error = str(e)
_logger.warning(
f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
)
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
comparison_experiment_name=comparison_experiment_name,
scores=score_summary,
metrics=metric_summary,
scores_fetch_error=scores_fetch_error,
)

def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
"""Summary of the experiment's scores."""
metrics: dict[str, MetricSummary]
"""Summary of the experiment's metrics."""
scores_fetch_error: str | None = None
"""If set, fetching the score/metric summary from the server failed with this error, and
`scores`/`metrics` are empty as a result of that failure -- not because the experiment has
no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
treating an empty `scores` dict as "the experiment has no scores"."""

def __str__(self):
comparison_line = ""
if self.comparison_experiment_name:
comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
fetch_error_line = ""
if self.scores_fetch_error:
fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
return (
f"""\n=========================SUMMARY=========================\n{comparison_line}"""
f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
+ "\n".join([str(score) for score in self.scores.values()])
+ ("\n\n" if self.scores else "")
+ "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down
29 changes: 29 additions & 0 deletions py/src/braintrust/test_framework.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
)


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
"""A failure to fetch experiment-comparison2 must be distinguishable from a genuine
empty-scores result, not silently collapsed into the same `scores == {}` shape.

See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
"""
exp = init_test_exp("test-evaluator", "test-project")
mock_conn = MagicMock()

def get_json(path, args=None):
if path == "v1/experiment/base-exp-id":
return {"name": "base-exp"}
if path == "experiment-comparison2":
raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
raise AssertionError(f"Unexpected get_json call: {path}, {args}")

mock_conn.get_json.side_effect = get_json

with patch.object(exp.state, "api_conn", return_value=mock_conn):
summary = exp.summarize(comparison_experiment_id="base-exp-id")

assert summary.scores == {}
assert summary.metrics == {}
assert summary.scores_fetch_error is not None
assert "502 Bad Gateway" in summary.scores_fetch_error
assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down
99 changes: 99 additions & 0 deletions py/src/braintrust/test_http.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
"""HTTP handler that returns a transient error status for the first N requests.

Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
HTTP response rather than raising a connection-level exception -- this does not enter
the exception-handling retry path, only a status-code-aware one.
"""

request_count = 0
fail_count = 1
error_status = 502

def log_message(self, format, *args):
pass

def do_GET(self):
TransientErrorStatusHandler.request_count += 1

if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
self.send_response(TransientErrorStatusHandler.error_status)
self.send_header("Content-Type", "text/html")
self.end_headers()
self.wfile.write(b"<html>Bad Gateway</html>")
return

self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
"""Fixture that creates a server returning an error status for the first request."""
TransientErrorStatusHandler.request_count = 0
TransientErrorStatusHandler.fail_count = 1
TransientErrorStatusHandler.error_status = 502

server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
server.daemon_threads = True
port = server.server_address[1]

thread = threading.Thread(target=server.serve_forever)
thread.daemon = True
thread.start()

yield f"http://127.0.0.1:{port}"

server.shutdown()
server.server_close()


class TestRetryOnHttpErrorStatus:
"""Tests that the adapter retries completed responses with a transient error status.

Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
a request that completes with a non-2xx status returns a normal Response rather than
raising, so it must be retried by inspecting the status code, not just by catching
connection-level exceptions.
"""

def test_adapter_retries_on_502(self, transient_error_server):
adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
session = requests.Session()
session.mount("http://", adapter)

resp = session.get(f"{transient_error_server}/experiment-comparison2")

assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
assert TransientErrorStatusHandler.request_count == 2

def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
TransientErrorStatusHandler.fail_count = 100 # always fails

adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
session = requests.Session()
session.mount("http://", adapter)

resp = session.get(f"{transient_error_server}/experiment-comparison2")

# 1 initial attempt + 2 retries, then the caller gets the last error response back.
assert resp.status_code == 502
assert TransientErrorStatusHandler.request_count == 3

def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
TransientErrorStatusHandler.fail_count = 100
TransientErrorStatusHandler.error_status = 404

adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
session = requests.Session()
session.mount("http://", adapter)

resp = session.get(f"{transient_error_server}/experiment-comparison2")

assert resp.status_code == 404
assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
"""Tests for HTTPConnection timeout configuration."""

Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
fix: surface experiment-comparison2 fetch failures and retry transient 5xx#640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

Praneeth Yenugutala (praneethy91)
wants to merge
1
commit into

from

 Closed


fix: surface experiment-comparison2 fetch failures and retry transient 5xx
#640
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


43 changes: 38 additions & 5 deletions
py/src/braintrust/logger.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -728,16 +728,23 @@ def set_http_adapter(adapter: HTTPAdapter) -> None:
 _state._api_conn._reset()


#: HTTP status codes that indicate a transient failure worth retrying, rather than a
#: client error that will fail identically on every attempt.
RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})


class RetryRequestExceptionsAdapter(HTTPAdapter):
 """An HTTP adapter that automatically retries requests on connection exceptions.

 This adapter extends requests' HTTPAdapter to add retry logic for common network-related
 exceptions including connection errors, timeouts, and other HTTP errors. It implements
 an exponential backoff strategy between retries to avoid overwhelming servers during
 intermittent connectivity issues.
 exceptions including connection errors, timeouts, and other HTTP errors, as well as
 responses that complete with a transient HTTP error status (see `RETRYABLE_STATUS_CODES`).
 It implements an exponential backoff strategy between retries to avoid overwhelming
 servers during intermittent connectivity issues.

 Attributes:
 base_num_retries: Maximum number of retries before giving up and re-raising the exception.
 base_num_retries: Maximum number of retries before giving up and re-raising the exception
 (or returning the last error response).
 backoff_factor: A multiplier used to determine the time to wait between retries.
 The actual wait time is calculated as: backoff_factor * (2 ** retry_count).
 default_timeout_secs: Default timeout in seconds for requests that don't specify one.
Expand DownExpand Up@@ -770,6 +777,21 @@ def send(self, *args, **kwargs):
 # downloading.
 if not response.is_redirect and response.content:
 pass
 if response.status_code in RETRYABLE_STATUS_CODES and num_prev_retries < self.base_num_retries:
 # Unlike connection-level failures, a completed response with an error
 # status doesn't raise -- retry it here so transient 5xx/429 responses
 # get the same backoff treatment as network exceptions.
 sleep_s = self.backoff_factor * (2**num_prev_retries)
 print(
 "Retrying request after HTTP",
 response.status_code,
 "response",
 file=sys.stderr,
 )
 print("Sleeping for", sleep_s, "seconds", file=sys.stderr)
 time.sleep(sleep_s)
 num_prev_retries += 1
 continue
 return response
 except (urllib3.exceptions.HTTPError, requests.exceptions.RequestException) as e:
 if num_prev_retries < self.base_num_retries:
Expand DownExpand Up@@ -4362,6 +4384,7 @@ def summarize(
 score_summary = {}
 metric_summary = {}
 comparison_experiment_name = None
 scores_fetch_error = None
 if summarize_scores:
 # Get the comparison experiment
 if comparison_experiment_id is None:
Expand All@@ -4385,6 +4408,7 @@ def summarize(
 },
 )
 except Exception as e:
 scores_fetch_error = str(e)
 _logger.warning(
 f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again."
 )
Expand DownExpand Up@@ -4413,6 +4437,7 @@ def summarize(
 comparison_experiment_name=comparison_experiment_name,
 scores=score_summary,
 metrics=metric_summary,
 scores_fetch_error=scores_fetch_error,
 )

 def export(self) -> str:
Expand DownExpand Up@@ -6002,13 +6027,21 @@ class ExperimentSummary(SerializableDataClass):
 """Summary of the experiment's scores."""
 metrics: dict[str, MetricSummary]
 """Summary of the experiment's metrics."""
 scores_fetch_error: str | None = None
 """If set, fetching the score/metric summary from the server failed with this error, and
 `scores`/`metrics` are empty as a result of that failure -- not because the experiment has
 no scores. Callers that gate automation (e.g. CI) on `scores` should check this field before
 treating an empty `scores` dict as "the experiment has no scores"."""

 def __str__(self):
 comparison_line = ""
 if self.comparison_experiment_name:
 comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n"""
 fetch_error_line = ""
 if self.scores_fetch_error:
 fetch_error_line = f"WARNING: failed to fetch scores and metrics ({self.scores_fetch_error}). The summary below does not reflect the experiment's actual scores.\n\n"
 return (
 f"""\n=========================SUMMARY=========================\n{comparison_line}"""
 f"""\n=========================SUMMARY=========================\n{fetch_error_line}{comparison_line}"""
 + "\n".join([str(score) for score in self.scores.values()])
 + ("\n\n" if self.scores else "")
 + "\n".join([str(metric) for metric in self.metrics.values()])
Expand Down

29 changes: 29 additions & 0 deletions
py/src/braintrust/test_framework.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@

import pytest
from braintrust.logger import BraintrustState
from braintrust.util import AugmentedHTTPError

from .framework import (
 Eval,
Expand DownExpand Up@@ -164,6 +165,34 @@ def get_json(path, args=None):
 )


def test_experiment_summarize_surfaces_scores_fetch_error(with_memory_logger, with_simulate_login):
 """A failure to fetch experiment-comparison2 must be distinguishable from a genuine
 empty-scores result, not silently collapsed into the same `scores == {}` shape.

 See https://github.com/braintrustdata/braintrust-sdk-python/issues/639.
 """
 exp = init_test_exp("test-evaluator", "test-project")
 mock_conn = MagicMock()

 def get_json(path, args=None):
 if path == "v1/experiment/base-exp-id":
 return {"name": "base-exp"}
 if path == "experiment-comparison2":
 raise AugmentedHTTPError("<html>502 Bad Gateway</html>")
 raise AssertionError(f"Unexpected get_json call: {path}, {args}")

 mock_conn.get_json.side_effect = get_json

 with patch.object(exp.state, "api_conn", return_value=mock_conn):
 summary = exp.summarize(comparison_experiment_id="base-exp-id")

 assert summary.scores == {}
 assert summary.metrics == {}
 assert summary.scores_fetch_error is not None
 assert "502 Bad Gateway" in summary.scores_fetch_error
 assert "WARNING" in str(summary)


@pytest.mark.asyncio
@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
async def test_run_evaluator_exposes_validated_parameter_values_to_hooks():
Expand Down

99 changes: 99 additions & 0 deletions
py/src/braintrust/test_http.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -174,6 +174,105 @@ def test_adapter_resets_pool_on_timeout(self, hanging_server):
 assert HangingConnectionHandler.request_count >= 2


class TransientErrorStatusHandler(http.server.BaseHTTPRequestHandler):
 """HTTP handler that returns a transient error status for the first N requests.

 Simulates a load balancer/proxy hiccup (502/503/504) that completes with a normal
 HTTP response rather than raising a connection-level exception -- this does not enter
 the exception-handling retry path, only a status-code-aware one.
 """

 request_count = 0
 fail_count = 1
 error_status = 502

 def log_message(self, format, *args):
 pass

 def do_GET(self):
 TransientErrorStatusHandler.request_count += 1

 if TransientErrorStatusHandler.request_count <= TransientErrorStatusHandler.fail_count:
 self.send_response(TransientErrorStatusHandler.error_status)
 self.send_header("Content-Type", "text/html")
 self.end_headers()
 self.wfile.write(b"<html>Bad Gateway</html>")
 return

 self.send_response(200)
 self.send_header("Content-Type", "application/json")
 self.end_headers()
 self.wfile.write(b'{"status": "ok"}')


@pytest.fixture
def transient_error_server():
 """Fixture that creates a server returning an error status for the first request."""
 TransientErrorStatusHandler.request_count = 0
 TransientErrorStatusHandler.fail_count = 1
 TransientErrorStatusHandler.error_status = 502

 server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), TransientErrorStatusHandler)
 server.daemon_threads = True
 port = server.server_address[1]

 thread = threading.Thread(target=server.serve_forever)
 thread.daemon = True
 thread.start()

 yield f"http://127.0.0.1:{port}"

 server.shutdown()
 server.server_close()


class TestRetryOnHttpErrorStatus:
 """Tests that the adapter retries completed responses with a transient error status.

 Regression coverage for https://github.com/braintrustdata/braintrust-sdk-python/issues/639:
 a request that completes with a non-2xx status returns a normal Response rather than
 raising, so it must be retried by inspecting the status code, not just by catching
 connection-level exceptions.
 """

 def test_adapter_retries_on_502(self, transient_error_server):
 adapter = RetryRequestExceptionsAdapter(base_num_retries=3, backoff_factor=0.05)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 200
 assert resp.json() == {"status": "ok"}
 assert TransientErrorStatusHandler.request_count == 2

 def test_adapter_gives_up_after_base_num_retries(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100 # always fails

 adapter = RetryRequestExceptionsAdapter(base_num_retries=2, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 # 1 initial attempt + 2 retries, then the caller gets the last error response back.
 assert resp.status_code == 502
 assert TransientErrorStatusHandler.request_count == 3

 def test_adapter_does_not_retry_non_retryable_status(self, transient_error_server):
 TransientErrorStatusHandler.fail_count = 100
 TransientErrorStatusHandler.error_status = 404

 adapter = RetryRequestExceptionsAdapter(base_num_retries=5, backoff_factor=0.01)
 session = requests.Session()
 session.mount("http://", adapter)

 resp = session.get(f"{transient_error_server}/experiment-comparison2")

 assert resp.status_code == 404
 assert TransientErrorStatusHandler.request_count == 1


class TestHTTPConnection:
 """Tests for HTTPConnection timeout configuration."""

Expand Down