test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lucasjia-aws merged 3 commits into from
Jul 23, 2026
Merged

test(integ): let exhausted IAM throttling fail instead of skipping #6094

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ccbdf95
test(integ): let exhausted IAM throttling fail instead of skipping
jam-jee Jul 23, 2026
0965d40
Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip
jam-jee Jul 23, 2026
79120aa
test(integ): remove throttle skip hook from train & mlops; fix serve …
jam-jee Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 5 additions & 39 deletions sagemaker-mlops/tests/integ/conftest.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
os.environ[key] = value


def _is_simulate_policy_throttle(exc):
"""Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
from botocore.exceptions import ClientError

if not isinstance(exc, ClientError):
return False
error_code = exc.response.get("Error", {}).get("Code", "")
operation = getattr(exc, "operation_name", "") or ""
return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
survives the adaptive retries under heavy concurrency. Applies to setup and
call phases, since role resolution frequently happens in fixture setup."""
outcome = yield
report = outcome.get_result()

if report.when not in ("setup", "call") or not report.failed:
return

exc = call.excinfo.value if call.excinfo else None
# Walk the exception chain so a wrapped ClientError is still detected.
while exc is not None:
if _is_simulate_policy_throttle(exc):
report.outcome = "skipped"
report.wasxfail = (
"iam:SimulatePrincipalPolicy throttled under concurrent test load"
)
break
exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down
47 changes: 4 additions & 43 deletions sagemaker-serve/tests/integ/conftest.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
retries, convert the failure into a skip so a transient rate limit never reds
the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
os.environ.pop(key, None)
else:
os.environ[key] = value


def _is_simulate_policy_throttle(exc):
"""Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
from botocore.exceptions import ClientError

if not isinstance(exc, ClientError):
return False
error_code = exc.response.get("Error", {}).get("Code", "")
operation = getattr(exc, "operation_name", "") or ""
return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
survives the adaptive retries under heavy concurrency. Applies to setup and
call phases, since role resolution frequently happens in fixture setup."""
outcome = yield
report = outcome.get_result()

if report.when not in ("setup", "call") or not report.failed:
return

exc = call.excinfo.value if call.excinfo else None
# Walk the exception chain so a wrapped ClientError is still detected.
while exc is not None:
if _is_simulate_policy_throttle(exc):
report.outcome = "skipped"
report.wasxfail = (
"iam:SimulatePrincipalPolicy throttled under concurrent test load"
)
break
exc = exc.__cause__ or exc.__context__
48 changes: 4 additions & 44 deletions sagemaker-train/tests/integ/conftest.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
its own). ``adaptive`` mode adds client-side rate limiting so bursts of
``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
retries, convert the failure into a skip so a transient rate limit never reds
the build. Genuine failures (and throttling on any other operation) are
untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
os.environ.pop(key, None)
else:
os.environ[key] = value


def _is_simulate_policy_throttle(exc):
"""Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
from botocore.exceptions import ClientError

if not isinstance(exc, ClientError):
return False
error_code = exc.response.get("Error", {}).get("Code", "")
operation = getattr(exc, "operation_name", "") or ""
return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
survives the adaptive retries under heavy concurrency. Applies to setup and
call phases, since role resolution frequently happens in fixture setup."""
outcome = yield
report = outcome.get_result()

if report.when not in ("setup", "call") or not report.failed:
return

exc = call.excinfo.value if call.excinfo else None
# Walk the exception chain so a wrapped ClientError is still detected.
while exc is not None:
if _is_simulate_policy_throttle(exc):
report.outcome = "skipped"
report.wasxfail = (
"iam:SimulatePrincipalPolicy throttled under concurrent test load"
)
break
exc = exc.__cause__ or exc.__context__
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
test(integ): let exhausted IAM throttling fail instead of skipping#6094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

lucasjia-aws
merged 3 commits into

from

Jul 23, 2026
 Merged


test(integ): let exhausted IAM throttling fail instead of skipping
#6094
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
ccbdf95

test(integ): let exhausted IAM throttling fail instead of skipping


jam-jee Jul 23, 2026
0965d40

Merge branch 'master' into fix/integ-iam-throttle-fail-not-skip


jam-jee Jul 23, 2026
79120aa

test(integ): remove throttle skip hook from train & mlops; fix serve …


jam-jee Jul 23, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


44 changes: 5 additions & 39 deletions
sagemaker-mlops/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,6 +69,11 @@
# ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
# test-harness concurrency fix and is intentionally identical to the block in
# the sagemaker-train / sagemaker-serve integ conftests.
#
# Throttling that still exhausts the adaptive retry budget is deliberately left
# to fail the test loudly (rather than being converted to a skip), so a
# persistent rate-limit regression stays visible instead of silently
# disappearing from the results.
# ---------------------------------------------------------------------------

# botocore adaptive retry settings for throttling-prone IAM validation calls.
Expand All@@ -77,11 +82,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__


# ---------------------------------------------------------------------------
# CLI options
# ---------------------------------------------------------------------------
Expand Down

47 changes: 4 additions & 43 deletions
sagemaker-serve/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,10 +30,10 @@
 default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
 limiting to smooth bursts.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures are untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -47,11 +47,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__

48 changes: 4 additions & 44 deletions
sagemaker-train/tests/integ/conftest.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,11 +31,10 @@
 its own). ``adaptive`` mode adds client-side rate limiting so bursts of
 ``SimulatePrincipalPolicy`` calls ride out transient throttling.

* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
 ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
 retries, convert the failure into a skip so a transient rate limit never reds
 the build. Genuine failures (and throttling on any other operation) are
 untouched.
Throttling that still exhausts the adaptive retry budget is deliberately left to
fail the test loudly (rather than being converted to a skip), so a persistent
rate-limit regression stays visible instead of silently disappearing from the
results.
"""
from __future__ import absolute_import

Expand All@@ -49,11 +48,6 @@
_RETRY_MODE = "adaptive"
_MAX_ATTEMPTS = "10"

# Only tolerate throttling on the IAM permission simulation used during role
# validation, so unrelated throttling still fails loudly.
_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
_SIMULATE_OP = "SimulatePrincipalPolicy"


@pytest.fixture(autouse=True, scope="session")
def _configure_boto_adaptive_retries():
Expand All@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
 os.environ.pop(key, None)
 else:
 os.environ[key] = value


def _is_simulate_policy_throttle(exc):
 """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
 from botocore.exceptions import ClientError

 if not isinstance(exc, ClientError):
 return False
 error_code = exc.response.get("Error", {}).get("Code", "")
 operation = getattr(exc, "operation_name", "") or ""
 return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
 """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
 survives the adaptive retries under heavy concurrency. Applies to setup and
 call phases, since role resolution frequently happens in fixture setup."""
 outcome = yield
 report = outcome.get_result()

 if report.when not in ("setup", "call") or not report.failed:
 return

 exc = call.excinfo.value if call.excinfo else None
 # Walk the exception chain so a wrapped ClientError is still detected.
 while exc is not None:
 if _is_simulate_policy_throttle(exc):
 report.outcome = "skipped"
 report.wasxfail = (
 "iam:SimulatePrincipalPolicy throttled under concurrent test load"
 )
 break
 exc = exc.__cause__ or exc.__context__
Loading