fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jariy17 merged 1 commit into from
May 6, 2026
Merged

fix: correct AB test execution role IAM policy and promote stability #1126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions e2e-tests/ab-test-config-bundle.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
await retry(
async () => {
const result = await run(['deploy', '--yes', '--json']);
if (result.exitCode !== 0) {
console.log('Initial deploy stdout:', result.stdout);
console.log('Initial deploy stderr:', result.stderr);
}
expect(result.exitCode, `Initial deploy failed`).toBe(0);
const json = parseJsonOutput(result.stdout) as { success: boolean };
expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
it.skipIf(!canRun)(
'adds config-bundle AB test with 90/10 split',
async () => {
// Config bundles reference ARNs from deployed resources.
// Use placeholder bundle ARNs — the deploy step will validate or create them.
const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
// Use placeholder bundle ARNs that satisfy the service format constraints.
// Real config bundles would be created separately; these test the AB test wiring.
const region = process.env.AWS_REGION ?? 'us-east-1';
const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

const result = await run([
'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
'--control-bundle',
controlBundle,
'--control-version',
'v1',
'00000000-0000-0000-0000-000000000001',
'--treatment-bundle',
treatmentBundle,
'--treatment-version',
'v1',
'00000000-0000-0000-0000-000000000002',
'--control-weight',
'90',
'--treatment-weight',
Expand Down
34 changes: 25 additions & 9 deletions e2e-tests/ab-test-target-based.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
await retry(
async () => {
const result = await run(['deploy', '--yes', '--json']);
if (result.exitCode !== 0) {
console.log('Deploy stdout:', result.stdout);
console.log('Deploy stderr:', result.stderr);
}
expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
const json = parseJsonOutput(result.stdout) as { success: boolean };
expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
600000
);

it.skipIf(!canRun)(
'AB test reaches RUNNING status after deploy',
async () => {
await retry(
async () => {
const result = await run(['ab-test', abTestName, '--json']);
expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
const json = parseJsonOutput(result.stdout) as { executionStatus: string };
expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
},
12,
15000
);
},
300000
);

it.skipIf(!canRun)(
'status shows all resources deployed',
async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

const json = parseJsonOutput(result.stdout) as {
success: boolean;
resources: { resourceType: string; name: string; deploymentState: string }[];
resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
};
expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
expect(agent!.deploymentState).toBe('deployed');

// Gateway should be deployed
const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
// AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
expect(abTest!.deploymentState).toBe('deployed');
// invocationUrl proves the HTTP gateway was deployed and wired up correctly
expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
},
3,
15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
'promotes AB test (updates agentcore.json)',
async () => {
const result = await run(['promote', 'ab-test', abTestName, '--json']);
expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
expect(json).toHaveProperty('success', true);
expect(json).toHaveProperty('promoted', true);
Expand Down
112 changes: 112 additions & 0 deletions scripts/run-e2e-local.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
if [[ "$arg" == "--all" ]]; then
RUN_ALL=true
else
TEST_FILES+=("$arg")
fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
--role-arn "$ROLE_ARN" \
--role-session-name "local-e2e-$(date +%s)" \
--duration-seconds 3600 \
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
--output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
--secret-id "$SECRET_ARN" \
--region "$AWS_REGION" \
--query SecretString \
--output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
echo "Running full e2e suite"
npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
echo "Running: ${TEST_FILES[*]}"
npx vitest run --project e2e "${TEST_FILES[@]}"
else
echo "Running default: e2e-tests/strands-bedrock.test.ts"
npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi
59 changes: 59 additions & 0 deletions src/cli/commands/pause/__tests__/promote.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
getABTest: (...args: unknown[]) => mockGetABTest(...args),
updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
beforeEach(() => {
vi.clearAllMocks();
mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
});

it('stops immediately when already RUNNING', async () => {
mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

expect(mockGetABTest).toHaveBeenCalledTimes(1);
expect(mockUpdateABTest).toHaveBeenCalledWith({
region: 'us-east-1',
abTestId: 'abt-123',
executionStatus: 'STOPPED',
});
});

it('polls until RUNNING then stops', async () => {
mockGetABTest
.mockResolvedValueOnce({ executionStatus: 'UPDATING' })
.mockResolvedValueOnce({ executionStatus: 'UPDATING' })
.mockResolvedValueOnce({ executionStatus: 'RUNNING' });

await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

expect(mockGetABTest).toHaveBeenCalledTimes(3);
expect(mockUpdateABTest).toHaveBeenCalledOnce();
});

it('throws if AB test never reaches RUNNING', async () => {
mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
'did not reach RUNNING state'
);

expect(mockGetABTest).toHaveBeenCalledTimes(3);
expect(mockUpdateABTest).not.toHaveBeenCalled();
});

it('includes current status in the error message', async () => {
mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
});
});
8 changes: 2 additions & 6 deletions src/cli/commands/pause/command.tsx
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
process.exit(1);
}

// Stop the AB test
const result = await updateABTest({
region,
abTestId,
executionStatus: 'STOPPED',
});
const result = await waitForRunningThenStop(region, abTestId, name);

// Apply promotion to agentcore.json
const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down
28 changes: 28 additions & 0 deletions src/cli/commands/pause/promote-utils.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
* Poll until the AB test reaches RUNNING status, then stop it.
* Throws if the test never reaches RUNNING within the allotted attempts.
*/
export async function waitForRunningThenStop(
region: string,
abTestId: string,
name: string,
maxAttempts = 12,
delayMs = 10_000
): Promise<UpdateABTestResult> {
let currentStatus: string | undefined;
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const current = await getABTest({ region, abTestId });
currentStatus = current.executionStatus;
if (currentStatus === 'RUNNING') break;
await new Promise(resolve => setTimeout(resolve, delayMs));
}
if (currentStatus !== 'RUNNING') {
throw new Error(
`AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
);
}
return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}
22 changes: 11 additions & 11 deletions src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
expect(trustPolicy.Statement).toHaveLength(1);
expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

// Second call: PutRolePolicyCommand with inline policy
const putPolicyCall = mockIAMSend.mock.calls[1]![0];
const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
expect(sids).toContain('GatewayRuleStatement');
expect(sids).toContain('GatewayReadStatement');
expect(sids).toContain('GatewayListStatement');
expect(sids).toContain('OnlineEvaluationConfigStatement');
expect(sids).toContain('ConfigurationBundleReadStatement');
expect(sids).toContain('CloudWatchLogReadStatement');
expect(sids).toContain('CloudWatchIndexPolicyStatement');

// ListGateways must use wildcard resource (can't be scoped)
const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
expect(listGatewayStmt.Resource).toEqual(['*']);
expect(sids).toContain('AgentCoreResources');
expect(sids).toContain('CloudWatchLogs');

// AgentCoreResources must include all required actions
const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
});
});

Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
fix: correct AB test execution role IAM policy and promote stability#1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

jariy17
merged 1 commit into

from

May 6, 2026
 Merged


fix: correct AB test execution role IAM policy and promote stability
#1126
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 8 additions & 10 deletions
e2e-tests/ab-test-config-bundle.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -105,10 +105,6 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Initial deploy stdout:', result.stdout);
 console.log('Initial deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Initial deploy failed`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -123,10 +119,12 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 it.skipIf(!canRun)(
 'adds config-bundle AB test with 90/10 split',
 async () => {
 // Config bundles reference ARNs from deployed resources.
 // Use placeholder bundle ARNs — the deploy step will validate or create them.
 const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`;
 // Use placeholder bundle ARNs that satisfy the service format constraints.
 // Real config bundles would be created separately; these test the AB test wiring.
 const region = process.env.AWS_REGION ?? 'us-east-1';
 const account = process.env.AWS_ACCOUNT_ID ?? '000000000000';
 const controlBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/control-bundle-AbCdEfGhIj`;
 const treatmentBundle = `arn:aws:bedrock-agentcore:${region}:${account}:configuration-bundle/treatment-bundle-AbCdEfGhIj`;

 const result = await run([
 'add',
Expand All@@ -140,11 +138,11 @@ describe.sequential('e2e: config-bundle AB test lifecycle', () => {
 '--control-bundle',
 controlBundle,
 '--control-version',
 'v1',
 '00000000-0000-0000-0000-000000000001',
 '--treatment-bundle',
 treatmentBundle,
 '--treatment-version',
 'v1',
 '00000000-0000-0000-0000-000000000002',
 '--control-weight',
 '90',
 '--treatment-weight',
Expand Down

34 changes: 25 additions & 9 deletions
e2e-tests/ab-test-target-based.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -195,10 +195,6 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 await retry(
 async () => {
 const result = await run(['deploy', '--yes', '--json']);
 if (result.exitCode !== 0) {
 console.log('Deploy stdout:', result.stdout);
 console.log('Deploy stderr:', result.stderr);
 }
 expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { success: boolean };
 expect(json.success).toBe(true);
Expand All@@ -210,6 +206,23 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 600000
 );

 it.skipIf(!canRun)(
 'AB test reaches RUNNING status after deploy',
 async () => {
 await retry(
 async () => {
 const result = await run(['ab-test', abTestName, '--json']);
 expect(result.exitCode, `ab-test lookup failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as { executionStatus: string };
 expect(json.executionStatus, 'AB test should be RUNNING after deploy').toBe('RUNNING');
 },
 12,
 15000
 );
 },
 300000
 );

 it.skipIf(!canRun)(
 'status shows all resources deployed',
 async () => {
Expand All@@ -220,7 +233,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {

 const json = parseJsonOutput(result.stdout) as {
 success: boolean;
 resources: { resourceType: string; name: string; deploymentState: string }[];
 resources: { resourceType: string; name: string; deploymentState: string; invocationUrl?: string }[];
 };
 expect(json.success).toBe(true);

Expand All@@ -229,9 +242,12 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined();
 expect(agent!.deploymentState).toBe('deployed');

 // Gateway should be deployed
 const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`);
 expect(gateway, 'HTTP gateway should appear in status').toBeDefined();
 // AB test should be deployed (HTTP gateways are not surfaced as top-level status resources)
 const abTest = json.resources.find(r => r.resourceType === 'ab-test' && r.name === abTestName);
 expect(abTest, `AB test "${abTestName}" should appear in status`).toBeDefined();
 expect(abTest!.deploymentState).toBe('deployed');
 // invocationUrl proves the HTTP gateway was deployed and wired up correctly
 expect(abTest!.invocationUrl, 'AB test should have a gateway invocation URL').toBeTruthy();
 },
 3,
 15000
Expand DownExpand Up@@ -280,7 +296,7 @@ describe.sequential('e2e: target-based AB test lifecycle', () => {
 'promotes AB test (updates agentcore.json)',
 async () => {
 const result = await run(['promote', 'ab-test', abTestName, '--json']);
 expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0);
 expect(result.exitCode, `Promote failed: ${result.stdout} ${result.stderr}`).toBe(0);
 const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
 expect(json).toHaveProperty('success', true);
 expect(json).toHaveProperty('promoted', true);
Expand Down

112 changes: 112 additions & 0 deletions
scripts/run-e2e-local.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Run E2E tests locally, replicating the GitHub Actions e2e-tests.yml workflow.
#
# Required env vars:
# E2E_ROLE_ARN — IAM role ARN to assume (grants access to the test account)
# E2E_SECRET_ARN — Secrets Manager ARN containing ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY
#
# Optional env vars:
# AWS_REGION — defaults to us-east-1
#
# Usage:
# export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role>
# export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>
# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default)
# ./scripts/run-e2e-local.sh --all # runs the full e2e suite
# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file
#
# Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq

set -euo pipefail

ROLE_ARN="${E2E_ROLE_ARN:-}"
SECRET_ARN="${E2E_SECRET_ARN:-}"
AWS_REGION="${AWS_REGION:-us-east-1}"

if [[ -z "$ROLE_ARN" ]]; then
 echo "❌ E2E_ROLE_ARN is not set. Export it before running this script:"
 echo " export E2E_ROLE_ARN=arn:aws:iam::<account>:role/<role-name>"
 exit 1
fi

if [[ -z "$SECRET_ARN" ]]; then
 echo "❌ E2E_SECRET_ARN is not set. Export it before running this script:"
 echo " export E2E_SECRET_ARN=arn:aws:secretsmanager:<region>:<account>:secret:<name>"
 exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# ── Parse arguments ────────────────────────────────────────────────────────────
RUN_ALL=false
TEST_FILES=()
for arg in "$@"; do
 if [[ "$arg" == "--all" ]]; then
 RUN_ALL=true
 else
 TEST_FILES+=("$arg")
 fi
done

echo "=== Assuming IAM role ==="
CREDS=$(aws sts assume-role \
 --role-arn "$ROLE_ARN" \
 --role-session-name "local-e2e-$(date +%s)" \
 --duration-seconds 3600 \
 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
 --output text)

export AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk '{print $1}')
export AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk '{print $2}')
export AWS_SESSION_TOKEN=$(echo "$CREDS" | awk '{print $3}')
export AWS_REGION

echo "✅ Assumed role successfully"

echo "=== Fetching API keys from Secrets Manager ==="
SECRET_JSON=$(aws secretsmanager get-secret-value \
 --secret-id "$SECRET_ARN" \
 --region "$AWS_REGION" \
 --query SecretString \
 --output text)

# Mirror the GitHub workflow: parse-json-secrets maps keys to E2E_<KEY> then
# the workflow maps them to the bare names the tests expect.
export ANTHROPIC_API_KEY=$(echo "$SECRET_JSON" | jq -r '.ANTHROPIC_API_KEY // empty')
export OPENAI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.OPENAI_API_KEY // empty')
export GEMINI_API_KEY=$(echo "$SECRET_JSON" | jq -r '.GEMINI_API_KEY // empty')

echo "✅ Secrets loaded (keys present: $(echo "$SECRET_JSON" | jq -r 'keys | join(", ")')"

echo "=== Setting AWS account env var ==="
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "✅ AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID AWS_REGION=$AWS_REGION"

echo "=== Configuring git (required for agentcore create) ==="
git config --global user.email "ci@local" 2>/dev/null || true
git config --global user.name "Local E2E" 2>/dev/null || true

cd "$REPO_ROOT"

echo "=== Installing dependencies ==="
npm ci

echo "=== Building CLI ==="
npm run build

echo "=== Installing CLI globally ==="
TARBALL=$(npm pack | tail -1)
npm install -g "$TARBALL"
echo "✅ Installed: $(agentcore --version)"

echo "=== Running E2E tests ==="
if [[ "$RUN_ALL" == "true" ]]; then
 echo "Running full e2e suite"
 npx vitest run --project e2e
elif [[ ${#TEST_FILES[@]} -gt 0 ]]; then
 echo "Running: ${TEST_FILES[*]}"
 npx vitest run --project e2e "${TEST_FILES[@]}"
else
 echo "Running default: e2e-tests/strands-bedrock.test.ts"
 npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts
fi

59 changes: 59 additions & 0 deletions
src/cli/commands/pause/__tests__/promote.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
import { waitForRunningThenStop } from '../promote-utils.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';

const mockGetABTest = vi.fn();
const mockUpdateABTest = vi.fn();

vi.mock('../../../aws/agentcore-ab-tests', () => ({
 getABTest: (...args: unknown[]) => mockGetABTest(...args),
 updateABTest: (...args: unknown[]) => mockUpdateABTest(...args),
}));

describe('waitForRunningThenStop', () => {
 beforeEach(() => {
 vi.clearAllMocks();
 mockUpdateABTest.mockResolvedValue({ executionStatus: 'STOPPED' });
 });

 it('stops immediately when already RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(1);
 expect(mockUpdateABTest).toHaveBeenCalledWith({
 region: 'us-east-1',
 abTestId: 'abt-123',
 executionStatus: 'STOPPED',
 });
 });

 it('polls until RUNNING then stops', async () => {
 mockGetABTest
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'UPDATING' })
 .mockResolvedValueOnce({ executionStatus: 'RUNNING' });

 await waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 5, 0);

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).toHaveBeenCalledOnce();
 });

 it('throws if AB test never reaches RUNNING', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'UPDATING' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 3, 0)).rejects.toThrow(
 'did not reach RUNNING state'
 );

 expect(mockGetABTest).toHaveBeenCalledTimes(3);
 expect(mockUpdateABTest).not.toHaveBeenCalled();
 });

 it('includes current status in the error message', async () => {
 mockGetABTest.mockResolvedValue({ executionStatus: 'STOPPED' });

 await expect(waitForRunningThenStop('us-east-1', 'abt-123', 'MyTest', 2, 0)).rejects.toThrow('current: STOPPED');
 });
});

8 changes: 2 additions & 6 deletions
src/cli/commands/pause/command.tsx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@ import type { OnlineEvalActionOptions } from '../../operations/eval';
import { COMMAND_DESCRIPTIONS } from '../../tui/copy';
import { requireProject } from '../../tui/guards';
import { getRegion } from '../shared/region-utils';
import { waitForRunningThenStop } from './promote-utils';
import type { Command } from '@commander-js/extra-typings';
import { Text, render } from 'ink';
import React from 'react';
Expand DownExpand Up@@ -274,12 +275,7 @@ export const registerPromote = (program: Command) => {
 process.exit(1);
 }

 // Stop the AB test
 const result = await updateABTest({
 region,
 abTestId,
 executionStatus: 'STOPPED',
 });
 const result = await waitForRunningThenStop(region, abTestId, name);

 // Apply promotion to agentcore.json
 const { promoteABTestConfig } = await import('../../operations/ab-test/promote');
Expand Down

28 changes: 28 additions & 0 deletions
src/cli/commands/pause/promote-utils.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
import { getABTest, updateABTest } from '../../aws/agentcore-ab-tests';
import type { UpdateABTestResult } from '../../aws/agentcore-ab-tests';

/**
 * Poll until the AB test reaches RUNNING status, then stop it.
 * Throws if the test never reaches RUNNING within the allotted attempts.
 */
export async function waitForRunningThenStop(
 region: string,
 abTestId: string,
 name: string,
 maxAttempts = 12,
 delayMs = 10_000
): Promise<UpdateABTestResult> {
 let currentStatus: string | undefined;
 for (let attempt = 0; attempt < maxAttempts; attempt++) {
 const current = await getABTest({ region, abTestId });
 currentStatus = current.executionStatus;
 if (currentStatus === 'RUNNING') break;
 await new Promise(resolve => setTimeout(resolve, delayMs));
 }
 if (currentStatus !== 'RUNNING') {
 throw new Error(
 `AB test "${name}" did not reach RUNNING state after waiting (current: ${currentStatus}). Cannot promote.`
 );
 }
 return updateABTest({ region, abTestId, executionStatus: 'STOPPED' });
}

22 changes: 11 additions & 11 deletions
src/cli/operations/deploy/__tests__/post-deploy-ab-tests.test.ts

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -517,22 +517,22 @@ describe('setupABTests', () => {
 const trustPolicy = JSON.parse(createRoleCall.input.AssumeRolePolicyDocument);
 expect(trustPolicy.Statement).toHaveLength(1);
 expect(trustPolicy.Statement[0].Principal.Service).toBe('bedrock-agentcore.amazonaws.com');
 expect(trustPolicy.Statement[0].Condition.StringEquals['aws:SourceAccount']).toBeDefined();
 expect(trustPolicy.Statement[0].Condition.ArnLike['aws:SourceArn']).toContain('ab-test/*');

 // Second call: PutRolePolicyCommand with inline policy
 const putPolicyCall = mockIAMSend.mock.calls[1]![0];
 const policy = JSON.parse(putPolicyCall.input.PolicyDocument);
 const sids = policy.Statement.map((s: { Sid: string }) => s.Sid);
 expect(sids).toContain('GatewayRuleStatement');
 expect(sids).toContain('GatewayReadStatement');
 expect(sids).toContain('GatewayListStatement');
 expect(sids).toContain('OnlineEvaluationConfigStatement');
 expect(sids).toContain('ConfigurationBundleReadStatement');
 expect(sids).toContain('CloudWatchLogReadStatement');
 expect(sids).toContain('CloudWatchIndexPolicyStatement');

 // ListGateways must use wildcard resource (can't be scoped)
 const listGatewayStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'GatewayListStatement');
 expect(listGatewayStmt.Resource).toEqual(['*']);
 expect(sids).toContain('AgentCoreResources');
 expect(sids).toContain('CloudWatchLogs');

 // AgentCoreResources must include all required actions
 const agentCoreStmt = policy.Statement.find((s: { Sid: string }) => s.Sid === 'AgentCoreResources');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetEvaluator');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetGateway');
 expect(agentCoreStmt.Action).toContain('bedrock-agentcore:GetOnlineEvaluationConfig');
 expect(agentCoreStmt.Condition.StringEquals['aws:ResourceAccount']).toBeDefined();
 });
 });

Expand Down
Loading
Loading