') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); test(e2e): export a fully-featured harness in-project and by ARN by padmak30 · Pull Request #1641 · aws/agentcore-cli · GitHub
Skip to content

test(e2e): export a fully-featured harness in-project and by ARN - #1641

Merged
padmak30 merged 2 commits into
mainfrom
test/export-harness-e2e
Jun 25, 2026
Merged

test(e2e): export a fully-featured harness in-project and by ARN#1641
padmak30 merged 2 commits into
mainfrom
test/export-harness-e2e

Conversation

@padmak30

@padmak30padmak30 commented Jun 25, 2026•
edited
Loading

Copy link
Copy Markdown
Contributor

Description

Adds end-to-end coverage for agentcore export harness across both source modes — in-project (--name) and out-of-project (--arn) — proving each exported Strands runtime agent actually works at runtime, not just that the spec/wiring is generated.

The source harness attaches following export surface together:

  • an existing project memory (referenced by name)
  • an agentcore_code_interpreter tool (managed default)
  • a public GitHub skill (cloned at runtime, no credential)
  • an MCP gateway tool (in-project gateway + mcp-server target)

Flow (e2e-tests/export-harness-full.test.ts):

  1. create --no-agent + add memory + add gateway (mcp-server target)
  2. deploy build(deps): bump diff and @aws-cdk/cloudformation-diff #1 — provisions memory + gateway (ARNs now exist)
  3. add harness attaching the memory (by name) + gateway (by --gateway-arn) + code-interpreter tool + git skill
  4. deploy chore: Add 3rd party licenses #2 — harness with all surfaces
  5. invoke the harness; assert the code interpreter runs
  6. export --name → deploy → verify the in-project agent (memory wired via discovery env var; gateway + CI as connections)
  7. export --arn into a fresh project → deploy → verify the out-of-project agent (every resource external → all wired as connections)

Each exported agent is behaviorally verified via four invokes:

  • code interpreter: exact factorial value (the model can't fabricate it)
  • gateway tool: lists a gateway-prefixed / Exa MCP tool (provider-specific token, not generic prose)
  • skill: references the cloned returns-policy skill
  • memory: same-session round-trip recall

The test is skipIf(!canRun)-gated (AWS creds + npm + git), tears down both projects in afterAll, and uses E2e-prefixed, per-run-unique project names so a failed teardown is still swept by global-setup's stale-stack GC.

Type of Change

  • Other: test-only (adds e2e coverage; no product code changes)

Testing

  • I ran npm run typecheck (clean)
  • I ran npm run lint (clean)
  • Verified live end-to-end against a real AWS account (us-east-1): 7/7 steps pass; both exported agents (in-project + by-ARN) deploy and pass all four capability checks; resources auto-torn-down.

Checklist

  • I have added any necessary tests that prove the feature works
  • My changes generate no new warnings

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

@padmak30
test(e2e): export a fully-featured harness in-project and by ARN
db529f7
End-to-end coverage for `export harness` across both source modes, proving each
exported agent works at runtime (not just that the spec/wiring is generated).
The source harness attaches every export surface: an existing project memory
(by name), an agentcore_code_interpreter tool, a public GitHub skill, and an MCP
gateway tool. Flow: deploy memory+gateway → create the harness attaching both →
deploy → invoke the harness → export --name (in-project) → deploy → verify →
export --arn (new empty project) → deploy → verify.
Each exported agent is behaviorally verified via four invokes: code interpreter
(exact factorial value), gateway tool (assert the gateway-prefixed/Exa MCP tool
is listed), skill (assert the returns-policy skill is referenced), and memory
(same-session round-trip recall). Verified live: 7/7 steps pass; both projects
torn down in afterAll; project names are E2e-prefixed + per-run-unique so a
failed teardown is still swept by global-setup's stale-stack GC.
@padmak30
padmak30 requested a review from a teamJune 25, 2026 16:10
@github-actionsgithub-actionsBot added the size/l PR size: L label Jun 25, 2026
@agentcore-devx-automationagentcore-devx-automationBot added the claude-security-reviewing Claude Code /security-review in progress label Jun 25, 2026
@github-actionsgithub-actionsBot added the agentcore-harness-reviewing AgentCore Harness review in progress label Jun 25, 2026
@agentcore-devx-automation

agentcore-devx-automationBot commented Jun 25, 2026

Copy link
Copy Markdown
Contributor

Claude Security Review: no high-confidence findings. (run)

@agentcore-devx-automationagentcore-devx-automationBot removed the claude-security-reviewing Claude Code /security-review in progress label Jun 25, 2026
@github-actions

github-actionsBot commented Jun 25, 2026

Copy link
Copy Markdown
Contributor

Package Tarball

aws-agentcore-0.21.0.tgz

How to install

gh release download pr-1641-tarball --repo aws/agentcore-cli --pattern "*.tgz" --dir /tmp/pr-tarball
npm install -g /tmp/pr-tarball/aws-agentcore-0.21.0.tgz

agentcore-cli-automation
agentcore-cli-automation suggested changes Jun 25, 2026
View reviewed changes

@agentcore-cli-automationagentcore-cli-automation left a comment

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Test is well-structured and the per-capability behavioral assertions are a clear improvement over "did it return 200" checks. One concern about flakiness needs to be addressed before merging.

Comment threade2e-tests/export-harness-full.test.ts
@github-actionsgithub-actionsBot removed the agentcore-harness-reviewing AgentCore Harness review in progress label Jun 25, 2026
@github-actions

github-actionsBot commented Jun 25, 2026•
edited
Loading

Copy link
Copy Markdown
Contributor

Coverage Report

StatusCategoryPercentageCovered / Total
🔵Lines37.42%13771 / 36800
🔵Statements36.69%14646 / 39910
🔵Functions32%2356 / 7362
🔵Branches31.46%9180 / 29173
Generated in workflow #3839 for commit e9a124e by the Vitest Coverage Report Action

@padmak30
test(e2e): retry capability content assertions to avoid flakiness
e9a124e
The per-capability checks asserted on a single LLM response outside the retry —
only invokeAndExpectSuccess's CLI-`success` check was retried. That risks
intermittent CI failures from (a) memory same-session write/read visibility lag
on the recall turn and (b) LLM phrasing nondeterminism on the gateway-tool and
skill checks (a `success: true` response that happens not to name the expected
token).
Add an optional `verify` predicate to invokeAndExpectSuccess that runs INSIDE the
retried unit, and move every content assertion (factorial value, gateway tool
token, skill reference, memory recall) into it — so a flaky sample re-invokes
instead of failing. Mirrors the retry pattern already used in harness-e2e-helper.
@github-actionsgithub-actionsBot added size/l PR size: L and removed size/l PR size: L labels Jun 25, 2026
@agentcore-devx-automationagentcore-devx-automationBot added the claude-security-reviewing Claude Code /security-review in progress label Jun 25, 2026
@agentcore-devx-automation

agentcore-devx-automationBot commented Jun 25, 2026

Copy link
Copy Markdown
Contributor

Claude Security Review: no high-confidence findings. (run)

@agentcore-devx-automationagentcore-devx-automationBot removed the claude-security-reviewing Claude Code /security-review in progress label Jun 25, 2026
@padmak30
padmak30 merged commit aba397a into mainJun 25, 2026
33 checks passed
@padmak30
padmak30 deleted the test/export-harness-e2e branch June 25, 2026 17:41
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@avi-alpertavi-alpertavi-alpert approved these changes
+1 more reviewer
@agentcore-cli-automationagentcore-cli-automationagentcore-cli-automation requested changes
Reviewers whose approvals may not affect merge requirements

Assignees

No one assigned

Labels

size/lPR size: L

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

3 participants

@padmak30@avi-alpert@agentcore-cli-automation