feat(tests): add E2E and integration tests for Harness primitive

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});

.github/workflows/e2e-tests.yml

@@ -101,6 +101,7 @@ jobs:
  BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
  CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
  | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+ | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \
  | tr '\n' ' ')
  echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
  echo "Changed e2e tests: ${CHANGED:-none}"
@@ -113,5 +114,7 @@ jobs:
  OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
  GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
  CDK_TARBALL: ${{ env.CDK_TARBALL }}
- # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
- run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}
+ # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR
+ run:
+ npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{
+ steps.changed.outputs.extra_tests }}

e2e-tests/harness-bedrock.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'bedrock' });

e2e-tests/harness-e2e-helper.ts

@@ -0,0 +1,163 @@
+import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
+import {
+ cleanupStaleCredentialProviders,
+ installCdkTarball,
+ runAgentCoreCLI,
+ teardownE2EProject,
+ writeAwsTargets,
+} from './e2e-helper.js';
+import { randomUUID } from 'node:crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const baseCanRun = prereqs.npm && prereqs.git && hasAws;
+
+interface HarnessE2EConfig {
+ modelProvider: 'bedrock' | 'open_ai' | 'gemini';
+ requiredEnvVar?: string;
+ skipMemory?: boolean;
+}
+
+export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
+ const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar];
+ const canRun = baseCanRun && hasRequiredVar;
+
+ const providerLabel =
+ cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock';
+
+ describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => {
+ let testDir: string;
+ let projectPath: string;
+ let harnessName: string;
+
+ beforeAll(async () => {
+ if (!canRun) return;
+
+ await cleanupStaleCredentialProviders();
+
+ testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
+ await mkdir(testDir, { recursive: true });
+
+ const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4);
+ harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`;
+
+ const createArgs = [
+ 'create',
+ '--name',
+ harnessName,
+ '--model-provider',
+ cfg.modelProvider,
+ '--json',
+ '--skip-git',
+ ];
+
+ if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) {
+ createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!);
+ }
+
+ if (cfg.skipMemory) {
+ createArgs.push('--no-harness-memory');
+ }
+
+ const result = await runAgentCoreCLI(createArgs, testDir);
+
+ expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
+ const json = parseJsonOutput(result.stdout) as { projectPath: string };
+ projectPath = json.projectPath;
+
+ await writeAwsTargets(projectPath);
+ installCdkTarball(projectPath);
+ }, 300000);
+
+ afterAll(async () => {
+ if (projectPath && hasAws) {
+ await teardownE2EProject(projectPath, harnessName, cfg.modelProvider);
+ }
+ if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+ }, 600000);
+
+ it.skipIf(!canRun)(
+ 'deploys to AWS successfully',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+
+ if (result.exitCode !== 0) {
+ console.log('Deploy stdout:', result.stdout);
+ console.log('Deploy stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Deploy should report success').toBe(true);
+ },
+ 1,
+ 30000
+ );
+ },
+ 600000
+ );
+
+ it.skipIf(!canRun)(
+ 'invokes the deployed harness',
+ async () => {
+ expect(projectPath, 'Project should have been created').toBeTruthy();
+
+ await retry(
+ async () => {
+ const result = await runAgentCoreCLI(
+ ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'],
+ projectPath
+ );
+
+ if (result.exitCode !== 0) {
+ console.log('Invoke stdout:', result.stdout);
+ console.log('Invoke stderr:', result.stderr);
+ }
+
+ expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(result.stdout) as { success: boolean };
+ expect(json.success, 'Invoke should report success').toBe(true);
+ },
+ 3,
+ 15000
+ );
+ },
+ 180000
+ );
+
+ it.skipIf(!canRun)(
+ 'status shows the deployed harness',
+ async () => {
+ const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath);
+
+ expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0);
+
+ const json = parseJsonOutput(statusResult.stdout) as {
+ success: boolean;
+ resources: {
+ resourceType: string;
+ name: string;
+ deploymentState: string;
+ identifier?: string;
+ }[];
+ };
+ expect(json.success).toBe(true);
+
+ const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined();
+ expect(harness!.deploymentState).toBe('deployed');
+ expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy();
+ },
+ 120000
+ );
+ });
+}

e2e-tests/harness-gemini.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true });

e2e-tests/harness-openai.test.ts

@@ -0,0 +1,3 @@
+import { createHarnessE2ESuite } from './harness-e2e-helper.js';
+
+createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });

integ-tests/add-remove-harness.test.ts

@@ -0,0 +1,204 @@
+import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js';
+import type { TestProject } from '../src/test-utils/index.js';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+async function readHarnessSpec(projectPath: string, harnessName: string) {
+ return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8'));
+}
+
+describe('integration: harness add/remove lifecycle', () => {
+ let project: TestProject;
+ const harnessName = 'TestHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds a harness with defaults', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy();
+ expect(harness!.path).toBe(`app/${harnessName}`);
+ });
+
+ it('creates harness.json with correct model config', async () => {
+ const spec = await readHarnessSpec(project.projectPath, harnessName);
+ expect(spec.model).toBeDefined();
+ expect(spec.model.provider).toBe('bedrock');
+ expect(spec.model.modelId).toBeTruthy();
+ });
+
+ it('creates system-prompt.md', async () => {
+ const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`);
+ expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true);
+ });
+
+ it('auto-creates memory resource', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const memories = config.memories ?? [];
+ expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0);
+ });
+
+ it('rejects duplicate harness name', async () => {
+ const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('removes the harness', async () => {
+ const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+ const json = JSON.parse(result.stdout);
+ expect(json.success).toBe(true);
+
+ const config = await readProjectConfig(project.projectPath);
+ const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy();
+ });
+});
+
+describe('integration: harness configuration options', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('adds harness with truncation strategy', async () => {
+ const name = 'TruncHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.truncation?.strategy).toBe('sliding_window');
+ });
+
+ it('adds harness with lifecycle config', async () => {
+ const name = 'LifecycleHarness';
+ const result = await runCLI(
+ ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300);
+ expect(spec.lifecycleConfig?.maxLifetime).toBe(3600);
+ });
+
+ it('adds harness without memory when --no-memory is set', async () => {
+ const name = 'NoMemHarness';
+ const configBefore = await readProjectConfig(project.projectPath);
+ const memoriesBefore = (configBefore.memories ?? []).length;
+
+ const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath);
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const configAfter = await readProjectConfig(project.projectPath);
+ const memoriesAfter = (configAfter.memories ?? []).length;
+ expect(memoriesAfter).toBe(memoriesBefore);
+ });
+
+ it('adds harness with non-bedrock model provider', async () => {
+ const name = 'OpenAIHarness';
+ const result = await runCLI(
+ [
+ 'add',
+ 'harness',
+ '--name',
+ name,
+ '--model-provider',
+ 'open_ai',
+ '--model-id',
+ 'gpt-5',
+ '--api-key-arn',
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key',
+ '--json',
+ ],
+ project.projectPath
+ );
+
+ expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
+
+ const spec = await readHarnessSpec(project.projectPath, name);
+ expect(spec.model.provider).toBe('open_ai');
+ expect(spec.model.modelId).toBe('gpt-5');
+ expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key');
+ });
+});
+
+describe('integration: harness validation errors', () => {
+ let project: TestProject;
+
+ beforeAll(async () => {
+ project = await createTestProject({ noAgent: true });
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('rejects invalid harness name with special characters', async () => {
+ const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects harness name starting with a number', async () => {
+ const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+
+ it('rejects add harness without --name when --json is passed', async () => {
+ const result = await runCLI(['add', 'harness', '--json'], project.projectPath);
+ expect(result.exitCode).not.toBe(0);
+ });
+});
+
+describe('integration: create project with harness', () => {
+ let project: TestProject;
+ const harnessName = 'CreateHarness';
+
+ beforeAll(async () => {
+ project = await createTestProject({ name: harnessName, noAgent: true });
+ await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath);
+ });
+
+ afterAll(async () => {
+ await project.cleanup();
+ });
+
+ it('has correct project scaffolding', async () => {
+ expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true);
+ expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true);
+ expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true);
+ });
+
+ it('has harness registered in project config', async () => {
+ const config = await readProjectConfig(project.projectPath);
+ const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName);
+ expect(harness).toBeTruthy();
+ });
+});