feat(eval): ab-test target-based run (create)

src/core/eval.tsx

@@ -76,6 +76,7 @@ import {
  type EvaluationReferenceInput,
  type EvaluationResultContent,
  type EvaluationTarget,
+ type CreateABTestRequest,
  type CreateABTestResponse,
  type GetABTestResponse,
  type ListABTestsResponse,
@@ -123,6 +124,7 @@ import type {
  CoreEvalClient,
  CreateConfigurationBundleInput,
  CreateConfigBundleABTestInput,
+ CreateTargetBasedABTestInput,
  CreateDatasetInput,
  CreateOnlineEvalInput,
  CreateOnlineInsightInput,
@@ -481,65 +483,38 @@ export class EvalClient implements CoreEvalClient {
  .send(new DeleteABTestCommand({ abTestId: id }));
  }
 
- async createConfigBundleABTest(
- input: CreateConfigBundleABTestInput,
+ private async createABTest(
+ name: string,
+ gateway: string,
+ callerRoleArn: string | undefined,
+ build: (context: {
+ gatewayArn: string;
+ accountId: string;
+ roleArn: string;
+ }) => CreateABTestRequest,
  options: CoreOptions,
  ): Promise<CreateABTestResponse> {
  const control = this.clients.control(toClientConfig(options));
- const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway }));
-const gatewayArn = gateway.gatewayArn!;
+ const gatewayArn = (await control.send(new GetGatewayCommand({ gatewayIdentifier: gateway })))
+.gatewayArn!;
  const accountId = accountIdFromArn(gatewayArn);
 
- const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`;
- const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`;
- const onlineEvaluationConfigArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`;
-
- const treatmentWeight = input.treatmentWeight ?? 50;
- const variants = [
- {
- name: "C",
- weight: 100 - treatmentWeight,
- variantConfiguration: {
- configurationBundle: {
- bundleArn: controlBundleArn,
- bundleVersion: input.control.bundleVersion,
- },
- },
- },
- {
- name: "T1",
- weight: treatmentWeight,
- variantConfiguration: {
- configurationBundle: {
- bundleArn: treatmentBundleArn,
- bundleVersion: input.treatment.bundleVersion,
- },
- },
- },
- ];
-
- let roleArn = input.roleArn;
+ let roleArn = callerRoleArn;
  let provisionedRoleArn: string | undefined;
  if (!roleArn) {
- const iam = this.clients.iam({ region: options.region });
- const provisioned = await provisionAbTestRole(iam, input.name, gatewayArn, options.region);
+ const provisioned = await provisionAbTestRole(
+ this.clients.iam({ region: options.region }),
+ name,
+ gatewayArn,
+ options.region,
+ );
  roleArn = provisioned.roleArn;
  if (provisioned.created) provisionedRoleArn = provisioned.roleArn;
  }
 
- const command = new CreateABTestCommand({
- name: input.name,
- gatewayArn,
- variants,
- evaluationConfig: { onlineEvaluationConfigArn },
- roleArn,
- gatewayFilter: input.gatewayFilter,
- enableOnCreate: input.enableOnCreate ?? true,
- clientToken: randomUUID(),
- });
-
+ const command = new CreateABTestCommand(build({ gatewayArn, accountId, roleArn }));
  try {
- return input.roleArn
+ return callerRoleArn
  ? await this.clients.data(toClientConfig(options)).send(command)
  : await retryWhileRolePropagates(() =>
  this.clients.data(toClientConfig(options)).send(command),
@@ -556,6 +531,99 @@ export class EvalClient implements CoreEvalClient {
  }
  }
 
+ async createConfigBundleABTest(
+ input: CreateConfigBundleABTestInput,
+ options: CoreOptions,
+ ): Promise<CreateABTestResponse> {
+ const treatmentWeight = input.treatmentWeight ?? 50;
+ return this.createABTest(
+ input.name,
+ input.gateway,
+ input.roleArn,
+ ({ gatewayArn, accountId, roleArn }) => {
+ const bundleArn = (id: string) =>
+ `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${id}`;
+ return {
+ name: input.name,
+ gatewayArn,
+ variants: [
+ {
+ name: "C",
+ weight: 100 - treatmentWeight,
+ variantConfiguration: {
+ configurationBundle: {
+ bundleArn: bundleArn(input.control.configBundle),
+ bundleVersion: input.control.bundleVersion,
+ },
+ },
+ },
+ {
+ name: "T1",
+ weight: treatmentWeight,
+ variantConfiguration: {
+ configurationBundle: {
+ bundleArn: bundleArn(input.treatment.configBundle),
+ bundleVersion: input.treatment.bundleVersion,
+ },
+ },
+ },
+ ],
+ evaluationConfig: {
+ onlineEvaluationConfigArn: `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`,
+ },
+ roleArn,
+ gatewayFilter: input.gatewayFilter,
+ enableOnCreate: input.enableOnCreate ?? true,
+ clientToken: randomUUID(),
+ };
+ },
+ options,
+ );
+ }
+
+ async createTargetBasedABTest(
+ input: CreateTargetBasedABTestInput,
+ options: CoreOptions,
+ ): Promise<CreateABTestResponse> {
+ const treatmentWeight = input.treatmentWeight ?? 50;
+ return this.createABTest(
+ input.name,
+ input.gateway,
+ input.roleArn,
+ ({ gatewayArn, accountId, roleArn }) => {
+ const evalArn = (id: string) =>
+ `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${id}`;
+ return {
+ name: input.name,
+ gatewayArn,
+ variants: [
+ {
+ name: "C",
+ weight: 100 - treatmentWeight,
+ variantConfiguration: { target: { name: input.control.gatewayTarget } },
+ },
+ {
+ name: "T1",
+ weight: treatmentWeight,
+ variantConfiguration: { target: { name: input.treatment.gatewayTarget } },
+ },
+ ],
+ evaluationConfig: {
+ perVariantOnlineEvaluationConfig: [
+ { name: "C", onlineEvaluationConfigArn: evalArn(input.control.onlineEval) },
+ { name: "T1", onlineEvaluationConfigArn: evalArn(input.treatment.onlineEval) },
+ ],
+ },
+ roleArn,
+ gatewayFilter: input.gatewayFilter,
+ enableOnCreate: input.enableOnCreate ?? true,
+ clientToken: randomUUID(),
+ };
+ },
+ options,
+ );
+ }
+
  async listBatchInsights(
  nextToken: string | undefined,
  maxResults: number | undefined,

src/handlers/eval/ab-test/ab-test.test.tsx

@@ -79,9 +79,12 @@ describe("eval ab-test command hierarchy", () => {
  "stop",
  "delete",
  "config-bundle",
+ "target-based",
  ]);
  const cb = group?.children().find((c) => c.name() === "config-bundle");
  expect(cb?.children().map((c) => c.name())).toEqual(["run"]);
+ const tb = group?.children().find((c) => c.name() === "target-based");
+ expect(tb?.children().map((c) => c.name())).toEqual(["run"]);
  });
 });
 
@@ -264,3 +267,86 @@ describe("eval ab-test config-bundle run validation", () => {
  });
  });
 });
+
+describe("eval ab-test target-based run validation", () => {
+ const TB_BASE = [
+ "eval",
+ "ab-test",
+ "target-based",
+ "run",
+ "--name",
+ "orders-v2-canary",
+ "--gateway",
+ "orders-gateway-abc123",
+ "--control",
+ '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}',
+ "--treatment",
+ '{"gateway-target":"orders-v2-target","online-eval":"v2-quality"}',
+ "--json",
+ ];
+
+ test.each(["name", "gateway", "control", "treatment"] as const)(
+ "requires --%s",
+ async (missing) => {
+ const args = TB_BASE.filter(
+ (a, i) => a !== `--${missing}` && TB_BASE[i - 1] !== `--${missing}`,
+ );
+ await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`));
+ },
+ );
+
+ test("rejects a mis-shaped --control object", async () => {
+ const args = TB_BASE.map((a) =>
+ a === '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}'
+ ? '{"wrong":"shape"}'
+ : a,
+ );
+ await expect(run(args)).rejects.toThrow(/--control must be/);
+ });
+
+ test("rejects identical control/treatment targets", async () => {
+ const same = '{"gateway-target":"t","online-eval":"e"}';
+ await expect(
+ run([
+ "eval",
+ "ab-test",
+ "target-based",
+ "run",
+ "--name",
+ "x",
+ "--gateway",
+ "g",
+ "--control",
+ same,
+ "--treatment",
+ same,
+ "--json",
+ ]),
+ ).rejects.toThrow(/different gateway targets/);
+ });
+
+ test("maps flags to a createTargetBasedABTest call", async () => {
+ const { core } = await run([...TB_BASE, "--treatment-weight", "20"], (c) =>
+ c.eval.setAbTestCreateResponse({
+ abTestId: "x",
+ abTestArn: ARN,
+ name: "x",
+ status: "CREATING",
+ executionStatus: "RUNNING",
+ createdAt: new Date("2026-08-26T10:00:00.000Z"),
+ }),
+ );
+ const call = core.eval.calls.find((c) => c.method === "createTargetBasedABTest");
+ expect(call).toBeDefined();
+ expect(call!.args[0]).toEqual({
+ name: "orders-v2-canary",
+ gateway: "orders-gateway-abc123",
+ control: { gatewayTarget: "orders-prod-target", onlineEval: "prod-quality" },
+ treatment: { gatewayTarget: "orders-v2-target", onlineEval: "v2-quality" },
+ treatmentWeight: 20,
+ gatewayFilter: undefined,
+ roleArn: undefined,
+ enableOnCreate: undefined,
+ });
+ });
+});

src/handlers/eval/ab-test/index.tsx

@@ -10,6 +10,7 @@ import { createResumeAbTestHandler } from "./resume";
 import { createStopAbTestHandler } from "./stop";
 import { createDeleteAbTestHandler } from "./delete";
 import { createConfigBundleAbTestHandler } from "./config-bundle";
+import { createTargetBasedAbTestHandler } from "./target-based";
 
 export function createAbTestHandler(core: Core, io: AppIO): Router {
  return new Router("ab-test", "inspect AgentCore A/B tests")
@@ -22,7 +23,8 @@ export function createAbTestHandler(core: Core, io: AppIO): Router {
  .handler(createResumeAbTestHandler(core))
  .handler(createStopAbTestHandler(core))
  .handler(createDeleteAbTestHandler(core))
- .handler(createConfigBundleAbTestHandler(core, io));
+ .handler(createConfigBundleAbTestHandler(core, io))
+ .handler(createTargetBasedAbTestHandler(core, io));
 }
 
 export { AbTestScreen } from "./screen.tsx";

src/handlers/eval/ab-test/target-based/index.tsx

@@ -0,0 +1,10 @@
+import { Router } from "../../../../router";
+import type { AppIO } from "../../../../io";
+import type { Core } from "../../../types";
+import { createTargetBasedRunHandler } from "./run";
+
+export function createTargetBasedAbTestHandler(core: Core, io: AppIO): Router {
+ return new Router("target-based", "target-based A/B tests").handler(
+ createTargetBasedRunHandler(core, io),
+ );
+}