feat(eval): read-only ab-test commands + TUI

src/components/AbTestPicker.tsx

@@ -0,0 +1,74 @@
+import type { ABTestSummary } from "@aws-sdk/client-bedrock-agentcore";
+import { useNavigate } from "react-router";
+import type { ScreenProps } from "../handlers/types";
+import { coreOptsFromCtx } from "../handlers/utils";
+import { formatTimestamp } from "./formatTimestamp";
+import { PaginatedTablePicker } from "./PaginatedTablePicker";
+import type { DataTableColumn } from "./ui/data-table";
+
+interface AbTestRow extends Record<string, unknown> {
+ abTestId: string;
+ name: string;
+ status: string;
+ executionStatus: string;
+ updatedAt: string;
+}
+
+export const abTestColumns = [
+ { key: "name", header: "name", flex: true },
+ { key: "status", header: "status", width: 14 },
+ { key: "executionStatus", header: "execution", width: 12 },
+ { key: "updatedAt", header: "updated UTC", width: 16, render: formatTimestamp },
+] satisfies DataTableColumn<AbTestRow>[];
+
+function toRow(summary: ABTestSummary): AbTestRow {
+ const id = summary.abTestId ?? "";
+ return {
+ abTestId: id,
+ name: summary.name ?? id,
+ status: summary.status ?? "-",
+ executionStatus: summary.executionStatus ?? "-",
+ updatedAt: summary.updatedAt?.toISOString() ?? "-",
+ };
+}
+
+export interface AbTestPickerProps extends ScreenProps {
+ breadcrumb: string[];
+ description?: string;
+ onSelect: (abTestId: string) => void;
+ onEscape?: () => void;
+}
+
+export function AbTestPicker({
+ ctx,
+ core,
+ breadcrumb,
+ description,
+ onSelect,
+ onEscape,
+}: AbTestPickerProps) {
+ const opts = coreOptsFromCtx(ctx);
+ const navigate = useNavigate();
+ const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/")));
+
+ return (
+ <PaginatedTablePicker
+ breadcrumb={breadcrumb}
+ description={description}
+ queryKey={["ab-tests", opts.region]}
+ loadPage={async (token, pageSize) => {
+ const response = await core.eval.listABTests(token, pageSize, opts);
+ return { items: response.abTests ?? [], nextToken: response.nextToken };
+ }}
+ toRow={toRow}
+ columns={abTestColumns}
+ getValue={(row) => row.abTestId}
+ onSelect={onSelect}
+ onBack={goBack}
+ loadingMessage="Loading A/B tests…"
+ errorMessage={(error) => `Error: ${error.message}`}
+ emptyMessage="No A/B tests found in this Region."
+ emptyPageMessage="No A/B tests on this page."
+ />
+ );
+}

src/components/Root.tsx

@@ -62,6 +62,9 @@ import { ConfigBundleListScreen } from "../handlers/eval/config-bundle/list/scre
 import { ConfigBundleGetScreen } from "../handlers/eval/config-bundle/get/screen.tsx";
 import { ConfigBundleVersionScreen } from "../handlers/eval/config-bundle/version/screen.tsx";
 import { ConfigBundleVersionListScreen } from "../handlers/eval/config-bundle/version/list/screen.tsx";
+import { AbTestScreen } from "../handlers/eval/ab-test/screen.tsx";
+import { AbTestListScreen } from "../handlers/eval/ab-test/list/screen.tsx";
+import { AbTestGetScreen, AbTestGetJsonScreen } from "../handlers/eval/ab-test/get/screen.tsx";
 import { MemoryEventScreen } from "../handlers/memory/event/screen.tsx";
 import { MemoryEventGetScreen } from "../handlers/memory/event/get/screen.tsx";
 import { MemoryEventListScreen } from "../handlers/memory/event/list/screen.tsx";
@@ -552,6 +555,23 @@ export function Root({ path, ctx, core, queryClient }: RootProps) {
  path="agentcore/eval/batch-evaluation/get/:batchEvaluationId"
  element={<BatchEvaluationGetJsonScreen ctx={ctx} core={core} />}
  />
+ <Route path="agentcore/eval/ab-test" element={<AbTestScreen ctx={ctx} core={core} />} />
+ <Route
+ path="agentcore/eval/ab-test/list"
+ element={<AbTestListScreen ctx={ctx} core={core} />}
+ />
+ <Route
+ path="agentcore/eval/ab-test/get"
+ element={<Navigate to="/agentcore/eval/ab-test/list" replace />}
+ />
+ <Route
+ path="agentcore/eval/ab-test/get/:abTestId"
+ element={<AbTestGetScreen ctx={ctx} core={core} />}
+ />
+ <Route
+ path="agentcore/eval/ab-test/get/:abTestId/json"
+ element={<AbTestGetJsonScreen ctx={ctx} core={core} />}
+ />
  <Route
  path="agentcore/eval/batch-insights"
  element={<BatchInsightsScreen ctx={ctx} core={core} />}

src/core/eval.tsx

@@ -59,13 +59,22 @@ import {
 } from "@aws-sdk/client-bedrock-agentcore-control";
 import {
  EvaluateCommand,
+ GetABTestCommand,
+ ListABTestsCommand,
+ UpdateABTestCommand,
+ DeleteABTestCommand,
  GetBatchEvaluationCommand,
  ListBatchEvaluationsCommand,
  StartBatchEvaluationCommand,
  type EvaluationReferenceInput,
  type EvaluationResultContent,
  type EvaluationTarget,
  type BatchEvaluationSummary,
+ type GetABTestResponse,
+ type ListABTestsResponse,
+ type ABTestExecutionStatus,
+ type UpdateABTestResponse,
+ type DeleteABTestResponse,
  type ListBatchEvaluationsResponse,
  type StartBatchEvaluationResponse,
  type DataSourceConfig as DataPlaneDataSourceConfig,
@@ -385,6 +394,36 @@ export class EvalClient implements CoreEvalClient {
  .send(new ListBatchEvaluationsCommand({ nextToken, maxResults }));
  }
 
+ async getABTest(id: string, options: CoreOptions): Promise<GetABTestResponse> {
+ return this.clients.data(toClientConfig(options)).send(new GetABTestCommand({ abTestId: id }));
+ }
+
+ async listABTests(
+ nextToken: string | undefined,
+ maxResults: number | undefined,
+ options: CoreOptions,
+ ): Promise<ListABTestsResponse> {
+ return this.clients
+ .data(toClientConfig(options))
+ .send(new ListABTestsCommand({ nextToken, maxResults }));
+ }
+
+ async setABTestExecutionStatus(
+ id: string,
+ executionStatus: ABTestExecutionStatus,
+ options: CoreOptions,
+ ): Promise<UpdateABTestResponse> {
+ return this.clients
+ .data(toClientConfig(options))
+ .send(new UpdateABTestCommand({ abTestId: id, executionStatus }));
+ }
+
+ async deleteABTest(id: string, options: CoreOptions): Promise<DeleteABTestResponse> {
+ return this.clients
+ .data(toClientConfig(options))
+ .send(new DeleteABTestCommand({ abTestId: id }));
+ }
+
  async listBatchInsights(
  nextToken: string | undefined,
  maxResults: number | undefined,

src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.856b449fac91d3fc.json

@@ -0,0 +1,88 @@
+{
+ "abTestId": "abvfylatest_abtargettest-a5f5674e07",
+ "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07",
+ "name": "abvfylatest_abtargettest",
+ "status": "ACTIVE",
+ "executionStatus": "STOPPED",
+ "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi",
+ "variants": [
+ {
+ "name": "C",
+ "weight": 50,
+ "variantConfiguration": {
+ "target": {
+ "name": "prod"
+ }
+ }
+ },
+ {
+ "name": "T1",
+ "weight": 50,
+ "variantConfiguration": {
+ "target": {
+ "name": "staging"
+ }
+ }
+ }
+ ],
+ "evaluationConfig": {
+ "perVariantOnlineEvaluationConfig": [
+ {
+ "name": "C",
+ "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG"
+ },
+ {
+ "name": "T1",
+ "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9"
+ }
+ ]
+ },
+ "createdAt": {
+ "$date": "2026-06-17T22:10:40.198Z"
+ },
+ "updatedAt": {
+ "$date": "2026-07-01T22:11:16.000Z"
+ },
+ "description": "0.20.0 target-based AB",
+ "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-ABTestABTargetTest-111a51f2",
+ "currentRunId": "c9e913a7-9ee2-48fe-9a6b-820f7db1662e",
+ "startedAt": {
+ "$date": "2026-06-17T22:10:43.667Z"
+ },
+ "stoppedAt": {
+ "$date": "2026-07-01T22:11:16.749Z"
+ },
+ "maxDurationExpiresAt": {
+ "$date": "2026-07-01T22:10:43.667Z"
+ },
+ "results": {
+ "evaluatorMetrics": [
+ {
+ "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+ "controlStats": {
+ "variantName": "C",
+ "sampleSize": 4,
+ "mean": 0.6275000000000001
+ },
+ "variantResults": [
+ {
+ "variantName": "T1",
+ "sampleSize": 8,
+ "mean": 0.6475000000000001,
+ "isSignificant": false,
+ "absoluteChange": 0.020000000000000018,
+ "percentChange": 3.1872509960159388,
+ "pValue": 0.9687271479378647,
+ "confidenceInterval": {
+ "lower": -0.1078568731042645,
+ "upper": 0.14785687310426454
+ }
+ }
+ ]
+ }
+ ],
+ "analysisTimestamp": {
+ "$date": "2026-06-17T22:36:26.738Z"
+ }
+ }
+}

src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.dffd183fd711c7e0.json

@@ -0,0 +1,6 @@
+{
+ "$error": {
+ "name": "ResourceNotFoundException",
+ "message": "AB test not found: abTestId=missing-abtest-0000000000"
+ }
+}

src/handlers/eval/ab-test/__fixtures__/ListABTestsCommand.c9327252700186eb.json

@@ -0,0 +1,49 @@
+{
+ "abTests": [
+ {
+ "abTestId": "abtestval_cs_3p_abtest-fd3ed89cb0",
+ "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abtestval_cs_3p_abtest-fd3ed89cb0",
+ "name": "abtestval_cs_3p_abtest",
+ "status": "ACTIVE",
+ "executionStatus": "STOPPED",
+ "createdAt": {
+ "$date": "2026-06-24T23:02:52.192Z"
+ },
+ "updatedAt": {
+ "$date": "2026-07-08T23:03:42.000Z"
+ },
+ "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abtestval-cs-3p-abtest-gw-pywgl6xbpy"
+ },
+ {
+ "abTestId": "abvfylatest_abtargettest-a5f5674e07",
+ "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07",
+ "name": "abvfylatest_abtargettest",
+ "status": "ACTIVE",
+ "executionStatus": "STOPPED",
+ "createdAt": {
+ "$date": "2026-06-17T22:10:40.198Z"
+ },
+ "updatedAt": {
+ "$date": "2026-07-01T22:11:16.000Z"
+ },
+ "description": "0.20.0 target-based AB",
+ "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi"
+ },
+ {
+ "abTestId": "abvfyprerelease_abtargettest-3d4c25ff6c",
+ "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfyprerelease_abtargettest-3d4c25ff6c",
+ "name": "abvfyprerelease_abtargettest",
+ "status": "ACTIVE",
+ "executionStatus": "STOPPED",
+ "createdAt": {
+ "$date": "2026-06-17T20:29:28.252Z"
+ },
+ "updatedAt": {
+ "$date": "2026-07-01T20:30:07.000Z"
+ },
+ "description": "Prerelease target-based AB: prod vs staging",
+ "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfyprerelease-abgateway-oetvoualer"
+ }
+ ],
+ "nextToken": "eyJhd3NBY2NvdW50SWQiOiB7IlMiOiAiNzI1NDc2OTY0OTE3In0sICJhYlRlc3RJZCI6IHsiUyI6ICJhYnZmeXByZXJlbGVhc2VfYWJ0YXJnZXR0ZXN0LTNkNGMyNWZmNmMifSwgInVwZGF0ZWRBdCI6IHsiUyI6ICIyMDI2LTA3LTAxVDIwOjMwOjA3WiJ9fQ=="
+}

src/handlers/eval/ab-test/__fixtures__/get.golden.json

@@ -0,0 +1,76 @@
+{
+ "abTestId": "abvfylatest_abtargettest-a5f5674e07",
+ "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07",
+ "name": "abvfylatest_abtargettest",
+ "status": "ACTIVE",
+ "executionStatus": "STOPPED",
+ "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi",
+ "variants": [
+ {
+ "name": "C",
+ "weight": 50,
+ "variantConfiguration": {
+ "target": {
+ "name": "prod"
+ }
+ }
+ },
+ {
+ "name": "T1",
+ "weight": 50,
+ "variantConfiguration": {
+ "target": {
+ "name": "staging"
+ }
+ }
+ }
+ ],
+ "evaluationConfig": {
+ "perVariantOnlineEvaluationConfig": [
+ {
+ "name": "C",
+ "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG"
+ },
+ {
+ "name": "T1",
+ "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9"
+ }
+ ]
+ },
+ "createdAt": "2026-06-17T22:10:40.198Z",
+ "updatedAt": "2026-07-01T22:11:16.000Z",
+ "description": "0.20.0 target-based AB",
+ "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-ABTestABTargetTest-111a51f2",
+ "currentRunId": "c9e913a7-9ee2-48fe-9a6b-820f7db1662e",
+ "startedAt": "2026-06-17T22:10:43.667Z",
+ "stoppedAt": "2026-07-01T22:11:16.749Z",
+ "maxDurationExpiresAt": "2026-07-01T22:10:43.667Z",
+ "results": {
+ "evaluatorMetrics": [
+ {
+ "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+ "controlStats": {
+ "variantName": "C",
+ "sampleSize": 4,
+ "mean": 0.6275000000000001
+ },
+ "variantResults": [
+ {
+ "variantName": "T1",
+ "sampleSize": 8,
+ "mean": 0.6475000000000001,
+ "isSignificant": false,
+ "absoluteChange": 0.020000000000000018,
+ "percentChange": 3.1872509960159388,
+ "pValue": 0.9687271479378647,
+ "confidenceInterval": {
+ "lower": -0.1078568731042645,
+ "upper": 0.14785687310426454
+ }
+ }
+ ]
+ }
+ ],
+ "analysisTimestamp": "2026-06-17T22:36:26.738Z"
+ }
+}