diff --git a/src/components/AbTestPicker.tsx b/src/components/AbTestPicker.tsx new file mode 100644 index 000000000..6e06e716e --- /dev/null +++ b/src/components/AbTestPicker.tsx @@ -0,0 +1,74 @@ +import type { ABTestSummary } from "@aws-sdk/client-bedrock-agentcore"; +import { useNavigate } from "react-router"; +import type { ScreenProps } from "../handlers/types"; +import { coreOptsFromCtx } from "../handlers/utils"; +import { formatTimestamp } from "./formatTimestamp"; +import { PaginatedTablePicker } from "./PaginatedTablePicker"; +import type { DataTableColumn } from "./ui/data-table"; + +interface AbTestRow extends Record { + abTestId: string; + name: string; + status: string; + executionStatus: string; + updatedAt: string; +} + +export const abTestColumns = [ + { key: "name", header: "name", flex: true }, + { key: "status", header: "status", width: 14 }, + { key: "executionStatus", header: "execution", width: 12 }, + { key: "updatedAt", header: "updated UTC", width: 16, render: formatTimestamp }, +] satisfies DataTableColumn[]; + +function toRow(summary: ABTestSummary): AbTestRow { + const id = summary.abTestId ?? ""; + return { + abTestId: id, + name: summary.name ?? id, + status: summary.status ?? "-", + executionStatus: summary.executionStatus ?? "-", + updatedAt: summary.updatedAt?.toISOString() ?? "-", + }; +} + +export interface AbTestPickerProps extends ScreenProps { + breadcrumb: string[]; + description?: string; + onSelect: (abTestId: string) => void; + onEscape?: () => void; +} + +export function AbTestPicker({ + ctx, + core, + breadcrumb, + description, + onSelect, + onEscape, +}: AbTestPickerProps) { + const opts = coreOptsFromCtx(ctx); + const navigate = useNavigate(); + const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/"))); + + return ( + { + const response = await core.eval.listABTests(token, pageSize, opts); + return { items: response.abTests ?? [], nextToken: response.nextToken }; + }} + toRow={toRow} + columns={abTestColumns} + getValue={(row) => row.abTestId} + onSelect={onSelect} + onBack={goBack} + loadingMessage="Loading A/B tests…" + errorMessage={(error) => `Error: ${error.message}`} + emptyMessage="No A/B tests found in this Region." + emptyPageMessage="No A/B tests on this page." + /> + ); +} diff --git a/src/components/Root.tsx b/src/components/Root.tsx index 2b876a958..da3ce473f 100644 --- a/src/components/Root.tsx +++ b/src/components/Root.tsx @@ -62,6 +62,9 @@ import { ConfigBundleListScreen } from "../handlers/eval/config-bundle/list/scre import { ConfigBundleGetScreen } from "../handlers/eval/config-bundle/get/screen.tsx"; import { ConfigBundleVersionScreen } from "../handlers/eval/config-bundle/version/screen.tsx"; import { ConfigBundleVersionListScreen } from "../handlers/eval/config-bundle/version/list/screen.tsx"; +import { AbTestScreen } from "../handlers/eval/ab-test/screen.tsx"; +import { AbTestListScreen } from "../handlers/eval/ab-test/list/screen.tsx"; +import { AbTestGetScreen, AbTestGetJsonScreen } from "../handlers/eval/ab-test/get/screen.tsx"; import { MemoryEventScreen } from "../handlers/memory/event/screen.tsx"; import { MemoryEventGetScreen } from "../handlers/memory/event/get/screen.tsx"; import { MemoryEventListScreen } from "../handlers/memory/event/list/screen.tsx"; @@ -552,6 +555,23 @@ export function Root({ path, ctx, core, queryClient }: RootProps) { path="agentcore/eval/batch-evaluation/get/:batchEvaluationId" element={} /> + } /> + } + /> + } + /> + } + /> + } + /> } diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 849054732..aac8f8618 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -59,6 +59,10 @@ import { } from "@aws-sdk/client-bedrock-agentcore-control"; import { EvaluateCommand, + GetABTestCommand, + ListABTestsCommand, + UpdateABTestCommand, + DeleteABTestCommand, GetBatchEvaluationCommand, ListBatchEvaluationsCommand, StartBatchEvaluationCommand, @@ -66,6 +70,11 @@ import { type EvaluationResultContent, type EvaluationTarget, type BatchEvaluationSummary, + type GetABTestResponse, + type ListABTestsResponse, + type ABTestExecutionStatus, + type UpdateABTestResponse, + type DeleteABTestResponse, type ListBatchEvaluationsResponse, type StartBatchEvaluationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, @@ -385,6 +394,36 @@ export class EvalClient implements CoreEvalClient { .send(new ListBatchEvaluationsCommand({ nextToken, maxResults })); } + async getABTest(id: string, options: CoreOptions): Promise { + return this.clients.data(toClientConfig(options)).send(new GetABTestCommand({ abTestId: id })); + } + + async listABTests( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new ListABTestsCommand({ nextToken, maxResults })); + } + + async setABTestExecutionStatus( + id: string, + executionStatus: ABTestExecutionStatus, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new UpdateABTestCommand({ abTestId: id, executionStatus })); + } + + async deleteABTest(id: string, options: CoreOptions): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new DeleteABTestCommand({ abTestId: id })); + } + async listBatchInsights( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.856b449fac91d3fc.json b/src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.856b449fac91d3fc.json new file mode 100644 index 000000000..806da3b8a --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.856b449fac91d3fc.json @@ -0,0 +1,88 @@ +{ + "abTestId": "abvfylatest_abtargettest-a5f5674e07", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07", + "name": "abvfylatest_abtargettest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi", + "variants": [ + { + "name": "C", + "weight": 50, + "variantConfiguration": { + "target": { + "name": "prod" + } + } + }, + { + "name": "T1", + "weight": 50, + "variantConfiguration": { + "target": { + "name": "staging" + } + } + } + ], + "evaluationConfig": { + "perVariantOnlineEvaluationConfig": [ + { + "name": "C", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG" + }, + { + "name": "T1", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9" + } + ] + }, + "createdAt": { + "$date": "2026-06-17T22:10:40.198Z" + }, + "updatedAt": { + "$date": "2026-07-01T22:11:16.000Z" + }, + "description": "0.20.0 target-based AB", + "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-ABTestABTargetTest-111a51f2", + "currentRunId": "c9e913a7-9ee2-48fe-9a6b-820f7db1662e", + "startedAt": { + "$date": "2026-06-17T22:10:43.667Z" + }, + "stoppedAt": { + "$date": "2026-07-01T22:11:16.749Z" + }, + "maxDurationExpiresAt": { + "$date": "2026-07-01T22:10:43.667Z" + }, + "results": { + "evaluatorMetrics": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "controlStats": { + "variantName": "C", + "sampleSize": 4, + "mean": 0.6275000000000001 + }, + "variantResults": [ + { + "variantName": "T1", + "sampleSize": 8, + "mean": 0.6475000000000001, + "isSignificant": false, + "absoluteChange": 0.020000000000000018, + "percentChange": 3.1872509960159388, + "pValue": 0.9687271479378647, + "confidenceInterval": { + "lower": -0.1078568731042645, + "upper": 0.14785687310426454 + } + } + ] + } + ], + "analysisTimestamp": { + "$date": "2026-06-17T22:36:26.738Z" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.dffd183fd711c7e0.json b/src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.dffd183fd711c7e0.json new file mode 100644 index 000000000..9a8caed0b --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetABTestCommand.dffd183fd711c7e0.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "AB test not found: abTestId=missing-abtest-0000000000" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/ListABTestsCommand.c9327252700186eb.json b/src/handlers/eval/ab-test/__fixtures__/ListABTestsCommand.c9327252700186eb.json new file mode 100644 index 000000000..a7e904ee0 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/ListABTestsCommand.c9327252700186eb.json @@ -0,0 +1,49 @@ +{ + "abTests": [ + { + "abTestId": "abtestval_cs_3p_abtest-fd3ed89cb0", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abtestval_cs_3p_abtest-fd3ed89cb0", + "name": "abtestval_cs_3p_abtest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "createdAt": { + "$date": "2026-06-24T23:02:52.192Z" + }, + "updatedAt": { + "$date": "2026-07-08T23:03:42.000Z" + }, + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abtestval-cs-3p-abtest-gw-pywgl6xbpy" + }, + { + "abTestId": "abvfylatest_abtargettest-a5f5674e07", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07", + "name": "abvfylatest_abtargettest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "createdAt": { + "$date": "2026-06-17T22:10:40.198Z" + }, + "updatedAt": { + "$date": "2026-07-01T22:11:16.000Z" + }, + "description": "0.20.0 target-based AB", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi" + }, + { + "abTestId": "abvfyprerelease_abtargettest-3d4c25ff6c", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfyprerelease_abtargettest-3d4c25ff6c", + "name": "abvfyprerelease_abtargettest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "createdAt": { + "$date": "2026-06-17T20:29:28.252Z" + }, + "updatedAt": { + "$date": "2026-07-01T20:30:07.000Z" + }, + "description": "Prerelease target-based AB: prod vs staging", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfyprerelease-abgateway-oetvoualer" + } + ], + "nextToken": "eyJhd3NBY2NvdW50SWQiOiB7IlMiOiAiNzI1NDc2OTY0OTE3In0sICJhYlRlc3RJZCI6IHsiUyI6ICJhYnZmeXByZXJlbGVhc2VfYWJ0YXJnZXR0ZXN0LTNkNGMyNWZmNmMifSwgInVwZGF0ZWRBdCI6IHsiUyI6ICIyMDI2LTA3LTAxVDIwOjMwOjA3WiJ9fQ==" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/get.golden.json b/src/handlers/eval/ab-test/__fixtures__/get.golden.json new file mode 100644 index 000000000..cf40553d8 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/get.golden.json @@ -0,0 +1,76 @@ +{ + "abTestId": "abvfylatest_abtargettest-a5f5674e07", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07", + "name": "abvfylatest_abtargettest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi", + "variants": [ + { + "name": "C", + "weight": 50, + "variantConfiguration": { + "target": { + "name": "prod" + } + } + }, + { + "name": "T1", + "weight": 50, + "variantConfiguration": { + "target": { + "name": "staging" + } + } + } + ], + "evaluationConfig": { + "perVariantOnlineEvaluationConfig": [ + { + "name": "C", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG" + }, + { + "name": "T1", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9" + } + ] + }, + "createdAt": "2026-06-17T22:10:40.198Z", + "updatedAt": "2026-07-01T22:11:16.000Z", + "description": "0.20.0 target-based AB", + "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-ABTestABTargetTest-111a51f2", + "currentRunId": "c9e913a7-9ee2-48fe-9a6b-820f7db1662e", + "startedAt": "2026-06-17T22:10:43.667Z", + "stoppedAt": "2026-07-01T22:11:16.749Z", + "maxDurationExpiresAt": "2026-07-01T22:10:43.667Z", + "results": { + "evaluatorMetrics": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "controlStats": { + "variantName": "C", + "sampleSize": 4, + "mean": 0.6275000000000001 + }, + "variantResults": [ + { + "variantName": "T1", + "sampleSize": 8, + "mean": 0.6475000000000001, + "isSignificant": false, + "absoluteChange": 0.020000000000000018, + "percentChange": 3.1872509960159388, + "pValue": 0.9687271479378647, + "confidenceInterval": { + "lower": -0.1078568731042645, + "upper": 0.14785687310426454 + } + } + ] + } + ], + "analysisTimestamp": "2026-06-17T22:36:26.738Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/list.golden.json b/src/handlers/eval/ab-test/__fixtures__/list.golden.json new file mode 100644 index 000000000..91933bf8f --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/list.golden.json @@ -0,0 +1,37 @@ +{ + "abTests": [ + { + "abTestId": "abtestval_cs_3p_abtest-fd3ed89cb0", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abtestval_cs_3p_abtest-fd3ed89cb0", + "name": "abtestval_cs_3p_abtest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "createdAt": "2026-06-24T23:02:52.192Z", + "updatedAt": "2026-07-08T23:03:42.000Z", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abtestval-cs-3p-abtest-gw-pywgl6xbpy" + }, + { + "abTestId": "abvfylatest_abtargettest-a5f5674e07", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfylatest_abtargettest-a5f5674e07", + "name": "abvfylatest_abtargettest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "createdAt": "2026-06-17T22:10:40.198Z", + "updatedAt": "2026-07-01T22:11:16.000Z", + "description": "0.20.0 target-based AB", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfylatest-abgateway-t4w4fdbovi" + }, + { + "abTestId": "abvfyprerelease_abtargettest-3d4c25ff6c", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:ab-test/abvfyprerelease_abtargettest-3d4c25ff6c", + "name": "abvfyprerelease_abtargettest", + "status": "ACTIVE", + "executionStatus": "STOPPED", + "createdAt": "2026-06-17T20:29:28.252Z", + "updatedAt": "2026-07-01T20:30:07.000Z", + "description": "Prerelease target-based AB: prod vs staging", + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:gateway/abvfyprerelease-abgateway-oetvoualer" + } + ], + "nextToken": "eyJhd3NBY2NvdW50SWQiOiB7IlMiOiAiNzI1NDc2OTY0OTE3In0sICJhYlRlc3RJZCI6IHsiUyI6ICJhYnZmeXByZXJlbGVhc2VfYWJ0YXJnZXR0ZXN0LTNkNGMyNWZmNmMifSwgInVwZGF0ZWRBdCI6IHsiUyI6ICIyMDI2LTA3LTAxVDIwOjMwOjA3WiJ9fQ==" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx new file mode 100644 index 000000000..9f8bb97fe --- /dev/null +++ b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with: RECORD=1 bun test src/handlers/eval/ab-test/ab-test.fixture.test.tsx +// +// A/B tests are READ-ONLY here, so — like the batch-evaluation fixture suite — +// this pins pre-existing tests in the fixture account rather than creating one. +// Re-recording requires these ids to still exist; repoint them if they age out. +// +// Exercises the real seam end to end: parsing → handler → CoreClient → +// GetABTest / ListABTest (data plane). GetABTest returns the per-evaluator +// statistical results inline, so there is no CloudWatch seam to record. +const FIXTURE_ABTEST_ID = "abvfylatest_abtargettest-a5f5674e07"; + +// A well-formed but absent id, to reach the not-found path. +const MISSING_ABTEST_ID = "missing-abtest-0000000000"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval ab-test (fixture-backed)", () => { + test("get returns the test with per-evaluator metrics inline", async () => { + const stdout = await run(["eval", "ab-test", "get", "--id", FIXTURE_ABTEST_ID, "--json"]); + + matchGolden(FIXTURES, "get.golden.json", stdout); + const detail = JSON.parse(stdout); + expect(detail.abTestId).toBe(FIXTURE_ABTEST_ID); + expect(detail.status).toBeTruthy(); + expect(detail.executionStatus).toBeTruthy(); + expect(Array.isArray(detail.results.evaluatorMetrics)).toBe(true); + }); + + test("list returns the service page", async () => { + const stdout = await run(["eval", "ab-test", "list", "--max-results", "3", "--json"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(Array.isArray(JSON.parse(stdout).abTests)).toBe(true); + }); + + test("get surfaces a not-found error for an absent test", async () => { + await expect( + run(["eval", "ab-test", "get", "--id", MISSING_ABTEST_ID, "--json"]), + ).rejects.toThrow(); + }); +}); diff --git a/src/handlers/eval/ab-test/ab-test.screen.test.tsx b/src/handlers/eval/ab-test/ab-test.screen.test.tsx new file mode 100644 index 000000000..6aa3923bb --- /dev/null +++ b/src/handlers/eval/ab-test/ab-test.screen.test.tsx @@ -0,0 +1,199 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import type { GetABTestResponse, ABTestSummary } from "@aws-sdk/client-bedrock-agentcore"; +import { + cleanupScreens, + renderScreen, + TestCoreClient, + waitFor, + waitForText, +} from "../../../testing"; + +afterEach(cleanupScreens); + +function summary(overrides: Partial = {}): ABTestSummary { + return { + abTestId: "ab-test-1", + abTestArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:ab-test/ab-test-1", + name: "orders-v2", + status: "ACTIVE", + executionStatus: "RUNNING", + gatewayArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:gateway/orders", + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + createdAt: new Date("2026-07-19T01:02:03.000Z"), + ...overrides, + }; +} + +function getResponse(overrides: Partial = {}): GetABTestResponse { + return { + abTestId: "ab-test-1", + abTestArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:ab-test/ab-test-1", + name: "orders-v2", + status: "ACTIVE", + executionStatus: "RUNNING", + gatewayArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:gateway/orders", + variants: [ + { name: "C", weight: 50, variantConfiguration: { target: { name: "orders-prod-target" } } }, + { name: "T1", weight: 50, variantConfiguration: { target: { name: "orders-v2-target" } } }, + ], + evaluationConfig: { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/quality", + }, + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + results: { + evaluatorMetrics: [ + { + evaluatorArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:evaluator/quality", + controlStats: { variantName: "C", sampleSize: 100, mean: 0.8 }, + variantResults: [{ variantName: "T1", sampleSize: 100, mean: 0.86, isSignificant: true }], + }, + ], + }, + ...overrides, + } satisfies GetABTestResponse; +} + +function coreWithTests(tests: ABTestSummary[]): TestCoreClient { + const core = new TestCoreClient(); + core.eval.setAbTestListResponse({ abTests: tests }); + return core; +} + +describe("ab-test menu", () => { + test("lists only the read commands, not the write commands", async () => { + const r = renderScreen("/agentcore/eval/ab-test"); + + await waitForText(r.lastFrame, "list A/B tests"); + const frame = r.lastFrame()!; + expect(frame).toContain("get an A/B test by id"); + for (const write of [ + "pause a running A/B test", + "resume a paused A/B test", + "stop an A/B test", + "delete a stopped A/B test", + ]) { + expect(frame).not.toContain(write); + } + }); +}); + +describe("ab-test picker", () => { + test("renders columns, id, name, status, execution, and update time", async () => { + const core = coreWithTests([ + summary({ + abTestId: "orders-AbCdEf1234", + name: "orders", + status: "ACTIVE", + executionStatus: "STOPPED", + updatedAt: new Date("2026-07-19T01:02:03.000Z"), + }), + ]); + const r = renderScreen("/agentcore/eval/ab-test/list", { core }); + + await waitForText(r.lastFrame, "orders"); + const frame = r.lastFrame()!; + expect(frame).toContain("name"); + expect(frame).toContain("status"); + expect(frame).toContain("execution"); + expect(frame).toContain("updated UTC"); + expect(frame).toContain("STOPPED"); + expect(frame).toContain("2026-07-19 01:02"); + }); + + test("calls listABTests once with exact Core options", async () => { + const core = coreWithTests([summary()]); + renderScreen("/agentcore/eval/ab-test/list", { core }); + + await waitFor(() => core.eval.calls.some((call) => call.method === "listABTests")); + expect(core.eval.calls.filter((call) => call.method === "listABTests")).toEqual([ + { method: "listABTests", args: [undefined, expect.any(Number), { region: "us-east-1" }] }, + ]); + }); + + test("shows the first-page empty state", async () => { + const r = renderScreen("/agentcore/eval/ab-test/list"); + await waitForText(r.lastFrame, "No A/B tests found in this Region."); + }); + + test("bare get redirects to the picker", async () => { + const core = coreWithTests([summary({ name: "redirected" })]); + const r = renderScreen("/agentcore/eval/ab-test/get", { core }); + + await waitForText(r.lastFrame, "redirected"); + expect(core.eval.calls[0]?.method).toBe("listABTests"); + }); + + test("selection opens the matching A/B test hub", async () => { + const core = coreWithTests([summary({ abTestId: "ab-test-1", name: "encoded" })]); + core.eval.setAbTestGetResponse(getResponse()); + const r = renderScreen("/agentcore/eval/ab-test/list", { core }); + + await waitForText(r.lastFrame, "encoded"); + await r.press("return"); + await waitForText(r.lastFrame, "agentcore → eval → ab-test → get → ab-test-1"); + await waitFor(() => + core.eval.calls.some((call) => call.method === "getABTest" && call.args[0] === "ab-test-1"), + ); + }); +}); + +describe("ab-test hub", () => { + test("fetches the route id with exact Core options and renders its summary", async () => { + const core = new TestCoreClient(); + core.eval.setAbTestGetResponse(getResponse()); + const r = renderScreen("/agentcore/eval/ab-test/get/ab-test-1", { core }); + + await waitForText(r.lastFrame, "RUNNING"); + const frame = r.lastFrame()!; + expect(frame).toContain("ab-test-1"); + expect(frame).toContain("orders-v2"); + expect(frame).toMatch(/variants\s+C 50% \/ T1 50%/); + expect(frame).not.toContain("failureReason"); + expect(core.eval.calls.find((call) => call.method === "getABTest")).toEqual({ + method: "getABTest", + args: ["ab-test-1", { region: "us-east-1" }], + }); + }); + + test("shows the error details only when the service provides them", async () => { + const core = new TestCoreClient(); + core.eval.setAbTestGetResponse( + getResponse({ status: "CREATE_FAILED", errorDetails: ["gateway not deployed"] }), + ); + const r = renderScreen("/agentcore/eval/ab-test/get/ab-test-1", { core }); + + await waitForText(r.lastFrame, "gateway not deployed"); + expect(r.lastFrame()).toMatch(/errors\s+gateway not deployed/); + }); + + test("detail action opens the full JSON with inline metrics", async () => { + const core = new TestCoreClient(); + core.eval.setAbTestGetResponse(getResponse()); + const r = renderScreen("/agentcore/eval/ab-test/get/ab-test-1", { core }); + + await waitForText( + r.lastFrame, + "show the full JSON definition, including per-evaluator metrics", + ); + await r.press("return"); + await waitForText(r.lastFrame, "agentcore → eval → ab-test → get → ab-test-1 → json"); + expect(r.lastFrame()).toContain('"abTestId"'); + expect(r.lastFrame()).toContain('"evaluatorMetrics"'); + }); + + test("retries a failed hub query without leaving the route", async () => { + const core = new TestCoreClient(); + core.eval.setError(new Error("ab-test unavailable")); + const r = renderScreen("/agentcore/eval/ab-test/get/ab-test-1", { core }); + + await waitForText(r.lastFrame, "ab-test unavailable"); + expect(r.lastFrame()).toContain("[r] retry"); + + core.eval.setError(undefined); + core.eval.setAbTestGetResponse(getResponse()); + await r.write("r"); + await waitForText(r.lastFrame, "RUNNING"); + }); +}); diff --git a/src/handlers/eval/ab-test/ab-test.write.test.tsx b/src/handlers/eval/ab-test/ab-test.write.test.tsx new file mode 100644 index 000000000..31116a1a5 --- /dev/null +++ b/src/handlers/eval/ab-test/ab-test.write.test.tsx @@ -0,0 +1,88 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +describe("eval ab-test command hierarchy", () => { + test("registers get, list, pause, resume, stop, delete", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ab-test"); + expect(group?.children().map((c) => c.name())).toEqual([ + "get", + "list", + "pause", + "resume", + "stop", + "delete", + ]); + }); +}); + +describe("eval ab-test transitions", () => { + test.each([ + ["pause", "PAUSED"], + ["resume", "RUNNING"], + ["stop", "STOPPED"], + ] as const)("%s sets executionStatus %s via Core", async (command, status) => { + const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => + c.eval.setAbTestUpdateResponse({ + abTestId: "ab-test-1", + abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1", + status: "ACTIVE", + executionStatus: status, + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + }), + ); + expect(core.eval.calls).toEqual([ + { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] }, + ]); + }); + + test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => { + await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/); + }); +}); + +describe("eval ab-test delete", () => { + test("deletes by id via Core", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], + (c) => + c.eval.setAbTestDeleteResponse({ + abTestId: "ab-test-1", + abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1", + status: "DELETING", + }), + ); + expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); + expect(core.eval.calls).toEqual([ + { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] }, + ]); + }); + + test("requires --id", async () => { + await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/); + }); +}); diff --git a/src/handlers/eval/ab-test/delete/index.tsx b/src/handlers/eval/ab-test/delete/index.tsx new file mode 100644 index 000000000..d3b1ddca1 --- /dev/null +++ b/src/handlers/eval/ab-test/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteAbTestHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete a stopped A/B test", + flags: [flag("id", "the ID of the A/B test", z.string().optional())], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteABTest(id, coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/ab-test/get/index.tsx b/src/handlers/eval/ab-test/get/index.tsx new file mode 100644 index 000000000..39b402055 --- /dev/null +++ b/src/handlers/eval/ab-test/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetAbTestHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "get", + description: "get an A/B test by id, with per-evaluator comparison metrics", + flags: [flag("id", "the ID of the A/B test", z.string().optional())], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + const detail = await core.eval.getABTest(id, coreOptsFromCtx(ctx)); + ctx.require(JsonRendererKey).renderJson(detail); + }, + }); diff --git a/src/handlers/eval/ab-test/get/screen.tsx b/src/handlers/eval/ab-test/get/screen.tsx new file mode 100644 index 000000000..3db67ce37 --- /dev/null +++ b/src/handlers/eval/ab-test/get/screen.tsx @@ -0,0 +1,74 @@ +import { useQuery } from "@tanstack/react-query"; +import { useNavigate, useParams } from "react-router"; +import { JsonDetail } from "../../../../components/JsonDetail"; +import { ResourceDetailScreen } from "../../../../components/ResourceDetailScreen"; +import type { ScreenProps } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +function useAbTestDetail({ ctx, core }: ScreenProps, abTestId: string | undefined) { + const opts = coreOptsFromCtx(ctx); + return useQuery({ + queryKey: ["ab-test", opts.region, abTestId], + queryFn: () => core.eval.getABTest(abTestId!, opts), + enabled: abTestId !== undefined, + }); +} + +export function AbTestGetScreen(props: ScreenProps) { + const navigate = useNavigate(); + const { abTestId } = useParams(); + const detail = useAbTestDetail(props, abTestId); + const test = detail.data; + + const variants = (test?.variants ?? []) + .map((variant) => `${variant.name ?? "?"} ${variant.weight ?? 0}%`) + .join(" / "); + + return ( + + navigate(`/agentcore/eval/ab-test/get/${encodeURIComponent(abTestId)}/json`), + }, + ] + : [] + } + loadingLabel="Loading A/B test…" + onRetry={() => void detail.refetch()} + /> + ); +} + +export function AbTestGetJsonScreen(props: ScreenProps) { + const { abTestId } = useParams(); + const detail = useAbTestDetail(props, abTestId); + + return ( + void detail.refetch()} + /> + ); +} diff --git a/src/handlers/eval/ab-test/index.tsx b/src/handlers/eval/ab-test/index.tsx new file mode 100644 index 000000000..6bccf91fc --- /dev/null +++ b/src/handlers/eval/ab-test/index.tsx @@ -0,0 +1,26 @@ +import { Router } from "../../../router"; +import { renderTui } from "../../../tui"; +import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createGetAbTestHandler } from "./get"; +import { createListAbTestsHandler } from "./list"; +import { createPauseAbTestHandler } from "./pause"; +import { createResumeAbTestHandler } from "./resume"; +import { createStopAbTestHandler } from "./stop"; +import { createDeleteAbTestHandler } from "./delete"; + +export function createAbTestHandler(core: Core, io: AppIO): Router { + return new Router("ab-test", "inspect AgentCore A/B tests") + .use(withTuiOnEmptyFlagsAndArgs(core, io)) + .default(renderTui(core, io)) + .supportedTuiCommands("get", "list") + .handler(createGetAbTestHandler(core, io)) + .handler(createListAbTestsHandler(core)) + .handler(createPauseAbTestHandler(core)) + .handler(createResumeAbTestHandler(core)) + .handler(createStopAbTestHandler(core)) + .handler(createDeleteAbTestHandler(core)); +} + +export { AbTestScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/ab-test/list/index.tsx b/src/handlers/eval/ab-test/list/index.tsx new file mode 100644 index 000000000..0937ac76c --- /dev/null +++ b/src/handlers/eval/ab-test/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListAbTestsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list A/B tests", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listABTests( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/ab-test/list/screen.tsx b/src/handlers/eval/ab-test/list/screen.tsx new file mode 100644 index 000000000..31899c045 --- /dev/null +++ b/src/handlers/eval/ab-test/list/screen.tsx @@ -0,0 +1,17 @@ +import { useNavigate } from "react-router"; +import { AbTestPicker } from "../../../../components/AbTestPicker"; +import type { ScreenProps } from "../../../types"; + +export function AbTestListScreen(props: ScreenProps) { + const navigate = useNavigate(); + + return ( + + navigate(`/agentcore/eval/ab-test/get/${encodeURIComponent(abTestId)}`) + } + /> + ); +} diff --git a/src/handlers/eval/ab-test/pause/index.tsx b/src/handlers/eval/ab-test/pause/index.tsx new file mode 100644 index 000000000..82bf51f71 --- /dev/null +++ b/src/handlers/eval/ab-test/pause/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createPauseAbTestHandler = (core: Core) => + createHandler({ + name: "pause", + description: "pause a running A/B test", + flags: [flag("id", "the ID of the A/B test", z.string().optional())], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.setABTestExecutionStatus(id, "PAUSED", coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/ab-test/resume/index.tsx b/src/handlers/eval/ab-test/resume/index.tsx new file mode 100644 index 000000000..ac7f5be11 --- /dev/null +++ b/src/handlers/eval/ab-test/resume/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createResumeAbTestHandler = (core: Core) => + createHandler({ + name: "resume", + description: "resume a paused A/B test", + flags: [flag("id", "the ID of the A/B test", z.string().optional())], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.setABTestExecutionStatus(id, "RUNNING", coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/ab-test/screen.tsx b/src/handlers/eval/ab-test/screen.tsx new file mode 100644 index 000000000..278f42ee2 --- /dev/null +++ b/src/handlers/eval/ab-test/screen.tsx @@ -0,0 +1,6 @@ +import { RouterScreen } from "../../../components/RouterScreen"; +import type { ScreenProps } from "../../types"; + +export function AbTestScreen(props: ScreenProps) { + return ; +} diff --git a/src/handlers/eval/ab-test/stop/index.tsx b/src/handlers/eval/ab-test/stop/index.tsx new file mode 100644 index 000000000..a7a2af304 --- /dev/null +++ b/src/handlers/eval/ab-test/stop/index.tsx @@ -0,0 +1,24 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createStopAbTestHandler = (core: Core) => + createHandler({ + name: "stop", + description: "stop an A/B test (terminal)", + flags: [flag("id", "the ID of the A/B test", z.string().optional())], + handle: async (ctx, flags) => { + const id = flags["id"]; + if (!id) throw new InputValidationError("required option '--id ' not specified"); + // TODO: after stopping, print a suggested (never executed) update-gateway-rule + // command routing production traffic to the treatment. Shape depends on mode + // (config-bundle: swap bundleVersion; target-based: routeToTarget); fall back to + // create-gateway-rule when no prod rule exists, or list candidates when ambiguous. + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.setABTestExecutionStatus(id, "STOPPED", coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 2e41a78f5..4db27094a 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -10,6 +10,7 @@ import { createBatchEvaluationHandler } from "./batch-evaluation"; import { createBatchInsightsHandler } from "./batch-insights"; import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; +import { createAbTestHandler } from "./ab-test"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -21,7 +22,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createBatchEvaluationHandler(core, io)) .handler(createBatchInsightsHandler(core, io)) .handler(createOnDemandHandler(core, io)) - .handler(createConfigBundleHandler(core, io)); + .handler(createConfigBundleHandler(core, io)) + .handler(createAbTestHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index f34d4a081..72cfc0832 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -30,6 +30,11 @@ import type { UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { + GetABTestResponse, + ListABTestsResponse, + ABTestExecutionStatus, + UpdateABTestResponse, + DeleteABTestResponse, GetBatchEvaluationResponse, ListBatchEvaluationsResponse, StartBatchEvaluationResponse, @@ -344,6 +349,19 @@ export interface CoreEvalClient { maxResults: number | undefined, options: CoreOptions, ): Promise; + + getABTest(id: string, options: CoreOptions): Promise; + listABTests( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + setABTestExecutionStatus( + id: string, + executionStatus: ABTestExecutionStatus, + options: CoreOptions, + ): Promise; + deleteABTest(id: string, options: CoreOptions): Promise; // startBatchEvaluation submits an async, service-side evaluation over sessions // the service gathers from the resolved data source. Returns the durable job id // + RUNNING status; poll with getBatchEvaluation. diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index b835ded7f..bb4061942 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -73,6 +73,11 @@ import type { UpdateHarnessResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { + GetABTestResponse, + ListABTestsResponse, + ABTestExecutionStatus, + UpdateABTestResponse, + DeleteABTestResponse, GetBatchEvaluationResponse, GetEventInput, GetEventOutput, @@ -264,6 +269,10 @@ const DEFAULT_DELETE_DATASET_RESPONSE = {} as DeleteDatasetResponse; const DEFAULT_PUBLISH_DATASET_RESPONSE = {} as CreateDatasetVersionResponse; const DEFAULT_GET_BATCH_EVAL_RESPONSE = {} as GetBatchEvaluationResponse; const DEFAULT_LIST_BATCH_EVALS_RESPONSE: ListBatchEvaluationsResponse = { batchEvaluations: [] }; +const DEFAULT_GET_ABTEST_RESPONSE = {} as GetABTestResponse; +const DEFAULT_LIST_ABTESTS_RESPONSE: ListABTestsResponse = { abTests: [] }; +const DEFAULT_UPDATE_ABTEST_RESPONSE = {} as UpdateABTestResponse; +const DEFAULT_DELETE_ABTEST_RESPONSE = {} as DeleteABTestResponse; const DEFAULT_START_BATCH_EVAL_RESPONSE = { batchEvaluationId: "batch-eval-test", status: "RUNNING", @@ -1403,6 +1412,10 @@ export class TestEvalClient implements CoreEvalClient { private batchEvalGetResponse: GetBatchEvaluationResponse = DEFAULT_GET_BATCH_EVAL_RESPONSE; private batchEvalListResponses = new Map(); private batchInsightsListResponses = new Map(); + private abTestGetResponse: GetABTestResponse = DEFAULT_GET_ABTEST_RESPONSE; + private abTestListResponses = new Map(); + private abTestUpdateResponse: UpdateABTestResponse = DEFAULT_UPDATE_ABTEST_RESPONSE; + private abTestDeleteResponse: DeleteABTestResponse = DEFAULT_DELETE_ABTEST_RESPONSE; private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; @@ -1606,6 +1619,26 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setAbTestGetResponse(response: GetABTestResponse): this { + this.abTestGetResponse = response; + return this; + } + + setAbTestListResponse(response: ListABTestsResponse, forNextToken?: string): this { + this.abTestListResponses.set(forNextToken, response); + return this; + } + + setAbTestUpdateResponse(response: UpdateABTestResponse): this { + this.abTestUpdateResponse = response; + return this; + } + + setAbTestDeleteResponse(response: DeleteABTestResponse): this { + this.abTestDeleteResponse = response; + return this; + } + // setUpdateDatasetResult sets what updateDatasetExamples resolves to (when not // erroring). setUpdateDatasetResult(result: DatasetUpdateResult): this { @@ -1737,6 +1770,42 @@ export class TestEvalClient implements CoreEvalClient { return { ...response, batchEvaluations: response.batchEvaluations ?? [] }; } + async getABTest(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getABTest", args: [id, options] }); + if (this.error) throw this.error; + return this.abTestGetResponse; + } + + async listABTests( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listABTests", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + return ( + this.abTestListResponses.get(nextToken) ?? + this.abTestListResponses.get(undefined) ?? + DEFAULT_LIST_ABTESTS_RESPONSE + ); + } + + async setABTestExecutionStatus( + id: string, + executionStatus: ABTestExecutionStatus, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "setABTestExecutionStatus", args: [id, executionStatus, options] }); + if (this.error) throw this.error; + return this.abTestUpdateResponse; + } + + async deleteABTest(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "deleteABTest", args: [id, options] }); + if (this.error) throw this.error; + return this.abTestDeleteResponse; + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions,