feat(eval): ab-test target-based run (create)

src/core/eval.tsx

@@ -76,6 +76,7 @@ import {
   type EvaluationReferenceInput,
   type EvaluationResultContent,
   type EvaluationTarget,
+  type CreateABTestRequest,
   type CreateABTestResponse,
   type GetABTestResponse,
   type ListABTestsResponse,
@@ -123,6 +124,7 @@ import type {
   CoreEvalClient,
   CreateConfigurationBundleInput,
   CreateConfigBundleABTestInput,
+  CreateTargetBasedABTestInput,
   CreateDatasetInput,
   CreateOnlineEvalInput,
   CreateOnlineInsightInput,
@@ -481,65 +483,38 @@ export class EvalClient implements CoreEvalClient {
       .send(new DeleteABTestCommand({ abTestId: id }));
   }
 
-  async createConfigBundleABTest(
-    input: CreateConfigBundleABTestInput,
+  private async createABTest(
+    name: string,
+    gateway: string,
+    callerRoleArn: string | undefined,
+    build: (context: {
+      gatewayArn: string;
+      accountId: string;
+      roleArn: string;
+    }) => CreateABTestRequest,
     options: CoreOptions,
   ): Promise<CreateABTestResponse> {
     const control = this.clients.control(toClientConfig(options));
-    const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway }));
-    const gatewayArn = gateway.gatewayArn!;
+    const gatewayArn = (await control.send(new GetGatewayCommand({ gatewayIdentifier: gateway })))
+      .gatewayArn!;
     const accountId = accountIdFromArn(gatewayArn);
 
-    const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`;
-    const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`;
-    const onlineEvaluationConfigArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`;
-
-    const treatmentWeight = input.treatmentWeight ?? 50;
-    const variants = [
-      {
-        name: "C",
-        weight: 100 - treatmentWeight,
-        variantConfiguration: {
-          configurationBundle: {
-            bundleArn: controlBundleArn,
-            bundleVersion: input.control.bundleVersion,
-          },
-        },
-      },
-      {
-        name: "T1",
-        weight: treatmentWeight,
-        variantConfiguration: {
-          configurationBundle: {
-            bundleArn: treatmentBundleArn,
-            bundleVersion: input.treatment.bundleVersion,
-          },
-        },
-      },
-    ];
-
-    let roleArn = input.roleArn;
+    let roleArn = callerRoleArn;
     let provisionedRoleArn: string | undefined;
     if (!roleArn) {
-      const iam = this.clients.iam({ region: options.region });
-      const provisioned = await provisionAbTestRole(iam, input.name, gatewayArn, options.region);
+      const provisioned = await provisionAbTestRole(
+        this.clients.iam({ region: options.region }),
+        name,
+        gatewayArn,
+        options.region,
+      );
       roleArn = provisioned.roleArn;
       if (provisioned.created) provisionedRoleArn = provisioned.roleArn;
     }
 
-    const command = new CreateABTestCommand({
-      name: input.name,
-      gatewayArn,
-      variants,
-      evaluationConfig: { onlineEvaluationConfigArn },
-      roleArn,
-      gatewayFilter: input.gatewayFilter,
-      enableOnCreate: input.enableOnCreate ?? true,
-      clientToken: randomUUID(),
-    });
-
+    const command = new CreateABTestCommand(build({ gatewayArn, accountId, roleArn }));
     try {
-      return input.roleArn
+      return callerRoleArn
         ? await this.clients.data(toClientConfig(options)).send(command)
         : await retryWhileRolePropagates(() =>
             this.clients.data(toClientConfig(options)).send(command),
@@ -556,6 +531,99 @@ export class EvalClient implements CoreEvalClient {
     }
   }
 
+  async createConfigBundleABTest(
+    input: CreateConfigBundleABTestInput,
+    options: CoreOptions,
+  ): Promise<CreateABTestResponse> {
+    const treatmentWeight = input.treatmentWeight ?? 50;
+    return this.createABTest(
+      input.name,
+      input.gateway,
+      input.roleArn,
+      ({ gatewayArn, accountId, roleArn }) => {
+        const bundleArn = (id: string) =>
+          `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${id}`;
+        return {
+          name: input.name,
+          gatewayArn,
+          variants: [
+            {
+              name: "C",
+              weight: 100 - treatmentWeight,
+              variantConfiguration: {
+                configurationBundle: {
+                  bundleArn: bundleArn(input.control.configBundle),
+                  bundleVersion: input.control.bundleVersion,
+                },
+              },
+            },
+            {
+              name: "T1",
+              weight: treatmentWeight,
+              variantConfiguration: {
+                configurationBundle: {
+                  bundleArn: bundleArn(input.treatment.configBundle),
+                  bundleVersion: input.treatment.bundleVersion,
+                },
+              },
+            },
+          ],
+          evaluationConfig: {
+            onlineEvaluationConfigArn: `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`,
+          },
+          roleArn,
+          gatewayFilter: input.gatewayFilter,
+          enableOnCreate: input.enableOnCreate ?? true,
+          clientToken: randomUUID(),
+        };
+      },
+      options,
+    );
+  }
+
+  async createTargetBasedABTest(
+    input: CreateTargetBasedABTestInput,
+    options: CoreOptions,
+  ): Promise<CreateABTestResponse> {
+    const treatmentWeight = input.treatmentWeight ?? 50;
+    return this.createABTest(
+      input.name,
+      input.gateway,
+      input.roleArn,
+      ({ gatewayArn, accountId, roleArn }) => {
+        const evalArn = (id: string) =>
+          `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${id}`;
+        return {
+          name: input.name,
+          gatewayArn,
+          variants: [
+            {
+              name: "C",
+              weight: 100 - treatmentWeight,
+              variantConfiguration: { target: { name: input.control.gatewayTarget } },
+            },
+            {
+              name: "T1",
+              weight: treatmentWeight,
+              variantConfiguration: { target: { name: input.treatment.gatewayTarget } },
+            },
+          ],
+          evaluationConfig: {
+            perVariantOnlineEvaluationConfig: [
+              { name: "C", onlineEvaluationConfigArn: evalArn(input.control.onlineEval) },
+              { name: "T1", onlineEvaluationConfigArn: evalArn(input.treatment.onlineEval) },
+            ],
+          },
+          roleArn,
+          gatewayFilter: input.gatewayFilter,
+          enableOnCreate: input.enableOnCreate ?? true,
+          clientToken: randomUUID(),
+        };
+      },
+      options,
+    );
+  }
+
   async listBatchInsights(
     nextToken: string | undefined,
     maxResults: number | undefined,

src/handlers/eval/ab-test/ab-test.test.tsx

@@ -79,9 +79,12 @@ describe("eval ab-test command hierarchy", () => {
       "stop",
       "delete",
       "config-bundle",
+      "target-based",
     ]);
     const cb = group?.children().find((c) => c.name() === "config-bundle");
     expect(cb?.children().map((c) => c.name())).toEqual(["run"]);
+    const tb = group?.children().find((c) => c.name() === "target-based");
+    expect(tb?.children().map((c) => c.name())).toEqual(["run"]);
   });
 });
 
@@ -264,3 +267,86 @@ describe("eval ab-test config-bundle run validation", () => {
     });
   });
 });
+
+describe("eval ab-test target-based run validation", () => {
+  const TB_BASE = [
+    "eval",
+    "ab-test",
+    "target-based",
+    "run",
+    "--name",
+    "orders-v2-canary",
+    "--gateway",
+    "orders-gateway-abc123",
+    "--control",
+    '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}',
+    "--treatment",
+    '{"gateway-target":"orders-v2-target","online-eval":"v2-quality"}',
+    "--json",
+  ];
+
+  test.each(["name", "gateway", "control", "treatment"] as const)(
+    "requires --%s",
+    async (missing) => {
+      const args = TB_BASE.filter(
+        (a, i) => a !== `--${missing}` && TB_BASE[i - 1] !== `--${missing}`,
+      );
+      await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`));
+    },
+  );
+
+  test("rejects a mis-shaped --control object", async () => {
+    const args = TB_BASE.map((a) =>
+      a === '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}'
+        ? '{"wrong":"shape"}'
+        : a,
+    );
+    await expect(run(args)).rejects.toThrow(/--control must be/);
+  });
+
+  test("rejects identical control/treatment targets", async () => {
+    const same = '{"gateway-target":"t","online-eval":"e"}';
+    await expect(
+      run([
+        "eval",
+        "ab-test",
+        "target-based",
+        "run",
+        "--name",
+        "x",
+        "--gateway",
+        "g",
+        "--control",
+        same,
+        "--treatment",
+        same,
+        "--json",
+      ]),
+    ).rejects.toThrow(/different gateway targets/);
+  });
+
+  test("maps flags to a createTargetBasedABTest call", async () => {
+    const { core } = await run([...TB_BASE, "--treatment-weight", "20"], (c) =>
+      c.eval.setAbTestCreateResponse({
+        abTestId: "x",
+        abTestArn: ARN,
+        name: "x",
+        status: "CREATING",
+        executionStatus: "RUNNING",
+        createdAt: new Date("2026-08-26T10:00:00.000Z"),
+      }),
+    );
+    const call = core.eval.calls.find((c) => c.method === "createTargetBasedABTest");
+    expect(call).toBeDefined();
+    expect(call!.args[0]).toEqual({
+      name: "orders-v2-canary",
+      gateway: "orders-gateway-abc123",
+      control: { gatewayTarget: "orders-prod-target", onlineEval: "prod-quality" },
+      treatment: { gatewayTarget: "orders-v2-target", onlineEval: "v2-quality" },
+      treatmentWeight: 20,
+      gatewayFilter: undefined,
+      roleArn: undefined,
+      enableOnCreate: undefined,
+    });
+  });
+});

src/handlers/eval/ab-test/index.tsx

@@ -10,6 +10,7 @@ import { createResumeAbTestHandler } from "./resume";
 import { createStopAbTestHandler } from "./stop";
 import { createDeleteAbTestHandler } from "./delete";
 import { createConfigBundleAbTestHandler } from "./config-bundle";
+import { createTargetBasedAbTestHandler } from "./target-based";
 
 export function createAbTestHandler(core: Core, io: AppIO): Router {
   return new Router("ab-test", "inspect AgentCore A/B tests")
@@ -22,7 +23,8 @@ export function createAbTestHandler(core: Core, io: AppIO): Router {
     .handler(createResumeAbTestHandler(core))
     .handler(createStopAbTestHandler(core))
     .handler(createDeleteAbTestHandler(core))
-    .handler(createConfigBundleAbTestHandler(core, io));
+    .handler(createConfigBundleAbTestHandler(core, io))
+    .handler(createTargetBasedAbTestHandler(core, io));
 }
 
 export { AbTestScreen } from "./screen.tsx";

src/handlers/eval/ab-test/target-based/index.tsx

@@ -0,0 +1,10 @@
+import { Router } from "../../../../router";
+import type { AppIO } from "../../../../io";
+import type { Core } from "../../../types";
+import { createTargetBasedRunHandler } from "./run";
+
+export function createTargetBasedAbTestHandler(core: Core, io: AppIO): Router {
+  return new Router("target-based", "target-based A/B tests").handler(
+    createTargetBasedRunHandler(core, io),
+  );
+}