From 95a1b67c45f6ce4c8988f414bc196636df893cf1 Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Wed, 26 Aug 2026 17:37:44 +0000
Subject: [PATCH 1/7] feat(eval): add ab-test config-bundle run (create)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds `agentcore eval ab-test config-bundle run` — the first create in the
ab-test family. Runs an A/B test between two config-bundle versions on one
gateway. --control / --treatment / --gateway-filter each accept inline JSON,
file://<path>, or - (stdin) via SourceResolver (same shape as online-eval
create's --filters).

Core provisions an IAM execution role when --role-arn is omitted (mirrors
online-eval create + retryWhileRolePropagates), and rolls the role back if
CreateABTest fails. Validation is server-side (gateway READY, bundles,
online-eval enabled iff enableOnCreate) — the CLI resolves ids to ARNs and
surfaces the service's 4xx cleanly.

Note: --runtime dropped (not a CreateABTest field) and deviates from doc.
---
 src/core/abTestExecutionRole.tsx              | 149 +++++++++++++++
 src/core/eval.tsx                             |  80 +++++++++
 .../eval/ab-test/ab-test.create.test.tsx      | 169 ++++++++++++++++++
 .../eval/ab-test/ab-test.write.test.tsx       |   1 +
 .../eval/ab-test/config-bundle/index.tsx      |  10 ++
 .../eval/ab-test/config-bundle/run/index.tsx  | 115 ++++++++++++
 src/handlers/eval/ab-test/index.tsx           |   4 +-
 src/handlers/eval/types.tsx                   |  20 +++
 src/testing/TestCoreClient.tsx                |  18 ++
 9 files changed, 565 insertions(+), 1 deletion(-)
 create mode 100644 src/core/abTestExecutionRole.tsx
 create mode 100644 src/handlers/eval/ab-test/ab-test.create.test.tsx
 create mode 100644 src/handlers/eval/ab-test/config-bundle/index.tsx
 create mode 100644 src/handlers/eval/ab-test/config-bundle/run/index.tsx

diff --git a/src/core/abTestExecutionRole.tsx b/src/core/abTestExecutionRole.tsx
new file mode 100644
index 000000000..a4463f5f8
--- /dev/null
+++ b/src/core/abTestExecutionRole.tsx
@@ -0,0 +1,149 @@
+import {
+  CreateRoleCommand,
+  GetRoleCommand,
+  PutRolePolicyCommand,
+  DeleteRoleCommand,
+  DeleteRolePolicyCommand,
+  type IAMClient,
+} from "@aws-sdk/client-iam";
+import { createHash } from "node:crypto";
+
+const AB_TEST_POLICY_NAME = "ABTestExecutionPolicy";
+
+export function abTestExecutionRoleName(testName: string): string {
+  const hash = createHash("sha256").update(`ab-test:${testName}`).digest("hex").slice(0, 8);
+  const base = `AgentCoreABTest-${testName}`;
+  return `${base.slice(0, 55)}-${hash}`;
+}
+
+export function roleNameFromArn(roleArn: string): string {
+  const parts = roleArn.split("/");
+  return parts[parts.length - 1] ?? roleArn;
+}
+
+function trustPolicy(accountId: string, region: string): string {
+  return JSON.stringify({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Principal: { Service: "bedrock-agentcore.amazonaws.com" },
+        Action: "sts:AssumeRole",
+        Condition: {
+          StringEquals: { "aws:SourceAccount": accountId },
+          ArnLike: {
+            "aws:SourceArn": `arn:aws:bedrock-agentcore:${region}:${accountId}:ab-test/*`,
+          },
+        },
+      },
+    ],
+  });
+}
+
+function executionPolicy(accountId: string, region: string): string {
+  return JSON.stringify({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Sid: "AgentCoreResources",
+        Effect: "Allow",
+        Action: [
+          "bedrock-agentcore:GetGateway",
+          "bedrock-agentcore:GetGatewayTarget",
+          "bedrock-agentcore:ListGatewayTargets",
+          "bedrock-agentcore:CreateGatewayRule",
+          "bedrock-agentcore:UpdateGatewayRule",
+          "bedrock-agentcore:GetGatewayRule",
+          "bedrock-agentcore:DeleteGatewayRule",
+          "bedrock-agentcore:ListGatewayRules",
+          "bedrock-agentcore:GetOnlineEvaluationConfig",
+          "bedrock-agentcore:GetEvaluator",
+          "bedrock-agentcore:GetConfigurationBundle",
+          "bedrock-agentcore:GetConfigurationBundleVersion",
+          "bedrock-agentcore:ListConfigurationBundleVersions",
+        ],
+        Resource: `arn:aws:bedrock-agentcore:${region}:${accountId}:*`,
+        Condition: { StringEquals: { "aws:ResourceAccount": accountId } },
+      },
+      {
+        Sid: "CloudWatchLogsDescribe",
+        Effect: "Allow",
+        Action: ["logs:DescribeLogGroups"],
+        Resource: "*",
+      },
+      {
+        Sid: "CloudWatchLogs",
+        Effect: "Allow",
+        Action: [
+          "logs:DescribeIndexPolicies",
+          "logs:PutIndexPolicy",
+          "logs:StartQuery",
+          "logs:GetQueryResults",
+          "logs:StopQuery",
+          "logs:FilterLogEvents",
+          "logs:GetLogEvents",
+        ],
+        Resource: [
+          `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/evaluations/*`,
+          `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/runtimes/*`,
+          `arn:aws:logs:${region}:${accountId}:log-group:aws/spans`,
+          `arn:aws:logs:${region}:${accountId}:log-group:aws/spans:*`,
+        ],
+      },
+    ],
+  });
+}
+
+export async function provisionAbTestRole(
+  iam: IAMClient,
+  testName: string,
+  gatewayArn: string,
+  region: string,
+): Promise<{ roleArn: string; created: boolean }> {
+  const accountId = gatewayArn.split(":")[4] ?? "*";
+  const roleName = abTestExecutionRoleName(testName);
+
+  let roleArn: string;
+  let created = false;
+  try {
+    const existing = await iam.send(new GetRoleCommand({ RoleName: roleName }));
+    roleArn = existing.Role!.Arn!;
+  } catch (error) {
+    if ((error as Error).name !== "NoSuchEntityException") throw error;
+    const result = await iam.send(
+      new CreateRoleCommand({
+        RoleName: roleName,
+        AssumeRolePolicyDocument: trustPolicy(accountId, region),
+        Description: `Execution role for AgentCore A/B test "${testName}" (created by agentcore CLI)`,
+      }),
+    );
+    roleArn = result.Role!.Arn!;
+    created = true;
+  }
+
+  await iam.send(
+    new PutRolePolicyCommand({
+      RoleName: roleName,
+      PolicyName: AB_TEST_POLICY_NAME,
+      PolicyDocument: executionPolicy(accountId, region),
+    }),
+  );
+
+  return { roleArn, created };
+}
+
+export async function deleteAbTestRole(iam: IAMClient, roleArn: string): Promise<void> {
+  const roleName = roleNameFromArn(roleArn);
+  try {
+    await iam.send(
+      new DeleteRolePolicyCommand({ RoleName: roleName, PolicyName: AB_TEST_POLICY_NAME }),
+    );
+  } catch {
+    // best effort
+  }
+  try {
+    await iam.send(new DeleteRoleCommand({ RoleName: roleName }));
+  } catch {
+    // best effort
+  }
+}
diff --git a/src/core/eval.tsx b/src/core/eval.tsx
index fd51e04ee..b06308745 100644
--- a/src/core/eval.tsx
+++ b/src/core/eval.tsx
@@ -16,6 +16,7 @@ import {
   GetDatasetCommand,
   GetEvaluatorCommand,
   GetHarnessCommand,
+  GetGatewayCommand,
   GetOnlineEvaluationConfigCommand,
   ListConfigurationBundlesCommand,
   ListConfigurationBundleVersionsCommand,
@@ -60,6 +61,7 @@ import {
 import {
   DeleteRecommendationCommand,
   EvaluateCommand,
+  CreateABTestCommand,
   GetABTestCommand,
   ListABTestsCommand,
   UpdateABTestCommand,
@@ -74,6 +76,7 @@ import {
   type EvaluationReferenceInput,
   type EvaluationResultContent,
   type EvaluationTarget,
+  type CreateABTestResponse,
   type GetABTestResponse,
   type ListABTestsResponse,
   type ABTestExecutionStatus,
@@ -119,6 +122,7 @@ import type {
   RoleScopeWarning,
   CoreEvalClient,
   CreateConfigurationBundleInput,
+  CreateConfigBundleABTestInput,
   CreateDatasetInput,
   CreateOnlineEvalInput,
   CreateOnlineInsightInput,
@@ -165,6 +169,7 @@ import {
   revokeOnlineEvalScope,
   scopePolicyName,
 } from "./onlineEvalExecutionRole";
+import { deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole";
 
 const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT";
 const DEFAULT_INGESTION_WAIT_MS = 180_000;
@@ -476,6 +481,81 @@ export class EvalClient implements CoreEvalClient {
       .send(new DeleteABTestCommand({ abTestId: id }));
   }
 
+  async createConfigBundleABTest(
+    input: CreateConfigBundleABTestInput,
+    options: CoreOptions,
+  ): Promise<CreateABTestResponse> {
+    const control = this.clients.control(toClientConfig(options));
+    const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway }));
+    const gatewayArn = gateway.gatewayArn!;
+    const accountId = gatewayArn.split(":")[4] ?? "*";
+
+    const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`;
+    const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`;
+    const onlineEvaluationConfigArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`;
+
+    const treatmentWeight = input.treatmentWeight ?? 50;
+    const variants = [
+      {
+        name: "C",
+        weight: 100 - treatmentWeight,
+        variantConfiguration: {
+          configurationBundle: {
+            bundleArn: controlBundleArn,
+            bundleVersion: input.control.bundleVersion,
+          },
+        },
+      },
+      {
+        name: "T1",
+        weight: treatmentWeight,
+        variantConfiguration: {
+          configurationBundle: {
+            bundleArn: treatmentBundleArn,
+            bundleVersion: input.treatment.bundleVersion,
+          },
+        },
+      },
+    ];
+
+    let roleArn = input.roleArn;
+    let provisionedRoleArn: string | undefined;
+    if (!roleArn) {
+      const iam = this.clients.iam({ region: options.region });
+      const provisioned = await provisionAbTestRole(iam, input.name, gatewayArn, options.region);
+      roleArn = provisioned.roleArn;
+      if (provisioned.created) provisionedRoleArn = provisioned.roleArn;
+    }
+
+    const command = new CreateABTestCommand({
+      name: input.name,
+      gatewayArn,
+      variants,
+      evaluationConfig: { onlineEvaluationConfigArn },
+      roleArn,
+      gatewayFilter: input.gatewayFilter,
+      enableOnCreate: !input.disableOnCreate,
+      clientToken: randomUUID(),
+    });
+
+    try {
+      return input.roleArn
+        ? await this.clients.data(toClientConfig(options)).send(command)
+        : await retryWhileRolePropagates(() =>
+            this.clients.data(toClientConfig(options)).send(command),
+          );
+    } catch (error) {
+      if (provisionedRoleArn) {
+        try {
+          await deleteAbTestRole(this.clients.iam({ region: options.region }), provisionedRoleArn);
+        } catch {
+          // best effort
+        }
+      }
+      throw error;
+    }
+  }
+
   async listBatchInsights(
     nextToken: string | undefined,
     maxResults: number | undefined,
diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx
new file mode 100644
index 000000000..c96a31dd3
--- /dev/null
+++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx
@@ -0,0 +1,169 @@
+import { test, expect, describe } from "bun:test";
+import type { CreateABTestResponse } from "@aws-sdk/client-bedrock-agentcore";
+import { createRootHandler } from "../../index";
+import { createSilentLogger, TestCoreClient, testIO } from "../../../testing";
+import { TestGlobalConfigAccessor } from "../../../testing/";
+
+const OK: CreateABTestResponse = {
+  abTestId: "orders-v2-abc123",
+  abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/orders-v2-abc123",
+  name: "orders-v2",
+  status: "CREATING",
+  executionStatus: "NOT_STARTED",
+  createdAt: new Date("2026-08-26T10:00:00.000Z"),
+} satisfies CreateABTestResponse;
+
+async function run(args: string[], configure?: (core: TestCoreClient) => void) {
+  const core = new TestCoreClient();
+  core.eval.setAbTestCreateResponse(OK);
+  configure?.(core);
+  const io = testIO();
+  const root = createRootHandler(core, {
+    io: io.io,
+    logger: createSilentLogger(),
+    globalConfigAccessor: new TestGlobalConfigAccessor(),
+  });
+  await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]);
+  return { core, stdout: io.stdout() };
+}
+
+const BASE = [
+  "eval",
+  "ab-test",
+  "config-bundle",
+  "run",
+  "--name",
+  "orders-v2",
+  "--gateway",
+  "orders-gateway-abc123",
+  "--control",
+  '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}',
+  "--treatment",
+  '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}',
+  "--online-eval",
+  "online-eval-abc123",
+  "--json",
+];
+
+describe("eval ab-test config-bundle run", () => {
+  test("registers under ab-test → config-bundle", () => {
+    const io = testIO();
+    const root = createRootHandler(new TestCoreClient(), {
+      io: io.io,
+      logger: createSilentLogger(),
+      globalConfigAccessor: new TestGlobalConfigAccessor(),
+    });
+    const abTest = root
+      .children()
+      .find((c) => c.name() === "eval")
+      ?.children()
+      .find((c) => c.name() === "ab-test");
+    expect(abTest?.children().map((c) => c.name())).toContain("config-bundle");
+    const cb = abTest?.children().find((c) => c.name() === "config-bundle");
+    expect(cb?.children().map((c) => c.name())).toEqual(["run"]);
+  });
+
+  test("maps flags to a createConfigBundleABTest call", async () => {
+    const { core, stdout } = await run([...BASE, "--treatment-weight", "20"]);
+    expect(JSON.parse(stdout).abTestId).toBe("orders-v2-abc123");
+    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
+    expect(call?.args[0]).toEqual({
+      name: "orders-v2",
+      gateway: "orders-gateway-abc123",
+      control: { configBundle: "orders-prompt-abc", bundleVersion: "1111" },
+      treatment: { configBundle: "orders-prompt-abc", bundleVersion: "2222" },
+      onlineEval: "online-eval-abc123",
+      treatmentWeight: 20,
+      gatewayFilter: undefined,
+      roleArn: undefined,
+      disableOnCreate: false,
+    });
+    expect(call?.args[1]).toEqual({ region: "us-west-2" });
+  });
+
+  test("passes --gateway-filter through as a GatewayFilter", async () => {
+    const { core } = await run([
+      ...BASE,
+      "--gateway-filter",
+      '{"targetPaths":["/orders/checkout"]}',
+    ]);
+    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
+    expect(call).toBeDefined();
+    expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({
+      targetPaths: ["/orders/checkout"],
+    });
+  });
+
+  test("passes --disable-on-create and --role-arn through", async () => {
+    const { core } = await run([
+      ...BASE,
+      "--disable-on-create",
+      "--role-arn",
+      "arn:aws:iam::123456789012:role/customer-owned",
+    ]);
+    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
+    const input = call?.args[0] as { disableOnCreate?: boolean; roleArn?: string };
+    expect(input.disableOnCreate).toBe(true);
+    expect(input.roleArn).toBe("arn:aws:iam::123456789012:role/customer-owned");
+  });
+
+  test("rejects equal control/treatment bundle-versions", async () => {
+    await expect(
+      run([
+        "eval",
+        "ab-test",
+        "config-bundle",
+        "run",
+        "--name",
+        "x",
+        "--gateway",
+        "g",
+        "--control",
+        '{"config-bundle":"b","bundle-version":"same"}',
+        "--treatment",
+        '{"config-bundle":"b","bundle-version":"same"}',
+        "--online-eval",
+        "o",
+        "--json",
+      ]),
+    ).rejects.toThrow(/must differ/);
+  });
+
+  test("rejects --treatment-weight outside 1-99", async () => {
+    await expect(run([...BASE, "--treatment-weight", "0"])).rejects.toThrow(/1 and 99/);
+    await expect(run([...BASE, "--treatment-weight", "100"])).rejects.toThrow(/1 and 99/);
+  });
+
+  test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)(
+    "requires --%s",
+    async (missing) => {
+      const args = BASE.filter((_, i, arr) => {
+        const prev = arr[i - 1];
+        return prev !== `--${missing}` && arr[i] !== `--${missing}`;
+      });
+      await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`));
+    },
+  );
+
+  test("rejects a malformed control JSON shape", async () => {
+    await expect(
+      run([
+        "eval",
+        "ab-test",
+        "config-bundle",
+        "run",
+        "--name",
+        "x",
+        "--gateway",
+        "g",
+        "--control",
+        '{"wrong":"shape"}',
+        "--treatment",
+        '{"config-bundle":"b","bundle-version":"2"}',
+        "--online-eval",
+        "o",
+        "--json",
+      ]),
+    ).rejects.toThrow(/--control must be/);
+  });
+});
diff --git a/src/handlers/eval/ab-test/ab-test.write.test.tsx b/src/handlers/eval/ab-test/ab-test.write.test.tsx
index 31116a1a5..3a7bd3d9d 100644
--- a/src/handlers/eval/ab-test/ab-test.write.test.tsx
+++ b/src/handlers/eval/ab-test/ab-test.write.test.tsx
@@ -36,6 +36,7 @@ describe("eval ab-test command hierarchy", () => {
       "resume",
       "stop",
       "delete",
+      "config-bundle",
     ]);
   });
 });
diff --git a/src/handlers/eval/ab-test/config-bundle/index.tsx b/src/handlers/eval/ab-test/config-bundle/index.tsx
new file mode 100644
index 000000000..1c04c571c
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/index.tsx
@@ -0,0 +1,10 @@
+import { Router } from "../../../../router";
+import type { AppIO } from "../../../../io";
+import type { Core } from "../../../types";
+import { createConfigBundleRunHandler } from "./run";
+
+export function createConfigBundleAbTestHandler(core: Core, io: AppIO): Router {
+  return new Router("config-bundle", "config-bundle A/B tests").handler(
+    createConfigBundleRunHandler(core, io),
+  );
+}
diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx
new file mode 100644
index 000000000..c8016fbdc
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx
@@ -0,0 +1,115 @@
+import type { GatewayFilter } from "@aws-sdk/client-bedrock-agentcore";
+import z from "zod";
+import { createHandler, flag } from "../../../../../router";
+import { InputValidationError } from "../../../../../errors";
+import { JsonRendererKey } from "../../../../../tui";
+import { SourceResolver, type AppIO } from "../../../../../io";
+import type { Core } from "../../../../types";
+import type { BundleRef } from "../../../types";
+import { coreOptsFromCtx } from "../../../../utils";
+import { parseJsonFlag } from "../../../../utils";
+
+const bundleRefSchema = z
+  .object({
+    "config-bundle": z.string().min(1),
+    "bundle-version": z.string().min(1),
+  })
+  .strict();
+
+function toBundleRef(name: string, raw: unknown): BundleRef {
+  const parsed = bundleRefSchema.safeParse(raw);
+  if (!parsed.success) {
+    throw new InputValidationError(
+      `--${name} must be {"config-bundle": "<id>", "bundle-version": "<version>"}`,
+    );
+  }
+  return {
+    configBundle: parsed.data["config-bundle"],
+    bundleVersion: parsed.data["bundle-version"],
+  };
+}
+
+export const createConfigBundleRunHandler = (core: Core, io: AppIO) =>
+  createHandler({
+    name: "run",
+    description: "run an A/B test between two config-bundle versions on one gateway",
+    flags: [
+      flag("name", "the A/B test name", z.string().optional()),
+      flag("gateway", "deployed gateway id", z.string().optional()),
+      flag(
+        "control",
+        'control JSON {"config-bundle","bundle-version"} (inline, file://, or -)',
+        z.string().optional(),
+      ),
+      flag(
+        "treatment",
+        'treatment JSON {"config-bundle","bundle-version"} (inline, file://, or -)',
+        z.string().optional(),
+      ),
+      flag("online-eval", "online-evaluation config id", z.string().optional()),
+      flag(
+        "treatment-weight",
+        "1-99; control weight = 100 - this (default 50)",
+        z.number().optional(),
+      ),
+      flag(
+        "gateway-filter",
+        'GatewayFilter JSON, e.g. {"targetPaths":["/orders"]} (inline, file://, or -)',
+        z.string().optional(),
+      ),
+      flag(
+        "role-arn",
+        "execution-role override (default: auto-provisioned)",
+        z.string().optional(),
+      ),
+      flag("disable-on-create", "create without starting", z.boolean().optional()),
+    ],
+    handle: async (ctx, flags) => {
+      const required = ["name", "gateway", "control", "treatment", "online-eval"] as const;
+      for (const f of required) {
+        if (!flags[f]) throw new InputValidationError(`required option '--${f}' not specified`);
+      }
+
+      const source = new SourceResolver({ stdin: io.stdin });
+      const controlRaw = parseJsonFlag<unknown>(
+        "control",
+        await source.resolveText("control", flags["control"]),
+      );
+      const treatmentRaw = parseJsonFlag<unknown>(
+        "treatment",
+        await source.resolveText("treatment", flags["treatment"]),
+      );
+      const gatewayFilter = parseJsonFlag<GatewayFilter>(
+        "gateway-filter",
+        await source.resolveText("gateway-filter", flags["gateway-filter"]),
+      );
+
+      const control = toBundleRef("control", controlRaw);
+      const treatment = toBundleRef("treatment", treatmentRaw);
+      if (control.bundleVersion === treatment.bundleVersion) {
+        throw new InputValidationError("treatment bundle-version must differ from control");
+      }
+
+      const treatmentWeight = flags["treatment-weight"];
+      if (treatmentWeight !== undefined && (treatmentWeight < 1 || treatmentWeight > 99)) {
+        throw new InputValidationError("--treatment-weight must be between 1 and 99");
+      }
+
+      const result = await core.eval.createConfigBundleABTest(
+        {
+          name: flags["name"]!,
+          gateway: flags["gateway"]!,
+          control,
+          treatment,
+          onlineEval: flags["online-eval"]!,
+          treatmentWeight,
+          gatewayFilter,
+          roleArn: flags["role-arn"],
+          disableOnCreate: flags["disable-on-create"],
+        },
+        coreOptsFromCtx(ctx),
+      );
+
+      ctx.require(JsonRendererKey).renderJson(result);
+    },
+  });
diff --git a/src/handlers/eval/ab-test/index.tsx b/src/handlers/eval/ab-test/index.tsx
index 6bccf91fc..4e42ada3a 100644
--- a/src/handlers/eval/ab-test/index.tsx
+++ b/src/handlers/eval/ab-test/index.tsx
@@ -9,6 +9,7 @@ import { createPauseAbTestHandler } from "./pause";
 import { createResumeAbTestHandler } from "./resume";
 import { createStopAbTestHandler } from "./stop";
 import { createDeleteAbTestHandler } from "./delete";
+import { createConfigBundleAbTestHandler } from "./config-bundle";
 
 export function createAbTestHandler(core: Core, io: AppIO): Router {
   return new Router("ab-test", "inspect AgentCore A/B tests")
@@ -20,7 +21,8 @@ export function createAbTestHandler(core: Core, io: AppIO): Router {
     .handler(createPauseAbTestHandler(core))
     .handler(createResumeAbTestHandler(core))
     .handler(createStopAbTestHandler(core))
-    .handler(createDeleteAbTestHandler(core));
+    .handler(createDeleteAbTestHandler(core))
+    .handler(createConfigBundleAbTestHandler(core, io));
 }
 
 export { AbTestScreen } from "./screen.tsx";
diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx
index b1be77f4b..2ce7cd2fd 100644
--- a/src/handlers/eval/types.tsx
+++ b/src/handlers/eval/types.tsx
@@ -30,6 +30,8 @@ import type {
   UpdateOnlineEvaluationConfigResponse,
 } from "@aws-sdk/client-bedrock-agentcore-control";
 import type {
+  CreateABTestResponse,
+  GatewayFilter,
   GetABTestResponse,
   ListABTestsResponse,
   ABTestExecutionStatus,
@@ -231,6 +233,20 @@ export type RoleScopeWarning = {
   logGroupNames: string[];
 };
 
+export type BundleRef = { configBundle: string; bundleVersion: string };
+
+export type CreateConfigBundleABTestInput = {
+  name: string;
+  gateway: string;
+  control: BundleRef;
+  treatment: BundleRef;
+  onlineEval: string;
+  treatmentWeight?: number;
+  gatewayFilter?: GatewayFilter;
+  roleArn?: string;
+  disableOnCreate?: boolean;
+};
+
 export type CreateDatasetInput = CreateDatasetRequest;
 export type StartRecommendationInput = {
   name: string;
@@ -427,6 +443,10 @@ export interface CoreEvalClient {
     options: CoreOptions,
   ): Promise<UpdateABTestResponse>;
   deleteABTest(id: string, options: CoreOptions): Promise<DeleteABTestResponse>;
+  createConfigBundleABTest(
+    input: CreateConfigBundleABTestInput,
+    options: CoreOptions,
+  ): Promise<CreateABTestResponse>;
   // startBatchEvaluation submits an async, service-side evaluation over sessions
   // the service gathers from the resolved data source. Returns the durable job id
   // + RUNNING status; poll with getBatchEvaluation.
diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx
index 8fcce7a46..94a888c47 100644
--- a/src/testing/TestCoreClient.tsx
+++ b/src/testing/TestCoreClient.tsx
@@ -76,6 +76,7 @@ import type {
   GetABTestResponse,
   ListABTestsResponse,
   ABTestExecutionStatus,
+  CreateABTestResponse,
   UpdateABTestResponse,
   DeleteABTestResponse,
   DeleteRecommendationResponse,
@@ -137,6 +138,7 @@ import type {
   CodeBasedUpdate,
   CoreEvalClient,
   CreateConfigurationBundleInput,
+  CreateConfigBundleABTestInput,
   CreateDatasetInput,
   CreateOnlineEvalInput,
   CreateOnlineInsightInput,
@@ -281,6 +283,7 @@ const DEFAULT_GET_ABTEST_RESPONSE = {} as GetABTestResponse;
 const DEFAULT_LIST_ABTESTS_RESPONSE: ListABTestsResponse = { abTests: [] };
 const DEFAULT_UPDATE_ABTEST_RESPONSE = {} as UpdateABTestResponse;
 const DEFAULT_DELETE_ABTEST_RESPONSE = {} as DeleteABTestResponse;
+const DEFAULT_CREATE_ABTEST_RESPONSE = {} as CreateABTestResponse;
 const DEFAULT_START_BATCH_EVAL_RESPONSE = {
   batchEvaluationId: "batch-eval-test",
   status: "RUNNING",
@@ -1438,6 +1441,7 @@ export class TestEvalClient implements CoreEvalClient {
   private abTestListResponses = new Map<string | undefined, ListABTestsResponse>();
   private abTestUpdateResponse: UpdateABTestResponse = DEFAULT_UPDATE_ABTEST_RESPONSE;
   private abTestDeleteResponse: DeleteABTestResponse = DEFAULT_DELETE_ABTEST_RESPONSE;
+  private abTestCreateResponse: CreateABTestResponse = DEFAULT_CREATE_ABTEST_RESPONSE;
   private batchEvalResults: BatchEvaluationResultEntry[] = [];
   private batchEvalResultsError?: unknown;
   private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE;
@@ -1685,6 +1689,11 @@ export class TestEvalClient implements CoreEvalClient {
     return this;
   }
 
+  setAbTestCreateResponse(response: CreateABTestResponse): this {
+    this.abTestCreateResponse = response;
+    return this;
+  }
+
   // setUpdateDatasetResult sets what updateDatasetExamples resolves to (when not
   // erroring).
   setUpdateDatasetResult(result: DatasetUpdateResult): this {
@@ -1894,6 +1903,15 @@ export class TestEvalClient implements CoreEvalClient {
     return this.abTestDeleteResponse;
   }
 
+  async createConfigBundleABTest(
+    input: CreateConfigBundleABTestInput,
+    options: CoreOptions,
+  ): Promise<CreateABTestResponse> {
+    this.calls.push({ method: "createConfigBundleABTest", args: [input, options] });
+    if (this.error) throw this.error;
+    return this.abTestCreateResponse;
+  }
+
   async startBatchEvaluation(
     input: StartBatchEvaluationInput,
     options: CoreOptions,

From ac398b838bd2c340488df479fd6150fbcf0cfcfc Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Wed, 26 Aug 2026 17:45:09 +0000
Subject: [PATCH 2/7] fix(eval): address ab-test config-bundle run review

- Reject control/treatment only when the (config-bundle, bundle-version) pair
  is identical, not on version-string collision across different bundles.
- Retry CreateABTest on data-plane AccessDenied (403), not just the
  control-plane role-not-propagated phrasing, so a freshly provisioned role
  that is mid-propagation is retried.
- Extract accountId via a throwing helper instead of a silent '*' fallback.
- --treatment-weight must be an integer.
- Add unit tests for the execution-role module (name cap, trust + inline
  policy, create vs reuse).
---
 src/core/abTestExecutionRole.test.ts          | 100 ++++++++++++++++++
 src/core/abTestExecutionRole.tsx              |  12 ++-
 src/core/eval.tsx                             |  33 +++++-
 .../eval/ab-test/ab-test.create.test.tsx      |   2 +-
 .../eval/ab-test/config-bundle/run/index.tsx  |  11 +-
 5 files changed, 147 insertions(+), 11 deletions(-)
 create mode 100644 src/core/abTestExecutionRole.test.ts

diff --git a/src/core/abTestExecutionRole.test.ts b/src/core/abTestExecutionRole.test.ts
new file mode 100644
index 000000000..7409d61c7
--- /dev/null
+++ b/src/core/abTestExecutionRole.test.ts
@@ -0,0 +1,100 @@
+import { test, expect, describe } from "bun:test";
+import { CreateRoleCommand, GetRoleCommand, type IAMClient } from "@aws-sdk/client-iam";
+import {
+  abTestExecutionRoleName,
+  accountIdFromArn,
+  provisionAbTestRole,
+} from "./abTestExecutionRole";
+
+const GATEWAY_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders-gw";
+
+type Sent = { name: string; input: unknown };
+
+function fakeIam(onGet: "found" | "missing"): { iam: IAMClient; sent: Sent[] } {
+  const sent: Sent[] = [];
+  const iam = {
+    send: async (command: { constructor: { name: string }; input: unknown }) => {
+      sent.push({ name: command.constructor.name, input: command.input });
+      if (command instanceof GetRoleCommand) {
+        if (onGet === "missing") {
+          throw Object.assign(new Error("no such entity"), { name: "NoSuchEntityException" });
+        }
+        return {
+          Role: {
+            Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`,
+          },
+        };
+      }
+      if (command instanceof CreateRoleCommand) {
+        return {
+          Role: {
+            Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`,
+          },
+        };
+      }
+      return {};
+    },
+  } as unknown as IAMClient;
+  return { iam, sent };
+}
+
+describe("abTestExecutionRoleName", () => {
+  test("stays within IAM's 64-char limit and is deterministic", () => {
+    const long = abTestExecutionRoleName("x".repeat(120));
+    expect(long.length).toBeLessThanOrEqual(64);
+    expect(abTestExecutionRoleName("orders")).toBe(abTestExecutionRoleName("orders"));
+  });
+
+  test("distinct names for distinct tests", () => {
+    expect(abTestExecutionRoleName("a")).not.toBe(abTestExecutionRoleName("b"));
+  });
+});
+
+describe("accountIdFromArn", () => {
+  test("extracts the account segment", () => {
+    expect(accountIdFromArn(GATEWAY_ARN)).toBe("123456789012");
+  });
+  test("throws on a malformed ARN", () => {
+    expect(() => accountIdFromArn("not-an-arn")).toThrow(/account id/);
+  });
+});
+
+describe("provisionAbTestRole", () => {
+  test("creates the role + inline policy and reports created=true", async () => {
+    const { iam, sent } = fakeIam("missing");
+    const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2");
+
+    expect(result.created).toBe(true);
+    expect(result.roleArn).toContain(":role/");
+    expect(sent.map((s) => s.name)).toEqual([
+      "GetRoleCommand",
+      "CreateRoleCommand",
+      "PutRolePolicyCommand",
+    ]);
+
+    const create = sent.find((s) => s.name === "CreateRoleCommand")!.input as {
+      AssumeRolePolicyDocument: string;
+    };
+    const trust = JSON.parse(create.AssumeRolePolicyDocument);
+    expect(trust.Statement[0].Principal.Service).toBe("bedrock-agentcore.amazonaws.com");
+    expect(trust.Statement[0].Condition.StringEquals["aws:SourceAccount"]).toBe("123456789012");
+    expect(trust.Statement[0].Condition.ArnLike["aws:SourceArn"]).toContain(":ab-test/*");
+
+    const policy = sent.find((s) => s.name === "PutRolePolicyCommand")!.input as {
+      PolicyDocument: string;
+    };
+    const doc = JSON.parse(policy.PolicyDocument);
+    const actions = doc.Statement.flatMap((s: { Action: string[] }) => s.Action);
+    expect(actions).toContain("bedrock-agentcore:GetGateway");
+    expect(actions).toContain("bedrock-agentcore:GetConfigurationBundleVersion");
+    expect(actions).toContain("bedrock-agentcore:GetOnlineEvaluationConfig");
+  });
+
+  test("reuses an existing role and reports created=false", async () => {
+    const { iam, sent } = fakeIam("found");
+    const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2");
+
+    expect(result.created).toBe(false);
+    expect(sent.map((s) => s.name)).toEqual(["GetRoleCommand", "PutRolePolicyCommand"]);
+  });
+});
diff --git a/src/core/abTestExecutionRole.tsx b/src/core/abTestExecutionRole.tsx
index a4463f5f8..721f4d026 100644
--- a/src/core/abTestExecutionRole.tsx
+++ b/src/core/abTestExecutionRole.tsx
@@ -21,6 +21,12 @@ export function roleNameFromArn(roleArn: string): string {
   return parts[parts.length - 1] ?? roleArn;
 }
 
+export function accountIdFromArn(arn: string): string {
+  const accountId = arn.split(":")[4];
+  if (!accountId) throw new Error(`could not extract account id from ARN: ${arn}`);
+  return accountId;
+}
+
 function trustPolicy(accountId: string, region: string): string {
   return JSON.stringify({
     Version: "2012-10-17",
@@ -100,7 +106,7 @@ export async function provisionAbTestRole(
   gatewayArn: string,
   region: string,
 ): Promise<{ roleArn: string; created: boolean }> {
-  const accountId = gatewayArn.split(":")[4] ?? "*";
+  const accountId = accountIdFromArn(gatewayArn);
   const roleName = abTestExecutionRoleName(testName);
 
   let roleArn: string;
@@ -139,11 +145,11 @@ export async function deleteAbTestRole(iam: IAMClient, roleArn: string): Promise
       new DeleteRolePolicyCommand({ RoleName: roleName, PolicyName: AB_TEST_POLICY_NAME }),
     );
   } catch {
-    // best effort
+    void 0;
   }
   try {
     await iam.send(new DeleteRoleCommand({ RoleName: roleName }));
   } catch {
-    // best effort
+    void 0;
   }
 }
diff --git a/src/core/eval.tsx b/src/core/eval.tsx
index b06308745..6e282633b 100644
--- a/src/core/eval.tsx
+++ b/src/core/eval.tsx
@@ -169,7 +169,7 @@ import {
   revokeOnlineEvalScope,
   scopePolicyName,
 } from "./onlineEvalExecutionRole";
-import { deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole";
+import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole";
 
 const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT";
 const DEFAULT_INGESTION_WAIT_MS = 180_000;
@@ -488,7 +488,7 @@ export class EvalClient implements CoreEvalClient {
     const control = this.clients.control(toClientConfig(options));
     const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway }));
     const gatewayArn = gateway.gatewayArn!;
-    const accountId = gatewayArn.split(":")[4] ?? "*";
+    const accountId = accountIdFromArn(gatewayArn);
 
     const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`;
     const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`;
@@ -541,7 +541,7 @@ export class EvalClient implements CoreEvalClient {
     try {
       return input.roleArn
         ? await this.clients.data(toClientConfig(options)).send(command)
-        : await retryWhileRolePropagates(() =>
+        : await retryWhileRoleUnassumable(() =>
             this.clients.data(toClientConfig(options)).send(command),
           );
     } catch (error) {
@@ -549,7 +549,7 @@ export class EvalClient implements CoreEvalClient {
         try {
           await deleteAbTestRole(this.clients.iam({ region: options.region }), provisionedRoleArn);
         } catch {
-          // best effort
+          void 0;
         }
       }
       throw error;
@@ -2106,6 +2106,31 @@ function chunk<T>(items: T[], size: number): T[][] {
 const ROLE_NOT_PROPAGATED =
   /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i;
 
+// CreateABTest is on the data plane and surfaces a freshly-provisioned role that
+// has not propagated as a plain AccessDenied rather than the control-plane phrasing
+// ROLE_NOT_PROPAGATED matches, so the ab-test create path retries on that too.
+async function retryWhileRoleUnassumable<T>(send: () => Promise<T>): Promise<T> {
+  const delaysMs = [1_000, 2_000, 4_000, 8_000];
+  for (const delay of delaysMs) {
+    try {
+      return await send();
+    } catch (error) {
+      const err = error as {
+        name?: string;
+        message?: string;
+        $metadata?: { httpStatusCode?: number };
+      };
+      const retryable =
+        err.name === "AccessDeniedException" ||
+        err.$metadata?.httpStatusCode === 403 ||
+        ROLE_NOT_PROPAGATED.test(err.message ?? "");
+      if (!retryable) throw error;
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+  return send();
+}
+
 // retryWhileRolePropagates retries `send` while the service reports the execution
 // role as unusable, which is how a not-yet-propagated role or policy surfaces.
 // Bounded and short: propagation is normally a few seconds, and a role that is
diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx
index c96a31dd3..38ebd8fa6 100644
--- a/src/handlers/eval/ab-test/ab-test.create.test.tsx
+++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx
@@ -126,7 +126,7 @@ describe("eval ab-test config-bundle run", () => {
         "o",
         "--json",
       ]),
-    ).rejects.toThrow(/must differ/);
+    ).rejects.toThrow(/must reference a different/);
   });
 
   test("rejects --treatment-weight outside 1-99", async () => {
diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx
index c8016fbdc..1c852f421 100644
--- a/src/handlers/eval/ab-test/config-bundle/run/index.tsx
+++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx
@@ -50,7 +50,7 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) =>
       flag(
         "treatment-weight",
         "1-99; control weight = 100 - this (default 50)",
-        z.number().optional(),
+        z.number().int().optional(),
       ),
       flag(
         "gateway-filter",
@@ -86,8 +86,13 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) =>
 
       const control = toBundleRef("control", controlRaw);
       const treatment = toBundleRef("treatment", treatmentRaw);
-      if (control.bundleVersion === treatment.bundleVersion) {
-        throw new InputValidationError("treatment bundle-version must differ from control");
+      if (
+        control.configBundle === treatment.configBundle &&
+        control.bundleVersion === treatment.bundleVersion
+      ) {
+        throw new InputValidationError(
+          "control and treatment must reference a different config-bundle or bundle-version",
+        );
       }
 
       const treatmentWeight = flags["treatment-weight"];

From 653dc9fd710bda6f3163faf152eb7f7a05107312 Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Wed, 26 Aug 2026 17:54:25 +0000
Subject: [PATCH 3/7] feat(eval): use --enable-on-create for ab-test
 config-bundle run

Match online-eval create's flag ergonomics: replace the boolean opt-out
--disable-on-create with a value flag --enable-on-create <true|false>
(default true). Input carries enableOnCreate?: boolean; core sends
enableOnCreate ?? true.
---
 src/core/eval.tsx                              |  2 +-
 .../eval/ab-test/ab-test.create.test.tsx       | 18 +++++++++++++-----
 .../eval/ab-test/config-bundle/run/index.tsx   | 11 +++++++++--
 src/handlers/eval/types.tsx                    |  2 +-
 4 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/src/core/eval.tsx b/src/core/eval.tsx
index 6e282633b..825a4c2a7 100644
--- a/src/core/eval.tsx
+++ b/src/core/eval.tsx
@@ -534,7 +534,7 @@ export class EvalClient implements CoreEvalClient {
       evaluationConfig: { onlineEvaluationConfigArn },
       roleArn,
       gatewayFilter: input.gatewayFilter,
-      enableOnCreate: !input.disableOnCreate,
+      enableOnCreate: input.enableOnCreate ?? true,
       clientToken: randomUUID(),
     });
 
diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx
index 38ebd8fa6..a20ae75c2 100644
--- a/src/handlers/eval/ab-test/ab-test.create.test.tsx
+++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx
@@ -76,7 +76,7 @@ describe("eval ab-test config-bundle run", () => {
       treatmentWeight: 20,
       gatewayFilter: undefined,
       roleArn: undefined,
-      disableOnCreate: false,
+      enableOnCreate: undefined,
     });
     expect(call?.args[1]).toEqual({ region: "us-west-2" });
   });
@@ -94,19 +94,27 @@ describe("eval ab-test config-bundle run", () => {
     });
   });
 
-  test("passes --disable-on-create and --role-arn through", async () => {
+  test("passes --enable-on-create false and --role-arn through", async () => {
     const { core } = await run([
       ...BASE,
-      "--disable-on-create",
+      "--enable-on-create",
+      "false",
       "--role-arn",
       "arn:aws:iam::123456789012:role/customer-owned",
     ]);
     const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
-    const input = call?.args[0] as { disableOnCreate?: boolean; roleArn?: string };
-    expect(input.disableOnCreate).toBe(true);
+    const input = call?.args[0] as { enableOnCreate?: boolean; roleArn?: string };
+    expect(input.enableOnCreate).toBe(false);
     expect(input.roleArn).toBe("arn:aws:iam::123456789012:role/customer-owned");
   });
 
+  test("--enable-on-create true is passed through", async () => {
+    const { core } = await run([...BASE, "--enable-on-create", "true"]);
+    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
+    expect(call).toBeDefined();
+    expect((call!.args[0] as { enableOnCreate?: boolean }).enableOnCreate).toBe(true);
+  });
+
   test("rejects equal control/treatment bundle-versions", async () => {
     await expect(
       run([
diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx
index 1c852f421..a7abd1e5d 100644
--- a/src/handlers/eval/ab-test/config-bundle/run/index.tsx
+++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx
@@ -62,7 +62,11 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) =>
         "execution-role override (default: auto-provisioned)",
         z.string().optional(),
       ),
-      flag("disable-on-create", "create without starting", z.boolean().optional()),
+      flag(
+        "enable-on-create",
+        "whether to start the test immediately (default true; pass false to create it paused)",
+        z.enum(["true", "false"]).optional(),
+      ),
     ],
     handle: async (ctx, flags) => {
       const required = ["name", "gateway", "control", "treatment", "online-eval"] as const;
@@ -110,7 +114,10 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) =>
           treatmentWeight,
           gatewayFilter,
           roleArn: flags["role-arn"],
-          disableOnCreate: flags["disable-on-create"],
+          enableOnCreate:
+            flags["enable-on-create"] === undefined
+              ? undefined
+              : flags["enable-on-create"] === "true",
         },
         coreOptsFromCtx(ctx),
       );
diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx
index 2ce7cd2fd..b13c6f9cd 100644
--- a/src/handlers/eval/types.tsx
+++ b/src/handlers/eval/types.tsx
@@ -244,7 +244,7 @@ export type CreateConfigBundleABTestInput = {
   treatmentWeight?: number;
   gatewayFilter?: GatewayFilter;
   roleArn?: string;
-  disableOnCreate?: boolean;
+  enableOnCreate?: boolean;
 };
 
 export type CreateDatasetInput = CreateDatasetRequest;

From 0ac67548d84481c0b6673aa4d292e7c5d707f7cd Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Wed, 26 Aug 2026 22:49:09 +0000
Subject: [PATCH 4/7] refactor(eval): reuse retryWhileRolePropagates for
 ab-test create

Drop the duplicate retryWhileRoleUnassumable I added; broaden the existing
retryWhileRolePropagates to also retry on data-plane AccessDenied/403 (how a
freshly-provisioned role surfaces on CreateABTest) and reuse it. Removes the
cross-file name collision with harness's helper.
---
 src/core/eval.tsx | 24 ++----------------------
 1 file changed, 2 insertions(+), 22 deletions(-)

diff --git a/src/core/eval.tsx b/src/core/eval.tsx
index 825a4c2a7..859bfd215 100644
--- a/src/core/eval.tsx
+++ b/src/core/eval.tsx
@@ -541,7 +541,7 @@ export class EvalClient implements CoreEvalClient {
     try {
       return input.roleArn
         ? await this.clients.data(toClientConfig(options)).send(command)
-        : await retryWhileRoleUnassumable(() =>
+        : await retryWhileRolePropagates(() =>
             this.clients.data(toClientConfig(options)).send(command),
           );
     } catch (error) {
@@ -2106,10 +2106,7 @@ function chunk<T>(items: T[], size: number): T[][] {
 const ROLE_NOT_PROPAGATED =
   /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i;
 
-// CreateABTest is on the data plane and surfaces a freshly-provisioned role that
-// has not propagated as a plain AccessDenied rather than the control-plane phrasing
-// ROLE_NOT_PROPAGATED matches, so the ab-test create path retries on that too.
-async function retryWhileRoleUnassumable<T>(send: () => Promise<T>): Promise<T> {
+async function retryWhileRolePropagates<T>(send: () => Promise<T>): Promise<T> {
   const delaysMs = [1_000, 2_000, 4_000, 8_000];
   for (const delay of delaysMs) {
     try {
@@ -2131,23 +2128,6 @@ async function retryWhileRoleUnassumable<T>(send: () => Promise<T>): Promise<T>
   return send();
 }
 
-// retryWhileRolePropagates retries `send` while the service reports the execution
-// role as unusable, which is how a not-yet-propagated role or policy surfaces.
-// Bounded and short: propagation is normally a few seconds, and a role that is
-// genuinely misconfigured should fail fast rather than hang.
-async function retryWhileRolePropagates<T>(send: () => Promise<T>): Promise<T> {
-  const delaysMs = [1_000, 2_000, 4_000, 8_000];
-  for (const delay of delaysMs) {
-    try {
-      return await send();
-    } catch (error) {
-      if (!ROLE_NOT_PROPAGATED.test((error as Error).message)) throw error;
-      await new Promise((resolve) => setTimeout(resolve, delay));
-    }
-  }
-  return send();
-}
-
 // evaluatorKmsKeys collects the customer managed KMS keys of the referenced
 // evaluators. The service validates that the execution role can decrypt them when
 // the config is created, so a provisioned role has to grant kms:Decrypt on exactly

From f325a1a9d43d5c90daf3e2e4c8cf648ec6667f1f Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:06:41 +0000
Subject: [PATCH 5/7] test(eval): golden fixture for ab-test config-bundle run

Record a self-contained config-bundle run golden in account 685197708687
(matches the config-bundle fixtures): create a bundle (v1 -> v2), a paused
online-eval on a real runtime, then run the paused A/B test; afterAll tears
down the ab-test, online-eval, provisioned role, and bundle.

Broaden retryWhileRolePropagates to also retry ValidationException 'unable to
assume the provided IAM role' -- how CreateABTest surfaces a freshly
provisioned role mid-propagation. Drop the TestCoreClient happy-path mapping
tests the golden now covers; keep the local validation/error cases.
---
 src/core/eval.tsx                             |   5 +-
 .../eval/ab-test/ab-test.create.test.tsx      |  39 ----
 .../CreateABTestCommand.506cd57a7653b22c.json |   6 +
 .../CreateABTestCommand.a4666d7f80bc7cb0.json |  10 +
 ...urationBundleCommand.e8ee73bc166ad5e4.json |   8 +
 ...luationConfigCommand.ae1ee3532d19f571.json |  14 ++
 .../CreateRoleCommand.1fa2ac2a7f0f7fc0.json   |  12 ++
 .../CreateRoleCommand.7b030b47662eee32.json   |  12 ++
 .../DeleteRoleCommand.c6a8dc12fb95054d.json   |   1 +
 ...eteRolePolicyCommand.3826bd85235b40f0.json |   1 +
 ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json |  47 +++++
 ...urationBundleCommand.cf3c23a25f6bf298.json |  23 ++
 ...urationBundleCommand.ed835ef9d614b3e6.json |  23 ++
 .../GetEvaluatorCommand.716589b0884f35c0.json |  72 +++++++
 .../GetGatewayCommand.4216a59651bb046a.json   |  20 ++
 .../GetRoleCommand.2ca20231e1472584.json      |  15 ++
 .../GetRoleCommand.c6a8dc12fb95054d.json      |   6 +
 ...PutRolePolicyCommand.8bf1be5c5e52a2ec.json |   1 +
 ...PutRolePolicyCommand.bc0b72d6bad3afad.json |   1 +
 ...urationBundleCommand.aca9ba06670aa395.json |   8 +
 ...urationBundleCommand.de17ce40709b64eb.json |   8 +
 .../run-bundle-create.golden.json             |   6 +
 .../run-bundle-update.golden.json             |   6 +
 .../__fixtures__/run-online-eval.golden.json  |  12 ++
 .../__fixtures__/run.golden.json              |   8 +
 .../config-bundle-run.fixture.test.tsx        | 198 ++++++++++++++++++
 26 files changed, 521 insertions(+), 41 deletions(-)
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json
 create mode 100644 src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx

diff --git a/src/core/eval.tsx b/src/core/eval.tsx
index 859bfd215..439c67f4f 100644
--- a/src/core/eval.tsx
+++ b/src/core/eval.tsx
@@ -2104,10 +2104,10 @@ function chunk<T>(items: T[], size: number): T[][] {
 // is created. It surfaces as one of two messages depending on which part has not
 // propagated yet.
 const ROLE_NOT_PROPAGATED =
-  /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i;
+  /cannot be assumed|unable to assume|does not have permissions to (create log group|access the specified log groups)/i;
 
 async function retryWhileRolePropagates<T>(send: () => Promise<T>): Promise<T> {
-  const delaysMs = [1_000, 2_000, 4_000, 8_000];
+  const delaysMs = [2_000, 4_000, 8_000, 15_000];
   for (const delay of delaysMs) {
     try {
       return await send();
@@ -2120,6 +2120,7 @@ async function retryWhileRolePropagates<T>(send: () => Promise<T>): Promise<T> {
       const retryable =
         err.name === "AccessDeniedException" ||
         err.$metadata?.httpStatusCode === 403 ||
+        (err.name === "ValidationException" && /assume|role|trust/i.test(err.message ?? "")) ||
         ROLE_NOT_PROPAGATED.test(err.message ?? "");
       if (!retryable) throw error;
       await new Promise((resolve) => setTimeout(resolve, delay));
diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx
index a20ae75c2..ca0d4f519 100644
--- a/src/handlers/eval/ab-test/ab-test.create.test.tsx
+++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx
@@ -63,24 +63,6 @@ describe("eval ab-test config-bundle run", () => {
     expect(cb?.children().map((c) => c.name())).toEqual(["run"]);
   });
 
-  test("maps flags to a createConfigBundleABTest call", async () => {
-    const { core, stdout } = await run([...BASE, "--treatment-weight", "20"]);
-    expect(JSON.parse(stdout).abTestId).toBe("orders-v2-abc123");
-    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
-    expect(call?.args[0]).toEqual({
-      name: "orders-v2",
-      gateway: "orders-gateway-abc123",
-      control: { configBundle: "orders-prompt-abc", bundleVersion: "1111" },
-      treatment: { configBundle: "orders-prompt-abc", bundleVersion: "2222" },
-      onlineEval: "online-eval-abc123",
-      treatmentWeight: 20,
-      gatewayFilter: undefined,
-      roleArn: undefined,
-      enableOnCreate: undefined,
-    });
-    expect(call?.args[1]).toEqual({ region: "us-west-2" });
-  });
-
   test("passes --gateway-filter through as a GatewayFilter", async () => {
     const { core } = await run([
       ...BASE,
@@ -94,27 +76,6 @@ describe("eval ab-test config-bundle run", () => {
     });
   });
 
-  test("passes --enable-on-create false and --role-arn through", async () => {
-    const { core } = await run([
-      ...BASE,
-      "--enable-on-create",
-      "false",
-      "--role-arn",
-      "arn:aws:iam::123456789012:role/customer-owned",
-    ]);
-    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
-    const input = call?.args[0] as { enableOnCreate?: boolean; roleArn?: string };
-    expect(input.enableOnCreate).toBe(false);
-    expect(input.roleArn).toBe("arn:aws:iam::123456789012:role/customer-owned");
-  });
-
-  test("--enable-on-create true is passed through", async () => {
-    const { core } = await run([...BASE, "--enable-on-create", "true"]);
-    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
-    expect(call).toBeDefined();
-    expect((call!.args[0] as { enableOnCreate?: boolean }).enableOnCreate).toBe(true);
-  });
-
   test("rejects equal control/treatment bundle-versions", async () => {
     await expect(
       run([
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json
new file mode 100644
index 000000000..51aff6399
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json
@@ -0,0 +1,6 @@
+{
+  "$error": {
+    "name": "ValidationException",
+    "message": "Unable to assume the provided IAM role. Verify the role exists and its trust policy allows bedrock-agentcore.amazonaws.com to assume it."
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json
new file mode 100644
index 000000000..971f7115e
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json
@@ -0,0 +1,10 @@
+{
+  "abTestId": "agentcore_cli_abtest_run-8e10bf2f27",
+  "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_run-8e10bf2f27",
+  "status": "CREATING",
+  "executionStatus": "NOT_STARTED",
+  "createdAt": {
+    "$date": "2026-08-27T21:03:19.535Z"
+  },
+  "name": "agentcore_cli_abtest_run"
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json
new file mode 100644
index 000000000..265aebbc3
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json
@@ -0,0 +1,8 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5",
+  "createdAt": {
+    "$date": "2026-08-27T21:02:58.851Z"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json
new file mode 100644
index 000000000..1d2638815
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json
@@ -0,0 +1,14 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_run_eval-i7s3ryDRdt",
+  "onlineEvaluationConfigId": "agentcore_cli_abtest_run_eval-i7s3ryDRdt",
+  "createdAt": {
+    "$date": "2026-08-27T21:03:03.249Z"
+  },
+  "status": "CREATING",
+  "executionStatus": "DISABLED",
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_run_eval-i7s3ryDRdt"
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json
new file mode 100644
index 000000000..40fb878c0
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json
@@ -0,0 +1,12 @@
+{
+  "Role": {
+    "Path": "/",
+    "RoleName": "AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b",
+    "RoleId": "AROAZ7CHXJWHZBY6C35TH",
+    "Arn": "arn:aws:iam::685197708687:role/AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b",
+    "CreateDate": {
+      "$date": "2026-08-27T21:03:03.000Z"
+    },
+    "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%2C%22Condition%22%3A%7B%22StringEquals%22%3A%7B%22aws%3ASourceAccount%22%3A%22685197708687%22%7D%2C%22ArnLike%22%3A%7B%22aws%3ASourceArn%22%3A%22arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A685197708687%3Aab-test%2F%2A%22%7D%7D%7D%5D%7D"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json
new file mode 100644
index 000000000..1968c9437
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json
@@ -0,0 +1,12 @@
+{
+  "Role": {
+    "Path": "/",
+    "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_run_eval",
+    "RoleId": "AROAZ7CHXJWH3HARLZYI3",
+    "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_run_eval",
+    "CreateDate": {
+      "$date": "2026-08-27T21:01:46.000Z"
+    },
+    "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json
new file mode 100644
index 000000000..9e26dfeeb
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json
new file mode 100644
index 000000000..9e26dfeeb
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json
new file mode 100644
index 000000000..2d3b5e713
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json
@@ -0,0 +1,47 @@
+{
+  "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q",
+  "agentRuntimeName": "asdf_MyAgent",
+  "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q",
+  "agentRuntimeVersion": "1",
+  "createdAt": {
+    "$date": "2026-04-23T21:17:21.895Z"
+  },
+  "lastUpdatedAt": {
+    "$date": "2026-04-23T21:17:35.159Z"
+  },
+  "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK",
+  "networkConfiguration": {
+    "networkMode": "PUBLIC"
+  },
+  "status": "READY",
+  "lifecycleConfiguration": {
+    "idleRuntimeSessionTimeout": 900,
+    "maxLifetime": 28800
+  },
+  "description": "AgentCore Runtime: asdf_MyAgent",
+  "workloadIdentityDetails": {
+    "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q"
+  },
+  "agentRuntimeArtifact": {
+    "codeConfiguration": {
+      "code": {
+        "s3": {
+          "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2",
+          "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip"
+        }
+      },
+      "runtime": "PYTHON_3_13",
+      "entryPoint": [
+        "opentelemetry-instrument",
+        "main.py"
+      ]
+    }
+  },
+  "environmentVariables": {
+    "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE",
+    "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp"
+  },
+  "metadataConfiguration": {
+    "requireMMDSV2": true
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json
new file mode 100644
index 000000000..7876da211
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json
@@ -0,0 +1,23 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "bundleName": "agentcore_cli_abtest_run_bundle",
+  "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5",
+  "components": {
+    "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q": {
+      "configuration": {
+        "system_prompt": "A/B run fixture v1."
+      }
+    }
+  },
+  "createdAt": {
+    "$date": "2026-08-27T21:02:58.851Z"
+  },
+  "updatedAt": {
+    "$date": "2026-08-27T21:02:58.851Z"
+  },
+  "lineageMetadata": {
+    "parentVersionIds": [],
+    "branchName": "mainline"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json
new file mode 100644
index 000000000..489607a72
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json
@@ -0,0 +1,23 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-Fcuyfm8hxE",
+  "bundleId": "agentcore_cli_abtest_run_bundle-Fcuyfm8hxE",
+  "bundleName": "agentcore_cli_abtest_run_bundle",
+  "versionId": "ee7a2803-a60c-4b93-ac4d-5c20f32856da",
+  "components": {
+    "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q": {
+      "configuration": {
+        "system_prompt": "A/B run fixture v1."
+      }
+    }
+  },
+  "createdAt": {
+    "$date": "2026-08-27T21:01:43.000Z"
+  },
+  "updatedAt": {
+    "$date": "2026-08-27T21:01:43.000Z"
+  },
+  "lineageMetadata": {
+    "parentVersionIds": [],
+    "branchName": "mainline"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json
new file mode 100644
index 000000000..ebf428698
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json
@@ -0,0 +1,72 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+  "evaluatorId": "Builtin.Helpfulness",
+  "evaluatorName": "Builtin.Helpfulness",
+  "evaluatorConfig": {
+    "llmAsAJudge": {
+      "ratingScale": {
+        "numerical": [
+          {
+            "value": {
+              "string": "0.0",
+              "type": "bigDecimal"
+            },
+            "label": "Not helpful at all"
+          },
+          {
+            "value": {
+              "string": "1.0",
+              "type": "bigDecimal"
+            },
+            "label": "Very unhelpful"
+          },
+          {
+            "value": {
+              "string": "2.0",
+              "type": "bigDecimal"
+            },
+            "label": "Somewhat unhelpful"
+          },
+          {
+            "value": {
+              "string": "3.0",
+              "type": "bigDecimal"
+            },
+            "label": "Neutral/Mixed"
+          },
+          {
+            "value": {
+              "string": "4.0",
+              "type": "bigDecimal"
+            },
+            "label": "Somewhat helpful"
+          },
+          {
+            "value": {
+              "string": "5.0",
+              "type": "bigDecimal"
+            },
+            "label": "Very helpful"
+          },
+          {
+            "value": {
+              "string": "6.0",
+              "type": "bigDecimal"
+            },
+            "label": "Above and beyond"
+          }
+        ]
+      }
+    }
+  },
+  "level": "TRACE",
+  "status": "ACTIVE",
+  "createdAt": {
+    "$date": "2024-10-22T00:00:00.000Z"
+  },
+  "updatedAt": {
+    "$date": "2024-10-22T00:00:00.000Z"
+  },
+  "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is",
+  "lockedForModification": true
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json
new file mode 100644
index 000000000..1007a1d2f
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json
@@ -0,0 +1,20 @@
+{
+  "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:gateway/agentcore-cli-gateway-read-fixture-a-l6opkbe2kd",
+  "gatewayId": "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd",
+  "createdAt": {
+    "$date": "2026-07-29T22:19:37.409Z"
+  },
+  "updatedAt": {
+    "$date": "2026-07-29T22:19:37.971Z"
+  },
+  "status": "READY",
+  "name": "agentcore-cli-gateway-read-fixture-a",
+  "authorizerType": "NONE",
+  "gatewayUrl": "https://agentcore-cli-gateway-read-fixture-a-l6opkbe2kd.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp",
+  "description": "AgentCore CLI persistent Gateway read fixture",
+  "roleArn": "arn:aws:iam::685197708687:role/AgentCoreCliGatewayReadFixtureRole",
+  "protocolType": "MCP",
+  "workloadIdentityDetails": {
+    "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/agentcore-cli-gateway-read-fixture-a-l6opkbe2kd"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json
new file mode 100644
index 000000000..afe04004e
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json
@@ -0,0 +1,15 @@
+{
+  "Role": {
+    "Path": "/",
+    "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_run_eval",
+    "RoleId": "AROAZ7CHXJWH3HARLZYI3",
+    "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_run_eval",
+    "CreateDate": {
+      "$date": "2026-08-27T21:01:46.000Z"
+    },
+    "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D",
+    "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_abtest_run_eval\" (created by the agentcore CLI)",
+    "MaxSessionDuration": 3600,
+    "RoleLastUsed": {}
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json
new file mode 100644
index 000000000..8bda1283f
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json
@@ -0,0 +1,6 @@
+{
+  "$error": {
+    "name": "NoSuchEntityException",
+    "message": "The role with name AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b cannot be found."
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json
new file mode 100644
index 000000000..9e26dfeeb
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json
new file mode 100644
index 000000000..9e26dfeeb
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json
new file mode 100644
index 000000000..6284b216a
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json
@@ -0,0 +1,8 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "versionId": "d9379d05-8c1c-4764-afcb-8b5f852d6830",
+  "updatedAt": {
+    "$date": "2026-08-27T21:03:02.235Z"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json
new file mode 100644
index 000000000..8772cde4d
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json
@@ -0,0 +1,8 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-Fcuyfm8hxE",
+  "bundleId": "agentcore_cli_abtest_run_bundle-Fcuyfm8hxE",
+  "versionId": "e4d7ad4b-764b-41a5-a007-7b7f0ddac0df",
+  "updatedAt": {
+    "$date": "2026-08-27T21:01:46.324Z"
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json
new file mode 100644
index 000000000..af43b1ef8
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json
@@ -0,0 +1,6 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5",
+  "createdAt": "2026-08-27T21:02:58.851Z"
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json
new file mode 100644
index 000000000..f60044fae
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json
@@ -0,0 +1,6 @@
+{
+  "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6",
+  "versionId": "d9379d05-8c1c-4764-afcb-8b5f852d6830",
+  "updatedAt": "2026-08-27T21:03:02.235Z"
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json
new file mode 100644
index 000000000..654a825b1
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json
@@ -0,0 +1,12 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_run_eval-i7s3ryDRdt",
+  "onlineEvaluationConfigId": "agentcore_cli_abtest_run_eval-i7s3ryDRdt",
+  "createdAt": "2026-08-27T21:03:03.249Z",
+  "status": "CREATING",
+  "executionStatus": "DISABLED",
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_run_eval-i7s3ryDRdt"
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json
new file mode 100644
index 000000000..66b3dfd1e
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json
@@ -0,0 +1,8 @@
+{
+  "abTestId": "agentcore_cli_abtest_run-8e10bf2f27",
+  "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_run-8e10bf2f27",
+  "status": "CREATING",
+  "executionStatus": "NOT_STARTED",
+  "createdAt": "2026-08-27T21:03:19.535Z",
+  "name": "agentcore_cli_abtest_run"
+}
\ No newline at end of file
diff --git a/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx b/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx
new file mode 100644
index 000000000..3b54ca580
--- /dev/null
+++ b/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx
@@ -0,0 +1,198 @@
+import { afterAll, describe, expect, test } from "bun:test";
+import {
+  DeleteConfigurationBundleCommand,
+  GetConfigurationBundleCommand,
+  DeleteOnlineEvaluationConfigCommand,
+} from "@aws-sdk/client-bedrock-agentcore-control";
+import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore";
+import { join } from "node:path";
+import { CoreClient } from "../../../../core";
+import {
+  createSilentLogger,
+  fixtureFactories,
+  isRecording,
+  matchGolden,
+  TestGlobalConfigAccessor,
+  testIO,
+} from "../../../../testing";
+import { createControlClient, createDataClient, createIamClient } from "../../../../core/factories";
+import { abTestExecutionRoleName, deleteAbTestRole } from "../../../../core/abTestExecutionRole";
+import { createRootHandler } from "../../../index";
+
+const REGION = "us-west-2";
+const FIXTURES = join(import.meta.dir, "__fixtures__");
+
+const RUNTIME_ARN =
+  "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q";
+const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q";
+const EVALUATOR_ID = "Builtin.Helpfulness";
+const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd";
+const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle";
+const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval";
+
+const COMPONENTS_V1 = {
+  [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } },
+};
+const COMPONENTS_V2 = {
+  [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } },
+};
+
+function createFixtureCore(): CoreClient {
+  const { createControlClient, createDataClient, createIamClient, createLogsClient } =
+    fixtureFactories(FIXTURES);
+  return new CoreClient({
+    createControlClient,
+    createDataClient,
+    createIamClient,
+    createLogsClient,
+    logger: createSilentLogger(),
+  });
+}
+
+async function run(args: string[]): Promise<string> {
+  const io = testIO();
+  const root = createRootHandler(createFixtureCore(), {
+    io: io.io,
+    logger: createSilentLogger(),
+    globalConfigAccessor: new TestGlobalConfigAccessor(),
+  });
+  await root.route(["node", "agentcore", ...args, "--region", REGION]);
+  return io.stdout();
+}
+
+async function settle(): Promise<void> {
+  if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000));
+}
+
+const created: {
+  bundleId?: string;
+  v1?: string;
+  v2?: string;
+  onlineEvalId?: string;
+  abTestId?: string;
+} = {};
+
+afterAll(async () => {
+  if (!isRecording()) return;
+  const control = createControlClient({ region: REGION });
+  const data = createDataClient({ region: REGION });
+  if (created.abTestId) {
+    try {
+      await data.send(new DeleteABTestCommand({ abTestId: created.abTestId }));
+    } catch (error) {
+      console.error("cleanup ab-test:", error);
+    }
+    try {
+      await deleteAbTestRole(
+        createIamClient({ region: REGION }),
+        abTestExecutionRoleName("agentcore_cli_abtest_run"),
+      );
+    } catch (error) {
+      console.error("cleanup ab-test role:", error);
+    }
+  }
+  if (created.onlineEvalId) {
+    try {
+      await control.send(
+        new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }),
+      );
+    } catch (error) {
+      console.error("cleanup online-eval:", error);
+    }
+  }
+  if (created.bundleId) {
+    try {
+      await control.send(
+        new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }),
+      );
+      await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId }));
+    } catch (error) {
+      if ((error as Error).name !== "ResourceNotFoundException") {
+        console.error("cleanup bundle:", error);
+      }
+    }
+  }
+});
+
+describe("eval ab-test config-bundle run (fixture-backed)", () => {
+  test("provisions a bundle with two versions", async () => {
+    const v1 = await run([
+      "eval",
+      "config-bundle",
+      "create",
+      "--name",
+      BUNDLE_NAME,
+      "--components",
+      JSON.stringify(COMPONENTS_V1),
+    ]);
+    matchGolden(FIXTURES, "run-bundle-create.golden.json", v1);
+    const first = JSON.parse(v1);
+    created.bundleId = first.bundleId;
+    created.v1 = first.versionId;
+
+    await settle();
+
+    const v2 = await run([
+      "eval",
+      "config-bundle",
+      "update",
+      "--id",
+      created.bundleId!,
+      "--components",
+      JSON.stringify(COMPONENTS_V2),
+      "--commit-message",
+      "A/B run fixture v2",
+    ]);
+    matchGolden(FIXTURES, "run-bundle-update.golden.json", v2);
+    created.v2 = JSON.parse(v2).versionId;
+    expect(created.v2).not.toBe(created.v1);
+  }, 180_000);
+
+  test("provisions a paused online evaluation config", async () => {
+    const out = await run([
+      "eval",
+      "online-eval",
+      "create",
+      "--name",
+      ONLINE_EVAL_NAME,
+      "--agent",
+      AGENT_ID,
+      "--evaluator",
+      EVALUATOR_ID,
+      "--sampling-rate",
+      "100",
+      "--enable-on-create",
+      "false",
+    ]);
+    matchGolden(FIXTURES, "run-online-eval.golden.json", out);
+    created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId;
+  }, 180_000);
+
+  test("runs a paused config-bundle A/B test", async () => {
+    const out = await run([
+      "eval",
+      "ab-test",
+      "config-bundle",
+      "run",
+      "--name",
+      "agentcore_cli_abtest_run",
+      "--gateway",
+      GATEWAY_ID,
+      "--control",
+      JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }),
+      "--treatment",
+      JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }),
+      "--online-eval",
+      created.onlineEvalId!,
+      "--treatment-weight",
+      "20",
+      "--enable-on-create",
+      "false",
+    ]);
+    matchGolden(FIXTURES, "run.golden.json", out);
+    const abTest = JSON.parse(out);
+    created.abTestId = abTest.abTestId;
+    expect(abTest.abTestId).toBeString();
+    expect(abTest.executionStatus).toBe("NOT_STARTED");
+  }, 180_000);
+});

From f09cea01a1df96d5aa9954c55205559ac09c8623 Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:10:23 +0000
Subject: [PATCH 6/7] test(eval): consolidate ab-test command-flow + unhappy
 paths into one file

Merge ab-test.write.test.tsx + ab-test.create.test.tsx into a single
ab-test.test.tsx (mirrors batch-evaluation.test.tsx): hierarchy, get/list/
pause/resume/stop/delete happy paths, and every unhappy path in one place --
missing --id (now covers get, which regressed), Core-error surfacing per op
(not-found / invalid-transition / not-stopped), and config-bundle run
validation (required flags, malformed + mis-shaped JSON, identical variants,
weight bounds). Golden fixture files unchanged.
---
 .../eval/ab-test/ab-test.create.test.tsx      | 138 ---------
 src/handlers/eval/ab-test/ab-test.test.tsx    | 266 ++++++++++++++++++
 .../eval/ab-test/ab-test.write.test.tsx       |  89 ------
 3 files changed, 266 insertions(+), 227 deletions(-)
 delete mode 100644 src/handlers/eval/ab-test/ab-test.create.test.tsx
 create mode 100644 src/handlers/eval/ab-test/ab-test.test.tsx
 delete mode 100644 src/handlers/eval/ab-test/ab-test.write.test.tsx

diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx
deleted file mode 100644
index ca0d4f519..000000000
--- a/src/handlers/eval/ab-test/ab-test.create.test.tsx
+++ /dev/null
@@ -1,138 +0,0 @@
-import { test, expect, describe } from "bun:test";
-import type { CreateABTestResponse } from "@aws-sdk/client-bedrock-agentcore";
-import { createRootHandler } from "../../index";
-import { createSilentLogger, TestCoreClient, testIO } from "../../../testing";
-import { TestGlobalConfigAccessor } from "../../../testing/";
-
-const OK: CreateABTestResponse = {
-  abTestId: "orders-v2-abc123",
-  abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/orders-v2-abc123",
-  name: "orders-v2",
-  status: "CREATING",
-  executionStatus: "NOT_STARTED",
-  createdAt: new Date("2026-08-26T10:00:00.000Z"),
-} satisfies CreateABTestResponse;
-
-async function run(args: string[], configure?: (core: TestCoreClient) => void) {
-  const core = new TestCoreClient();
-  core.eval.setAbTestCreateResponse(OK);
-  configure?.(core);
-  const io = testIO();
-  const root = createRootHandler(core, {
-    io: io.io,
-    logger: createSilentLogger(),
-    globalConfigAccessor: new TestGlobalConfigAccessor(),
-  });
-  await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]);
-  return { core, stdout: io.stdout() };
-}
-
-const BASE = [
-  "eval",
-  "ab-test",
-  "config-bundle",
-  "run",
-  "--name",
-  "orders-v2",
-  "--gateway",
-  "orders-gateway-abc123",
-  "--control",
-  '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}',
-  "--treatment",
-  '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}',
-  "--online-eval",
-  "online-eval-abc123",
-  "--json",
-];
-
-describe("eval ab-test config-bundle run", () => {
-  test("registers under ab-test → config-bundle", () => {
-    const io = testIO();
-    const root = createRootHandler(new TestCoreClient(), {
-      io: io.io,
-      logger: createSilentLogger(),
-      globalConfigAccessor: new TestGlobalConfigAccessor(),
-    });
-    const abTest = root
-      .children()
-      .find((c) => c.name() === "eval")
-      ?.children()
-      .find((c) => c.name() === "ab-test");
-    expect(abTest?.children().map((c) => c.name())).toContain("config-bundle");
-    const cb = abTest?.children().find((c) => c.name() === "config-bundle");
-    expect(cb?.children().map((c) => c.name())).toEqual(["run"]);
-  });
-
-  test("passes --gateway-filter through as a GatewayFilter", async () => {
-    const { core } = await run([
-      ...BASE,
-      "--gateway-filter",
-      '{"targetPaths":["/orders/checkout"]}',
-    ]);
-    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
-    expect(call).toBeDefined();
-    expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({
-      targetPaths: ["/orders/checkout"],
-    });
-  });
-
-  test("rejects equal control/treatment bundle-versions", async () => {
-    await expect(
-      run([
-        "eval",
-        "ab-test",
-        "config-bundle",
-        "run",
-        "--name",
-        "x",
-        "--gateway",
-        "g",
-        "--control",
-        '{"config-bundle":"b","bundle-version":"same"}',
-        "--treatment",
-        '{"config-bundle":"b","bundle-version":"same"}',
-        "--online-eval",
-        "o",
-        "--json",
-      ]),
-    ).rejects.toThrow(/must reference a different/);
-  });
-
-  test("rejects --treatment-weight outside 1-99", async () => {
-    await expect(run([...BASE, "--treatment-weight", "0"])).rejects.toThrow(/1 and 99/);
-    await expect(run([...BASE, "--treatment-weight", "100"])).rejects.toThrow(/1 and 99/);
-  });
-
-  test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)(
-    "requires --%s",
-    async (missing) => {
-      const args = BASE.filter((_, i, arr) => {
-        const prev = arr[i - 1];
-        return prev !== `--${missing}` && arr[i] !== `--${missing}`;
-      });
-      await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`));
-    },
-  );
-
-  test("rejects a malformed control JSON shape", async () => {
-    await expect(
-      run([
-        "eval",
-        "ab-test",
-        "config-bundle",
-        "run",
-        "--name",
-        "x",
-        "--gateway",
-        "g",
-        "--control",
-        '{"wrong":"shape"}',
-        "--treatment",
-        '{"config-bundle":"b","bundle-version":"2"}',
-        "--online-eval",
-        "o",
-        "--json",
-      ]),
-    ).rejects.toThrow(/--control must be/);
-  });
-});
diff --git a/src/handlers/eval/ab-test/ab-test.test.tsx b/src/handlers/eval/ab-test/ab-test.test.tsx
new file mode 100644
index 000000000..3f75f8647
--- /dev/null
+++ b/src/handlers/eval/ab-test/ab-test.test.tsx
@@ -0,0 +1,266 @@
+import { test, expect, describe } from "bun:test";
+import type { GetABTestResponse, ListABTestsResponse } from "@aws-sdk/client-bedrock-agentcore";
+import { createRootHandler } from "../../index";
+import { createSilentLogger, TestCoreClient, testIO } from "../../../testing";
+import { TestGlobalConfigAccessor } from "../../../testing/";
+
+async function run(args: string[], configure?: (core: TestCoreClient) => void) {
+  const core = new TestCoreClient();
+  configure?.(core);
+  const io = testIO();
+  const root = createRootHandler(core, {
+    io: io.io,
+    logger: createSilentLogger(),
+    globalConfigAccessor: new TestGlobalConfigAccessor(),
+  });
+  await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]);
+  return { core, stdout: io.stdout() };
+}
+
+const ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1";
+
+const GET_RESPONSE = {
+  abTestId: "ab-test-1",
+  abTestArn: ARN,
+  name: "orders-v2",
+  status: "ACTIVE",
+  executionStatus: "RUNNING",
+  gatewayArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders",
+  variants: [],
+  evaluationConfig: {
+    onlineEvaluationConfigArn:
+      "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/x",
+  },
+  createdAt: new Date("2026-07-19T01:02:03.000Z"),
+  updatedAt: new Date("2026-07-20T12:34:56.000Z"),
+} satisfies GetABTestResponse;
+
+const LIST_RESPONSE = {
+  abTests: [{ abTestId: "ab-test-1", status: "ACTIVE" }],
+  nextToken: "next",
+} as ListABTestsResponse;
+
+const RUN_BASE = [
+  "eval",
+  "ab-test",
+  "config-bundle",
+  "run",
+  "--name",
+  "orders-v2",
+  "--gateway",
+  "orders-gateway-abc123",
+  "--control",
+  '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}',
+  "--treatment",
+  '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}',
+  "--online-eval",
+  "online-eval-abc123",
+  "--json",
+];
+
+describe("eval ab-test command hierarchy", () => {
+  test("registers get, list, pause, resume, stop, delete, config-bundle", () => {
+    const io = testIO();
+    const root = createRootHandler(new TestCoreClient(), {
+      io: io.io,
+      logger: createSilentLogger(),
+      globalConfigAccessor: new TestGlobalConfigAccessor(),
+    });
+    const group = root
+      .children()
+      .find((c) => c.name() === "eval")
+      ?.children()
+      .find((c) => c.name() === "ab-test");
+    expect(group?.children().map((c) => c.name())).toEqual([
+      "get",
+      "list",
+      "pause",
+      "resume",
+      "stop",
+      "delete",
+      "config-bundle",
+    ]);
+    const cb = group?.children().find((c) => c.name() === "config-bundle");
+    expect(cb?.children().map((c) => c.name())).toEqual(["run"]);
+  });
+});
+
+describe("eval ab-test get", () => {
+  test("returns the test by id", async () => {
+    const { core, stdout } = await run(
+      ["eval", "ab-test", "get", "--id", "ab-test-1", "--json"],
+      (c) => c.eval.setAbTestGetResponse(GET_RESPONSE),
+    );
+    expect(JSON.parse(stdout).abTestId).toBe("ab-test-1");
+    expect(core.eval.calls).toEqual([
+      { method: "getABTest", args: ["ab-test-1", { region: "us-west-2" }] },
+    ]);
+  });
+
+  test("requires --id", async () => {
+    await expect(run(["eval", "ab-test", "get", "--json"])).rejects.toThrow(/--id/);
+  });
+
+  test("surfaces a Core error", async () => {
+    await expect(
+      run(["eval", "ab-test", "get", "--id", "missing", "--json"], (c) =>
+        c.eval.setError(new Error("ResourceNotFound")),
+      ),
+    ).rejects.toThrow(/ResourceNotFound/);
+  });
+});
+
+describe("eval ab-test list", () => {
+  test("passes pagination through", async () => {
+    const { core, stdout } = await run(
+      ["eval", "ab-test", "list", "--max-results", "10", "--json"],
+      (c) => c.eval.setAbTestListResponse(LIST_RESPONSE),
+    );
+    expect(JSON.parse(stdout).nextToken).toBe("next");
+    expect(core.eval.calls[0]?.args).toEqual([undefined, 10, { region: "us-west-2" }]);
+  });
+
+  test("surfaces a Core error", async () => {
+    await expect(
+      run(["eval", "ab-test", "list", "--json"], (c) => c.eval.setError(new Error("boom"))),
+    ).rejects.toThrow(/boom/);
+  });
+});
+
+describe("eval ab-test transitions", () => {
+  test.each([
+    ["pause", "PAUSED"],
+    ["resume", "RUNNING"],
+    ["stop", "STOPPED"],
+  ] as const)("%s sets executionStatus %s via Core", async (command, status) => {
+    const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) =>
+      c.eval.setAbTestUpdateResponse({
+        abTestId: "ab-test-1",
+        abTestArn: ARN,
+        status: "ACTIVE",
+        executionStatus: status,
+        updatedAt: new Date("2026-07-20T12:34:56.000Z"),
+      }),
+    );
+    expect(core.eval.calls).toEqual([
+      { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] },
+    ]);
+  });
+
+  test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => {
+    await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/);
+  });
+
+  test.each(["pause", "resume", "stop"] as const)("%s surfaces a Core error", async (command) => {
+    await expect(
+      run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) =>
+        c.eval.setError(new Error("invalid transition")),
+      ),
+    ).rejects.toThrow(/invalid transition/);
+  });
+});
+
+describe("eval ab-test delete", () => {
+  test("deletes by id via Core", async () => {
+    const { core, stdout } = await run(
+      ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"],
+      (c) =>
+        c.eval.setAbTestDeleteResponse({
+          abTestId: "ab-test-1",
+          abTestArn: ARN,
+          status: "DELETING",
+        }),
+    );
+    expect(JSON.parse(stdout).abTestId).toBe("ab-test-1");
+    expect(core.eval.calls).toEqual([
+      { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] },
+    ]);
+  });
+
+  test("requires --id", async () => {
+    await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/);
+  });
+
+  test("surfaces a Core error (e.g. not stopped)", async () => {
+    await expect(
+      run(["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], (c) =>
+        c.eval.setError(new Error("must be stopped")),
+      ),
+    ).rejects.toThrow(/must be stopped/);
+  });
+});
+
+describe("eval ab-test config-bundle run validation", () => {
+  test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)(
+    "requires --%s",
+    async (missing) => {
+      const args = RUN_BASE.filter(
+        (a, i) => a !== `--${missing}` && RUN_BASE[i - 1] !== `--${missing}`,
+      );
+      await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`));
+    },
+  );
+
+  test("rejects malformed --control JSON", async () => {
+    const args = RUN_BASE.map((a) =>
+      a === '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}' ? "notjson" : a,
+    );
+    await expect(run(args)).rejects.toThrow(/Invalid JSON/);
+  });
+
+  test("rejects a mis-shaped --control object", async () => {
+    const args = RUN_BASE.map((a) =>
+      a === '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}'
+        ? '{"wrong":"shape"}'
+        : a,
+    );
+    await expect(run(args)).rejects.toThrow(/--control must be/);
+  });
+
+  test("rejects identical control/treatment", async () => {
+    const same = '{"config-bundle":"b","bundle-version":"same"}';
+    await expect(
+      run([
+        "eval",
+        "ab-test",
+        "config-bundle",
+        "run",
+        "--name",
+        "x",
+        "--gateway",
+        "g",
+        "--control",
+        same,
+        "--treatment",
+        same,
+        "--online-eval",
+        "o",
+        "--json",
+      ]),
+    ).rejects.toThrow(/must reference a different/);
+  });
+
+  test.each(["0", "100"])("rejects --treatment-weight %s", async (w) => {
+    await expect(run([...RUN_BASE, "--treatment-weight", w])).rejects.toThrow(/1 and 99/);
+  });
+
+  test("passes --gateway-filter through as a GatewayFilter", async () => {
+    const { core } = await run(
+      [...RUN_BASE, "--gateway-filter", '{"targetPaths":["/orders/checkout"]}'],
+      (c) =>
+        c.eval.setAbTestCreateResponse({
+          abTestId: "x",
+          abTestArn: ARN,
+          name: "x",
+          status: "CREATING",
+          executionStatus: "NOT_STARTED",
+          createdAt: new Date("2026-08-26T10:00:00.000Z"),
+        }),
+    );
+    const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest");
+    expect(call).toBeDefined();
+    expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({
+      targetPaths: ["/orders/checkout"],
+    });
+  });
+});
diff --git a/src/handlers/eval/ab-test/ab-test.write.test.tsx b/src/handlers/eval/ab-test/ab-test.write.test.tsx
deleted file mode 100644
index 3a7bd3d9d..000000000
--- a/src/handlers/eval/ab-test/ab-test.write.test.tsx
+++ /dev/null
@@ -1,89 +0,0 @@
-import { test, expect, describe } from "bun:test";
-import { createRootHandler } from "../../index";
-import { createSilentLogger, TestCoreClient, testIO } from "../../../testing";
-import { TestGlobalConfigAccessor } from "../../../testing/";
-
-async function run(args: string[], configure?: (core: TestCoreClient) => void) {
-  const core = new TestCoreClient();
-  configure?.(core);
-  const io = testIO();
-  const root = createRootHandler(core, {
-    io: io.io,
-    logger: createSilentLogger(),
-    globalConfigAccessor: new TestGlobalConfigAccessor(),
-  });
-  await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]);
-  return { core, stdout: io.stdout() };
-}
-
-describe("eval ab-test command hierarchy", () => {
-  test("registers get, list, pause, resume, stop, delete", () => {
-    const io = testIO();
-    const root = createRootHandler(new TestCoreClient(), {
-      io: io.io,
-      logger: createSilentLogger(),
-      globalConfigAccessor: new TestGlobalConfigAccessor(),
-    });
-    const group = root
-      .children()
-      .find((c) => c.name() === "eval")
-      ?.children()
-      .find((c) => c.name() === "ab-test");
-    expect(group?.children().map((c) => c.name())).toEqual([
-      "get",
-      "list",
-      "pause",
-      "resume",
-      "stop",
-      "delete",
-      "config-bundle",
-    ]);
-  });
-});
-
-describe("eval ab-test transitions", () => {
-  test.each([
-    ["pause", "PAUSED"],
-    ["resume", "RUNNING"],
-    ["stop", "STOPPED"],
-  ] as const)("%s sets executionStatus %s via Core", async (command, status) => {
-    const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) =>
-      c.eval.setAbTestUpdateResponse({
-        abTestId: "ab-test-1",
-        abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1",
-        status: "ACTIVE",
-        executionStatus: status,
-        updatedAt: new Date("2026-07-20T12:34:56.000Z"),
-      }),
-    );
-    expect(core.eval.calls).toEqual([
-      { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] },
-    ]);
-  });
-
-  test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => {
-    await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/);
-  });
-});
-
-describe("eval ab-test delete", () => {
-  test("deletes by id via Core", async () => {
-    const { core, stdout } = await run(
-      ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"],
-      (c) =>
-        c.eval.setAbTestDeleteResponse({
-          abTestId: "ab-test-1",
-          abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1",
-          status: "DELETING",
-        }),
-    );
-    expect(JSON.parse(stdout).abTestId).toBe("ab-test-1");
-    expect(core.eval.calls).toEqual([
-      { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] },
-    ]);
-  });
-
-  test("requires --id", async () => {
-    await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/);
-  });
-});

From 0f31b84f9a48544af20de4f7fb86864818cbb655 Mon Sep 17 00:00:00 2001
From: jariy17 <tjariy+jariy17@users.noreply.github.com>
Date: Thu, 27 Aug 2026 21:21:29 +0000
Subject: [PATCH 7/7] test(eval): fold config-bundle run golden into
 ab-test.fixture.test.tsx

---
 .../CreateABTestCommand.506cd57a7653b22c.json |   0
 .../CreateABTestCommand.a4666d7f80bc7cb0.json |   0
 ...urationBundleCommand.e8ee73bc166ad5e4.json |   0
 ...luationConfigCommand.ae1ee3532d19f571.json |   0
 .../CreateRoleCommand.1fa2ac2a7f0f7fc0.json   |   0
 .../CreateRoleCommand.7b030b47662eee32.json   |   0
 .../DeleteRoleCommand.c6a8dc12fb95054d.json   |   0
 ...eteRolePolicyCommand.3826bd85235b40f0.json |   0
 ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json |   0
 ...urationBundleCommand.cf3c23a25f6bf298.json |   0
 ...urationBundleCommand.ed835ef9d614b3e6.json |   0
 .../GetEvaluatorCommand.716589b0884f35c0.json |   0
 .../GetGatewayCommand.4216a59651bb046a.json   |   0
 .../GetRoleCommand.2ca20231e1472584.json      |   0
 .../GetRoleCommand.c6a8dc12fb95054d.json      |   0
 ...PutRolePolicyCommand.8bf1be5c5e52a2ec.json |   0
 ...PutRolePolicyCommand.bc0b72d6bad3afad.json |   0
 ...urationBundleCommand.aca9ba06670aa395.json |   0
 ...urationBundleCommand.de17ce40709b64eb.json |   0
 .../run-bundle-create.golden.json             |   0
 .../run-bundle-update.golden.json             |   0
 .../__fixtures__/run-online-eval.golden.json  |   0
 .../__fixtures__/run.golden.json              |   0
 .../eval/ab-test/ab-test.fixture.test.tsx     | 185 ++++++++++++++--
 .../config-bundle-run.fixture.test.tsx        | 198 ------------------
 25 files changed, 170 insertions(+), 213 deletions(-)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateRoleCommand.7b030b47662eee32.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetGatewayCommand.4216a59651bb046a.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetRoleCommand.2ca20231e1472584.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run-bundle-create.golden.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run-bundle-update.golden.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run-online-eval.golden.json (100%)
 rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run.golden.json (100%)
 delete mode 100644 src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx

diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json
rename to src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json
rename to src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json b/src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json
rename to src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json
rename to src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json
rename to src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json
rename to src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json
rename to src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json b/src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json
rename to src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json
rename to src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json
rename to src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json
rename to src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json
rename to src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json b/src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json
rename to src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json
rename to src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json
rename to src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json
rename to src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json
rename to src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json
rename to src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json
rename to src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json
rename to src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json
rename to src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json
rename to src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json
diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json b/src/handlers/eval/ab-test/__fixtures__/run.golden.json
similarity index 100%
rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json
rename to src/handlers/eval/ab-test/__fixtures__/run.golden.json
diff --git a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx
index 9f8bb97fe..f6f28a6c5 100644
--- a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx
+++ b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx
@@ -1,32 +1,52 @@
-import { describe, expect, test } from "bun:test";
+import { afterAll, describe, expect, test } from "bun:test";
+import {
+  DeleteConfigurationBundleCommand,
+  GetConfigurationBundleCommand,
+  DeleteOnlineEvaluationConfigCommand,
+} from "@aws-sdk/client-bedrock-agentcore-control";
+import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore";
 import { join } from "node:path";
 import { CoreClient } from "../../../core";
 import {
   createSilentLogger,
   fixtureFactories,
+  isRecording,
   matchGolden,
   TestGlobalConfigAccessor,
   testIO,
 } from "../../../testing";
+import { createControlClient, createDataClient, createIamClient } from "../../../core/factories";
+import { abTestExecutionRoleName, deleteAbTestRole } from "../../../core/abTestExecutionRole";
 import { createRootHandler } from "../../index";
 
 const REGION = "us-west-2";
 const FIXTURES = join(import.meta.dir, "__fixtures__");
 
-// Record with: RECORD=1 bun test src/handlers/eval/ab-test/ab-test.fixture.test.tsx
-//
-// A/B tests are READ-ONLY here, so — like the batch-evaluation fixture suite —
-// this pins pre-existing tests in the fixture account rather than creating one.
-// Re-recording requires these ids to still exist; repoint them if they age out.
-//
-// Exercises the real seam end to end: parsing → handler → CoreClient →
-// GetABTest / ListABTest (data plane). GetABTest returns the per-evaluator
-// statistical results inline, so there is no CloudWatch seam to record.
+// The read describe records against account 725476964917 (a pre-existing target
+// based test); the create describe records against 685197708687 (self-created,
+// matching the config-bundle fixtures). Replay is offline and account-agnostic;
+// re-record each describe under its own account:
+//   RECORD=1 bun test -t "fixture-backed reads"
+//   RECORD=1 bun test -t "config-bundle run"
 const FIXTURE_ABTEST_ID = "abvfylatest_abtargettest-a5f5674e07";
-
-// A well-formed but absent id, to reach the not-found path.
 const MISSING_ABTEST_ID = "missing-abtest-0000000000";
 
+const RUNTIME_ARN =
+  "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q";
+const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q";
+const EVALUATOR_ID = "Builtin.Helpfulness";
+const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd";
+const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle";
+const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval";
+const AB_TEST_NAME = "agentcore_cli_abtest_run";
+
+const COMPONENTS_V1 = {
+  [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } },
+};
+const COMPONENTS_V2 = {
+  [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } },
+};
+
 function createFixtureCore(): CoreClient {
   const { createControlClient, createDataClient, createIamClient, createLogsClient } =
     fixtureFactories(FIXTURES);
@@ -50,10 +70,13 @@ async function run(args: string[]): Promise<string> {
   return io.stdout();
 }
 
-describe("eval ab-test (fixture-backed)", () => {
+async function settle(): Promise<void> {
+  if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000));
+}
+
+describe("eval ab-test fixture-backed reads", () => {
   test("get returns the test with per-evaluator metrics inline", async () => {
     const stdout = await run(["eval", "ab-test", "get", "--id", FIXTURE_ABTEST_ID, "--json"]);
-
     matchGolden(FIXTURES, "get.golden.json", stdout);
     const detail = JSON.parse(stdout);
     expect(detail.abTestId).toBe(FIXTURE_ABTEST_ID);
@@ -64,7 +87,6 @@ describe("eval ab-test (fixture-backed)", () => {
 
   test("list returns the service page", async () => {
     const stdout = await run(["eval", "ab-test", "list", "--max-results", "3", "--json"]);
-
     matchGolden(FIXTURES, "list.golden.json", stdout);
     expect(Array.isArray(JSON.parse(stdout).abTests)).toBe(true);
   });
@@ -75,3 +97,136 @@ describe("eval ab-test (fixture-backed)", () => {
     ).rejects.toThrow();
   });
 });
+
+const created: {
+  bundleId?: string;
+  v1?: string;
+  v2?: string;
+  onlineEvalId?: string;
+  abTestId?: string;
+} = {};
+
+afterAll(async () => {
+  if (!isRecording()) return;
+  const control = createControlClient({ region: REGION });
+  const data = createDataClient({ region: REGION });
+  if (created.abTestId) {
+    try {
+      await data.send(new DeleteABTestCommand({ abTestId: created.abTestId }));
+    } catch (error) {
+      console.error("cleanup ab-test:", error);
+    }
+    try {
+      await deleteAbTestRole(
+        createIamClient({ region: REGION }),
+        abTestExecutionRoleName(AB_TEST_NAME),
+      );
+    } catch (error) {
+      console.error("cleanup ab-test role:", error);
+    }
+  }
+  if (created.onlineEvalId) {
+    try {
+      await control.send(
+        new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }),
+      );
+    } catch (error) {
+      console.error("cleanup online-eval:", error);
+    }
+  }
+  if (created.bundleId) {
+    try {
+      await control.send(
+        new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }),
+      );
+      await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId }));
+    } catch (error) {
+      if ((error as Error).name !== "ResourceNotFoundException") {
+        console.error("cleanup bundle:", error);
+      }
+    }
+  }
+});
+
+describe("eval ab-test config-bundle run", () => {
+  test("provisions a bundle with two versions", async () => {
+    const v1 = await run([
+      "eval",
+      "config-bundle",
+      "create",
+      "--name",
+      BUNDLE_NAME,
+      "--components",
+      JSON.stringify(COMPONENTS_V1),
+    ]);
+    matchGolden(FIXTURES, "run-bundle-create.golden.json", v1);
+    const first = JSON.parse(v1);
+    created.bundleId = first.bundleId;
+    created.v1 = first.versionId;
+
+    await settle();
+
+    const v2 = await run([
+      "eval",
+      "config-bundle",
+      "update",
+      "--id",
+      created.bundleId!,
+      "--components",
+      JSON.stringify(COMPONENTS_V2),
+      "--commit-message",
+      "A/B run fixture v2",
+    ]);
+    matchGolden(FIXTURES, "run-bundle-update.golden.json", v2);
+    created.v2 = JSON.parse(v2).versionId;
+    expect(created.v2).not.toBe(created.v1);
+  }, 180_000);
+
+  test("provisions a paused online evaluation config", async () => {
+    const out = await run([
+      "eval",
+      "online-eval",
+      "create",
+      "--name",
+      ONLINE_EVAL_NAME,
+      "--agent",
+      AGENT_ID,
+      "--evaluator",
+      EVALUATOR_ID,
+      "--sampling-rate",
+      "100",
+      "--enable-on-create",
+      "false",
+    ]);
+    matchGolden(FIXTURES, "run-online-eval.golden.json", out);
+    created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId;
+  }, 180_000);
+
+  test("runs a paused config-bundle A/B test", async () => {
+    const out = await run([
+      "eval",
+      "ab-test",
+      "config-bundle",
+      "run",
+      "--name",
+      AB_TEST_NAME,
+      "--gateway",
+      GATEWAY_ID,
+      "--control",
+      JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }),
+      "--treatment",
+      JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }),
+      "--online-eval",
+      created.onlineEvalId!,
+      "--treatment-weight",
+      "20",
+      "--enable-on-create",
+      "false",
+    ]);
+    matchGolden(FIXTURES, "run.golden.json", out);
+    const abTest = JSON.parse(out);
+    created.abTestId = abTest.abTestId;
+    expect(abTest.abTestId).toBeString();
+    expect(abTest.executionStatus).toBe("NOT_STARTED");
+  }, 180_000);
+});
diff --git a/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx b/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx
deleted file mode 100644
index 3b54ca580..000000000
--- a/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx
+++ /dev/null
@@ -1,198 +0,0 @@
-import { afterAll, describe, expect, test } from "bun:test";
-import {
-  DeleteConfigurationBundleCommand,
-  GetConfigurationBundleCommand,
-  DeleteOnlineEvaluationConfigCommand,
-} from "@aws-sdk/client-bedrock-agentcore-control";
-import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore";
-import { join } from "node:path";
-import { CoreClient } from "../../../../core";
-import {
-  createSilentLogger,
-  fixtureFactories,
-  isRecording,
-  matchGolden,
-  TestGlobalConfigAccessor,
-  testIO,
-} from "../../../../testing";
-import { createControlClient, createDataClient, createIamClient } from "../../../../core/factories";
-import { abTestExecutionRoleName, deleteAbTestRole } from "../../../../core/abTestExecutionRole";
-import { createRootHandler } from "../../../index";
-
-const REGION = "us-west-2";
-const FIXTURES = join(import.meta.dir, "__fixtures__");
-
-const RUNTIME_ARN =
-  "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q";
-const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q";
-const EVALUATOR_ID = "Builtin.Helpfulness";
-const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd";
-const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle";
-const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval";
-
-const COMPONENTS_V1 = {
-  [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } },
-};
-const COMPONENTS_V2 = {
-  [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } },
-};
-
-function createFixtureCore(): CoreClient {
-  const { createControlClient, createDataClient, createIamClient, createLogsClient } =
-    fixtureFactories(FIXTURES);
-  return new CoreClient({
-    createControlClient,
-    createDataClient,
-    createIamClient,
-    createLogsClient,
-    logger: createSilentLogger(),
-  });
-}
-
-async function run(args: string[]): Promise<string> {
-  const io = testIO();
-  const root = createRootHandler(createFixtureCore(), {
-    io: io.io,
-    logger: createSilentLogger(),
-    globalConfigAccessor: new TestGlobalConfigAccessor(),
-  });
-  await root.route(["node", "agentcore", ...args, "--region", REGION]);
-  return io.stdout();
-}
-
-async function settle(): Promise<void> {
-  if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000));
-}
-
-const created: {
-  bundleId?: string;
-  v1?: string;
-  v2?: string;
-  onlineEvalId?: string;
-  abTestId?: string;
-} = {};
-
-afterAll(async () => {
-  if (!isRecording()) return;
-  const control = createControlClient({ region: REGION });
-  const data = createDataClient({ region: REGION });
-  if (created.abTestId) {
-    try {
-      await data.send(new DeleteABTestCommand({ abTestId: created.abTestId }));
-    } catch (error) {
-      console.error("cleanup ab-test:", error);
-    }
-    try {
-      await deleteAbTestRole(
-        createIamClient({ region: REGION }),
-        abTestExecutionRoleName("agentcore_cli_abtest_run"),
-      );
-    } catch (error) {
-      console.error("cleanup ab-test role:", error);
-    }
-  }
-  if (created.onlineEvalId) {
-    try {
-      await control.send(
-        new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }),
-      );
-    } catch (error) {
-      console.error("cleanup online-eval:", error);
-    }
-  }
-  if (created.bundleId) {
-    try {
-      await control.send(
-        new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }),
-      );
-      await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId }));
-    } catch (error) {
-      if ((error as Error).name !== "ResourceNotFoundException") {
-        console.error("cleanup bundle:", error);
-      }
-    }
-  }
-});
-
-describe("eval ab-test config-bundle run (fixture-backed)", () => {
-  test("provisions a bundle with two versions", async () => {
-    const v1 = await run([
-      "eval",
-      "config-bundle",
-      "create",
-      "--name",
-      BUNDLE_NAME,
-      "--components",
-      JSON.stringify(COMPONENTS_V1),
-    ]);
-    matchGolden(FIXTURES, "run-bundle-create.golden.json", v1);
-    const first = JSON.parse(v1);
-    created.bundleId = first.bundleId;
-    created.v1 = first.versionId;
-
-    await settle();
-
-    const v2 = await run([
-      "eval",
-      "config-bundle",
-      "update",
-      "--id",
-      created.bundleId!,
-      "--components",
-      JSON.stringify(COMPONENTS_V2),
-      "--commit-message",
-      "A/B run fixture v2",
-    ]);
-    matchGolden(FIXTURES, "run-bundle-update.golden.json", v2);
-    created.v2 = JSON.parse(v2).versionId;
-    expect(created.v2).not.toBe(created.v1);
-  }, 180_000);
-
-  test("provisions a paused online evaluation config", async () => {
-    const out = await run([
-      "eval",
-      "online-eval",
-      "create",
-      "--name",
-      ONLINE_EVAL_NAME,
-      "--agent",
-      AGENT_ID,
-      "--evaluator",
-      EVALUATOR_ID,
-      "--sampling-rate",
-      "100",
-      "--enable-on-create",
-      "false",
-    ]);
-    matchGolden(FIXTURES, "run-online-eval.golden.json", out);
-    created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId;
-  }, 180_000);
-
-  test("runs a paused config-bundle A/B test", async () => {
-    const out = await run([
-      "eval",
-      "ab-test",
-      "config-bundle",
-      "run",
-      "--name",
-      "agentcore_cli_abtest_run",
-      "--gateway",
-      GATEWAY_ID,
-      "--control",
-      JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }),
-      "--treatment",
-      JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }),
-      "--online-eval",
-      created.onlineEvalId!,
-      "--treatment-weight",
-      "20",
-      "--enable-on-create",
-      "false",
-    ]);
-    matchGolden(FIXTURES, "run.golden.json", out);
-    const abTest = JSON.parse(out);
-    created.abTestId = abTest.abTestId;
-    expect(abTest.abTestId).toBeString();
-    expect(abTest.executionStatus).toBe("NOT_STARTED");
-  }, 180_000);
-});