From a1f2f32ac066d7108aa08b4680127ca76d9105a1 Mon Sep 17 00:00:00 2001 From: Gitika <53349492+notgitika@users.noreply.github.com> Date: Thu, 27 Aug 2026 02:00:45 -0400 Subject: [PATCH 01/16] fix(deps): pin @aws-cdk/toolkit-lib yaml to v1 to fix 'yaml/types' resolution (#2122) @aws-cdk/toolkit-lib's yaml-cfn.js does require("yaml/types") at import time, a subpath that only exists in yaml v1. When an install topology resolves toolkit-lib's yaml to the hoisted yaml v2 (whose exports map blocks ./types), the module fails to load with ERR_PACKAGE_PATH_NOT_EXPORTED ("Cannot find module 'yaml/types'"). This bites bun compile, which inlines toolkit-lib and embeds whatever the build machine resolves. Add a nested override forcing toolkit-lib's yaml to ^1 so it always resolves the v1 nested copy that ships yaml/types. Co-authored-by: gitikavj --- bun.lock | 3 +++ package.json | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bun.lock b/bun.lock index a872b8204..183450cdf 100644 --- a/bun.lock +++ b/bun.lock @@ -51,6 +51,9 @@ }, }, "overrides": { + "@aws-cdk/toolkit-lib": { + "yaml": "^1", + }, "@opentelemetry/core": "^2.10.0", }, "packages": { diff --git a/package.json b/package.json index a26c178cb..d13e82d0e 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,9 @@ "zod": "^4.4.3" }, "overrides": { - "@opentelemetry/core": "^2.10.0" + "@opentelemetry/core": "^2.10.0", + "@aws-cdk/toolkit-lib": { + "yaml": "^1" + } } } From 55d04b6e2316db52fc383b878febb201fa2321d9 Mon Sep 17 00:00:00 2001 From: Gitika <53349492+notgitika@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:15:56 -0400 Subject: [PATCH 02/16] feat: persist minimal deploy state to deployed-state.json (#2105) * feat: persist minimal deploy state to top-level deployed-state.json Move the deploy-state file from agentcore/.cli/deployed-state.json to a committed agentcore/deployed-state.json, and stop storing a full snapshot of every resource. State is now keyed per target and holds only the deployed CloudFormation stack ARN (captured from the toolkit deploy result) plus the imperatively-created credential ARNs the synth step needs. Everything else is read live from CloudFormation, so the file never goes stale. Adds a DeployedState schema with readDeployedState/updateTargetState (merge-not-clobber, preserving sibling targets and unowned keys), surfaces stackArn from the CDK toolkit runner, and points the vended CDK app at the new path. * fix: harden deployed-state integrity (review feedback) Address review of the deployed-state work: - Validate any existing state before deploy, so a malformed file fails before AWS is mutated rather than after (leaving the new stack ARN unrecorded). The vended app likewise only treats a missing file as absent and surfaces a malformed one. - Require a stack ARN on a deploy result; a successful CDK deploy always has one, so its absence is malformed -- fail instead of silently skipping persistence. - Write the state file atomically (temp + rename) so an interruption can't leave unparseable JSON that blocks later deploys. - Passthrough the credential-entry schema so a stack-ARN-only rewrite doesn't strip fields a newer CLI records. - Qualify the merge guarantee: safe for sequential deploys, not concurrent. * refactor: keep deployed-state under agentcore/.cli/ Move the state file back under agentcore/.cli/ to match the released CLI's location, so a project created by an older CLI keeps reading the same path after upgrading (the vended app isn't re-vended on deploy). The scaffolded .gitignore ignores the rest of .cli/ but re-includes deployed-state.json, so the stack binding + credential ARNs stay committed and shared. --------- Co-authored-by: gitikavj --- src/assets/cdk/bin/cdk.ts | 10 +- .../templates/shared/gitignore.template | 6 +- src/core/project/backends/cdk.test.ts | 67 +++++++- src/core/project/backends/cdk.ts | 25 ++- .../backends/cdk/deployedState.test.ts | 152 ++++++++++++++++++ .../project/backends/cdk/deployedState.ts | 119 ++++++++++++++ src/core/project/backends/cdk/toolkit.test.ts | 13 +- src/core/project/backends/cdk/toolkit.ts | 17 +- 8 files changed, 390 insertions(+), 19 deletions(-) create mode 100644 src/core/project/backends/cdk/deployedState.test.ts create mode 100644 src/core/project/backends/cdk/deployedState.ts diff --git a/src/assets/cdk/bin/cdk.ts b/src/assets/cdk/bin/cdk.ts index 83e54bb4a..701339bce 100644 --- a/src/assets/cdk/bin/cdk.ts +++ b/src/assets/cdk/bin/cdk.ts @@ -124,12 +124,16 @@ async function main() { const connectorParametersByFile = resolveConnectorParametersByFile(specAny, projectRoot); const harnessConfigs = resolveHarnessConfigs(specAny, projectRoot); - // Read deployed state for credential ARNs (populated by pre-deploy identity setup) + // Read deployed state for credential ARNs (populated by pre-deploy identity setup). + // Under agentcore/.cli/ to match the released CLI's location. let deployedState: Record | undefined; try { deployedState = JSON.parse(fs.readFileSync(path.join(configRoot, '.cli', 'deployed-state.json'), 'utf8')); - } catch { - // Deployed state may not exist on first deploy + } catch (err) { + // A missing file is the normal first-deploy case. A malformed one is not: + // surface it rather than silently synthesizing without the credential ARNs + // it holds (which would drop them from the stack). + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err; } const app = new App(); diff --git a/src/assets/templates/shared/gitignore.template b/src/assets/templates/shared/gitignore.template index d00650afc..f50fc8b44 100644 --- a/src/assets/templates/shared/gitignore.template +++ b/src/assets/templates/shared/gitignore.template @@ -10,8 +10,10 @@ __pycache__/ # Node node_modules/ -# AgentCore CLI state -agentcore/.cli/ +# AgentCore CLI state (ignore local scratch like traces/logs, but commit the +# deployed-state binding so a target's stack + credential ARNs are shared) +agentcore/.cli/* +!agentcore/.cli/deployed-state.json # CDK agentcore/cdk/cdk.out/ diff --git a/src/core/project/backends/cdk.test.ts b/src/core/project/backends/cdk.test.ts index 221d4b181..288bb2199 100644 --- a/src/core/project/backends/cdk.test.ts +++ b/src/core/project/backends/cdk.test.ts @@ -1,11 +1,13 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync } from "node:fs"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; import type { DeployResult, Project, ProjectEvent } from "../../../handlers/project/types"; import { ProjectSpecSchema } from "../../../projectSchemas/project"; import { createSilentLogger } from "../../../testing"; import { CdkBackend } from "./cdk"; +import { DEPLOYED_STATE_RELATIVE_PATH } from "./cdk/deployedState"; import type { BootstrapState } from "./cdk/environment"; import type { CdkCredentialProvider, CdkOperation, CdkOutputs, CdkRunOptions } from "./cdk/toolkit"; @@ -78,6 +80,8 @@ type HarnessOptions = { account?: string; bootstrap?: BootstrapState; outputs?: CdkOutputs; + stackArn?: string; + omitStackArn?: boolean; template?: boolean; failOperation?: CdkOperation["kind"]; bootstrapError?: Error; @@ -124,7 +128,19 @@ function harness(options: HarnessOptions = {}) { if (operation.kind === options.failOperation) { throw new Error(`${operation.kind} failed`); } - return operation.kind === "deploy" ? (options.outputs ?? {}) : {}; + if (operation.kind !== "deploy") return { outputs: {} }; + return { + outputs: options.outputs ?? {}, + // A real deploy always carries a stack ARN; default one so tests exercise + // the persistence path, and use `omitStackArn` to test its absence. + ...(options.omitStackArn + ? {} + : { + stackArn: + options.stackArn ?? + "arn:aws:cloudformation:us-east-1:111122223333:stack/AgentCore-example-default/deployed", + }), + }; }, loadBootstrapTemplate: async () => { templateLoads++; @@ -239,6 +255,53 @@ describe("CdkBackend.deploy", () => { expect(subject.templateLoads()).toBe(0); }); + test("persists the deployed stack ARN under the target", async () => { + const input = await project(); + await writeAssembly(input, [TARGET.name]); + const subject = harness({ + outputs: { RuntimeArn: "arn:runtime" }, + stackArn: "arn:aws:cloudformation:us-east-1:111122223333:stack/AgentCore-example-default/abc", + }); + + await collectDeploy(subject.backend.deploy(input, { target: TARGET })); + + const statePath = join(input.rootPath, DEPLOYED_STATE_RELATIVE_PATH); + expect(JSON.parse(await Bun.file(statePath).text())).toEqual({ + targets: { + default: { + stackArn: + "arn:aws:cloudformation:us-east-1:111122223333:stack/AgentCore-example-default/abc", + }, + }, + }); + }); + + test("fails a deploy whose result carries no stack ARN, recording nothing", async () => { + const input = await project(); + await writeAssembly(input, [TARGET.name]); + const subject = harness({ outputs: { RuntimeArn: "arn:runtime" }, omitStackArn: true }); + + await expect(collectDeploy(subject.backend.deploy(input, { target: TARGET }))).rejects.toThrow( + /without a stack ARN/, + ); + expect(existsSync(join(input.rootPath, DEPLOYED_STATE_RELATIVE_PATH))).toBe(false); + }); + + test("fails before touching AWS when the existing state file is malformed", async () => { + const input = await project(); + const statePath = join(input.rootPath, DEPLOYED_STATE_RELATIVE_PATH); + await mkdir(dirname(statePath), { recursive: true }); + await writeFile(statePath, "{ not valid json"); + const subject = harness({ outputs: { RuntimeArn: "arn:runtime" } }); + + await expect( + collectDeploy(subject.backend.deploy(input, { target: TARGET })), + ).rejects.toThrow(); + // Validated before synth/bootstrap/deploy, so nothing ran against AWS. + expect(subject.commands).toEqual([]); + expect(subject.runs).toEqual([]); + }); + test.each([ ["absent", { kind: "absent" } as const], ["outdated", { kind: "outdated", version: 29 } as const], diff --git a/src/core/project/backends/cdk.ts b/src/core/project/backends/cdk.ts index 334d3d373..ee160263e 100644 --- a/src/core/project/backends/cdk.ts +++ b/src/core/project/backends/cdk.ts @@ -1,6 +1,6 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; -import { ProjectStateError } from "../../../errors/errors"; +import { MalformedServiceResponseError, ProjectStateError } from "../../../errors/errors"; import type { DeployResult, Project, ProjectEvent } from "../../../handlers/project/types"; import { FsReadWriteJson, @@ -12,6 +12,7 @@ import { import type { Logger } from "../../../logging"; import type { DeployBackendInput, ProjectBackend } from "./types"; import { stackArtifactIdForTarget } from "./cdk/assembly"; +import { readDeployedState, updateTargetState } from "./cdk/deployedState"; import { probeBootstrap, resolveAwsAccount, @@ -100,6 +101,11 @@ export class CdkBackend implements ProjectBackend { ); } + // Validate any existing deployed state before mutating AWS. A malformed file + // must fail here — not after bootstrap/deploy — so we never leave AWS changed + // with the new stack ARN unrecorded because the post-deploy write can't parse it. + await readDeployedState(this.json, project.rootPath); + yield* this.build(project); const assemblyDirectory = this.assemblyDirectory(project); const stackArtifactId = await stackArtifactIdForTarget( @@ -138,7 +144,22 @@ export class CdkBackend implements ProjectBackend { } yield { message: `Deploying ${stackArtifactId}` }; - const outputs = await this.cdk({ kind: "deploy", stackArtifactId }, options); + const { outputs, stackArn } = await this.cdk({ kind: "deploy", stackArtifactId }, options); + + // A successful deploy always has a stack ARN (CDK's DeployedStack requires + // it). Its absence means a malformed result; fail loudly rather than return + // success without recording the binding later commands need. + if (!stackArn) { + throw new MalformedServiceResponseError( + `The CDK Toolkit reported a successful deploy of '${stackArtifactId}' without a stack ARN.`, + ); + } + + // Persist the deployed stack's ARN so later commands read live resource state + // from CloudFormation. Merged per target, so deploying one target never drops + // another's recorded state. + await updateTargetState(this.json, project.rootPath, target.name, { stackArn }); + return { outputs }; } diff --git a/src/core/project/backends/cdk/deployedState.test.ts b/src/core/project/backends/cdk/deployedState.test.ts new file mode 100644 index 000000000..32bac0a33 --- /dev/null +++ b/src/core/project/backends/cdk/deployedState.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { FsReadWriteJson } from "../../../../io"; +import { createSilentLogger } from "../../../../testing"; +import { + DEPLOYED_STATE_RELATIVE_PATH, + readDeployedState, + updateTargetState, +} from "./deployedState"; + +const json = new FsReadWriteJson({ logger: createSilentLogger() }); + +const tempDirectories: string[] = []; +afterEach(async () => { + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function projectRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), "agentcore-deployed-state-")); + tempDirectories.push(root); + return root; +} + +function statePath(root: string): string { + return join(root, DEPLOYED_STATE_RELATIVE_PATH); +} + +async function readRaw(root: string): Promise { + return JSON.parse(await Bun.file(statePath(root)).text()); +} + +describe("readDeployedState", () => { + test("returns an empty state when the file does not exist", async () => { + const root = await projectRoot(); + + expect(await readDeployedState(json, root)).toEqual({ targets: {} }); + expect(existsSync(statePath(root))).toBe(false); + }); + + test("round-trips a previously written state", async () => { + const root = await projectRoot(); + + await updateTargetState(json, root, "default", { stackArn: "arn:stack:default" }); + + expect(await readDeployedState(json, root)).toEqual({ + targets: { default: { stackArn: "arn:stack:default" } }, + }); + }); + + test("resolves the file under agentcore/.cli/", () => { + expect(DEPLOYED_STATE_RELATIVE_PATH).toBe(join("agentcore", ".cli", "deployed-state.json")); + }); +}); + +describe("updateTargetState", () => { + test("creates the file with the target entry", async () => { + const root = await projectRoot(); + + await updateTargetState(json, root, "default", { stackArn: "arn:stack:default" }); + + expect(await readRaw(root)).toEqual({ + targets: { default: { stackArn: "arn:stack:default" } }, + }); + }); + + test("preserves other targets", async () => { + const root = await projectRoot(); + + await updateTargetState(json, root, "default", { stackArn: "arn:stack:default" }); + await updateTargetState(json, root, "prod", { stackArn: "arn:stack:prod" }); + + expect(await readRaw(root)).toEqual({ + targets: { + default: { stackArn: "arn:stack:default" }, + prod: { stackArn: "arn:stack:prod" }, + }, + }); + }); + + test("merges resources without dropping the stack ARN", async () => { + const root = await projectRoot(); + const credentials = { "openai-key": { credentialProviderArn: "arn:apikey:openai-key" } }; + + await updateTargetState(json, root, "default", { stackArn: "arn:stack:default" }); + await updateTargetState(json, root, "default", { resources: { credentials } }); + + expect(await readRaw(root)).toEqual({ + targets: { default: { stackArn: "arn:stack:default", resources: { credentials } } }, + }); + }); + + test("replaces a resource kind's map wholesale but keeps other kinds", async () => { + const root = await projectRoot(); + + await updateTargetState(json, root, "default", { + resources: { + credentials: { old: { credentialProviderArn: "arn:apikey:old" } }, + // A resource kind the CLI does not own must survive the merge. + runtimes: { main: { runtimeArn: "arn:runtime:main" } }, + }, + }); + await updateTargetState(json, root, "default", { + resources: { credentials: { fresh: { credentialProviderArn: "arn:apikey:fresh" } } }, + }); + + expect(await readRaw(root)).toEqual({ + targets: { + default: { + resources: { + credentials: { fresh: { credentialProviderArn: "arn:apikey:fresh" } }, + runtimes: { main: { runtimeArn: "arn:runtime:main" } }, + }, + }, + }, + }); + }); + + test("preserves unknown fields inside a credential entry across an update", async () => { + const root = await projectRoot(); + // A field a newer CLI (or the CDK app) records that this code doesn't model. + await Bun.write( + statePath(root), + JSON.stringify({ + targets: { + default: { + resources: { + credentials: { k: { credentialProviderArn: "arn:a", futureField: "keep" } }, + }, + }, + }, + }), + ); + + await updateTargetState(json, root, "default", { stackArn: "arn:stack:default" }); + + expect(await readRaw(root)).toEqual({ + targets: { + default: { + stackArn: "arn:stack:default", + resources: { + credentials: { k: { credentialProviderArn: "arn:a", futureField: "keep" } }, + }, + }, + }, + }); + }); +}); diff --git a/src/core/project/backends/cdk/deployedState.ts b/src/core/project/backends/cdk/deployedState.ts new file mode 100644 index 000000000..b1edc215a --- /dev/null +++ b/src/core/project/backends/cdk/deployedState.ts @@ -0,0 +1,119 @@ +import { existsSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { z } from "zod"; +import { atomicWrite, type ReadWriteJson } from "../../../../io"; + +/** + * Project-relative path of the state file the synthesized CDK app reads. + * + * Under `agentcore/.cli/` to match the released CLI's location, so a project + * created by an older CLI keeps reading the same path after upgrading. It holds + * a target's stack binding and its imperatively created credential ARNs; the + * scaffolded `.gitignore` keeps this one file committed while ignoring the rest + * of `.cli/`. + */ +export const DEPLOYED_STATE_RELATIVE_PATH = join("agentcore", ".cli", "deployed-state.json"); + +// Passthrough like the levels above it: a stack-ARN-only update reads and +// rewrites the whole file, so stripping unknown keys here would drop fields a +// newer CLI (or the CDK app) records inside a credential entry. +const CredentialStateSchema = z + .object({ + credentialProviderArn: z.string(), + clientSecretArn: z.string().optional(), + }) + .passthrough(); + +// Only the branches this CLI owns are modelled. Every other key the CDK app or +// the published @aws/agentcore-cdk DeployedStateSchema records under a target — +// runtimes, memories, and the rest — passes through untouched so a merge never +// drops state this code does not own. +const ResourceStateSchema = z + .object({ + credentials: z.record(z.string(), CredentialStateSchema).optional(), + }) + .passthrough(); + +const TargetStateSchema = z + .object({ + // The deployed CloudFormation stack's ARN, captured after a successful + // deploy. It embeds account + region + a unique id, so it both binds the + // target to an exact deployment and lets us detect a delete-and-recreate. + stackArn: z.string().optional(), + resources: ResourceStateSchema.optional(), + }) + .passthrough(); + +export const DeployedStateSchema = z + .object({ + targets: z.record(z.string(), TargetStateSchema).default({}), + }) + .passthrough(); + +export type DeployedState = z.infer; +export type TargetState = z.infer; + +function statePathFor(projectRoot: string): string { + return join(projectRoot, DEPLOYED_STATE_RELATIVE_PATH); +} + +/** + * Reads the deployed state for a project, returning an empty state when the + * file does not exist yet (the common case before the first deploy). Callers + * get a fully-shaped object either way, so they never special-case absence. + */ +export async function readDeployedState( + json: ReadWriteJson, + projectRoot: string, +): Promise { + const statePath = statePathFor(projectRoot); + if (!existsSync(statePath)) return { targets: {} }; + return json.read(statePath, DeployedStateSchema); +} + +/** + * Merges a patch into a single target's entry and writes the whole file back, + * preserving every other target and every resource kind this code does not own. + * + * The merge is shallow except for `resources`, which is merged one level deep so + * updating one resource kind (e.g. `credentials`) leaves the others in place. + * A resource map provided in the patch replaces the previous map for that kind + * wholesale, so a credential dropped from the spec stops being advertised. + * + * This read-modify-write is safe for sequential updates (one deploy at a time), + * which is the only supported case — concurrent deploys of the same project can + * still lose an update, since each reads the file before the other writes. + */ +export async function updateTargetState( + json: ReadWriteJson, + projectRoot: string, + targetName: string, + patch: Partial, +): Promise { + const statePath = statePathFor(projectRoot); + const state = await readDeployedState(json, projectRoot); + const previous = state.targets[targetName] ?? {}; + + const resources = + previous.resources || patch.resources + ? { ...previous.resources, ...patch.resources } + : undefined; + + const merged: TargetState = { + ...previous, + ...patch, + ...(resources && { resources }), + }; + + const next: DeployedState = { + ...state, + targets: { ...state.targets, [targetName]: merged }, + }; + + // Written atomically (temp file + rename) so an interruption or disk failure + // can't leave a half-written, unparseable state file that blocks later deploys. + await mkdir(dirname(statePath), { recursive: true }); + await atomicWrite(statePath, JSON.stringify(next, undefined, 2)); + return next; +} diff --git a/src/core/project/backends/cdk/toolkit.test.ts b/src/core/project/backends/cdk/toolkit.test.ts index 8519366fb..171e07fc3 100644 --- a/src/core/project/backends/cdk/toolkit.test.ts +++ b/src/core/project/backends/cdk/toolkit.test.ts @@ -114,7 +114,7 @@ describe("performCdkOperation", () => { { kind: "bootstrap", environments: ["aws://111122223333/us-east-1"] }, runOptions(), ), - ).toEqual({}); + ).toEqual({ outputs: {} }); expect(calls.map(({ method }) => method)).toEqual(["bootstrap"]); const [environments, options] = calls[0]!.args as [ @@ -153,13 +153,16 @@ describe("performCdkOperation", () => { test("deploys exactly one named stack from the synthesized assembly", async () => { const { calls, loaded } = loadedToolkit(); - const outputs = await performCdkOperation( + const result = await performCdkOperation( loaded, { kind: "deploy", stackArtifactId: "AgentCore-orders-default" }, runOptions({ assemblyDirectory: "/workspace/agentcore/cdk/cdk.out" }), ); - expect(outputs).toEqual({ RuntimeArn: "arn:runtime" }); + expect(result).toEqual({ + outputs: { RuntimeArn: "arn:runtime" }, + stackArn: DEPLOYED_STACK.stackArn, + }); expect(calls.map(({ method }) => method)).toEqual(["fromAssemblyDirectory", "deploy"]); expect(calls[0]!.args).toEqual(["/workspace/agentcore/cdk/cdk.out"]); expect(calls[1]!.args[1]).toMatchObject({ @@ -190,13 +193,13 @@ describe("performCdkOperation", () => { test("accepts a deployed stack that declares no outputs", async () => { const { loaded } = loadedToolkit([{ ...DEPLOYED_STACK, outputs: {} }]); - const outputs = await performCdkOperation( + const result = await performCdkOperation( loaded, { kind: "deploy", stackArtifactId: "AgentCore-orders-default" }, runOptions({ assemblyDirectory: "/workspace/agentcore/cdk/cdk.out" }), ); - expect(outputs).toEqual({}); + expect(result).toEqual({ outputs: {}, stackArn: DEPLOYED_STACK.stackArn }); }); }); diff --git a/src/core/project/backends/cdk/toolkit.ts b/src/core/project/backends/cdk/toolkit.ts index 243427c14..5bce6f503 100644 --- a/src/core/project/backends/cdk/toolkit.ts +++ b/src/core/project/backends/cdk/toolkit.ts @@ -28,7 +28,13 @@ export type CdkOutputs = Record; export type CdkCredentialProvider = SdkBaseConfig["credentialProvider"]; export type CdkCredentialResolver = (region: string) => Promise; -export type CdkRunner = (operation: CdkOperation, options: CdkRunOptions) => Promise; +/** + * Result of a CDK operation. `stackArn` is the ARN of the deployed stack (only + * a deploy produces one); bootstrap leaves it undefined. + */ +export type CdkRunResult = { outputs: CdkOutputs; stackArn?: string }; + +export type CdkRunner = (operation: CdkOperation, options: CdkRunOptions) => Promise; export type CdkToolkit = Pick; @@ -167,7 +173,7 @@ export async function performCdkOperation( { lib, toolkit }: LoadedCdkToolkit, operation: CdkOperation, options: CdkRunOptions, -): Promise { +): Promise { if (operation.kind === "bootstrap") { await toolkit.bootstrap(lib.BootstrapEnvironments.fromList(operation.environments), { parameters: lib.BootstrapStackParameters.withExisting({ @@ -177,7 +183,7 @@ export async function performCdkOperation( source: lib.BootstrapSource.customTemplate(operation.templateFile), }), }); - return {}; + return { outputs: {} }; } const source = await toolkit.fromAssemblyDirectory(options.assemblyDirectory); @@ -201,8 +207,9 @@ export async function performCdkOperation( ); } - // A stack that deployed but declares no outputs is legitimate. - return result.stacks[0]?.outputs ?? {}; + // A stack that deployed but declares no outputs is legitimate. The stack ARN + // is what the CLI persists to bind the target to this exact deployment. + return { outputs: result.stacks[0]?.outputs ?? {}, stackArn: result.stacks[0]?.stackArn }; } export function createCdkRunner( From 794ddbf30f1ce76e170c9b84b309a9ba4e412805 Mon Sep 17 00:00:00 2001 From: Gitika <53349492+notgitika@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:16:13 -0400 Subject: [PATCH 03/16] feat: read project stack state live from CloudFormation (#2112) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: read project stack state live from CloudFormation Adds a reader that describes a project's CloudFormation stack and classifies its lifecycle into not-deployed / in-progress / failed / ready, returning the stack outputs (resource ARNs/IDs) only when settled and successful. This is the source-of-truth side of the deploy-state refactor: resource details come from CloudFormation on demand rather than a local snapshot that can go stale. Generalizes the existing bootstrap not-found helper to isStackNotFound and reuses it. No command is wired to this yet; project status consumes it in a follow-up. * refactor: trim reader to the raw DescribeStacks API Per review, drop the stack-status classification (not-deployed / in-progress / failed / ready) and the StackState shape — that's a project status interface decision and belongs with whoever builds it, not baked in ahead of the consumer. Keep just describeStack: a DescribeStacks call that returns the stack or undefined when it doesn't exist. The CloudFormation call is injectable at the function seam (lazy-loaded like environment.ts), so it's unit-tested without a real client; wiring it through CoreClient/the project manager is left to the consumer. * fix: throw on an empty successful DescribeStacks response A missing stack is reported by a thrown ValidationError, so that stays the only not-found (undefined) signal. A successful response with no stack is malformed, not not-found; return undefined there would misreport a service problem as 'not deployed'. Throw MalformedServiceResponseError instead, matching the bootstrap reader. --------- Co-authored-by: gitikavj --- .../project/backends/cdk/environment.test.ts | 4 +- src/core/project/backends/cdk/environment.ts | 5 +- .../project/backends/cdk/stackReader.test.ts | 62 ++++++++++++++++++ src/core/project/backends/cdk/stackReader.ts | 65 +++++++++++++++++++ 4 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 src/core/project/backends/cdk/stackReader.test.ts create mode 100644 src/core/project/backends/cdk/stackReader.ts diff --git a/src/core/project/backends/cdk/environment.test.ts b/src/core/project/backends/cdk/environment.test.ts index 64d2c83a9..fddbbaa63 100644 --- a/src/core/project/backends/cdk/environment.test.ts +++ b/src/core/project/backends/cdk/environment.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { Stack } from "@aws-sdk/client-cloudformation"; -import { isBootstrapStackNotFound, probeBootstrap, readBootstrapState } from "./environment"; +import { isStackNotFound, probeBootstrap, readBootstrapState } from "./environment"; import type { CdkCredentialProvider } from "./toolkit"; const credentials: CdkCredentialProvider = async () => ({ @@ -69,7 +69,7 @@ describe("probeBootstrap", () => { name: "ValidationError", }); - expect(isBootstrapStackNotFound(notFound)).toBe(true); + expect(isStackNotFound(notFound)).toBe(true); expect( await probeBootstrap("us-east-1", credentials, async () => { throw notFound; diff --git a/src/core/project/backends/cdk/environment.ts b/src/core/project/backends/cdk/environment.ts index 8169e125c..55a6667b9 100644 --- a/src/core/project/backends/cdk/environment.ts +++ b/src/core/project/backends/cdk/environment.ts @@ -60,7 +60,8 @@ export function readBootstrapState(stacks?: Stack[]): Exclude ({ + accessKeyId: "access-key", + secretAccessKey: "secret-key", +}); + +function stack(name: string): Stack { + return { StackName: name, CreationTime: new Date(0), StackStatus: "CREATE_COMPLETE" }; +} + +describe("describeStack", () => { + test("returns the described stack, passing the stack name to the describer", async () => { + const names: string[] = []; + const describe: DescribeStacks = async (stackName) => { + names.push(stackName); + return [stack("AgentCore-example-prod")]; + }; + + const result = await describeStack( + "eu-west-1", + credentials, + "AgentCore-example-prod", + describe, + ); + + expect(result).toEqual(stack("AgentCore-example-prod")); + expect(names).toEqual(["AgentCore-example-prod"]); + }); + + test("returns undefined when CloudFormation reports the stack does not exist", async () => { + const notFound = Object.assign(new Error("Stack with id missing does not exist"), { + name: "ValidationError", + }); + const describe: DescribeStacks = async () => { + throw notFound; + }; + + expect(await describeStack("us-east-1", credentials, "missing", describe)).toBeUndefined(); + }); + + test("throws on an empty successful response — distinct from not-found", async () => { + const describe: DescribeStacks = async () => []; + await expect(describeStack("us-east-1", credentials, "empty", describe)).rejects.toThrow( + /returned no stack/, + ); + }); + + test("propagates errors other than not-found", async () => { + const failure = Object.assign(new Error("User is not authorized"), { + name: "AccessDeniedException", + }); + const describe: DescribeStacks = async () => { + throw failure; + }; + + await expect(describeStack("us-east-1", credentials, "denied", describe)).rejects.toBe(failure); + }); +}); diff --git a/src/core/project/backends/cdk/stackReader.ts b/src/core/project/backends/cdk/stackReader.ts new file mode 100644 index 000000000..a0a4bf88f --- /dev/null +++ b/src/core/project/backends/cdk/stackReader.ts @@ -0,0 +1,65 @@ +import type { Stack } from "@aws-sdk/client-cloudformation"; +import { MalformedServiceResponseError } from "../../../../errors/errors"; +import { isStackNotFound } from "./environment"; +import type { CdkCredentialProvider } from "./toolkit"; + +/** + * Runs a `DescribeStacks` for one stack, returning the matched stacks (or + * undefined). Injectable so callers/tests can supply the AWS call. + */ +export type DescribeStacks = (stackName: string) => Promise; + +// Real describer: lazily imports the SDK (kept off the CLI startup path, like +// environment.ts) and scopes a client to the target region + credentials. +function cloudFormationDescriber( + region: string, + credentials: CdkCredentialProvider, +): DescribeStacks { + return async (stackName) => { + const { CloudFormationClient, DescribeStacksCommand } = + await import("@aws-sdk/client-cloudformation"); + const client = new CloudFormationClient({ credentials, region }); + try { + const response = await client.send(new DescribeStacksCommand({ StackName: stackName })); + return response.Stacks; + } finally { + client.destroy(); + } + }; +} + +/** + * Describes a project's CloudFormation stack, returning it or undefined when it + * does not exist. Accepts a stack name or ARN. + * + * This is only the read: interpreting the stack's status and outputs (deployed + * vs. in-progress vs. failed, which outputs to surface) is left to the caller — + * e.g. `project status` — which owns that shape. + */ +export async function describeStack( + region: string, + credentials: CdkCredentialProvider, + stackName: string, + describe: DescribeStacks = cloudFormationDescriber(region, credentials), +): Promise { + let stacks: Stack[] | undefined; + try { + stacks = await describe(stackName); + } catch (error) { + // A missing stack is reported by a thrown ValidationError, not an empty + // result, so this is the only "not deployed" signal. Every other error + // (auth, throttling, malformed request) is real and propagates. + if (isStackNotFound(error)) return undefined; + throw error; + } + + const stack = stacks?.[0]; + if (!stack) { + // A *successful* DescribeStacks with no stack is malformed, not not-found; + // returning undefined would misreport a service problem as "not deployed". + throw new MalformedServiceResponseError( + `CloudFormation returned no stack after describing '${stackName}'`, + ); + } + return stack; +} From c105d4fd0ef082e54770c1a1613fb58d452ed404 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 27 Aug 2026 12:29:03 -0400 Subject: [PATCH 04/16] feat: support imperative eval recommendation command (#2111) * feat: support imperative eval recommendation command * chore: leverage shared utils + delete stale tests * fix: create recommendation input interface + update required flag handling --- src/core/eval.tsx | 43 +++ src/handlers/eval/index.tsx | 4 +- ...ecommendationCommand.117a4cc7f54d3397.json | 4 + ...ecommendationCommand.117a4cc7f54d3397.json | 87 ++++++ ...commendationsCommand.ec1869545cd08c49.json | 3 + ...ecommendationCommand.d33be02c79c64275.json | 87 ++++++ .../__fixtures__/delete.golden.json | 4 + .../__fixtures__/get.golden.json | 83 +++++ .../__fixtures__/list.golden.json | 3 + .../__fixtures__/start.golden.json | 83 +++++ .../eval/recommendation/delete/index.tsx | 20 ++ .../eval/recommendation/get/index.tsx | 20 ++ src/handlers/eval/recommendation/index.tsx | 16 + .../eval/recommendation/list/index.tsx | 37 +++ .../recommendation.fixture.test.tsx | 289 ++++++++++++++++++ .../eval/recommendation/start/index.tsx | 80 +++++ src/handlers/eval/types.tsx | 28 ++ src/testing/TestCoreClient.tsx | 84 +++++ src/testing/index.tsx | 2 +- src/testing/timing.tsx | 16 +- 20 files changed, 987 insertions(+), 6 deletions(-) create mode 100644 src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/delete.golden.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/get.golden.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/list.golden.json create mode 100644 src/handlers/eval/recommendation/__fixtures__/start.golden.json create mode 100644 src/handlers/eval/recommendation/delete/index.tsx create mode 100644 src/handlers/eval/recommendation/get/index.tsx create mode 100644 src/handlers/eval/recommendation/index.tsx create mode 100644 src/handlers/eval/recommendation/list/index.tsx create mode 100644 src/handlers/eval/recommendation/recommendation.fixture.test.tsx create mode 100644 src/handlers/eval/recommendation/start/index.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index ea59625cd..aa8323191 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -58,14 +58,19 @@ import { type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + DeleteRecommendationCommand, EvaluateCommand, GetABTestCommand, ListABTestsCommand, UpdateABTestCommand, DeleteABTestCommand, GetBatchEvaluationCommand, + GetRecommendationCommand, ListBatchEvaluationsCommand, + ListRecommendationsCommand, StartBatchEvaluationCommand, + StartRecommendationCommand, + type DeleteRecommendationResponse, type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, @@ -75,8 +80,12 @@ import { type ABTestExecutionStatus, type UpdateABTestResponse, type DeleteABTestResponse, + type GetRecommendationResponse, type ListBatchEvaluationsResponse, + type ListRecommendationsResponse, + type RecommendationStatus, type StartBatchEvaluationResponse, + type StartRecommendationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; @@ -133,6 +142,7 @@ import type { SpanRecord, StartBatchInsightsInput, StartBatchEvaluationInput, + StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -342,6 +352,39 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteEvaluatorCommand({ evaluatorId: id })); } + async startRecommendation( + input: StartRecommendationInput, + options: CoreOptions, + ): Promise { + return this.clients.data(toClientConfig(options)).send(new StartRecommendationCommand(input)); + } + + async getRecommendation(id: string, options: CoreOptions): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new GetRecommendationCommand({ recommendationId: id })); + } + + async listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new ListRecommendationsCommand({ nextToken, maxResults, statusFilter })); + } + + async deleteRecommendation( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .data(toClientConfig(options)) + .send(new DeleteRecommendationCommand({ recommendationId: id })); + } + // getBatchEvaluation returns the service-side job (status + evaluator summaries // + CloudWatch output config) and, by default, the per-session results read from // the job's CloudWatch stream once it is terminal. Batch evaluation lives on the diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index c28aaa503..0c610ed32 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -12,6 +12,7 @@ import { createBatchInsightsHandler } from "./batch-insights"; import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; import { createAbTestHandler } from "./ab-test"; +import { createRecommendationHandler } from "./recommendation"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -25,7 +26,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createBatchInsightsHandler(core, io)) .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)) - .handler(createAbTestHandler(core, io)); + .handler(createAbTestHandler(core, io)) + .handler(createRecommendationHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json b/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json new file mode 100644 index 000000000..e8cddf557 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/DeleteRecommendationCommand.117a4cc7f54d3397.json @@ -0,0 +1,4 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json b/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json new file mode 100644 index 000000000..78499ac27 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/GetRecommendationCommand.117a4cc7f54d3397.json @@ -0,0 +1,87 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "IN_PROGRESS", + "createdAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "updatedAt": { + "$date": "2026-08-26T14:53:00.933Z" + }, + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json b/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json new file mode 100644 index 000000000..fa5f4d49b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/ListRecommendationsCommand.ec1869545cd08c49.json @@ -0,0 +1,3 @@ +{ + "recommendationSummaries": [] +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json b/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json new file mode 100644 index 000000000..5273fad79 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/StartRecommendationCommand.d33be02c79c64275.json @@ -0,0 +1,87 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "PENDING", + "createdAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "updatedAt": { + "$date": "2026-08-26T14:53:00.323Z" + }, + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/delete.golden.json b/src/handlers/eval/recommendation/__fixtures__/delete.golden.json new file mode 100644 index 000000000..e8cddf557 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/delete.golden.json @@ -0,0 +1,4 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/get.golden.json b/src/handlers/eval/recommendation/__fixtures__/get.golden.json new file mode 100644 index 000000000..26577d77b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/get.golden.json @@ -0,0 +1,83 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "IN_PROGRESS", + "createdAt": "2026-08-26T14:53:00.323Z", + "updatedAt": "2026-08-26T14:53:00.933Z", + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/list.golden.json b/src/handlers/eval/recommendation/__fixtures__/list.golden.json new file mode 100644 index 000000000..fa5f4d49b --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/list.golden.json @@ -0,0 +1,3 @@ +{ + "recommendationSummaries": [] +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/__fixtures__/start.golden.json b/src/handlers/eval/recommendation/__fixtures__/start.golden.json new file mode 100644 index 000000000..66b4dc097 --- /dev/null +++ b/src/handlers/eval/recommendation/__fixtures__/start.golden.json @@ -0,0 +1,83 @@ +{ + "recommendationId": "agentcore_cli_recommendation_fixture-34E234DD5E", + "recommendationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:recommendation/agentcore_cli_recommendation_fixture-34E234DD5E", + "name": "agentcore_cli_recommendation_fixture", + "type": "SYSTEM_PROMPT_RECOMMENDATION", + "recommendationConfig": { + "systemPromptRecommendationConfig": { + "systemPrompt": { + "text": "You are a concise support assistant. Answer the user's question directly." + }, + "agentTraces": { + "sessionSpans": [ + { + "traceId": "0123456789abcdef0123456789abcdef", + "endTimeUnixNano": 1750000001000000000, + "resource": { + "attributes": { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent" + } + }, + "kind": "INTERNAL", + "flags": 256, + "durationNano": 1000000000, + "startTimeUnixNano": 1750000000000000000, + "body": { + "output": { + "messages": [ + { + "content": [ + { + "text": "Two plus two is four." + } + ], + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": [ + { + "text": "What is two plus two?" + } + ], + "role": "user" + } + ] + } + }, + "spanId": "0123456789abcdef", + "scope": { + "name": "agentcore-cli-fixture" + }, + "name": "invoke_agent AgentCore CLI fixture", + "attributes": { + "gen_ai.agent.name": "AgentCore CLI fixture", + "aws.genai.span_kind": "AGENT", + "gen_ai.operation.name": "invoke_agent", + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.system": "strands-agents" + }, + "status": { + "code": "OK" + } + } + ] + }, + "evaluationConfig": { + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness" + } + ] + } + } + }, + "status": "PENDING", + "createdAt": "2026-08-26T14:53:00.323Z", + "updatedAt": "2026-08-26T14:53:00.323Z", + "description": "Golden recommendation fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/recommendation/delete/index.tsx b/src/handlers/eval/recommendation/delete/index.tsx new file mode 100644 index 000000000..2eedde534 --- /dev/null +++ b/src/handlers/eval/recommendation/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteRecommendationHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete a recommendation by id", + flags: [flag("id", "the ID of the recommendation to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteRecommendation(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/recommendation/get/index.tsx b/src/handlers/eval/recommendation/get/index.tsx new file mode 100644 index 000000000..af1d2a622 --- /dev/null +++ b/src/handlers/eval/recommendation/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetRecommendationHandler = (core: Core) => + createHandler({ + name: "get", + description: "get a recommendation by id", + flags: [flag("id", "the ID of the recommendation", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getRecommendation(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/recommendation/index.tsx b/src/handlers/eval/recommendation/index.tsx new file mode 100644 index 000000000..917664075 --- /dev/null +++ b/src/handlers/eval/recommendation/index.tsx @@ -0,0 +1,16 @@ +import type { AppIO } from "../../../io"; +import { Router } from "../../../router"; +import type { Core } from "../../types"; +import { createDeleteRecommendationHandler } from "./delete"; +import { createGetRecommendationHandler } from "./get"; +import { createListRecommendationsHandler } from "./list"; +import { createStartRecommendationHandler } from "./start"; + +export function createRecommendationHandler(core: Core, io: AppIO): Router { + return new Router("recommendation", "manage AgentCore recommendations") + .supportedTuiCommands() + .handler(createStartRecommendationHandler(core, io)) + .handler(createGetRecommendationHandler(core)) + .handler(createListRecommendationsHandler(core)) + .handler(createDeleteRecommendationHandler(core)); +} diff --git a/src/handlers/eval/recommendation/list/index.tsx b/src/handlers/eval/recommendation/list/index.tsx new file mode 100644 index 000000000..3f2a86386 --- /dev/null +++ b/src/handlers/eval/recommendation/list/index.tsx @@ -0,0 +1,37 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const RECOMMENDATION_STATUSES = [ + "PENDING", + "IN_PROGRESS", + "COMPLETED", + "FAILED", + "DELETING", +] as const; + +export const createListRecommendationsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list recommendations", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + flag( + "status-filter", + `return only recommendations with this status (${RECOMMENDATION_STATUSES.join(" | ")})`, + z.enum(RECOMMENDATION_STATUSES).optional(), + ), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listRecommendations( + flags["next-token"], + flags["max-results"], + flags["status-filter"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/recommendation/recommendation.fixture.test.tsx b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx new file mode 100644 index 000000000..746c5fe95 --- /dev/null +++ b/src/handlers/eval/recommendation/recommendation.fixture.test.tsx @@ -0,0 +1,289 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + DeleteRecommendationCommand, + GetRecommendationCommand, + ListRecommendationsCommand, + type BedrockAgentCoreClient, + type RecommendationConfig, + type RecommendationStatus, +} from "@aws-sdk/client-bedrock-agentcore"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { createDataClient } from "../../../core/factories"; +import { + createSilentLogger, + fixtureFactories, + isRecording, + matchGolden, + settle, + TestGlobalConfigAccessor, + testIO, + waitFor, + WaitForTimeoutError, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); +const RECOMMENDATION_NAME = "agentcore_cli_recommendation_fixture"; +const RECORDING_TIMEOUT_MS = 10 * 60_000; +const POLL_INTERVAL_MS = 5_000; +const CONFIG: RecommendationConfig = { + systemPromptRecommendationConfig: { + systemPrompt: { + text: "You are a concise support assistant. Answer the user's question directly.", + }, + agentTraces: { + sessionSpans: [ + { + resource: { + attributes: { + "service.name": "agentcore-cli-fixture", + "aws.service.type": "gen_ai_agent", + }, + }, + traceId: "0123456789abcdef0123456789abcdef", + spanId: "0123456789abcdef", + flags: 256, + name: "invoke_agent AgentCore CLI fixture", + kind: "INTERNAL", + startTimeUnixNano: 1_750_000_000_000_000_000, + endTimeUnixNano: 1_750_000_001_000_000_000, + durationNano: 1_000_000_000, + scope: { name: "agentcore-cli-fixture" }, + attributes: { + "session.id": "00000000-0000-4000-8000-000000000001", + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "AgentCore CLI fixture", + "gen_ai.system": "strands-agents", + "aws.genai.span_kind": "AGENT", + }, + status: { code: "OK" }, + body: { + input: { + messages: [{ role: "user", content: [{ text: "What is two plus two?" }] }], + }, + output: { + messages: [{ role: "assistant", content: [{ text: "Two plus two is four." }] }], + }, + }, + }, + ], + }, + evaluationConfig: { + evaluators: [ + { + evaluatorArn: "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + }, + ], + }, + }, +}; + +// Record with: +// RECORD=1 bun test src/handlers/eval/recommendation/recommendation.fixture.test.tsx +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["bun", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +let recommendationId: string | undefined; +let deleted = false; + +function requireRecommendationId(): string { + if (!recommendationId) { + throw new Error("start fixture did not return a recommendation id"); + } + return recommendationId; +} + +function isNotFound(error: unknown): boolean { + return (error as Error).name === "ResourceNotFoundException"; +} + +async function waitForTerminal( + client: BedrockAgentCoreClient, + id: string, +): Promise { + let status: RecommendationStatus | undefined; + try { + await waitFor( + async () => { + status = (await client.send(new GetRecommendationCommand({ recommendationId: id }))).status; + return status === "COMPLETED" || status === "FAILED" || status === "DELETING"; + }, + RECORDING_TIMEOUT_MS, + POLL_INTERVAL_MS, + ); + } catch (error) { + if (!(error instanceof WaitForTimeoutError)) throw error; + throw new Error(`timed out waiting for recommendation ${id} to reach a terminal state`, { + cause: error, + }); + } + return status; +} + +async function waitUntilDeleted(client: BedrockAgentCoreClient, id: string): Promise { + try { + await waitFor( + async () => { + try { + await client.send(new GetRecommendationCommand({ recommendationId: id })); + return false; + } catch (error) { + if (isNotFound(error)) return true; + throw error; + } + }, + RECORDING_TIMEOUT_MS, + POLL_INTERVAL_MS, + ); + } catch (error) { + if (!(error instanceof WaitForTimeoutError)) throw error; + throw new Error(`timed out waiting for recommendation ${id} to be deleted`, { cause: error }); + } +} + +async function cleanupRecommendation(client: BedrockAgentCoreClient, id: string): Promise { + let status: RecommendationStatus | undefined; + try { + status = (await client.send(new GetRecommendationCommand({ recommendationId: id }))).status; + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + + if (status === "PENDING" || status === "IN_PROGRESS") { + status = await waitForTerminal(client, id); + } + if (status !== "DELETING") { + await client.send(new DeleteRecommendationCommand({ recommendationId: id })); + } + await waitUntilDeleted(client, id); +} + +beforeAll(async () => { + if (!isRecording()) return; + + const client = createDataClient({ region: REGION }); + const ids: string[] = []; + let nextToken: string | undefined; + do { + const page = await client.send(new ListRecommendationsCommand({ maxResults: 100, nextToken })); + ids.push( + ...(page.recommendationSummaries ?? []) + .filter((summary) => summary.name === RECOMMENDATION_NAME) + .flatMap((summary) => (summary.recommendationId ? [summary.recommendationId] : [])), + ); + nextToken = page.nextToken; + } while (nextToken); + + for (const id of ids) await cleanupRecommendation(client, id); +}, RECORDING_TIMEOUT_MS); + +afterAll(async () => { + if (!isRecording() || !recommendationId || deleted) return; + + try { + await cleanupRecommendation(createDataClient({ region: REGION }), recommendationId); + } catch (error) { + if (!isNotFound(error)) { + console.error(`could not clean up fixture recommendation ${recommendationId}:`, error); + } + } +}, RECORDING_TIMEOUT_MS); + +describe("eval recommendation against recorded responses", () => { + test("starts a recommendation", async () => { + const stdout = await run([ + "eval", + "recommendation", + "start", + "--name", + RECOMMENDATION_NAME, + "--description", + "Golden recommendation fixture", + "--type", + "SYSTEM_PROMPT_RECOMMENDATION", + "--recommendation-config", + JSON.stringify(CONFIG), + "--tags", + "suite=golden", + ]); + + matchGolden(FIXTURES, "start.golden.json", stdout); + const response = JSON.parse(stdout); + recommendationId = response.recommendationId; + expect(recommendationId).toBeString(); + expect(response.name).toBe(RECOMMENDATION_NAME); + expect(response.type).toBe("SYSTEM_PROMPT_RECOMMENDATION"); + }); + + test("gets the recommendation", async () => { + await settle(2_000); + const id = requireRecommendationId(); + + const stdout = await run(["eval", "recommendation", "get", "--id", id]); + + matchGolden(FIXTURES, "get.golden.json", stdout); + const response = JSON.parse(stdout); + expect(response.recommendationId).toBe(id); + expect(response.name).toBe(RECOMMENDATION_NAME); + expect(response.recommendationConfig).toEqual(CONFIG); + }, 60_000); + + test("lists recommendations with API pagination and filtering", async () => { + const stdout = await run([ + "eval", + "recommendation", + "list", + "--max-results", + "5", + "--status-filter", + "COMPLETED", + ]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).recommendationSummaries).toBeArray(); + }); + + test( + "deletes the recommendation", + async () => { + const id = requireRecommendationId(); + if (isRecording()) { + await waitForTerminal(createDataClient({ region: REGION }), id); + } + + const stdout = await run(["eval", "recommendation", "delete", "--id", id]); + + matchGolden(FIXTURES, "delete.golden.json", stdout); + expect(JSON.parse(stdout)).toMatchObject({ + recommendationId: id, + status: "DELETING", + }); + deleted = true; + }, + RECORDING_TIMEOUT_MS, + ); +}); diff --git a/src/handlers/eval/recommendation/start/index.tsx b/src/handlers/eval/recommendation/start/index.tsx new file mode 100644 index 000000000..d69a5b5b7 --- /dev/null +++ b/src/handlers/eval/recommendation/start/index.tsx @@ -0,0 +1,80 @@ +import type { RecommendationConfig } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { InputValidationError } from "../../../../errors"; +import { SourceResolver, type AppIO } from "../../../../io"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag, parseTags } from "../../../utils"; + +const RECOMMENDATION_TYPES = [ + "SYSTEM_PROMPT_RECOMMENDATION", + "TOOL_DESCRIPTION_RECOMMENDATION", +] as const; + +const REQUIRED_FLAGS = ["name", "type", "recommendation-config"] as const; +type RequiredFlag = (typeof REQUIRED_FLAGS)[number]; + +function assertRequiredFlags( + flags: Partial>, +): asserts flags is Record { + for (const name of REQUIRED_FLAGS) { + if (!flags[name]) { + throw new InputValidationError(`required option '--${name} <${name}>' not specified`); + } + } +} + +export const createStartRecommendationHandler = (core: Core, io: AppIO) => + createHandler({ + name: "start", + description: "start an asynchronous recommendation", + flags: [ + flag("name", "the name of the recommendation", z.string().optional()), + flag( + "type", + `the recommendation type (${RECOMMENDATION_TYPES.join(" | ")})`, + z.enum(RECOMMENDATION_TYPES).optional(), + ), + flag( + "recommendation-config", + "recommendation configuration (JSON inline, file://, or - for stdin)", + z.string().optional(), + { sensitive: true }, + ), + flag("description", "a description of the recommendation", z.string().optional()), + flag( + "kms-key-arn", + "customer managed KMS key ARN for recommendation data", + z.string().optional(), + ), + flag("tags", "tags as key=value (repeatable) or JSON object", z.array(z.string()).optional()), + ], + handle: async (ctx, flags) => { + assertRequiredFlags(flags); + + const source = new SourceResolver({ stdin: io.stdin }); + const recommendationConfig = parseJsonFlag( + "recommendation-config", + await source.resolveText("recommendation-config", flags["recommendation-config"]), + ); + if (!recommendationConfig) { + throw new InputValidationError( + "required option '--recommendation-config ' not specified", + ); + } + + const response = await core.eval.startRecommendation( + { + name: flags["name"], + type: flags["type"], + recommendationConfig, + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags: parseTags(flags["tags"]), + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 252dac569..6bcb6447f 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -35,8 +35,15 @@ import type { ABTestExecutionStatus, UpdateABTestResponse, DeleteABTestResponse, + DeleteRecommendationResponse, GetBatchEvaluationResponse, + GetRecommendationResponse, + ListRecommendationsResponse, ListBatchEvaluationsResponse, + RecommendationConfig, + RecommendationStatus, + RecommendationType, + StartRecommendationResponse, StartBatchEvaluationResponse, SessionMetadataShape, InlineGroundTruth, @@ -225,6 +232,14 @@ export type RoleScopeWarning = { }; export type CreateDatasetInput = CreateDatasetRequest; +export type StartRecommendationInput = { + name: string; + description?: string; + type: RecommendationType; + recommendationConfig: RecommendationConfig; + kmsKeyArn?: string; + tags?: Record; +}; export type CreateConfigurationBundleInput = Pick< CreateConfigurationBundleRequest, "bundleName" | "components" | "branchName" | "commitMessage" | "kmsKeyArn" @@ -365,6 +380,19 @@ export interface CoreEvalClient { ): Promise; deleteEvaluator(id: string, options: CoreOptions): Promise; + startRecommendation( + input: StartRecommendationInput, + options: CoreOptions, + ): Promise; + getRecommendation(id: string, options: CoreOptions): Promise; + listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise; + deleteRecommendation(id: string, options: CoreOptions): Promise; + // getBatchEvaluation returns the service-side job and, unless `includeResults` // is false, the per-session results read from its per-job CloudWatch stream once // terminal. A CloudWatch read failure is returned as `resultsError` (never diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 5414837c2..ef5064c21 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -78,7 +78,9 @@ import type { ABTestExecutionStatus, UpdateABTestResponse, DeleteABTestResponse, + DeleteRecommendationResponse, GetBatchEvaluationResponse, + GetRecommendationResponse, GetEventInput, GetEventOutput, GetMemoryRecordInput, @@ -96,9 +98,12 @@ import type { ListEventsOutput, ListMemoryRecordsInput, ListMemoryRecordsOutput, + ListRecommendationsResponse, ListSessionsInput, ListSessionsOutput, + RecommendationStatus, StartBatchEvaluationResponse, + StartRecommendationResponse, } from "@aws-sdk/client-bedrock-agentcore"; import type { Core } from "../handlers/types"; import type { CoreHarnessClient, CreateHarnessInput } from "../handlers/harness/types"; @@ -148,6 +153,7 @@ import type { SessionTrace, StartBatchInsightsInput, StartBatchEvaluationInput, + StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -279,6 +285,12 @@ const DEFAULT_START_BATCH_EVAL_RESPONSE = { batchEvaluationId: "batch-eval-test", status: "RUNNING", } as unknown as StartBatchEvaluationResponse; +const DEFAULT_START_RECOMMENDATION_RESPONSE = {} as StartRecommendationResponse; +const DEFAULT_GET_RECOMMENDATION_RESPONSE = {} as GetRecommendationResponse; +const DEFAULT_LIST_RECOMMENDATIONS_RESPONSE: ListRecommendationsResponse = { + recommendationSummaries: [], +}; +const DEFAULT_DELETE_RECOMMENDATION_RESPONSE = {} as DeleteRecommendationResponse; const DEFAULT_UPDATE_DATASET_RESULT: DatasetUpdateResult = { datasetId: "dataset-orders-abc123", added: 0, @@ -1370,6 +1382,13 @@ export class TestEvalClient implements CoreEvalClient { private updateResponse: UpdateEvaluatorResponse = DEFAULT_UPDATE_EVALUATOR_RESPONSE; private getResponse: GetEvaluatorResponse = DEFAULT_GET_EVALUATOR_RESPONSE; private deleteResponse: DeleteEvaluatorResponse = DEFAULT_DELETE_EVALUATOR_RESPONSE; + private startRecommendationResponse: StartRecommendationResponse = + DEFAULT_START_RECOMMENDATION_RESPONSE; + private getRecommendationResponse: GetRecommendationResponse = + DEFAULT_GET_RECOMMENDATION_RESPONSE; + private recommendationListResponses = new Map(); + private deleteRecommendationResponse: DeleteRecommendationResponse = + DEFAULT_DELETE_RECOMMENDATION_RESPONSE; // Online-eval responses, keyed the same way: listOnlineEvaluationConfigs pages // by nextToken, the rest are single canned values. private onlineEvalListResponses = new Map< @@ -1467,6 +1486,29 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setStartRecommendationResponse(response: StartRecommendationResponse): this { + this.startRecommendationResponse = response; + return this; + } + + setGetRecommendationResponse(response: GetRecommendationResponse): this { + this.getRecommendationResponse = response; + return this; + } + + setListRecommendationsResponse( + response: ListRecommendationsResponse, + forNextToken?: string, + ): this { + this.recommendationListResponses.set(forNextToken, response); + return this; + } + + setDeleteRecommendationResponse(response: DeleteRecommendationResponse): this { + this.deleteRecommendationResponse = response; + return this; + } + // setOnlineEvalListResponse sets what listOnlineEvaluationConfigs resolves to // (when not erroring). Pass `forNextToken` to serve a later page. setOnlineEvalListResponse( @@ -1710,6 +1752,48 @@ export class TestEvalClient implements CoreEvalClient { return this.deleteResponse; } + async startRecommendation( + input: StartRecommendationInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "startRecommendation", args: [input, options] }); + if (this.error) throw this.error; + return this.startRecommendationResponse; + } + + async getRecommendation(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getRecommendation", args: [id, options] }); + if (this.error) throw this.error; + return this.getRecommendationResponse; + } + + async listRecommendations( + nextToken: string | undefined, + maxResults: number | undefined, + statusFilter: RecommendationStatus | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ + method: "listRecommendations", + args: [nextToken, maxResults, statusFilter, options], + }); + if (this.error) throw this.error; + return ( + this.recommendationListResponses.get(nextToken) ?? + this.recommendationListResponses.get(undefined) ?? + DEFAULT_LIST_RECOMMENDATIONS_RESPONSE + ); + } + + async deleteRecommendation( + id: string, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "deleteRecommendation", args: [id, options] }); + if (this.error) throw this.error; + return this.deleteRecommendationResponse; + } + async getBatchEvaluation( id: string, options: CoreOptions, diff --git a/src/testing/index.tsx b/src/testing/index.tsx index fc313b4f5..548d3fd16 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -1,7 +1,7 @@ export { parse, stringify } from "./serialization"; export { fixtureFactories, fixtureFetch, isRecording, matchGolden, settle } from "./fixtures"; export { testIO, type TestIO } from "./testIO"; -export { tick, waitFor } from "./timing"; +export { tick, waitFor, WaitForTimeoutError } from "./timing"; export { TestCoreClient, TestGatewayClient, diff --git a/src/testing/timing.tsx b/src/testing/timing.tsx index 2d7b62494..52f392575 100644 --- a/src/testing/timing.tsx +++ b/src/testing/timing.tsx @@ -2,6 +2,13 @@ export function tick(ms = 0): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +export class WaitForTimeoutError extends Error { + constructor() { + super("waitFor: condition not met before timeout"); + this.name = "WaitForTimeoutError"; + } +} + /** * Polls {@link predicate} until it returns true or the timeout elapses, * ticking between attempts so pending renders/queries flush. @@ -11,17 +18,18 @@ export function tick(ms = 0): Promise { * * @param predicate - Condition to wait for. * @param timeoutMs - Maximum time to wait in milliseconds (default 1000). + * @param intervalMs - Delay between attempts in milliseconds (default 5). * @throws If the predicate is not satisfied before the timeout. */ export async function waitFor( predicate: () => boolean | Promise, timeoutMs = 1000, + intervalMs = 5, ): Promise { - const step = 5; let waited = 0; while (!(await predicate())) { - if (waited >= timeoutMs) throw new Error("waitFor: condition not met before timeout"); - await tick(step); - waited += step; + if (waited >= timeoutMs) throw new WaitForTimeoutError(); + await tick(intervalMs); + waited += intervalMs; } } From b78d3960616099389df45fb0e911a7755f315684 Mon Sep 17 00:00:00 2001 From: Trirmadura J Ariyawansa Date: Thu, 27 Aug 2026 12:57:51 -0400 Subject: [PATCH 05/16] =?UTF-8?q?feat(eval):=20batch=20simulate=20?= =?UTF-8?q?=E2=80=94=20--ingestion-wait-ms=20+=20per-example=20failures/se?= =?UTF-8?q?ssions=20(#2098)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(eval): batch simulate — --ingestion-wait-ms flag + per-example failures/sessions Follow-ups from the batch-evaluation simulate review: - Add --ingestion-wait-ms (default 180000, 0 skips); thread via InvokeDatasetInput.waitIngestionMs. Removes the SIMULATE_INGESTION_WAIT_MS env var — tests pass the value through the input. - runExamples now returns per-item failures (item + error), not a bare count + firstError; invokeDataset surfaces failures: [{ exampleId, error }] so a partial failure names which examples dropped and why. - batch simulate output renders sessions[] (exampleId <-> sessionId join key for a later get) and failures[] (omitted when empty). * chore: drop explanatory comments from batch simulate follow-up * fix: always render failures[] in batch simulate output * test(eval): simulate fixture golden via id seam + stream-aware recorder - inject newSessionId into EvalClient (default randomUUID) so replay fixtures + goldens are deterministic - teach makeRecordingSend to freeze/revive a streaming SDK response (InvokeAgentRuntime), which stringify couldn't serialize - add simulate fixture-golden case; move handler edges to batch-evaluation.test.tsx - split invokeDataset.test.ts into run.test.ts (pool) + load.test.ts (parse + GT-shape); delete it and simulate.test.tsx * test(eval): drop run.test.ts + load.test.ts * chore: drop explanatory comments from the simulate test refactor * test(eval): simulate fixture asserts via matchGolden only (drop redundant expects) --------- Co-authored-by: jariy17 --- src/core/eval.tsx | 42 +-- .../__snapshots__/invokeDataset.test.ts.snap | 172 ------------ .../eval/invokeDataset/invokeDataset.test.ts | 263 ------------------ src/core/eval/invokeDataset/run.ts | 17 +- src/core/index.tsx | 8 +- ...eAgentRuntimeCommand.46bef739bc7ae9c1.json | 8 + ...tchEvaluationCommand.9c92cd5b227a329e.json | 20 ++ .../__fixtures__/simulate-ds.jsonl | 1 + .../__fixtures__/simulate.golden.json | 13 + .../batch-evaluation.fixture.test.tsx | 48 ++++ .../batch-evaluation.test.tsx | 93 +++++++ .../__snapshots__/simulate.test.tsx.snap | 23 -- .../eval/batch-evaluation/simulate/index.tsx | 11 +- .../simulate/simulate.test.tsx | 169 ----------- src/handlers/eval/types.tsx | 7 +- src/testing/TestCoreClient.tsx | 1 + src/testing/fixtures.tsx | 32 ++- 17 files changed, 264 insertions(+), 664 deletions(-) delete mode 100644 src/core/eval/invokeDataset/__snapshots__/invokeDataset.test.ts.snap delete mode 100644 src/core/eval/invokeDataset/invokeDataset.test.ts create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/InvokeAgentRuntimeCommand.46bef739bc7ae9c1.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.9c92cd5b227a329e.json create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/simulate-ds.jsonl create mode 100644 src/handlers/eval/batch-evaluation/__fixtures__/simulate.golden.json delete mode 100644 src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap delete mode 100644 src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index aa8323191..a7c234ee3 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -168,6 +168,7 @@ import { } from "./onlineEvalExecutionRole"; const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +const DEFAULT_INGESTION_WAIT_MS = 180_000; const DATASET_EXAMPLES_BATCH_LIMIT = 1000; const DATASET_MUTATION_PAYLOAD_LIMIT_BYTES = 5 * 1024 * 1024; const DATASET_ACTIVE_TIMEOUT_MS = 60_000; @@ -218,6 +219,7 @@ export class EvalClient implements CoreEvalClient { private readonly fetch: CoreFetch = globalThis.fetch, // logger for batch-evaluation result-log diagnostics private readonly logger: Logger = noopLogger, + private readonly newSessionId: () => string = randomUUID, ) {} async createEvaluator( @@ -717,10 +719,10 @@ export class EvalClient implements CoreEvalClient { const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; const accountId = accountIdFromRuntimeArn(runtime.agentRuntimeArn); - const { ok, failed, firstError } = await runExamples(examples, async (example) => { + const { ok, failures } = await runExamples(examples, async (example) => { // One session per example; the id is a client-owned input per the AgentCore docs, // reused across turns so the conversation and its per-turn traces stay in order. - const sessionId = randomUUID(); + const sessionId = this.newSessionId(); const ctx: RunContext = { invokeOnce: async (payload) => { const response = await invokeRuntime( @@ -748,27 +750,22 @@ export class EvalClient implements CoreEvalClient { return { text }; }, }; - try { - const groundTruth = await example.run(ctx); - return { exampleId: example.exampleId, sessionId, groundTruth }; - } catch (error) { - // Enrich with the example identity so the dropped-invoke reason is self-describing - // in firstError, instead of a bare transport message logged separately. - const cause = error instanceof Error ? error : new Error(String(error)); - throw new Error( - `example "${example.exampleId}" (${example.schemaType}) failed to invoke: ${cause.message}`, - { cause }, - ); - } + const groundTruth = await example.run(ctx); + return { exampleId: example.exampleId, sessionId, groundTruth }; }); - if (failed > 0) { - this.logger.warn(`invokeDataset: ${failed} example(s) failed to invoke and were dropped`); + const invokeFailures = failures.map((f) => ({ + exampleId: f.item.exampleId, + error: f.error.message, + })); + if (invokeFailures.length > 0) { + this.logger.warn( + `invokeDataset: ${invokeFailures.length} example(s) failed to invoke and were dropped` + + `; first: ${invokeFailures[0]!.exampleId} — ${invokeFailures[0]!.error}`, + ); } - // AgentCore emits spans ~30s-3min after invoke; grade too early and it reads an empty - // log group and fails every session. Disabled via SIMULATE_INGESTION_WAIT_MS=0 (tests). - const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); + const waitMs = input.waitIngestionMs ?? DEFAULT_INGESTION_WAIT_MS; if (ok.length > 0 && waitMs > 0) { this.logger.info( `waiting ${Math.round(waitMs / 1000)}s for span ingestion before evaluating`, @@ -776,7 +773,12 @@ export class EvalClient implements CoreEvalClient { await sleep(waitMs, undefined, { signal }); } - return { sessions: ok, invoked: ok.length, failed, firstError }; + return { + sessions: ok, + invoked: ok.length, + failed: invokeFailures.length, + failures: invokeFailures, + }; } // Resolve a dataset ref to JSONL text: a local path directly, else download the id to a diff --git a/src/core/eval/invokeDataset/__snapshots__/invokeDataset.test.ts.snap b/src/core/eval/invokeDataset/__snapshots__/invokeDataset.test.ts.snap deleted file mode 100644 index c7a35def0..000000000 --- a/src/core/eval/invokeDataset/__snapshots__/invokeDataset.test.ts.snap +++ /dev/null @@ -1,172 +0,0 @@ -// Bun Snapshot v1, https://bun.sh/docs/test/snapshots - -exports[`EvalClient.invokeDataset golden: sessions + ground truth over representative datasets 1`] = ` -{ - "assertions + trajectory + sparse turns (full inline shape)": { - "failed": 0, - "invoked": 1, - "sessions": [ - { - "exampleId": "orders-1", - "groundTruth": { - "assertions": [ - { - "text": "stays polite", - }, - { - "text": "does not promise a date", - }, - ], - "expectedTrajectory": { - "toolNames": [ - "refund_lookup", - "refund_create", - ], - }, - "turns": [ - { - "input": { - "prompt": "I want a refund", - }, - }, - { - "expectedResponse": { - "text": "Refund started", - }, - "input": { - "prompt": "order 123", - }, - }, - ], - }, - "sessionId": "", - }, - ], - }, - "empty assertions/trajectory arrays are omitted": { - "failed": 0, - "invoked": 1, - "sessions": [ - { - "exampleId": "e4", - "groundTruth": { - "turns": [ - { - "expectedResponse": { - "text": "r1", - }, - "input": { - "prompt": "t1", - }, - }, - ], - }, - "sessionId": "", - }, - ], - }, - "empty expected_response is treated as no expectation": { - "failed": 0, - "invoked": 1, - "sessions": [ - { - "exampleId": "e3", - "groundTruth": undefined, - "sessionId": "", - }, - ], - }, - "legacy scenario_id fallback + unicode id": { - "failed": 0, - "invoked": 1, - "sessions": [ - { - "exampleId": "café-日本-🎉", - "groundTruth": { - "turns": [ - { - "expectedResponse": { - "text": "ok", - }, - "input": { - "prompt": "1", - }, - }, - ], - }, - "sessionId": "", - }, - ], - }, - "multi-turn, sparse expectation keeps its turn position": { - "failed": 0, - "invoked": 1, - "sessions": [ - { - "exampleId": "e2", - "groundTruth": { - "turns": [ - { - "input": { - "prompt": "t1", - }, - }, - { - "input": { - "prompt": "t2", - }, - }, - { - "expectedResponse": { - "text": "42", - }, - "input": { - "prompt": "t3", - }, - }, - ], - }, - "sessionId": "", - }, - ], - }, - "single turn, no ground truth": { - "failed": 0, - "invoked": 1, - "sessions": [ - { - "exampleId": "e1", - "groundTruth": undefined, - "sessionId": "", - }, - ], - }, - "tolerates blank lines and CRLF between multiple rows": { - "failed": 0, - "invoked": 2, - "sessions": [ - { - "exampleId": "a", - "groundTruth": undefined, - "sessionId": "", - }, - { - "exampleId": "b", - "groundTruth": { - "turns": [ - { - "expectedResponse": { - "text": "ok", - }, - "input": { - "prompt": "2", - }, - }, - ], - }, - "sessionId": "", - }, - ], - }, -} -`; diff --git a/src/core/eval/invokeDataset/invokeDataset.test.ts b/src/core/eval/invokeDataset/invokeDataset.test.ts deleted file mode 100644 index 5247ed431..000000000 --- a/src/core/eval/invokeDataset/invokeDataset.test.ts +++ /dev/null @@ -1,263 +0,0 @@ -// Disables the post-invoke span-ingestion wait so the replay returns immediately. -process.env.SIMULATE_INGESTION_WAIT_MS = "0"; - -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, writeFileSync } from "node:fs"; -import { rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { GetAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore-control"; -import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; -import { EvalClient } from "../../eval"; -import type { AwsClients, CoreFetch } from "../../types"; -import type { InvokedSession } from "../../../handlers/eval/types"; - -// End-to-end coverage of EvalClient.invokeDataset over a fake AWS layer. Exercising the real -// method also exercises its consumers — DatasetLoader, the Example classes, runExamples, -// renderJsonTemplate, and invokeRuntime's IAM path — so those need no separate unit tests. - -const OPTIONS = { region: "us-west-2" }; -const RUNTIME_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/rt-1"; -const row = (o: object) => JSON.stringify(o); - -async function* replyBytes(text: string): AsyncGenerator { - yield new TextEncoder().encode(text); -} - -// A fake AWS layer: control resolves the runtime, data answers each invoke. Records every -// payload it was asked to send, and per `opts` can fail or delay specific invokes. -function fakeClients(opts: { fail?: (payload: string) => boolean; delayMs?: number } = {}): { - clients: AwsClients; - payloads: string[]; - peak: () => number; -} { - const payloads: string[] = []; - let inFlight = 0; - let peak = 0; - const send = async (command: unknown) => { - if (command instanceof GetAgentRuntimeCommand) return { agentRuntimeArn: RUNTIME_ARN }; - if (command instanceof InvokeAgentRuntimeCommand) { - const payload = new TextDecoder().decode(command.input.payload as Uint8Array); - payloads.push(payload); - if (opts.fail?.(payload)) throw new Error(`invoke failed for ${payload}`); - inFlight++; - peak = Math.max(peak, inFlight); - if (opts.delayMs) await new Promise((r) => setTimeout(r, opts.delayMs)); - inFlight--; - return { statusCode: 200, contentType: "application/json", response: replyBytes("ok") }; - } - throw new Error( - `unexpected command: ${(command as { constructor: { name: string } }).constructor.name}`, - ); - }; - const client = { send } as never; - return { - clients: { control: () => client, data: () => client, iam: () => client, logs: () => client }, - payloads, - peak: () => peak, - }; -} - -const dirs: string[] = []; -afterEach(async () => { - await Promise.all(dirs.splice(0).map((d) => rm(d, { recursive: true, force: true }))); -}); - -function datasetFile(jsonl: string): string { - const dir = mkdtempSync(join(tmpdir(), "agentcore-invoke-ds-")); - dirs.push(dir); - const path = join(dir, "dataset.jsonl"); - writeFileSync(path, jsonl); - return path; -} - -function invokeDataset(jsonl: string, clients: AwsClients) { - const fetch = (() => { - throw new Error("fetch is only used on the CUSTOM_JWT path, which these tests do not exercise"); - }) as unknown as CoreFetch; - return new EvalClient(clients, fetch).invokeDataset( - { runtimeId: "rt-1", payloadTemplate: '{"prompt":"{input}"}', dataset: datasetFile(jsonl) }, - OPTIONS, - ); -} - -// sessionId is a fresh UUID per example, so pin it to compare shapes; sort so completion -// order (which is nondeterministic under concurrency) doesn't churn the golden. -function normalize(sessions: InvokedSession[]) { - return [...sessions] - .sort((a, b) => a.exampleId.localeCompare(b.exampleId)) - .map((s) => ({ ...s, sessionId: "" })); -} - -const GOLDEN_FIXTURES: { name: string; jsonl: string }[] = [ - { - name: "single turn, no ground truth", - jsonl: row({ example_id: "e1", turns: [{ input: "hi" }] }), - }, - { - name: "multi-turn, sparse expectation keeps its turn position", - jsonl: row({ - example_id: "e2", - turns: [{ input: "t1" }, { input: "t2" }, { input: "t3", expected_response: "42" }], - }), - }, - { - name: "empty expected_response is treated as no expectation", - jsonl: row({ example_id: "e3", turns: [{ input: "t1", expected_response: "" }] }), - }, - { - name: "assertions + trajectory + sparse turns (full inline shape)", - jsonl: row({ - example_id: "orders-1", - turns: [ - { input: "I want a refund" }, - { input: "order 123", expected_response: "Refund started" }, - ], - assertions: ["stays polite", "does not promise a date"], - expected_trajectory: ["refund_lookup", "refund_create"], - }), - }, - { - name: "empty assertions/trajectory arrays are omitted", - jsonl: row({ - example_id: "e4", - turns: [{ input: "t1", expected_response: "r1" }], - assertions: [], - expected_trajectory: [], - }), - }, - { - name: "legacy scenario_id fallback + unicode id", - jsonl: row({ scenario_id: "café-日本-🎉", turns: [{ input: "1", expected_response: "ok" }] }), - }, - { - name: "tolerates blank lines and CRLF between multiple rows", - jsonl: - row({ example_id: "a", turns: [{ input: "1" }] }) + - "\r\n\r\n" + - row({ example_id: "b", turns: [{ input: "2", expected_response: "ok" }] }) + - "\r\n", - }, -]; - -const THROW_FIXTURES: { name: string; jsonl: string; error: RegExp }[] = [ - { - name: "both turns and actor_profile", - jsonl: row({ example_id: "x", turns: [{ input: "a" }], actor_profile: {} }), - error: /both 'turns' and 'actor_profile'/, - }, - { - name: "neither turns nor actor_profile", - jsonl: row({ example_id: "x" }), - error: /neither 'turns' nor 'actor_profile'/, - }, - { - name: "simulated example not supported yet", - jsonl: row({ example_id: "x", actor_profile: { goal: "g" } }), - error: /simulated example/, - }, - { - name: "duplicate example ids", - jsonl: [ - row({ example_id: "a", turns: [{ input: "1" }] }), - row({ example_id: "a", turns: [{ input: "2" }] }), - ].join("\n"), - error: /duplicate example_id: "a"/, - }, - { - name: "missing example id", - jsonl: row({ turns: [{ input: "hi" }] }), - error: /missing 'example_id'/, - }, - { name: "invalid JSON line", jsonl: "{not json", error: /not valid JSON/ }, - { name: "non-object row (null)", jsonl: "null", error: /not a JSON object/ }, - { name: "empty dataset", jsonl: "\n \n", error: /no examples/ }, - { name: "empty turns array", jsonl: row({ example_id: "x", turns: [] }), error: /has no turns/ }, - { - name: "non-object turn entry", - jsonl: row({ example_id: "x", turns: [null] }), - error: /turn 1 is not an object/, - }, - { - name: "non-array assertions", - jsonl: row({ example_id: "x", turns: [{ input: "a" }], assertions: "nope" }), - error: /assertions must be an array of strings/, - }, - { - name: "non-array expected_trajectory", - jsonl: row({ example_id: "x", turns: [{ input: "a" }], expected_trajectory: "nope" }), - error: /expected_trajectory must be an array of strings/, - }, -]; - -describe("EvalClient.invokeDataset", () => { - // One golden block over representative datasets: locks the created sessions + the exact - // inline ground-truth shape handed to the grader, across every ground-truth variation. - test("golden: sessions + ground truth over representative datasets", async () => { - const results: Record = {}; - for (const f of GOLDEN_FIXTURES) { - const r = await invokeDataset(f.jsonl, fakeClients().clients); - results[f.name] = { invoked: r.invoked, failed: r.failed, sessions: normalize(r.sessions) }; - } - expect(results).toMatchSnapshot(); - }); - - test("rejects a payload-template without the {input} placeholder before invoking", async () => { - const { clients, payloads } = fakeClients(); - const fetch = (() => { - throw new Error("unused"); - }) as unknown as CoreFetch; - await expect( - new EvalClient(clients, fetch).invokeDataset( - { - runtimeId: "rt-1", - payloadTemplate: '{"prompt":"static"}', - dataset: datasetFile(row({ example_id: "x", turns: [{ input: "hi" }] })), - }, - OPTIONS, - ), - ).rejects.toThrow(/\{input\} placeholder/); - expect(payloads).toEqual([]); - }); - - test.each(THROW_FIXTURES)("rejects and invokes nothing: $name", async ({ jsonl, error }) => { - const { clients, payloads } = fakeClients(); - await expect(invokeDataset(jsonl, clients)).rejects.toThrow(error); - expect(payloads).toEqual([]); - }); - - test("a failed invoke is counted and dropped; the rest still run", async () => { - const jsonl = [ - row({ example_id: "ok1", turns: [{ input: "hi" }] }), - row({ example_id: "bad", turns: [{ input: "FAIL" }] }), - row({ example_id: "ok2", turns: [{ input: "yo" }] }), - ].join("\n"); - const r = await invokeDataset(jsonl, fakeClients({ fail: (p) => p.includes("FAIL") }).clients); - expect(r.invoked).toBe(2); - expect(r.failed).toBe(1); - expect(r.sessions.map((s) => s.exampleId).sort()).toEqual(["ok1", "ok2"]); - expect(r.firstError?.message).toMatch(/invoke failed/); - }); - - test("invokes each turn exactly once across all examples, rendered through the template", async () => { - const jsonl = [ - row({ example_id: "a", turns: [{ input: "a1" }, { input: "a2" }] }), - row({ example_id: "b", turns: [{ input: "b1" }] }), - ].join("\n"); - const { clients, payloads } = fakeClients(); - await invokeDataset(jsonl, clients); - expect(payloads.sort()).toEqual( - ['{"prompt":"a1"}', '{"prompt":"a2"}', '{"prompt":"b1"}'].sort(), - ); - }); - - test("runs examples concurrently but never past the pool bound", async () => { - const jsonl = Array.from({ length: 12 }, (_, i) => - row({ example_id: `e${i}`, turns: [{ input: `p${i}` }] }), - ).join("\n"); - const { clients, peak } = fakeClients({ delayMs: 5 }); - await invokeDataset(jsonl, clients); - expect(peak()).toBeLessThanOrEqual(5); // runExamples default concurrency - expect(peak()).toBeGreaterThanOrEqual(2); // proves it did not run serially - }); -}); diff --git a/src/core/eval/invokeDataset/run.ts b/src/core/eval/invokeDataset/run.ts index 6f82a98b9..5304d09f0 100644 --- a/src/core/eval/invokeDataset/run.ts +++ b/src/core/eval/invokeDataset/run.ts @@ -1,15 +1,15 @@ -// A failed worker is counted and dropped, not thrown — the caller reports all-failed via -// firstError. Bounded concurrency because each item invokes a live runtime. -export type ExampleRun = { ok: Result[]; failed: number; firstError?: Error }; +export type ExampleRun = { + ok: Result[]; + failures: { item: Item; error: Error }[]; +}; export async function runExamples( items: Item[], worker: (item: Item) => Promise, concurrency = 5, -): Promise> { +): Promise> { const ok: Result[] = []; - let failed = 0; - let firstError: Error | undefined; + const failures: { item: Item; error: Error }[] = []; let next = 0; const run = async (): Promise => { while (next < items.length) { @@ -17,11 +17,10 @@ export async function runExamples( try { ok.push(await worker(item)); } catch (error) { - failed++; - if (!firstError) firstError = error instanceof Error ? error : new Error(String(error)); + failures.push({ item, error: error instanceof Error ? error : new Error(String(error)) }); } } }; await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run)); - return { ok, failed, firstError }; + return { ok, failures }; } diff --git a/src/core/index.tsx b/src/core/index.tsx index 34b9b4795..d7275b912 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -38,6 +38,7 @@ type CoreClientConfig = { createLogsClient: CreateLogsClient; logger: Logger; fetch?: CoreFetch; + newSessionId?: () => string; }; // CoreClient is the single entry point to the Bedrock AgentCore APIs. It owns the @@ -78,7 +79,12 @@ export class CoreClient implements AwsClients { // EvalClient shares the injected fetch: dataset content is served from a // presigned S3 URL, outside the SDK seam the other operations use. The logger // is used for batch-evaluation result-log diagnostics. - this.eval = new EvalClient(this, fetch, this.logger.child({ module: "eval" })); + this.eval = new EvalClient( + this, + fetch, + this.logger.child({ module: "eval" }), + config.newSessionId, + ); this.projectManager = new FsProjectManager({ logger: this.logger.child({ module: "projectManager" }), diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/InvokeAgentRuntimeCommand.46bef739bc7ae9c1.json b/src/handlers/eval/batch-evaluation/__fixtures__/InvokeAgentRuntimeCommand.46bef739bc7ae9c1.json new file mode 100644 index 000000000..aa033fb90 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/InvokeAgentRuntimeCommand.46bef739bc7ae9c1.json @@ -0,0 +1,8 @@ +{ + "contentType": "text/event-stream; charset=utf-8", + "runtimeSessionId": "00000000-0000-4000-8000-000000000001", + "response": { + "$stream": "data: \"Hello! How can I help you today\"\n\ndata: \"?\"\n\n" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.9c92cd5b227a329e.json b/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.9c92cd5b227a329e.json new file mode 100644 index 000000000..1aaf75008 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.9c92cd5b227a329e.json @@ -0,0 +1,20 @@ +{ + "batchEvaluationId": "golden_batch_simulate_fixture1-86183b0ccc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_simulate_fixture1-86183b0ccc", + "batchEvaluationName": "golden_batch_simulate_fixture1", + "status": "PENDING", + "createdAt": { + "$date": "2026-08-26T22:07:26.049Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-golden_batch_simulate_fixture1-86183b0ccc" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/simulate-ds.jsonl b/src/handlers/eval/batch-evaluation/__fixtures__/simulate-ds.jsonl new file mode 100644 index 000000000..39fe36263 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/simulate-ds.jsonl @@ -0,0 +1 @@ +{"example_id":"e1","turns":[{"input":"hi"}],"assertions":["stays polite"]} diff --git a/src/handlers/eval/batch-evaluation/__fixtures__/simulate.golden.json b/src/handlers/eval/batch-evaluation/__fixtures__/simulate.golden.json new file mode 100644 index 000000000..841535721 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/__fixtures__/simulate.golden.json @@ -0,0 +1,13 @@ +{ + "batchEvaluationId": "golden_batch_simulate_fixture1-86183b0ccc", + "status": "PENDING", + "examplesInvoked": 1, + "examplesFailed": 0, + "sessions": [ + { + "exampleId": "e1", + "sessionId": "00000000-0000-4000-8000-000000000001" + } + ], + "failures": [] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx index 8494e9a53..034e3ac4d 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx @@ -42,6 +42,9 @@ const MISSING_JOB_ID = "missing-batch-eval-0000000000"; const FIXTURE_EVAL_AGENT = "asdf_MyAgent-3s5axvBC6Q"; const FIXTURE_EVAL_NAME = "golden_batch_evaluate_fixture685"; +const FIXTURE_SIMULATE_NAME = "golden_batch_simulate_fixture1"; +const FIXTURE_SIMULATE_DATASET = join(FIXTURES, "simulate-ds.jsonl"); + function createFixtureCore(): CoreClient { const { createControlClient, createDataClient, createIamClient, createLogsClient } = fixtureFactories(FIXTURES); @@ -136,4 +139,49 @@ describe("eval batch-evaluation (fixture-backed)", () => { expect(job.batchEvaluationId).toBeTruthy(); expect(job.status).toBeTruthy(); }); + + test("simulate replays a dataset, then submits a batch job over the created sessions", async () => { + let n = 0; + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + const core = new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + newSessionId: () => `00000000-0000-4000-8000-${String(++n).padStart(12, "0")}`, + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + FIXTURE_EVAL_AGENT, + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + FIXTURE_SIMULATE_DATASET, + "--evaluator", + "Builtin.Helpfulness", + "--name", + FIXTURE_SIMULATE_NAME, + "--ingestion-wait-ms", + "0", + "--json", + "--region", + REGION, + ]); + + matchGolden(FIXTURES, "simulate.golden.json", io.stdout()); + }, 180_000); }); diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index b710f529c..e76da4395 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -160,3 +160,96 @@ describe("eval batch-evaluation list", () => { expect(core.eval.calls[0]?.args).toEqual([undefined, 10, { region: "us-west-2" }]); }); }); + +describe("eval batch-evaluation simulate", () => { + const BASE = [ + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + "--name", + "sim-1", + ]; + + test.each<[RegExp, string[]]>([ + [ + /--runtime-id/, + ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + ], + [ + /--payload-template/, + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + ], + [ + /--dataset/, + ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"], + ], + [ + /--evaluator/, + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--name", + "n", + ], + ], + [ + /--name/, + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + ], + ], + ])("rejects when a required flag is missing (%s)", async (expected, args) => { + await expect(run(["eval", "batch-evaluation", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("refuses to grade when nothing was invoked, naming the first failure", async () => { + await expect( + run(BASE, (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [], + invoked: 0, + failed: 3, + failures: [ + { exampleId: "e1", error: "HTTP 500" }, + { exampleId: "e2", error: "HTTP 500" }, + { exampleId: "e3", error: "HTTP 500" }, + ], + }), + ), + ).rejects.toThrow(/no examples could be invoked \(3 failed\).*first error: e1 — HTTP 500/); + }); + + test("passes --ingestion-wait-ms through to invokeDataset and renders failures", async () => { + const { core, stdout } = await run([...BASE, "--ingestion-wait-ms", "0"], (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [{ exampleId: "ok1", sessionId: "s1" }], + invoked: 1, + failed: 1, + failures: [{ exampleId: "bad", error: "HTTP 500" }], + }), + ); + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(invoke).toBeDefined(); + expect((invoke!.args[0] as { waitIngestionMs?: number }).waitIngestionMs).toBe(0); + expect(JSON.parse(stdout).failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]); + }); +}); diff --git a/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap b/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap deleted file mode 100644 index ddc632555..000000000 --- a/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap +++ /dev/null @@ -1,23 +0,0 @@ -// Bun Snapshot v1, https://bun.sh/docs/test/snapshots - -exports[`eval batch-evaluation simulate builds the sessionMetadata ground-truth shape [golden] 1`] = ` -[ - { - "groundTruth": { - "inline": { - "assertions": [ - { - "text": "polite", - }, - ], - }, - }, - "sessionId": "s1", - "testScenarioId": "e1", - }, - { - "sessionId": "s2", - "testScenarioId": "e2", - }, -] -`; diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index 72dddd6d0..4246021fe 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -34,6 +34,11 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => flag("name", "batch evaluation name (unique in the account)", z.string().optional()), flag("description", "description for the batch evaluation", z.string().optional()), flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + flag( + "ingestion-wait-ms", + "ms to wait for span ingestion before grading (default 180000; 0 to skip)", + z.coerce.number().int().nonnegative().optional(), + ), ], handle: async (ctx, flags) => { if (!flags["runtime-id"]) @@ -69,12 +74,14 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => userId: flags["user-id"], dataset: flags["dataset"], datasetVersion: flags["dataset-version"], + waitIngestionMs: flags["ingestion-wait-ms"], }, opts, controller.signal, ); if (r.invoked === 0) { - const detail = r.firstError ? `; first error: ${r.firstError.message}` : ""; + const first = r.failures[0]; + const detail = first ? `; first error: ${first.exampleId} — ${first.error}` : ""; throw new InputValidationError( `no examples could be invoked (${r.failed} failed) — nothing to evaluate${detail}`, ); @@ -107,6 +114,8 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => status: job.status, examplesInvoked: r.invoked, examplesFailed: r.failed, + sessions: r.sessions.map((s) => ({ exampleId: s.exampleId, sessionId: s.sessionId })), + failures: r.failures, }); } finally { process.off("SIGINT", interrupt); diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx deleted file mode 100644 index 8c6215fd9..000000000 --- a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx +++ /dev/null @@ -1,169 +0,0 @@ -import { test, expect, describe } from "bun:test"; -import { createRootHandler } from "../../../index"; -import { - createSilentLogger, - TestCoreClient, - testIO, - TestGlobalConfigAccessor, -} from "../../../../testing"; -import type { InvokeDatasetResult } from "../../types"; - -// Two invoked sessions; the handler feeds these into startBatchEvaluation and renders -// the job it returns (DEFAULT_START_BATCH_EVAL_RESPONSE: batch-eval-test / RUNNING). -const INVOKE_RESULT: InvokeDatasetResult = { - sessions: [ - { exampleId: "e1", sessionId: "s1", groundTruth: { assertions: [{ text: "polite" }] } }, - { exampleId: "e2", sessionId: "s2" }, - ], - invoked: 2, - failed: 0, -}; - -async function run(args: string[], configure?: (core: TestCoreClient) => void) { - const core = new TestCoreClient(); - core.eval.setInvokeDatasetResponse(INVOKE_RESULT); - configure?.(core); - const io = testIO(); - const root = createRootHandler(core, { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); - return { core, stdout: io.stdout() }; -} - -const BASE = [ - "eval", - "batch-evaluation", - "simulate", - "--runtime-id", - "r-1", - "--payload-template", - '{"prompt":"{input}"}', - "--dataset", - "/tmp/ds.jsonl", - "--evaluator", - "Builtin.Helpfulness", - "--name", - "sim-1", -]; - -describe("eval batch-evaluation simulate", () => { - test("registered under batch-evaluation", () => { - const io = testIO(); - const root = createRootHandler(new TestCoreClient(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - const group = root - .children() - .find((c) => c.name() === "eval") - ?.children() - .find((c) => c.name() === "batch-evaluation"); - expect(group?.children().map((c) => c.name())).toContain("simulate"); - }); - - test.each([ - [ - [ - "--payload-template", - '{"prompt":"{input}"}', - "--dataset", - "/tmp/ds.jsonl", - "--evaluator", - "E", - "--name", - "n", - ], - /--runtime-id/, - ], - [ - ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], - /--payload-template/, - ], - [ - ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"], - /--dataset/, - ], - [ - [ - "--runtime-id", - "r-1", - "--payload-template", - "{}", - "--dataset", - "/tmp/ds.jsonl", - "--name", - "n", - ], - /--evaluator/, - ], - [ - [ - "--runtime-id", - "r-1", - "--payload-template", - "{}", - "--dataset", - "/tmp/ds.jsonl", - "--evaluator", - "E", - ], - /--name/, - ], - ])("rejects missing required flag", async (args, expected) => { - await expect(run(["eval", "batch-evaluation", "simulate", ...args])).rejects.toThrow(expected); - }); - - test("composes startBatchEvaluation over the created sessions + wrapped ground truth", async () => { - const { core, stdout } = await run(BASE); - - // Rendered output is the batch job + invoked/failed counts. - expect(JSON.parse(stdout)).toEqual({ - batchEvaluationId: "batch-eval-test", - status: "RUNNING", - examplesInvoked: 2, - examplesFailed: 0, - }); - - const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); - expect(start?.args[0]).toMatchObject({ - name: "sim-1", - evaluatorIds: ["Builtin.Helpfulness"], - source: { origin: "agent", agent: "r-1", sessionIds: ["s1", "s2"] }, - // e1's inline GT is wrapped; e2 (no GT) omits the member. - groundTruth: [ - { - sessionId: "s1", - testScenarioId: "e1", - groundTruth: { inline: { assertions: [{ text: "polite" }] } }, - }, - { sessionId: "s2", testScenarioId: "e2" }, - ], - }); - - // Handler threads the Ctrl-C AbortSignal into the replay (invokeDataset) call. - const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); - expect(invoke?.args[2]).toBeInstanceOf(AbortSignal); - }); - - // Golden: the exact evaluationMetadata (sessionMetadata) the handler builds from the - // invoked sessions. Locks the `{ inline: gt }` wrapping and the omitted-member case for - // a session with no ground truth — the wire shape the batch service reads. - test("builds the sessionMetadata ground-truth shape [golden]", async () => { - const { core } = await run(BASE); - const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); - const input = start!.args[0] as { groundTruth: unknown }; - expect(input.groundTruth).toMatchSnapshot(); - }); - - test("refuses to grade when nothing was invoked", async () => { - await expect( - run(BASE, (core) => - core.eval.setInvokeDatasetResponse({ sessions: [], invoked: 0, failed: 3 }), - ), - ).rejects.toThrow(/no examples could be invoked \(3 failed\)/); - }); -}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 6bcb6447f..b1be77f4b 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -286,8 +286,11 @@ export type InvokeDatasetInput = { userId?: string; dataset: string; // local JSONL path or a dataset id datasetVersion?: string; + waitIngestionMs?: number; }; +export type InvokeFailure = { exampleId: string; error: string }; + // InvokedSession is one replayed example: the session created for it plus its neutral // ground truth. Grader-agnostic — the batch handler wraps `groundTruth` as // SessionMetadataShape; a future ondemand handler adapts it to EvaluationReferenceInput. @@ -297,13 +300,11 @@ export type InvokedSession = { groundTruth?: InlineGroundTruth; }; -// InvokeDatasetResult reports the created sessions plus how many examples were invoked -// vs dropped (a failed invoke is skipped, not fatal). firstError explains a total failure. export type InvokeDatasetResult = { sessions: InvokedSession[]; invoked: number; failed: number; - firstError?: Error; + failures: InvokeFailure[]; }; export type SpanRecord = Record; diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index ef5064c21..e58ef5209 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1450,6 +1450,7 @@ export class TestEvalClient implements CoreEvalClient { sessions: [], invoked: 0, failed: 0, + failures: [], }; private error?: Error; diff --git a/src/testing/fixtures.tsx b/src/testing/fixtures.tsx index 51b6a47a7..a6a4d39c6 100644 --- a/src/testing/fixtures.tsx +++ b/src/testing/fixtures.tsx @@ -145,6 +145,31 @@ function reviveError(tagged: TaggedError): Error { return error; } +const STREAM_TAG = "$stream"; + +async function freezeStream(response: unknown): Promise { + const stream = (response as { response?: { transformToString?: () => Promise } }) + ?.response; + if (typeof stream?.transformToString !== "function") return response; + return { + ...(response as Record), + response: { [STREAM_TAG]: await stream.transformToString() }, + }; +} + +async function* streamOf(text: string): AsyncGenerator { + yield new TextEncoder().encode(text); +} + +function reviveStream(recorded: unknown): unknown { + const stream = (recorded as { response?: Record })?.response; + if (!stream || typeof stream !== "object" || !(STREAM_TAG in stream)) return recorded; + return { + ...(recorded as Record), + response: streamOf(stream[STREAM_TAG] as string), + }; +} + // makeRecordingSend returns a `.send()` that records to / replays from `dir`. // In record mode it delegates to the real client, saves the response (or the // service error), and propagates it; otherwise it reads the fixture, failing @@ -168,8 +193,9 @@ function makeRecordingSend Promise }>( writeFileSync(path, stringify(sanitizePresignedUrls(tagged))); throw error; } - writeFileSync(path, stringify(sanitizePresignedUrls(response))); - return response; + const frozen = await freezeStream(response); + writeFileSync(path, stringify(sanitizePresignedUrls(frozen))); + return reviveStream(frozen); } if (!existsSync(path)) { @@ -180,7 +206,7 @@ function makeRecordingSend Promise }>( } const recorded = parse(readFileSync(path, "utf8")); if (isTaggedError(recorded)) throw reviveError(recorded); - return recorded; + return reviveStream(recorded); }; } From 2595b9df4e09344b48d4f4fdf7012ac344e0055f Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:00:21 -0400 Subject: [PATCH 06/16] feat(templates): support rendering strands http template. (#2099) * feat(templates): add support for dynamic templates * docs(assets): update readme reccomended commands * feat(runtime): wire up runtime version * fix(templates): use key function to key templates * docs(templates): fix comment with missing the * docs(templates): fix incorrect wording on template types * fix(manager): render in correct directory * test(template): add a snapshot test for the new template * fix(template): avoid double nesting runtimes * fix(templates): remove dead parameters * refactor(harness): adapt harness to leverage template resolver * refactor(manager): rename project template to project tree to avoid confusion * fix(renderer): add missing handlebar helpers * fix(templates): add harnesses to merge entries * fix(handlebars): add missing helpers from upstream * fix(harness): address hardcoded path * fix(hello-world): reject non-HTTP protocol on template * docs(strands-python): remove non-existent command from readme * feat(template): install runtime dependencies in scaffolding * fix(python): normalize names before being sent to python templates * fix(templates): validate dockerfile for harness/ * fix(handlers): add strands as handler input * fix(spec): avoid writing windows paths --- .../templates/strands-http-python/README.md | 40 + .../strands-http-python/gitignore.template | 41 + .../templates/strands-http-python/main.py | 714 ++++++++++++++++++ .../mcp_client/__init__.py | 1 + .../strands-http-python/mcp_client/client.py | 116 +++ .../strands-http-python/model/__init__.py | 1 + .../strands-http-python/model/load.py | 239 ++++++ .../model/mantle_compat.py | 21 + .../strands-http-python/pyproject.toml | 31 + .../strands-http-python/skills/fetcher.py | 279 +++++++ .../__snapshots__/manager.test.ts.snap | 42 ++ src/core/project/manager.test.ts | 32 +- src/core/project/manager.tsx | 195 +++-- src/core/project/templates.ts | 134 ---- .../project/{ => templates}/fsTree.test.ts | 4 +- src/core/project/{ => templates}/fsTree.ts | 26 +- src/core/project/templates/harness.ts | 53 ++ src/core/project/templates/project.ts | 73 ++ src/core/project/templates/renderer.ts | 52 ++ src/core/project/templates/runtime.ts | 136 ++++ src/core/project/templates/types.ts | 30 + .../project/add/harness/index.test.ts | 2 +- .../project/add/runtime/index.test.ts | 122 ++- src/handlers/project/add/runtime/index.ts | 4 +- src/handlers/project/create/index.ts | 4 +- src/handlers/project/project.test.ts | 11 +- src/handlers/project/types.ts | 36 +- 27 files changed, 2192 insertions(+), 247 deletions(-) create mode 100644 src/assets/templates/strands-http-python/README.md create mode 100644 src/assets/templates/strands-http-python/gitignore.template create mode 100644 src/assets/templates/strands-http-python/main.py create mode 100644 src/assets/templates/strands-http-python/mcp_client/__init__.py create mode 100644 src/assets/templates/strands-http-python/mcp_client/client.py create mode 100644 src/assets/templates/strands-http-python/model/__init__.py create mode 100644 src/assets/templates/strands-http-python/model/load.py create mode 100644 src/assets/templates/strands-http-python/model/mantle_compat.py create mode 100644 src/assets/templates/strands-http-python/pyproject.toml create mode 100644 src/assets/templates/strands-http-python/skills/fetcher.py delete mode 100644 src/core/project/templates.ts rename src/core/project/{ => templates}/fsTree.test.ts (96%) rename src/core/project/{ => templates}/fsTree.ts (75%) create mode 100644 src/core/project/templates/harness.ts create mode 100644 src/core/project/templates/project.ts create mode 100644 src/core/project/templates/renderer.ts create mode 100644 src/core/project/templates/runtime.ts create mode 100644 src/core/project/templates/types.ts diff --git a/src/assets/templates/strands-http-python/README.md b/src/assets/templates/strands-http-python/README.md new file mode 100644 index 000000000..eafaa1ec0 --- /dev/null +++ b/src/assets/templates/strands-http-python/README.md @@ -0,0 +1,40 @@ +This is a project generated by the AgentCore CLI! + +# Layout + +The generated application code lives at the agent root directory. At the root, there is a `.gitignore` file, an +`agentcore/` folder which represents the configurations and state associated with this project. Other `agentcore` +commands like `deploy`, `dev`, and `invoke` rely on the configuration stored here. + +## Agent Root + +The main entrypoint to your app is defined in `main.py`. Using the AgentCore SDK `@app.entrypoint` decorator, this +file defines a Starlette ASGI app with the chosen Agent framework SDK running within. + +`model/load.py` instantiates your chosen model provider. + +## Input Validation + +Validate invocation input before forwarding it to Strands. Keep plain prompts typed as strings. If the app accepts a +caller-supplied message history, retain `strip_trailing_tool_use()`, which normalizes the history tail before +invoking the agent. + +## Environment Variables + +| Variable | Required | Description | +| --- | --- | --- | +{{#if hasIdentity}}| `{{identityProviders.[0].envVarName}}` | Yes | {{modelProvider}} API key (local) or Identity provider name (deployed) | +{{/if}}| `LOCAL_DEV` | No | Set to `1` to use `.env.local` instead of AgentCore Identity | + +# Developing locally + +If installation was successful, a virtual environment is already created with dependencies installed. + +Activate the environment with `source .venv/bin/activate` on macOS/Linux, `.venv\Scripts\activate.bat` in Windows +Command Prompt, or `.\.venv\Scripts\activate.ps1` in Windows PowerShell. + +`agentcore project dev` will start a local server on 0.0.0.0:8080. + +# Deployment + +After providing credentials, `agentcore project deploy` will deploy your project into Amazon Bedrock AgentCore. diff --git a/src/assets/templates/strands-http-python/gitignore.template b/src/assets/templates/strands-http-python/gitignore.template new file mode 100644 index 000000000..f36f968a0 --- /dev/null +++ b/src/assets/templates/strands-http-python/gitignore.template @@ -0,0 +1,41 @@ +# Environment variables +.env + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/src/assets/templates/strands-http-python/main.py b/src/assets/templates/strands-http-python/main.py new file mode 100644 index 000000000..69dca7e8e --- /dev/null +++ b/src/assets/templates/strands-http-python/main.py @@ -0,0 +1,714 @@ +from typing import Any +from collections import OrderedDict +{{#if inlineFunctionTools}} +import json + +from strands.tools.tools import PythonAgentTool +from strands.types.tools import ToolResult, ToolUse +{{/if}} +from strands import Agent, tool +{{#if hasSkillsFetcher}} +from strands import AgentSkills +{{#if hasFetchedSkills}} +from skills.fetcher import resolve_s3_skills, resolve_git_skills +{{/if}} +{{#if (some gitSkills "credentialArn")}} +from bedrock_agentcore.services.identity import IdentityClient +{{/if}} +{{/if}} +import asyncio +{{#if hasShell}} +import subprocess +{{/if}} +{{#if hasFileOperations}} +import os +{{/if}} +{{#if hasExecutionLimits}} +from strands.tools.executors import SequentialToolExecutor +from strands.types.exceptions import EventLoopException +from hooks.execution_limits import ExecutionLimitExceeded, ExecutionLimitsHook +{{/if}} +{{#if hasConfigBundle}} +from strands.hooks import HookProvider, HookRegistry, BeforeInvocationEvent, BeforeToolCallEvent +{{/if}} +{{#if truncationStrategy}} +{{#if (eq truncationStrategy "sliding_window")}} +from strands.agent.conversation_manager.sliding_window_conversation_manager import SlidingWindowConversationManager +{{/if}} +{{#if (eq truncationStrategy "summarization")}} +from strands.agent.conversation_manager.summarizing_conversation_manager import SummarizingConversationManager +{{/if}} +{{else}} +from strands.agent.conversation_manager.null_conversation_manager import NullConversationManager +{{/if}} +{{#if hasConfigBundle}} +from bedrock_agentcore.runtime.context import BedrockAgentCoreContext +{{/if}} +{{#if hasBrowser}} +from strands_tools.browser import AgentCoreBrowser +{{/if}} +{{#if hasCodeInterpreter}} +from strands_tools.code_interpreter import AgentCoreCodeInterpreter +{{/if}} +from bedrock_agentcore.runtime import BedrockAgentCoreApp +from model.load import load_model +{{#if hasGateway}} +from mcp_client.client import get_all_gateway_mcp_clients +{{/if}} +{{#if remoteMcpTools}} +from mcp_client.client import get_all_remote_mcp_clients +{{/if}} +{{#unless (or hasGateway remoteMcpTools)}} +{{#unless isExportHarness}} +from mcp_client.client import get_streamable_http_mcp_client +{{/unless}} +{{/unless}} +{{#if hasMemory}} +from memory.session import get_memory_session_manager +{{/if}} +{{#unless hasFileOperations}} +{{#if (or needsOs browserIdentifierEnvVar codeInterpreterIdentifierEnvVar (some gitSkills "credentialArn"))}} +import os +{{/if}} +{{/unless}} +{{#if hasPayment}} +from capabilities.payments.payments import create_payments_plugin, PAYMENT_SYSTEM_PROMPT +{{/if}} + +app = BedrockAgentCoreApp() +log = app.logger + +{{#if (or hasGateway remoteMcpTools)}} +# Define MCP clients for all configured MCP servers (gateways and/or remote MCP) +mcp_clients = [] +{{#if hasGateway}} +mcp_clients += get_all_gateway_mcp_clients() +{{/if}} +{{#if remoteMcpTools}} +mcp_clients += get_all_remote_mcp_clients() +{{/if}} +{{else}} +{{#unless isExportHarness}} +# Define a Streamable HTTP MCP Client +mcp_clients = [get_streamable_http_mcp_client()] +{{/unless}} +{{/if}} + +{{#if systemPromptText}} +DEFAULT_SYSTEM_PROMPT = """{{escapePyStr systemPromptText}}""" +{{else}} +DEFAULT_SYSTEM_PROMPT = """ +You are a helpful assistant. Use tools when appropriate. +{{#if needsOs}}{{#unless isExportHarness}} +You have access to the following mounted filesystems. Use file_read, file_write, and list_files with full absolute paths: +{{#if sessionStorageMountPath}}- {{sessionStorageMountPath}}: ephemeral session storage (lost when session ends) +{{/if}}{{#each efsMounts}}- {{mountPath}}: EFS persistent storage (persists across sessions and agent restarts) +{{/each}}{{#each s3Mounts}}- {{mountPath}}: S3 Files persistent storage (durable, backed by S3) +{{/each}}{{/unless}}{{/if}} +""" +{{/if}} + +{{#if hasConfigBundle}} +DEFAULT_TOOL_DESC = "Return the sum of two numbers" +{{/if}} + +# Define a collection of tools used by the model +tools = [] + +{{#if inlineFunctionTools}} +# Inline function tools — stop the agent loop so the tool call streams back to the caller +def _make_inline_tool(name: str, spec: dict) -> PythonAgentTool: + def _handler(tool: ToolUse, **kwargs: Any) -> ToolResult: + kwargs.get("request_state", {})["stop_event_loop"] = True + return {"toolUseId": tool["toolUseId"], "status": "success", "content": [{"text": " "}]} + _handler.__name__ = name + return PythonAgentTool(tool_name=name, tool_spec=spec, tool_func=_handler) + +{{#each inlineFunctionTools}} +_INLINE_SPEC_{{snakeCase name}} = { + "name": "{{name}}", + "description": {{safeJson description}}, + "inputSchema": {"json": json.loads({{pyJsonStr inputSchema}}) }, +} +tools.append(_make_inline_tool("{{name}}", _INLINE_SPEC_{{snakeCase name}})) +{{/each}} + +_INLINE_FUNCTION_NAMES = { {{#each inlineFunctionTools}}"{{name}}"{{#unless @last}}, {{/unless}}{{/each}} } + +{{else}} +_INLINE_FUNCTION_NAMES = set() + +{{#unless isExportHarness}} +# Define a simple function tool +{{#if hasConfigBundle}} +@tool(description=DEFAULT_TOOL_DESC) +{{else}} +@tool +{{/if}} +def add_numbers(a: int, b: int) -> int: + """Return the sum of two numbers""" + return a+b +tools.append(add_numbers) + +{{/unless}} +{{/if}} +{{#if hasBrowser}} +{{#if browserIdentifierEnvVar}} +_browser_id = os.getenv("{{browserIdentifierEnvVar}}") +tools.append(AgentCoreBrowser(**({"identifier": _browser_id} if _browser_id else {})).browser) +{{else}} +tools.append(AgentCoreBrowser().browser) +{{/if}} +{{/if}} +{{#if hasCodeInterpreter}} +{{#if codeInterpreterIdentifierEnvVar}} +_code_interpreter_id = os.getenv("{{codeInterpreterIdentifierEnvVar}}") +tools.append(AgentCoreCodeInterpreter(**({"identifier": _code_interpreter_id} if _code_interpreter_id else {})).code_interpreter) +{{else}} +tools.append(AgentCoreCodeInterpreter().code_interpreter) +{{/if}} +{{/if}} +{{#if hasShell}} +@tool +def shell(command: str, timeout: int = 300) -> dict: + """Execute a bash command and return the results. + + Args: + command: The bash command to execute + timeout: Timeout in seconds (default: 300) + + Returns: + Dict with stdout, stderr, and exit_code + """ + result = subprocess.run( + command, shell=True, capture_output=True, text=True, timeout=timeout + ) + return {"stdout": result.stdout, "stderr": result.stderr, "exit_code": result.returncode} + +tools.append(shell) +{{/if}} +{{#if hasFileOperations}} +@tool +def file_operations( + command: str, + path: str, + old_str: str = None, + new_str: str = None, + file_text: str = None, + insert_line: int = None, + view_range: list = None, +) -> str: + """Text editor tool for viewing and modifying files. + + Args: + command: The command to execute ("view", "str_replace", "create", "insert") + path: Path to the file or directory + old_str: Text to replace (for str_replace command) + new_str: Replacement text (for str_replace and insert commands) + file_text: Content for new file (for create command) + insert_line: Line number to insert after (for insert command) + view_range: [start_line, end_line] for viewing specific lines (for view command) + + Returns: + Result of the operation + """ + try: + if command == "view": + if not os.path.exists(path): + return f"Error: Path '{path}' does not exist" + if os.path.isdir(path): + return "\n".join(os.listdir(path)) + with open(path) as f: + lines = f.read().splitlines() + if view_range: + start, end = view_range + start_idx = max(0, start - 1) + end_idx = len(lines) if end == -1 else min(len(lines), end) + lines = lines[start_idx:end_idx] + start_num = start_idx + 1 + else: + start_num = 1 + return "\n".join(f"{start_num + i}: {line}" for i, line in enumerate(lines)) + elif command == "str_replace": + if old_str is None or new_str is None: + return "Error: str_replace requires both old_str and new_str parameters" + if not os.path.exists(path): + return f"Error: File '{path}' does not exist" + content = open(path).read() + if old_str not in content: + return "Error: Text not found in file" + count = content.count(old_str) + if count > 1: + return f"Error: Text appears {count} times in file. Please be more specific." + open(path, "w").write(content.replace(old_str, new_str, 1)) + return f"Successfully replaced text in '{path}'" + elif command == "create": + if file_text is None: + return "Error: create requires file_text parameter" + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + open(path, "w").write(file_text) + return f"Successfully created file '{path}'" + elif command == "insert": + if new_str is None or insert_line is None: + return "Error: insert requires both new_str and insert_line parameters" + if not os.path.exists(path): + return f"Error: File '{path}' does not exist" + lines = open(path).read().splitlines(True) + if insert_line == 0: + lines.insert(0, new_str + "\n") + elif insert_line >= len(lines): + lines.append(new_str + "\n") + else: + lines.insert(insert_line, new_str + "\n") + open(path, "w").write("".join(lines)) + return f"Successfully inserted text in '{path}' at line {insert_line + 1}" + else: + return f"Error: Unknown command '{command}'" + except Exception as e: + return f"Error: {e}" + +tools.append(file_operations) +{{/if}} +{{#if needsOs}}{{#unless isExportHarness}} +_MOUNT_PATHS = [ + {{#if sessionStorageMountPath}}"{{sessionStorageMountPath}}",{{/if}} + {{#each efsMounts}}"{{mountPath}}",{{/each}} + {{#each s3Mounts}}"{{mountPath}}",{{/each}} +] + +def _safe_resolve(path: str) -> str: + resolved = os.path.realpath(path) + if not any(resolved == os.path.realpath(m) or resolved.startswith(os.path.realpath(m) + os.sep) for m in _MOUNT_PATHS): + raise ValueError(f"Path '{path}' is not within any configured mount ({', '.join(_MOUNT_PATHS)})") + return resolved + +@tool +def file_read(path: str) -> str: + """Read a file from a mounted filesystem. Use the absolute path (e.g. /mnt/tools/data.txt).""" + try: + full_path = _safe_resolve(path) + with open(full_path) as f: + return f.read() + except ValueError as e: + return str(e) + except OSError as e: + return f"Error reading '{path}': {e.strerror}" + +@tool +def file_write(path: str, content: str) -> str: + """Write a file to a mounted filesystem. Use the absolute path (e.g. /mnt/tools/data.txt).""" + try: + full_path = _safe_resolve(path) + parent = os.path.dirname(full_path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(full_path, "w") as f: + f.write(content) + return f"Written to {path}" + except ValueError as e: + return str(e) + except OSError as e: + return f"Error writing '{path}': {e.strerror}" + +@tool +def list_files(path: str) -> str: + """List files in a mounted filesystem directory. Use the absolute path (e.g. /mnt/tools).""" + try: + full_path = _safe_resolve(path) + entries = os.listdir(full_path) + return "\n".join(entries) if entries else "(empty directory)" + except ValueError as e: + return str(e) + except OSError as e: + return f"Error listing '{path}': {e.strerror}" + +tools.extend([file_read, file_write, list_files]) +{{/unless}}{{/if}} + +{{#if (or hasGateway remoteMcpTools)}} +# Add MCP clients to tools +for mcp_client in mcp_clients: + if mcp_client: + tools.append(mcp_client) +{{else}} +{{#unless isExportHarness}} +# Add MCP client to tools if available +for mcp_client in mcp_clients: + if mcp_client: + tools.append(mcp_client) +{{/unless}} +{{/if}} + +{{#if hasConfigBundle}} + +class ConfigBundleHook(HookProvider): + """Injects config bundle values (system prompt, tool descriptions) before each invocation. + + BedrockAgentCoreContext.get_config_bundle() fetches the component configuration + for the current runtime ARN from the config bundle service. The SDK caches the + result and refreshes on bundle version changes. + """ + + def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: + registry.add_callback(BeforeInvocationEvent, self._inject_system_prompt) + registry.add_callback(BeforeToolCallEvent, self._override_tool_desc) + + def _inject_system_prompt(self, event: BeforeInvocationEvent) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + prompt = config.get("systemPrompt", DEFAULT_SYSTEM_PROMPT) + + if prompt != event.agent.system_prompt: + event.agent.system_prompt = prompt + + def _override_tool_desc(self, event: BeforeToolCallEvent) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + tool_descs = config.get("toolDescriptions", {}) + + tool_name = event.tool_use["name"] + override = tool_descs.get(tool_name) + if override and event.selected_tool: + spec = event.selected_tool.tool_spec + if spec and "description" in spec: + spec["description"] = override + +{{/if}} + +def _make_conversation_manager(): +{{#if truncationStrategy}} +{{#if (eq truncationStrategy "sliding_window")}} +{{#if truncationConfig}} + return SlidingWindowConversationManager(**{{safeJson truncationConfig}}, per_turn=True) +{{else}} + return SlidingWindowConversationManager(per_turn=True) +{{/if}} +{{else}} +{{#if truncationConfig}} + return SummarizingConversationManager(**{{safeJson truncationConfig}}) +{{else}} + return SummarizingConversationManager() +{{/if}} +{{/if}} +{{else}} + return NullConversationManager() +{{/if}} + +{{#if hasMemory}} +{{#unless hasPayment}} +def agent_factory(): + cache = {} + def get_or_create_agent(session_id, user_id{{#if hasSkillsFetcher}}, skill_plugins=None{{/if}}): + {{#if actorId}} + _actor_id = "{{actorId}}" + {{else}} + _actor_id = user_id + {{/if}} + key = f"{session_id}/{_actor_id}" + if key not in cache: + cache[key] = Agent( + model=load_model(), + session_manager=get_memory_session_manager(session_id, _actor_id), + conversation_manager=_make_conversation_manager(), + system_prompt=DEFAULT_SYSTEM_PROMPT, + tools=tools, + {{#if hasSkillsFetcher}} + plugins=skill_plugins or None, + {{/if}} + {{#if hasExecutionLimits}} + tool_executor=SequentialToolExecutor(), + callback_handler=None, + {{/if}} + hooks=[ + {{#if hasExecutionLimits}} + ExecutionLimitsHook( + {{#if maxIterations}}max_iterations={{maxIterations}},{{/if}} + {{#if maxTokens}}max_tokens={{maxTokens}},{{/if}} + {{#if timeoutSeconds}}timeout_seconds={{timeoutSeconds}},{{/if}} + ), + {{/if}} + {{#if hasConfigBundle}} + ConfigBundleHook(), + {{/if}} + ], + ) + return cache[key] + return get_or_create_agent +get_or_create_agent = agent_factory() +{{/unless}} +{{else}} +{{#unless hasPayment}} +# Reuses one Agent per session_id so each session keeps its own in-process +# conversation history (best-effort; resets on cold start). The cache is bounded +# to 128 sessions with LRU eviction (least-recently-used is dropped and its +# history reset) so a single process serving many sessions cannot leak history +# between them or grow without limit. For durable history, attach a session manager. +def agent_factory(): + cache = OrderedDict() + def get_or_create_agent(session_id{{#if hasSkillsFetcher}}, skill_plugins=None{{/if}}): + if session_id in cache: + cache.move_to_end(session_id) + return cache[session_id] + if len(cache) >= 128: + cache.popitem(last=False) + cache[session_id] = Agent( + model=load_model(), + system_prompt=DEFAULT_SYSTEM_PROMPT, + tools=tools, + conversation_manager=_make_conversation_manager(), + {{#if hasSkillsFetcher}} + plugins=skill_plugins or None, + {{/if}} + {{#if hasExecutionLimits}} + tool_executor=SequentialToolExecutor(), + callback_handler=None, + {{/if}} + hooks=[ + {{#if hasExecutionLimits}} + ExecutionLimitsHook( + {{#if maxIterations}}max_iterations={{maxIterations}},{{/if}} + {{#if maxTokens}}max_tokens={{maxTokens}},{{/if}} + {{#if timeoutSeconds}}timeout_seconds={{timeoutSeconds}},{{/if}} + ), + {{/if}} + {{#if hasConfigBundle}} + ConfigBundleHook(), + {{/if}} + ], + ) + return cache[session_id] + return get_or_create_agent +get_or_create_agent = agent_factory() +{{/unless}} +{{/if}} + + +def strip_trailing_tool_use(messages: Any) -> list[dict]: + """Strip toolUse blocks from the tail until the last message has none.""" + if not isinstance(messages, list): + raise ValueError("messages must be a list") + + messages = list(messages) + while messages: + last = messages[-1] + if not isinstance(last, dict): + raise ValueError("each message must be an object") + original_content = last.get("content", []) + if not isinstance(original_content, list) or not all(isinstance(block, dict) for block in original_content): + raise ValueError("each message content value must be a list of content blocks") + + content = [block for block in original_content if "toolUse" not in block] + if len(content) == len(original_content): + break + if content: + messages[-1] = {**last, "content": content} + break + messages.pop() + + return messages + + +def _extract_prompt(payload: dict): + """Accept validated harness messages, tool results, or a plain prompt string.""" + if not isinstance(payload, dict): + raise ValueError("payload must be a JSON object") + if "messages" in payload: + return strip_trailing_tool_use(payload["messages"]) + if "tool_results" in payload: + tool_results = payload["tool_results"] + if not isinstance(tool_results, list) or not all( + isinstance(tool_result, dict) and isinstance(tool_result.get("toolUseId"), str) + for tool_result in tool_results + ): + raise ValueError("tool_results must contain objects with a toolUseId string") + return [{"role": "user", "content": [{"toolResult": { + "toolUseId": tr["toolUseId"], + "status": tr.get("status", "success"), + "content": tr.get("content", []), + }} for tr in tool_results]}] + prompt = payload.get("prompt", "") + if not isinstance(prompt, str): + raise ValueError("prompt must be a string") + return prompt + + +def _has_inline_function_call(messages) -> bool: + """Return True if messages contains an assistant toolUse for an inline function tool.""" + if not _INLINE_FUNCTION_NAMES or not isinstance(messages, list): + return False + for msg in messages: + if msg.get("role") == "assistant": + for block in msg.get("content", []): + if isinstance(block, dict) and block.get("toolUse", {}).get("name") in _INLINE_FUNCTION_NAMES: + return True + return False + + +def _is_inline_function_call(event: dict) -> bool: + """Check if a contentBlockStart event is for an inline function tool.""" + if not _INLINE_FUNCTION_NAMES: + return False + cbs = event.get("contentBlockStart", {}) + start = cbs.get("start", {}) + tool_use = start.get("toolUse") if isinstance(start, dict) else None + return tool_use is not None and tool_use.get("name") in _INLINE_FUNCTION_NAMES + + + +@app.entrypoint +async def invoke(payload, context): + log.info("Invoking Agent.....") + +{{#if hasPayment}} + user_id = payload.get("user_id") or getattr(context, "user_id", "default-user") + instrument_id = payload.get("payment_instrument_id") + session_id = payload.get("payment_session_id") + payments_plugin = create_payments_plugin(user_id, instrument_id, session_id) + plugins = [payments_plugin] if payments_plugin else [] +{{/if}} +{{#if hasSkillsFetcher}} + skill_paths = [{{#each pathSkills}}{{safeJson this}}{{#unless @last}}, {{/unless}}{{/each}}] + {{#if s3Skills}} + s3_skill_sources = [{{#each s3Skills}}{{safeJson this}}{{#unless @last}}, {{/unless}}{{/each}}] + skill_paths.extend(await asyncio.to_thread(resolve_s3_skills, s3_skill_sources, None)) + {{/if}} + {{#if gitSkills}} + git_skill_sources = [ + {{#each gitSkills}} + dict(url={{safeJson this.url}}{{#if this.path}}, path={{safeJson this.path}}{{/if}}{{#if this.credentialArn}}, credentialArn={{safeJson this.credentialArn}}{{#if this.username}}, username={{safeJson this.username}}{{/if}}{{/if}}), + {{/each}} + ] + {{#if (some gitSkills "credentialArn")}} + _git_identity_client = IdentityClient(os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-east-1"))) + {{else}} + _git_identity_client = None + {{/if}} + skill_paths.extend(await asyncio.to_thread(resolve_git_skills, git_skill_sources, _git_identity_client)) + {{/if}} + _skill_plugins = [AgentSkills(skills=skill_paths)] if skill_paths else [] +{{/if}} + +{{#if hasMemory}} +{{#if hasPayment}} + mem_session_id = getattr(context, 'session_id', 'default-session') + {{#if actorId}} + mem_user_id = "{{actorId}}" + {{else}} + mem_user_id = getattr(context, 'user_id', 'default-user') + {{/if}} + agent = Agent( + model=load_model(), + session_manager=get_memory_session_manager(mem_session_id, mem_user_id), + system_prompt=DEFAULT_SYSTEM_PROMPT + PAYMENT_SYSTEM_PROMPT, + tools=tools, + plugins=plugins{{#if hasSkillsFetcher}} + _skill_plugins{{/if}},{{#if hasConfigBundle}} + hooks=[ConfigBundleHook()],{{/if}} + ) +{{else}} + session_id = getattr(context, 'session_id', 'default-session') + {{#if actorId}} + user_id = "{{actorId}}" + {{else}} + user_id = getattr(context, 'user_id', 'default-user') + {{/if}} + agent = get_or_create_agent(session_id, user_id{{#if hasSkillsFetcher}}, _skill_plugins{{/if}}) +{{/if}} +{{else}} +{{#if hasPayment}} + agent = Agent( + model=load_model(), + system_prompt=DEFAULT_SYSTEM_PROMPT + PAYMENT_SYSTEM_PROMPT, + tools=tools, + plugins=plugins{{#if hasSkillsFetcher}} + _skill_plugins{{/if}},{{#if hasConfigBundle}} + hooks=[ConfigBundleHook()],{{/if}} + ) +{{else}} + session_id = getattr(context, 'session_id', 'default-session') + agent = get_or_create_agent(session_id{{#if hasSkillsFetcher}}, _skill_plugins{{/if}}) +{{/if}} +{{/if}} + + prompt = _extract_prompt(payload) + + {{#if inlineFunctionTools}} + # If Turn 2 carries the harness-style assistant(toolUse)+user(toolResult) pair, + # strip the placeholder turn Strands stored during Turn 1 so the real toolResult + # is injected cleanly — same protocol as the harness runtime. + if _has_inline_function_call(prompt): + msgs = agent.messages + if len(msgs) >= 2 and any("toolResult" in b for b in msgs[-1].get("content", [])): + del msgs[-2:] + {{/if}} + + {{#if hasExecutionLimits}} + timeout_seconds = {{#if timeoutSeconds}}{{timeoutSeconds}}{{else}}None{{/if}} + timeout_fired = False + watchdog_task = None + if timeout_seconds is not None: + async def _timeout_watchdog(): + nonlocal timeout_fired + await asyncio.sleep(timeout_seconds) + timeout_fired = True + agent.cancel() + watchdog_task = asyncio.create_task(_timeout_watchdog()) + + try: + {{#if inlineFunctionTools}} + hit_inline_function = False + {{/if}} + async for event in agent.stream_async( + prompt, + ): + if not isinstance(event, dict) or "event" not in event: + continue + cbs = event["event"].get("contentBlockStart") + if cbs is not None and not cbs.get("start"): + continue + {{#if inlineFunctionTools}} + if not hit_inline_function: + hit_inline_function = _is_inline_function_call(event["event"]) + {{/if}} + yield event + {{#if inlineFunctionTools}} + if hit_inline_function and "messageStop" in event["event"]: + return + {{/if}} + + if timeout_fired: + yield {"event": {"messageStop": {"stopReason": "timeout_exceeded"}}} + except EventLoopException as e: + if isinstance(e.original_exception, ExecutionLimitExceeded): + yield {"event": {"messageStop": {"stopReason": str(e.original_exception)}}} + return + raise + finally: + if watchdog_task is not None: + watchdog_task.cancel() + try: + await watchdog_task + except asyncio.CancelledError: + pass + {{else}} + {{#if inlineFunctionTools}} + hit_inline_function = False + {{/if}} + async for event in agent.stream_async( + prompt, + ): + if not isinstance(event, dict) or "event" not in event: + continue + cbs = event["event"].get("contentBlockStart") + if cbs is not None and not cbs.get("start"): + continue + {{#if inlineFunctionTools}} + if not hit_inline_function: + hit_inline_function = _is_inline_function_call(event["event"]) + {{/if}} + yield event + {{#if inlineFunctionTools}} + if hit_inline_function and "messageStop" in event["event"]: + return + {{/if}} + {{/if}} + + +if __name__ == "__main__": + app.run() diff --git a/src/assets/templates/strands-http-python/mcp_client/__init__.py b/src/assets/templates/strands-http-python/mcp_client/__init__.py new file mode 100644 index 000000000..0e632e10c --- /dev/null +++ b/src/assets/templates/strands-http-python/mcp_client/__init__.py @@ -0,0 +1 @@ +# Package marker diff --git a/src/assets/templates/strands-http-python/mcp_client/client.py b/src/assets/templates/strands-http-python/mcp_client/client.py new file mode 100644 index 000000000..4de07e43a --- /dev/null +++ b/src/assets/templates/strands-http-python/mcp_client/client.py @@ -0,0 +1,116 @@ +import os +import logging +from mcp.client.streamable_http import streamablehttp_client +from strands.tools.mcp.mcp_client import MCPClient + +logger = logging.getLogger(__name__) + +{{#if hasGateway}} +{{#if (includes gatewayAuthTypes "AWS_IAM")}} +from mcp_proxy_for_aws.client import aws_iam_streamablehttp_client +{{/if}} +{{#if (includes gatewayAuthTypes "CUSTOM_JWT")}} +from bedrock_agentcore.identity import requires_access_token +{{/if}} + +{{#each gatewayProviders}} +{{#if (eq authType "CUSTOM_JWT")}} +@requires_access_token( + provider_name="{{credentialProviderName}}", + scopes=[{{#if scopes}}"{{scopes}}"{{/if}}], + auth_flow="{{#if authFlow}}{{authFlow}}{{else}}M2M{{/if}}", +{{#if customParameters}} + custom_parameters={{safeJson customParameters}}, +{{/if}} +) +def _get_bearer_token_{{snakeCase name}}(*, access_token: str): + """Obtain OAuth access token via AgentCore Identity for {{name}}.""" + return access_token + +{{/if}} +{{/each}} +{{#each gatewayProviders}} +def get_{{snakeCase name}}_mcp_client() -> MCPClient | None: + """Returns an MCP Client connected to the {{name}} gateway.""" + {{#if hardcodedUrl}} + url = {{safeJson hardcodedUrl}} + {{else}} + url = os.environ.get("{{envVarName}}") + if not url: + logger.warning("{{envVarName}} not set — {{name}} gateway tools unavailable") + return None + {{/if}} + {{#if (eq authType "AWS_IAM")}} + return MCPClient(lambda: aws_iam_streamablehttp_client(url, aws_service="bedrock-agentcore", aws_region=os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION"))), prefix="{{snakeCase name}}") + {{else if (eq authType "CUSTOM_JWT")}} + token = _get_bearer_token_{{snakeCase name}}() + headers = {"Authorization": f"Bearer {token}"} if token else {} + return MCPClient(lambda: streamablehttp_client(url, headers=headers), prefix="{{snakeCase name}}") + {{else}} + return MCPClient(lambda: streamablehttp_client(url), prefix="{{snakeCase name}}") + {{/if}} + +{{/each}} +def get_all_gateway_mcp_clients() -> list[MCPClient]: + """Returns MCP clients for all configured gateways.""" + clients = [] + {{#each gatewayProviders}} + client = get_{{snakeCase name}}_mcp_client() + if client: + clients.append(client) + {{/each}} + return clients +{{/if}} +{{#if remoteMcpTools}} +{{#if (some remoteMcpTools "headerCredentials")}} +from bedrock_agentcore.identity.auth import requires_api_key +{{/if}} +{{#each remoteMcpTools}} +{{#if headerCredentials}} +{{#each headerCredentials}} +@requires_api_key(provider_name="{{credentialName}}") +def _get_{{snakeCase ../name}}_{{snakeCase headerKey}}_key(api_key: str) -> str: + """Fetch {{headerKey}} credential for {{../name}} from AgentCore Identity.""" + return api_key + +{{/each}} +{{/if}} +def get_{{snakeCase name}}_mcp_client() -> MCPClient | None: + """Returns an MCP Client for the {{name}} remote MCP server.""" + url = {{safeJson url}} + {{#if headerCredentials}} + if os.getenv("LOCAL_DEV") == "1": + headers = { {{#each headerCredentials}}{{safeJson headerKey}}: os.environ.get("{{envVarName}}", ""){{#unless @last}}, {{/unless}}{{/each}} } + else: + headers = { {{#each headerCredentials}}{{safeJson headerKey}}: _get_{{snakeCase ../name}}_{{snakeCase headerKey}}_key(){{#unless @last}}, {{/unless}}{{/each}} } + return MCPClient(lambda: streamablehttp_client(url, headers=headers)) + {{else}} + return MCPClient(lambda: streamablehttp_client(url)) + {{/if}} + +{{/each}} +def get_all_remote_mcp_clients() -> list[MCPClient]: + """Returns all configured remote MCP clients.""" + clients = [{{#each remoteMcpTools}}get_{{snakeCase name}}_mcp_client(){{#unless @last}}, {{/unless}}{{/each}}] + return [c for c in clients if c is not None] +{{/if}} +{{#unless (or hasGateway remoteMcpTools)}} +{{#if isVpc}} +# VPC mode: external MCP endpoints are not reachable without a NAT gateway. +# Add an AgentCore Gateway with `agentcore add gateway`, or configure your own endpoint below. + +def get_streamable_http_mcp_client() -> MCPClient | None: + """No MCP server configured. Add a gateway with `agentcore add gateway`.""" + return None +{{else}} +{{#unless isExportHarness}} +# ExaAI provides information about code through web searches, crawling and code context searches through their platform. Requires no authentication +EXAMPLE_MCP_ENDPOINT = "https://mcp.exa.ai/mcp" + +def get_streamable_http_mcp_client() -> MCPClient: + """Returns an MCP Client compatible with Strands""" + # to use an MCP server that supports bearer authentication, add headers={"Authorization": f"Bearer {access_token}"} + return MCPClient(lambda: streamablehttp_client(EXAMPLE_MCP_ENDPOINT)) +{{/unless}} +{{/if}} +{{/unless}} diff --git a/src/assets/templates/strands-http-python/model/__init__.py b/src/assets/templates/strands-http-python/model/__init__.py new file mode 100644 index 000000000..0e632e10c --- /dev/null +++ b/src/assets/templates/strands-http-python/model/__init__.py @@ -0,0 +1 @@ +# Package marker diff --git a/src/assets/templates/strands-http-python/model/load.py b/src/assets/templates/strands-http-python/model/load.py new file mode 100644 index 000000000..0b3b23eac --- /dev/null +++ b/src/assets/templates/strands-http-python/model/load.py @@ -0,0 +1,239 @@ +{{#if (eq modelProvider "Bedrock")}} +{{#if bedrockMantle}} +import os + +from aws_bedrock_token_generator import provide_token +{{#if (eq mantleApiFormat "chat_completions")}} +from strands.models.openai import OpenAIModel +{{else}} +{{#if mantleProprietary}} +from strands.models.openai_responses import OpenAIResponsesModel +{{else}} +from model.mantle_compat import MantleCompatResponsesModel +{{/if}} +{{/if}} + +MODEL_ID = "{{modelId}}" + + +def load_model(): + """ + Get a Bedrock Mantle model client. These OpenAI-compatible models (e.g. openai.gpt-5.5, + openai.gpt-oss-120b) are served via the Bedrock Mantle endpoint, NOT the Converse API — so they + are invoked through an OpenAI-style client authenticated with a short-lived Bedrock bearer token. + Region is read from AWS_REGION (set by the AgentCore runtime). + """ + region = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-east-1")) + token = provide_token(region=region) + {{#if mantleProprietary}} + # Proprietary OpenAI models only work on the /openai/v1 Mantle path. + base_url = f"https://bedrock-mantle.{region}.api.aws/openai/v1" + {{else}} + # Open-source OpenAI models (gpt-oss-*) only work on the /v1 Mantle path. + base_url = f"https://bedrock-mantle.{region}.api.aws/v1" + {{/if}} + client_args = {"api_key": token, "base_url": base_url} + + params = {} + {{#if modelMaxTokens}} + {{#if (eq mantleApiFormat "chat_completions")}} + params["max_completion_tokens"] = {{modelMaxTokens}} + {{else}} + params["max_output_tokens"] = {{modelMaxTokens}} + {{/if}} + {{/if}} + {{#if modelTemperature}} + params["temperature"] = {{modelTemperature}} + {{/if}} + {{#if modelTopP}} + params["top_p"] = {{modelTopP}} + {{/if}} + {{#if (eq mantleApiFormat "chat_completions")}} + return OpenAIModel(client_args=client_args, model_id=MODEL_ID, params=params) + {{else}} + # Responses API: Mantle does not persist responses, so disable server-side storage. + params["store"] = False + {{#if mantleProprietary}} + return OpenAIResponsesModel(client_args=client_args, model_id=MODEL_ID, params=params) + {{else}} + return MantleCompatResponsesModel(client_args=client_args, model_id=MODEL_ID, params=params) + {{/if}} + {{/if}} +{{else}} +from strands.models.bedrock import BedrockModel + + +def load_model() -> BedrockModel: + """Get Bedrock model client using IAM credentials.""" + return BedrockModel(model_id="{{#if modelId}}{{modelId}}{{else}}global.anthropic.claude-sonnet-4-5-20250929-v1:0{{/if}}"{{#if modelMaxTokens}}, max_tokens={{modelMaxTokens}}{{/if}}) +{{/if}} +{{/if}} +{{#if (eq modelProvider "Anthropic")}} +import os + +from strands.models.anthropic import AnthropicModel +from bedrock_agentcore.identity.auth import requires_api_key + +IDENTITY_PROVIDER_NAME = "{{identityProviders.[0].name}}" +IDENTITY_ENV_VAR = "{{identityProviders.[0].envVarName}}" + + +@requires_api_key(provider_name=IDENTITY_PROVIDER_NAME) +def _agentcore_identity_api_key_provider(api_key: str) -> str: + """Fetch API key from AgentCore Identity.""" + return api_key + + +def _get_api_key() -> str: + """ + Uses AgentCore Identity for API key management in deployed environments. + For local development, run via 'agentcore dev' which loads agentcore/.env. + """ + if os.getenv("LOCAL_DEV") == "1": + api_key = os.getenv(IDENTITY_ENV_VAR) + if not api_key: + raise RuntimeError( + f"{IDENTITY_ENV_VAR} not found. Add {IDENTITY_ENV_VAR}=your-key to .env.local" + ) + return api_key + return _agentcore_identity_api_key_provider() + + +def load_model() -> AnthropicModel: + """Get authenticated Anthropic model client.""" + return AnthropicModel( + client_args={"api_key": _get_api_key()}, + model_id="claude-sonnet-4-5-20250929", + max_tokens=5000, + ) +{{/if}} +{{#if (eq modelProvider "OpenAI")}} +import os + +from strands.models.openai import OpenAIModel +from bedrock_agentcore.identity.auth import requires_api_key + +IDENTITY_PROVIDER_NAME = "{{identityProviders.[0].name}}" +IDENTITY_ENV_VAR = "{{identityProviders.[0].envVarName}}" + + +@requires_api_key(provider_name=IDENTITY_PROVIDER_NAME) +def _agentcore_identity_api_key_provider(api_key: str) -> str: + """Fetch API key from AgentCore Identity.""" + return api_key + + +def _get_api_key() -> str: + """ + Uses AgentCore Identity for API key management in deployed environments. + For local development, run via 'agentcore dev' which loads agentcore/.env. + """ + if os.getenv("LOCAL_DEV") == "1": + api_key = os.getenv(IDENTITY_ENV_VAR) + if not api_key: + raise RuntimeError( + f"{IDENTITY_ENV_VAR} not found. Add {IDENTITY_ENV_VAR}=your-key to .env.local" + ) + return api_key + return _agentcore_identity_api_key_provider() + + +def load_model() -> OpenAIModel: + """Get authenticated OpenAI model client.""" + return OpenAIModel( + client_args={"api_key": _get_api_key()}, + model_id="{{#if modelId}}{{modelId}}{{else}}gpt-4.1{{/if}}", + ) +{{/if}} +{{#if (eq modelProvider "Gemini")}} +import os + +from strands.models.gemini import GeminiModel +from bedrock_agentcore.identity.auth import requires_api_key + +IDENTITY_PROVIDER_NAME = "{{identityProviders.[0].name}}" +IDENTITY_ENV_VAR = "{{identityProviders.[0].envVarName}}" + + +@requires_api_key(provider_name=IDENTITY_PROVIDER_NAME) +def _agentcore_identity_api_key_provider(api_key: str) -> str: + """Fetch API key from AgentCore Identity.""" + return api_key + + +def _get_api_key() -> str: + """ + Uses AgentCore Identity for API key management in deployed environments. + For local development, run via 'agentcore dev' which loads agentcore/.env. + """ + if os.getenv("LOCAL_DEV") == "1": + api_key = os.getenv(IDENTITY_ENV_VAR) + if not api_key: + raise RuntimeError( + f"{IDENTITY_ENV_VAR} not found. Add {IDENTITY_ENV_VAR}=your-key to .env.local" + ) + return api_key + return _agentcore_identity_api_key_provider() + + +def load_model() -> GeminiModel: + """Get authenticated Gemini model client.""" + return GeminiModel( + client_args={"api_key": _get_api_key()}, + model_id="{{#if modelId}}{{modelId}}{{else}}gemini-2.5-flash{{/if}}", + ) +{{/if}} +{{#if (eq modelProvider "LiteLLM")}} +import os +{{#if litellmAdditionalParams}} +import json +{{/if}} + +from strands.models.litellm import LiteLLMModel +{{#if identityProviders.[0].name}} +from bedrock_agentcore.identity.auth import requires_api_key + +IDENTITY_PROVIDER_NAME = "{{identityProviders.[0].name}}" +IDENTITY_ENV_VAR = "{{identityProviders.[0].envVarName}}" + + +@requires_api_key(provider_name=IDENTITY_PROVIDER_NAME) +def _agentcore_identity_api_key_provider(api_key: str) -> str: + """Fetch API key from AgentCore Identity.""" + return api_key + + +def _get_api_key() -> str: + """ + Uses AgentCore Identity for API key management in deployed environments. + For local development, run via 'agentcore dev' which loads agentcore/.env. + """ + if os.getenv("LOCAL_DEV") == "1": + api_key = os.getenv(IDENTITY_ENV_VAR) + if not api_key: + raise RuntimeError( + f"{IDENTITY_ENV_VAR} not found. Add {IDENTITY_ENV_VAR}=your-key to .env.local" + ) + return api_key + return _agentcore_identity_api_key_provider() +{{/if}} + + + + +def load_model() -> LiteLLMModel: + """Get a LiteLLM model client (proxies to the provider encoded in model_id).""" + client_args = {} + {{#if identityProviders.[0].name}} + client_args["api_key"] = _get_api_key() + {{/if}} + {{#if litellmApiBase}} + client_args["api_base"] = {{safeJson litellmApiBase}} + {{/if}} + params = {{#if litellmAdditionalParams}}json.loads({{pyJsonStr litellmAdditionalParams}}){{else}}{}{{/if}} + return LiteLLMModel( + client_args=client_args, + model_id="{{#if modelId}}{{modelId}}{{else}}bedrock/us.anthropic.claude-sonnet-4-5-20250514-v1:0{{/if}}", + params=params, + ) +{{/if}} diff --git a/src/assets/templates/strands-http-python/model/mantle_compat.py b/src/assets/templates/strands-http-python/model/mantle_compat.py new file mode 100644 index 000000000..4607a3517 --- /dev/null +++ b/src/assets/templates/strands-http-python/model/mantle_compat.py @@ -0,0 +1,21 @@ +from strands.models.openai_responses import OpenAIResponsesModel + + +class MantleCompatResponsesModel(OpenAIResponsesModel): + """Workaround for Bedrock Mantle rejecting output_text in EasyInputMessage content arrays. + + Mantle's Pydantic validation only accepts content as a plain string for assistant messages, while + real OpenAI accepts both formats. Flatten assistant content arrays to strings so multi-turn works. + Used for open-source OpenAI models (gpt-oss-*) on the /v1 Mantle path; proprietary models use the + plain OpenAIResponsesModel on /openai/v1. + """ + + @classmethod + def _format_request_messages(cls, messages): + formatted = super()._format_request_messages(messages) + for msg in formatted: + if msg.get("role") == "assistant" and isinstance(msg.get("content"), list): + msg["content"] = "".join( + part.get("text", "") for part in msg["content"] if part.get("type") == "output_text" + ) + return formatted diff --git a/src/assets/templates/strands-http-python/pyproject.toml b/src/assets/templates/strands-http-python/pyproject.toml new file mode 100644 index 000000000..26d4055ea --- /dev/null +++ b/src/assets/templates/strands-http-python/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["hatchling ~= 1.27.0"] +build-backend = "hatchling.build" + +[project] +name = "{{ name }}" +version = "0.1.0" +description = "AgentCore Runtime Application using Strands SDK" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + {{#if (eq modelProvider "Anthropic")}}"anthropic ~= 0.30.0", + {{/if}}"aws-opentelemetry-distro ~= 0.17.0", + "bedrock-agentcore ~= 1.9.1", + "botocore[crt] ~= 1.43.0", + {{#if (eq modelProvider "Gemini")}}"google-genai ~= 1.0.0", + {{/if}}"mcp ~= 1.24.0", + {{#if (eq modelProvider "OpenAI")}}"openai ~= 1.0.0", + {{/if}}{{#if (eq modelProvider "LiteLLM")}}"litellm ~= 1.0.0", + {{/if}}{{#if bedrockMantle}}"openai ~= 1.0.0", + "aws-bedrock-token-generator ~= 1.0.0", + {{/if}}"strands-agents ~= 1.15.0", + {{#if (or hasBrowser hasCodeInterpreter)}}"strands-agents-tools ~= 0.1.0", + {{/if}}{{#if hasBrowser}}"nest-asyncio ~= 1.5.0", + "playwright ~= 1.42.0", + {{/if}}{{#if hasGateway}}{{#if (includes gatewayAuthTypes "AWS_IAM")}}"mcp-proxy-for-aws ~= 1.1.0", + {{/if}}{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/templates/strands-http-python/skills/fetcher.py b/src/assets/templates/strands-http-python/skills/fetcher.py new file mode 100644 index 000000000..2f82cd6c2 --- /dev/null +++ b/src/assets/templates/strands-http-python/skills/fetcher.py @@ -0,0 +1,279 @@ +"""Skill fetcher — downloads s3/git skills to local filesystem on first use. + +Resolved paths are passed to AgentSkills(skills=...) in main.py. +Cache directory: /.agents/skills/ — an absolute path under the system temp +directory (honors $TMPDIR, defaults to /tmp). The runtime working directory (e.g. +/var/task in a CodeZip runtime) is read-only, so the cache must live somewhere +guaranteed-writable. +""" + +import base64 +import hashlib +import json +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +_SKILLS_BASE = Path(tempfile.gettempdir()) / ".agents" / "skills" +_GIT_TIMEOUT = 60 +_S3_MAX_SIZE_BYTES = 1 * 1024 * 1024 * 1024 # 1 GB + + +def _stable_hash(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest()[:12] + + +def _cleanup(path: Path) -> None: + """Remove a partially-created skill directory so retries don't see stale state.""" + shutil.rmtree(path, ignore_errors=True) + + +def _read_map(type_dir: Path) -> dict: + map_file = type_dir / ".map.json" + return json.loads(map_file.read_text()) if map_file.exists() else {} + + +def _write_map(type_dir: Path, mapping: dict) -> None: + type_dir.mkdir(parents=True, exist_ok=True) + (type_dir / ".map.json").write_text(json.dumps(mapping)) + + +def _resolve_cached(type_dir: Path, source_hash: str) -> Optional[str]: + """Return the cached skill directory for a source hash, or None if not on disk.""" + mapping = _read_map(type_dir) + dir_name = mapping.get(source_hash) + if dir_name and (type_dir / dir_name).exists(): + return str(type_dir / dir_name) + return None + + +def _read_skill_name(skill_dir: Path) -> str: + """Extract the skill name from SKILL.md YAML frontmatter.""" + content = (skill_dir / "SKILL.md").read_text() + if not content.startswith("---"): + raise ValueError(f"SKILL.md in {skill_dir} has no YAML frontmatter (must start with ---)") + parts = content.split("---", 2) + if len(parts) < 3: + raise ValueError(f"SKILL.md in {skill_dir} has malformed frontmatter (missing closing ---)") + for line in parts[1].strip().splitlines(): + if line.startswith("name:"): + name = line[len("name:"):].strip().strip("\"'") + if name: + return name + raise ValueError(f"SKILL.md in {skill_dir} is missing a 'name' field in frontmatter") + + +def _pick_dir_name(type_dir: Path, name: str, source_hash: str) -> str: + """Pick a unique directory name, appending a hash suffix on collision.""" + if not (type_dir / name).exists(): + return name + return f"{name}-{source_hash[:8]}" + + +def _rename_and_cache_skill(type_dir: Path, temp_dir: Path, source_hash: str, skill_root: Path, + source_label: str = "") -> Path: + """Validate SKILL.md, rename the temp dir to the skill's declared name, and update the map. + + Raises ValueError if SKILL.md is missing or has invalid frontmatter. + """ + if not (skill_root / "SKILL.md").exists(): + _cleanup(temp_dir) + hint = f" (source: {source_label})" if source_label else "" + raise ValueError(f"No SKILL.md found in fetched skill{hint}") + + name = _read_skill_name(skill_root) + dir_name = _pick_dir_name(type_dir, name, source_hash) + final_dir = type_dir / dir_name + if final_dir != temp_dir: + temp_dir.rename(final_dir) + + mapping = _read_map(type_dir) + mapping[source_hash] = dir_name + _write_map(type_dir, mapping) + return final_dir + + +def _fetch_s3_skill(source: str, s3_client=None) -> Path: + """Download an s3:// skill prefix and return the local directory.""" + uri = source if source.endswith("/") else source + "/" + source_hash = _stable_hash(uri) + type_dir = _SKILLS_BASE / "s3" + + cached = _resolve_cached(type_dir, source_hash) + if cached: + return Path(cached) + + import boto3 + client = s3_client or boto3.client("s3") + bucket, _, prefix = uri[len("s3://"):].partition("/") + if not bucket: + raise ValueError(f"Invalid S3 URI (no bucket): {uri}") + + temp_dir = type_dir / source_hash + _cleanup(temp_dir) + temp_dir.mkdir(parents=True, exist_ok=True) + temp_root = temp_dir.resolve() + + paginator = client.get_paginator("list_objects_v2") + total = 0 + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + total += obj["Size"] + if total > _S3_MAX_SIZE_BYTES: + _cleanup(temp_dir) + raise ValueError(f"S3 skill {uri} exceeds 1 GB size limit") + rel = obj["Key"][len(prefix):].lstrip("/") + if not rel: + continue + dest = (temp_dir / rel).resolve() + if dest != temp_root and not str(dest).startswith(str(temp_root) + os.sep): + _cleanup(temp_dir) + raise ValueError(f"Path traversal detected in S3 key: {obj['Key']}") + dest.parent.mkdir(parents=True, exist_ok=True) + client.download_file(bucket, obj["Key"], str(dest)) + + if total == 0: + _cleanup(temp_dir) + raise ValueError(f"No files found at S3 URI: {uri}") + + return _rename_and_cache_skill(type_dir, temp_dir, source_hash, temp_dir, source_label=uri) + + +def _resolve_credential_arn(credential_arn: str, identity_client) -> str: + """Resolve a Token Vault API-key credential ARN to its secret value via AgentCore Identity. + + ARN format: arn:

:bedrock-agentcore:::token-vault//apikeycredentialprovider/ + """ + from bedrock_agentcore.runtime.context import BedrockAgentCoreContext # noqa: PLC0415 + + provider_name = credential_arn.rsplit("/", 1)[-1] + if not provider_name: + raise ValueError(f"Invalid credential ARN: {credential_arn}") + workload_token = BedrockAgentCoreContext.get_workload_access_token() + if not workload_token: + raise ValueError("Credential ARN resolution requires a workload access token") + api_key = identity_client.dp_client.get_resource_api_key( + resourceCredentialProviderName=provider_name, + workloadIdentityToken=workload_token, + )["apiKey"] + if not api_key: + raise ValueError(f"Identity returned empty API key for provider: {provider_name}") + return api_key + + +def _build_git_auth_env(credential_arn: Optional[str], username: Optional[str], identity_client=None) -> dict: + """Build GIT_CONFIG_* env vars for HTTP Basic auth using a Token Vault credential ARN. + + Uses env vars instead of -c args to avoid leaking credentials in /proc/*/cmdline, + and so auth propagates to sub-commands (e.g. sparse-checkout triggering a fetch). + """ + if not credential_arn or not identity_client: + return {} + password = _resolve_credential_arn(credential_arn, identity_client) + user = username or "oauth2" + encoded = base64.b64encode(f"{user}:{password}".encode()).decode() + return { + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "http.extraHeader", + "GIT_CONFIG_VALUE_0": f"Authorization: Basic {encoded}", + } + + +def _fetch_git_skill(url: str, skill_path: str = "", credential_arn: Optional[str] = None, + username: Optional[str] = None, identity_client=None) -> Path: + """Shallow-clone a git skill repository and return the local skill directory. + + Returns the directory containing SKILL.md (the subdir itself for sparse checkouts). + """ + if skill_path and (os.path.isabs(skill_path) or ".." in Path(skill_path).parts): + raise ValueError(f"Path traversal detected in skill path: {skill_path}") + + source_hash = _stable_hash(f"{url}:{skill_path}") + type_dir = _SKILLS_BASE / "git" + + cached = _resolve_cached(type_dir, source_hash) + if cached: + return Path(cached) / skill_path if skill_path else Path(cached) + + temp_dir = type_dir / source_hash + _cleanup(temp_dir) + temp_dir.mkdir(parents=True, exist_ok=True) + + extra_env = _build_git_auth_env(credential_arn, username, identity_client) + git_env = {**os.environ, **extra_env} if extra_env else None + + try: + if skill_path: + subprocess.run( + ["git", "clone", "--depth", "1", "--filter=blob:none", "--sparse", url, str(temp_dir)], + check=True, timeout=_GIT_TIMEOUT, capture_output=True, env=git_env, + ) + subprocess.run( + ["git", "sparse-checkout", "set", skill_path], + check=True, timeout=_GIT_TIMEOUT, capture_output=True, cwd=str(temp_dir), env=git_env, + ) + else: + subprocess.run( + ["git", "clone", "--depth", "1", url, str(temp_dir)], + check=True, timeout=_GIT_TIMEOUT, capture_output=True, env=git_env, + ) + except Exception: + _cleanup(temp_dir) + raise + + if skill_path and not (temp_dir / skill_path).exists(): + _cleanup(temp_dir) + raise ValueError(f"Skill path '{skill_path}' not found in repository '{url}'") + + # SKILL.md lives inside the subdir for sparse checkouts. + skill_root = temp_dir / skill_path if skill_path else temp_dir + label = f"{url}:{skill_path}" if skill_path else url + final_dir = _rename_and_cache_skill(type_dir, temp_dir, source_hash, skill_root, source_label=label) + return final_dir / skill_path if skill_path else final_dir + + +def resolve_s3_skills(sources: list, s3_client=None) -> list: + """Resolve s3:// skill URIs to local filesystem paths. + + Any fetch failure raises and fails the invocation — a partial skill set + would silently run the agent without capabilities the harness declared. + """ + paths = [] + for uri in sources: + try: + skill_dir = _fetch_s3_skill(uri, s3_client) + except Exception as e: + raise ValueError(f"Failed to resolve S3 skill '{uri}': {e}") from e + paths.append(str(skill_dir.resolve())) + return paths + + +def resolve_git_skills(sources: list, identity_client=None) -> list: + """Resolve git skill dicts to local filesystem paths. + + Each source is a dict with keys: url (required), path (optional), + credentialArn (optional), username (optional). + + Any fetch failure raises and fails the invocation — a partial skill set + would silently run the agent without capabilities the harness declared. + """ + paths = [] + for source in sources: + try: + skill_dir = _fetch_git_skill( + url=source["url"], + skill_path=source.get("path") or "", + credential_arn=source.get("credentialArn"), + username=source.get("username"), + identity_client=identity_client, + ) + except Exception as e: + raise ValueError(f"Failed to resolve git skill '{source.get('url', source)}': {e}") from e + paths.append(str(skill_dir.resolve())) + return paths diff --git a/src/core/project/__snapshots__/manager.test.ts.snap b/src/core/project/__snapshots__/manager.test.ts.snap index 0c6f46b2d..6fd45fcf4 100644 --- a/src/core/project/__snapshots__/manager.test.ts.snap +++ b/src/core/project/__snapshots__/manager.test.ts.snap @@ -22,3 +22,45 @@ exports[`FsProjectManager.create scaffolds the expected file tree into a fresh d "app/hello_world/pyproject.toml", ] `; + +exports[`FsProjectManager.create snapshots the Strands project manifest and runtime spec 1`] = ` +{ + "manifest": [ + ".gitignore", + "agentcore/.env.local", + "agentcore/agentcore.json", + "agentcore/aws-targets.json", + "agentcore/cdk/.gitignore", + "agentcore/cdk/.npmignore", + "agentcore/cdk/.prettierrc", + "agentcore/cdk/README.md", + "agentcore/cdk/bin/cdk.ts", + "agentcore/cdk/cdk.json", + "agentcore/cdk/jest.config.js", + "agentcore/cdk/lib/cdk-stack.ts", + "agentcore/cdk/package.json", + "agentcore/cdk/test/cdk.test.ts", + "agentcore/cdk/tsconfig.json", + "app/strands_agent/.gitignore", + "app/strands_agent/README.md", + "app/strands_agent/main.py", + "app/strands_agent/mcp_client/__init__.py", + "app/strands_agent/mcp_client/client.py", + "app/strands_agent/model/__init__.py", + "app/strands_agent/model/load.py", + "app/strands_agent/model/mantle_compat.py", + "app/strands_agent/pyproject.toml", + "app/strands_agent/skills/fetcher.py", + ], + "runtimes": [ + { + "build": "CodeZip", + "codeLocation": "app/strands_agent", + "entrypoint": "main.py", + "name": "strands_agent", + "protocol": "HTTP", + "runtimeVersion": "PYTHON_3_14", + }, + ], +} +`; diff --git a/src/core/project/manager.test.ts b/src/core/project/manager.test.ts index 955e88add..45997ff8e 100644 --- a/src/core/project/manager.test.ts +++ b/src/core/project/manager.test.ts @@ -18,6 +18,7 @@ import type { DeployBackendInput, ProjectBackend } from "./backends/types"; const HELLO_WORLD_PYTHON = RUNTIME_TEMPLATE_SHORTCUTS["hello-world-python"]; const HELLO_WORLD_PYTHON_CONTAINER = RUNTIME_TEMPLATE_SHORTCUTS["hello-world-python-container"]; +const STRANDS_PYTHON = RUNTIME_TEMPLATE_SHORTCUTS["strands-python"]; const originalCwd = process.cwd(); const tempDirectories: string[] = []; @@ -69,6 +70,13 @@ async function runCreate( } } +async function projectManifest(projectRoot: string): Promise { + return (await readdir(projectRoot, { recursive: true, withFileTypes: true })) + .filter((entry) => entry.isFile()) + .map((entry) => relative(projectRoot, join(entry.parentPath, entry.name)).replaceAll("\\", "/")) + .sort(); +} + describe("FsProjectManager.create", () => { test("scaffolds the expected file tree into a fresh directory", async () => { const directory = await inTempDirectory(); @@ -78,14 +86,22 @@ describe("FsProjectManager.create", () => { }); const projectRoot = join(directory, "example"); - const manifest = (await readdir(projectRoot, { recursive: true, withFileTypes: true })) - .filter((entry) => entry.isFile()) - .map((entry) => - relative(projectRoot, join(entry.parentPath, entry.name)).replaceAll("\\", "/"), - ) - .sort(); - - expect(manifest).toMatchSnapshot(); + expect(await projectManifest(projectRoot)).toMatchSnapshot(); + }); + + test("snapshots the Strands project manifest and runtime spec", async () => { + const directory = await inTempDirectory(); + await runCreate(manager().manager, { + name: "example", + scaffoldRuntimeInput: STRANDS_PYTHON, + }); + + const projectRoot = join(directory, "example"); + const spec = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); + expect({ + manifest: await projectManifest(projectRoot), + runtimes: spec.runtimes, + }).toMatchSnapshot(); }); test("writes a deploy-ready agentcore.json registering the template agent", async () => { diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 83cbff2c0..5b65372e8 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -1,6 +1,6 @@ import { existsSync } from "node:fs"; -import { copyFile, rm } from "node:fs/promises"; -import { join, relative } from "node:path"; +import { rm } from "node:fs/promises"; +import { dirname, join } from "node:path"; import type { AddResourceInput, CreateProjectInput, @@ -23,8 +23,14 @@ import { } from "../../io"; import { defaultSource, type AssetSource } from "./source"; import { ENV_LOCAL_RELATIVE_PATH, EnvLocalFile } from "./envLocal"; -import { createHarnessTreeFromSpec, createProjectTree } from "./templates"; +import { getHarnessTemplateResolver } from "./templates/harness"; +import { createProjectTree } from "./templates/project"; +import { getRuntimeTemplateResolver } from "./templates/runtime"; import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project"; +import { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; +import { CredentialSchema } from "../../projectSchemas/credential"; +import { MemorySchema } from "../../projectSchemas/memory"; +import { OnlineEvalConfigSchema } from "../../projectSchemas/online-eval-config"; import { enclosingProjectRoot } from "./fsUtils"; import { AgentCoreCLIError, @@ -32,21 +38,24 @@ import { NotImplementedError, ProjectStateError, } from "../../errors/errors"; -import type { HarnessSpecSchema } from "../../projectSchemas/harness"; import z from "zod"; import { CdkBackend } from "./backends/cdk"; import type { ProjectBackend } from "./backends/types"; import { AwsDeploymentTargetsSchema } from "../../projectSchemas/aws-targets"; +import type { RuntimeResourceConfig } from "../../handlers/project/add/runtime/types"; +import type { TemplateRenderer } from "./templates/types"; +import { HandlebarsTemplateRenderer } from "./templates/renderer"; const TARGETS_EXAMPLE = '[{ "name": "default", "account": "111122223333", "region": "us-east-1" }]'; type ProjectManagerConfig = { logger: Logger; - source?: AssetSource; // Bun executable or dist/assets depending on runtime - runner?: ProcessRunner; // injectable so tests never spawn real processes - checkTool?: typeof requireTool; // injectable so tests don't depend on the host's PATH - json?: ReadWriteJson; // injectable so tests read fixtures instead of disk + source?: AssetSource; + runner?: ProcessRunner; + checkTool?: typeof requireTool; + json?: ReadWriteJson; backends?: Partial>; + templateRenderer?: TemplateRenderer; }; /** @@ -54,7 +63,8 @@ type ProjectManagerConfig = { */ export class FsProjectManager implements ProjectManager { private readonly logger: Logger; - private readonly source: AssetSource; + private readonly assetSource: AssetSource; + private readonly templateRenderer: TemplateRenderer; private readonly runner: ProcessRunner; private readonly checkTool: typeof requireTool; private readonly json: ReadWriteJson; @@ -62,7 +72,7 @@ export class FsProjectManager implements ProjectManager { constructor(config: ProjectManagerConfig) { this.logger = config.logger; - this.source = config.source ?? defaultSource(); + this.assetSource = config.source ?? defaultSource(); this.runner = config.runner ?? runProcess; this.checkTool = config.checkTool ?? requireTool; this.json = config.json ?? new FsReadWriteJson({ logger: config.logger }); @@ -74,6 +84,7 @@ export class FsProjectManager implements ProjectManager { json: config.json, }), }; + this.templateRenderer = config.templateRenderer ?? new HandlebarsTemplateRenderer(); } public async resolve(input: ResolveProjectInput): Promise { @@ -102,8 +113,12 @@ export class FsProjectManager implements ProjectManager { const destination = join(process.cwd(), input.name); yield { message: "Creating project tree" }; - const tree = await createProjectTree(input.name, scaffoldRuntimeInput, this.source); - await tree.write(destination); + const projectTree = await createProjectTree( + { templateRenderer: this.templateRenderer, assetSource: this.assetSource }, + { projectName: input.name }, + { runtime: scaffoldRuntimeInput }, + ); + await projectTree.write(destination); // A failed step leaves the scaffolded files in place; the error tells the // user how to rerun the step by hand. @@ -113,14 +128,7 @@ export class FsProjectManager implements ProjectManager { await this.run(["npm", "install"], join(destination, "agentcore", "cdk")); const appDir = join(destination, "app", scaffoldRuntimeInput.runtimeName); - if (existsSync(join(appDir, "pyproject.toml"))) { - await this.checkTool( - "uv", - "Install uv: https://docs.astral.sh/uv/getting-started/installation/", - ); - yield { message: "Syncing Python dependencies with uv" }; - await this.run(["uv", "sync"], appDir); - } + yield* this.installRuntimeDependencies(appDir); } if (!input.skipGit) { @@ -145,66 +153,68 @@ export class FsProjectManager implements ProjectManager { project: Project, input: AddResourceInput, ): AsyncGenerator { - const { resourceType, resourceConfig } = input; const agentCoreSpecPath = this.getProjectSpecPath(project); - const projectSpecKey = toProjectSpecKey(resourceType); + const projectSpecKey = toProjectSpecKey(input.resourceType); yield { message: `Reading project spec file at '${agentCoreSpecPath}'` }; - const existingProjectSpec = await this.json.read(agentCoreSpecPath, ProjectSpecSchema); + const projectSpec = await this.json.read(agentCoreSpecPath, ProjectSpecSchema); - const existingResources = existingProjectSpec[projectSpecKey]; - if (resourceType === "gateway-target") { + const existingResources = projectSpec[projectSpecKey]; + if (input.resourceType === "gateway-target") { // Current L3 outputs are keyed only by Target name, so names must remain // project-unique until those outputs include the parent Gateway. - const gateway = existingProjectSpec.agentCoreGateways.find((candidate) => - candidate.targets.some((target) => target.name === resourceConfig.name), + const gateway = projectSpec.agentCoreGateways.find((candidate) => + candidate.targets.some((target) => target.name === input.resourceConfig.name), ); if (gateway) { throw new InputValidationError( - `a gateway target with name '${resourceConfig.name}' already exists in gateway '${gateway.name}'`, + `a gateway target with name '${input.resourceConfig.name}' already exists in gateway '${gateway.name}'`, ); } if ( - existingProjectSpec.unassignedTargets?.some((target) => target.name === resourceConfig.name) + projectSpec.unassignedTargets?.some((target) => target.name === input.resourceConfig.name) ) { throw new InputValidationError( - `an unassigned gateway target with name '${resourceConfig.name}' already exists`, + `an unassigned gateway target with name '${input.resourceConfig.name}' already exists`, ); } - } else if (existingResources.find((resource) => resource.name === resourceConfig.name)) { + } else if (existingResources.find((resource) => resource.name === input.resourceConfig.name)) { throw new InputValidationError( - `a ${resourceType} with name '${resourceConfig.name}' already exists`, + `a ${input.resourceType} with name '${input.resourceConfig.name}' already exists`, ); } - // Widened: arms push their own shapes; the whole-spec safeParse below validates. - const newResources: unknown[] = [...existingResources]; const scaffoldedPaths: string[] = []; - // Non-file work that a failed spec write must also reverse. let envFile: EnvLocalFile | undefined; switch (input.resourceType) { case "harness": { yield { message: `Scaffolding harness in project` }; - const outputPath = join(project.rootPath, "app", resourceConfig.name); + const outputPath = join(project.rootPath, "app", input.resourceConfig.name); scaffoldedPaths.push(outputPath); - const harnessPath = await this.scaffoldHarness(outputPath, input.resourceConfig); - newResources.push({ - name: input.resourceConfig.name, - path: relative(project.rootPath, harnessPath), - }); + const resolver = getHarnessTemplateResolver(); + const result = await resolver.resolve(input.resourceConfig); + await result.tree.write(dirname(outputPath)); + if (result.spec.harnesses) projectSpec.harnesses.push(...result.spec.harnesses); break; } case "runtime": { - throw new NotImplementedError( - "runtime case not yet implemented in FsProjectManager.addResource", - ); + yield { message: "Scaffolding runtime in project" }; + const outputPath = join(project.rootPath, "app", input.resourceConfig.name); + scaffoldedPaths.push(outputPath); + + const spec = await this.scaffoldRuntimeResources(outputPath, input.resourceConfig); + if (spec.runtimes) projectSpec.runtimes.push(...spec.runtimes); + if (spec.memories) projectSpec.memories.push(...spec.memories); + if (spec.credentials) projectSpec.credentials.push(...spec.credentials); + + yield* this.installRuntimeDependencies(outputPath); + break; } case "credential": { - // No file scaffolding; the secret placeholder is staged into .env.local - // and reversed with the spec write if that commit fails. - newResources.push(input.resourceConfig); + const credential = parseResource(CredentialSchema, input.resourceConfig); + projectSpec.credentials.push(credential); if (input.envEntries?.length) { envFile = new EnvLocalFile(project.rootPath); yield { message: `Updating secrets file at '${envFile.path}'` }; @@ -217,15 +227,26 @@ export class FsProjectManager implements ProjectManager { } break; } - case "config-bundle": + case "config-bundle": { + projectSpec.configBundles.push(parseResource(ConfigBundleSchema, input.resourceConfig)); + break; + } case "online-eval": - case "online-insight": - case "memory": + case "online-insight": { + projectSpec.onlineEvalConfigs.push( + parseResource(OnlineEvalConfigSchema, input.resourceConfig), + ); + break; + } + case "memory": { + projectSpec.memories.push(parseResource(MemorySchema, input.resourceConfig)); + break; + } case "gateway": - newResources.push(resourceConfig); + projectSpec.agentCoreGateways.push(input.resourceConfig); break; case "gateway-target": { - const gatewayIndex = existingProjectSpec.agentCoreGateways.findIndex( + const gatewayIndex = projectSpec.agentCoreGateways.findIndex( (gateway) => gateway.name === input.gatewayName, ); if (gatewayIndex < 0) { @@ -233,11 +254,7 @@ export class FsProjectManager implements ProjectManager { `gateway '${input.gatewayName}' does not exist in this project; check agentCoreGateways in agentcore.json`, ); } - const gateway = existingProjectSpec.agentCoreGateways[gatewayIndex]!; - newResources[gatewayIndex] = { - ...gateway, - targets: [...gateway.targets, resourceConfig], - }; + projectSpec.agentCoreGateways[gatewayIndex]!.targets.push(input.resourceConfig); break; } default: { @@ -248,15 +265,11 @@ export class FsProjectManager implements ProjectManager { yield { message: `Updating project spec file at '${agentCoreSpecPath}'` }; - const newSpec = { ...existingProjectSpec, [projectSpecKey]: newResources }; - - // Validate and write inside the same boundary so a rejected spec rolls back - // staged side effects (.env.local, scaffolded files) rather than leaving them. let newProjectSpec: z.infer; try { - const newSpecParseResult = ProjectSpecSchema.safeParse(newSpec); + const newSpecParseResult = ProjectSpecSchema.safeParse(projectSpec); if (!newSpecParseResult.success) - throw new InputValidationError(z.prettifyError(newSpecParseResult.error), { + throw new ProjectStateError(z.prettifyError(newSpecParseResult.error), { cause: newSpecParseResult.error, }); newProjectSpec = await this.json.write(agentCoreSpecPath, newSpecParseResult.data); @@ -337,26 +350,17 @@ export class FsProjectManager implements ProjectManager { }; } - private async scaffoldHarness( - outputPath: string, - harnessSpec: z.input, - ): Promise { - const harness = await createHarnessTreeFromSpec({ - ...harnessSpec, - dockerfile: harnessSpec.dockerfile ? "Dockerfile" : undefined, - }); - - if (harnessSpec.dockerfile) { - if (!existsSync(harnessSpec.dockerfile)) - throw new InputValidationError(`dockerfile not found: '${harnessSpec.dockerfile}'`); - } - - await harness.write(outputPath); - - if (harnessSpec.dockerfile) { - await copyFile(harnessSpec.dockerfile, join(outputPath, "Dockerfile")); - } - return outputPath; + private async scaffoldRuntimeResources(outputPath: string, input: RuntimeResourceConfig) { + const resolver = getRuntimeTemplateResolver( + { assetSource: this.assetSource, templateRenderer: this.templateRenderer }, + input, + ); + if (!resolver) + throw new InputValidationError(`unable to find template that matches given parameters`); + + const result = await resolver.resolve(input); + await result.tree.write(dirname(outputPath)); + return result.spec; } public async *build(project: Project): AsyncGenerator { @@ -408,6 +412,21 @@ export class FsProjectManager implements ProjectManager { return backend; } + /** + * Installs dependencies for a scaffolded runtime directory (e.g. `uv sync` + * for Python). No-ops if the runtime has no recognized dependency manifest. + */ + private async *installRuntimeDependencies(appDir: string): AsyncGenerator { + if (existsSync(join(appDir, "pyproject.toml"))) { + await this.checkTool( + "uv", + "Install uv: https://docs.astral.sh/uv/getting-started/installation/", + ); + yield { message: "Syncing Python dependencies with uv" }; + await this.run(["uv", "sync"], appDir); + } + } + // Runs a command with its output streamed to the file logger. private run(command: string[], cwd: string): Promise { return this.runner(command, { cwd, onOutput: (chunk) => this.logger.debug(chunk) }); @@ -437,3 +456,13 @@ function toProjectSpecKey(resourceType: ProjectResource) { return "agentCoreGateways"; } } + +function parseResource( + schema: TSchema, + input: z.input, +): z.output { + const result = schema.safeParse(input); + if (!result.success) + throw new InputValidationError(z.prettifyError(result.error), { cause: result.error }); + return result.data; +} diff --git a/src/core/project/templates.ts b/src/core/project/templates.ts deleted file mode 100644 index 8630a1d99..000000000 --- a/src/core/project/templates.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { ZodError, z } from "zod"; -import { HarnessSpecSchema } from "../../projectSchemas/harness"; -import { FsTreeNode } from "./fsTree"; -import type { AssetSource } from "./source"; -import { InputValidationError } from "../../errors/errors"; -import { - RUNTIME_TEMPLATE_SHORTCUTS, - type ScaffoldRuntimeInput, -} from "../../handlers/project/types"; - -type TemplateSpec = { - runtimes?: unknown[]; - memories?: unknown[]; - harnesses?: unknown[]; -}; - -/** - * A project template pairs the agent code scaffolded under app/ with the resource - * sections it registers in agentcore.json. Adding a template is one entry here plus its assets. - */ -type Template = { - /** Asset directory relative to the asset root, expanded into the app directory. */ - assetDir: string; - /** Resource sections this template contributes to agentcore.json. */ - spec: TemplateSpec; -}; - -const TEMPLATES: Record = { - [buildRuntimeTemplateKey(RUNTIME_TEMPLATE_SHORTCUTS["hello-world-python"])]: { - assetDir: "templates/hello-world-python", - spec: { - runtimes: [ - { - name: "hello_world", - build: "CodeZip", - entrypoint: "main.py", - codeLocation: "app/hello_world", - // Required for CodeZip builds: the CDK construct library rejects a - // CodeZip runtime with no runtimeVersion, and it is what selects the - // packager. Container builds take their version from the image. - runtimeVersion: "PYTHON_3_14", - }, - ], - }, - }, - [buildRuntimeTemplateKey(RUNTIME_TEMPLATE_SHORTCUTS["hello-world-python-container"])]: { - assetDir: "templates/hello-world-python-container", - spec: { - runtimes: [ - { - name: "hello_world", - build: "Container", - entrypoint: "main.py", - codeLocation: "app/hello_world", - dockerfile: "Dockerfile", - }, - ], - }, - }, -}; - -function buildRuntimeTemplateKey(input: ScaffoldRuntimeInput): string { - return `runtime_${input.build}_${input.framework}_${input.language}_${input.memory}_${input.modelProvider}`; -} - -function resolveTemplate(input: ScaffoldRuntimeInput): Template | undefined { - return TEMPLATES[buildRuntimeTemplateKey(input)]; -} - -/** Serializes a value as pretty-printed JSON with a trailing newline. */ -const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`; - -/** - * Builds the agentcore.json spec by adding the template's resource sections to the shared base. - * The base fields and template sections never overlap so this is a plain spread. - */ -function agentcoreSpec(name: string, template: Template): unknown { - return { - name, - version: 1, - managedBy: "CDK", - ...template.spec, - }; -} - -export async function createProjectTree( - name: string, - input: ScaffoldRuntimeInput, - src: AssetSource, -): Promise { - const template = resolveTemplate(input); - if (!template) - throw new InputValidationError(`unable to find template that matches given parameters`); - return FsTreeNode.createDirectory(".", [ - FsTreeNode.createFile(".gitignore", () => src.read("templates/shared/gitignore.template")), - FsTreeNode.createDirectory("agentcore", [ - await FsTreeNode.fromAssetSource(src, "cdk"), - FsTreeNode.createFile("agentcore.json", async () => json(agentcoreSpec(name, template))), - FsTreeNode.createFile("aws-targets.json", async () => json([])), - FsTreeNode.createFile(".env.local", () => src.read("templates/shared/env.local.template")), - ]), - FsTreeNode.createDirectory("app", [ - // TODO: replace this hardcoded "hello_world" with the runtime name once templates are more flexible. - await FsTreeNode.fromAssetSource(src, template.assetDir, "hello_world"), - ]), - ]); -} - -const DEFAULT_HARNESS_SYSTEM_PROMPT = "You are a helpful assistant"; - -export async function createHarnessTreeFromSpec( - spec: z.input, -): Promise { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { systemPrompt, ...rest } = spec; - // strip system prompt such that markdown file is source of truth. - const parsed = parseHarnessSpec(rest); - return FsTreeNode.createDirectory(".", [ - FsTreeNode.createFile("harness.json", async () => json(parsed)), - FsTreeNode.createFile( - "system-prompt.md", - async () => spec.systemPrompt ?? DEFAULT_HARNESS_SYSTEM_PROMPT, - ), - ]); -} - -function parseHarnessSpec(spec: z.input) { - try { - return HarnessSpecSchema.parse(spec); - } catch (err) { - if (err instanceof ZodError) throw new InputValidationError(z.prettifyError(err)); - throw err; - } -} diff --git a/src/core/project/fsTree.test.ts b/src/core/project/templates/fsTree.test.ts similarity index 96% rename from src/core/project/fsTree.test.ts rename to src/core/project/templates/fsTree.test.ts index f33e5f9fb..bdf4e0503 100644 --- a/src/core/project/fsTree.test.ts +++ b/src/core/project/templates/fsTree.test.ts @@ -2,8 +2,8 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { ProjectStateError } from "../../errors/errors"; -import type { AssetSource } from "./source"; +import { ProjectStateError } from "../../../errors/errors"; +import type { AssetSource } from "../source"; import { FsTreeNode } from "./fsTree"; const tempDirectories: string[] = []; diff --git a/src/core/project/fsTree.ts b/src/core/project/templates/fsTree.ts similarity index 75% rename from src/core/project/fsTree.ts rename to src/core/project/templates/fsTree.ts index 9c1b41dc4..d388cbe96 100644 --- a/src/core/project/fsTree.ts +++ b/src/core/project/templates/fsTree.ts @@ -1,9 +1,9 @@ import { existsSync } from "node:fs"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import type { AssetSource } from "./source"; -import { AgentCoreCLIError, ERROR_SOURCE } from "../../errors"; -import { ProjectStateError } from "../../errors/errors"; +import type { AssetSource } from "../source"; +import { AgentCoreCLIError, ERROR_SOURCE } from "../../../errors"; +import { InputValidationError, ProjectStateError } from "../../../errors/errors"; /** * FsTreeNode represents a tree of directories and files. @@ -58,6 +58,18 @@ export class FsTreeNode { return new FsTreeNode(name, true, children); } + /** + * A file node whose content is read from a local text file on disk at write time, + */ + static fromTextFile(name: string, sourcePath: string): FsTreeNode { + return FsTreeNode.createFile(name, async () => { + if (!existsSync(sourcePath)) { + throw new InputValidationError(`file not found: '${sourcePath}'`); + } + return readFile(sourcePath, "utf-8"); + }); + } + /** * Expands the flat asset listing under assetDir into a nested tree of nodes. */ @@ -65,6 +77,7 @@ export class FsTreeNode { src: AssetSource, assetDir: string, rootDirName?: string, + transform?: (content: string) => string, ): Promise { const paths = await src.list(assetDir); const root = FsTreeNode.createDirectory(rootDirName ?? assetDir, []); @@ -82,7 +95,10 @@ export class FsTreeNode { segments.forEach((segment, index) => { if (index === segments.length - 1) { parent.children.push( - FsTreeNode.createFile(renderName(segment), () => src.read(assetPath)), + FsTreeNode.createFile(renderName(segment), async () => { + const raw = await src.read(assetPath); + return transform ? transform(raw) : raw; + }), ); return; } diff --git a/src/core/project/templates/harness.ts b/src/core/project/templates/harness.ts new file mode 100644 index 000000000..c875291eb --- /dev/null +++ b/src/core/project/templates/harness.ts @@ -0,0 +1,53 @@ +import { existsSync } from "node:fs"; +import { ZodError, z } from "zod"; +import { HarnessSpecSchema } from "../../../projectSchemas/harness"; +import { FsTreeNode } from "./fsTree"; +import { InputValidationError } from "../../../errors/errors"; +import type { TemplateResolver } from "./types"; + +const DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant"; +const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`; + +/** Given a harness spec, resolve the {@link TemplateResolver} that renders its config directory **/ +export function getHarnessTemplateResolver(): TemplateResolver> { + return { + async resolve(spec) { + if (spec.dockerfile && !existsSync(spec.dockerfile)) { + throw new InputValidationError(`dockerfile not found: '${spec.dockerfile}'`); + } + + // strip system prompt from harness.json to keep file as source of truth. otherwise harness.json system prompt overrides. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { systemPrompt, ...rest } = spec; + const parsed = parseHarnessSpec({ + ...rest, + dockerfile: spec.dockerfile ? "Dockerfile" : undefined, + }); + + const tree = FsTreeNode.createDirectory(parsed.name, [ + FsTreeNode.createFile("harness.json", async () => json(parsed)), + FsTreeNode.createFile( + "system-prompt.md", + async () => systemPrompt ?? DEFAULT_SYSTEM_PROMPT, + ), + ...(spec.dockerfile ? [FsTreeNode.fromTextFile("Dockerfile", spec.dockerfile)] : []), + ]); + + return { + tree, + spec: { + harnesses: [{ name: parsed.name, path: `app/${parsed.name}` }], + }, + }; + }, + }; +} + +function parseHarnessSpec(spec: z.input) { + try { + return HarnessSpecSchema.parse(spec); + } catch (err) { + if (err instanceof ZodError) throw new InputValidationError(z.prettifyError(err)); + throw err; + } +} diff --git a/src/core/project/templates/project.ts b/src/core/project/templates/project.ts new file mode 100644 index 000000000..ff0a1ecaa --- /dev/null +++ b/src/core/project/templates/project.ts @@ -0,0 +1,73 @@ +import { FsTreeNode } from "./fsTree"; +import type { AssetSource } from "../source"; +import type { ScaffoldRuntimeInput } from "../../../handlers/project/types"; +import type { RuntimeResourceConfig } from "../../../handlers/project/add/runtime/types"; +import { InputValidationError } from "../../../errors/errors"; +import { getRuntimeTemplateResolver } from "./runtime"; +import type { SpecEntries, Template, TemplateRenderer } from "./types"; + +type CreateProjectConfig = { + assetSource: AssetSource; + templateRenderer: TemplateRenderer; +}; +/** Scaffold a project from scratch, with optional support for rendering a runtime with the project. **/ +export async function createProjectTree( + config: CreateProjectConfig, + input: { projectName: string }, + options?: { runtime?: ScaffoldRuntimeInput }, +): Promise { + const templates: Template[] = []; + if (options?.runtime) { + const runtimeConfig: RuntimeResourceConfig = { + name: options.runtime.runtimeName, + scaffoldRuntimeInput: options.runtime, + }; + + const resolver = getRuntimeTemplateResolver(config, runtimeConfig); + if (!resolver) + throw new InputValidationError(`unable to find template that matches given parameters`); + + templates.push(await resolver.resolve(runtimeConfig)); + } + + return FsTreeNode.createDirectory(".", [ + FsTreeNode.createFile(".gitignore", () => + config.assetSource.read("templates/shared/gitignore.template"), + ), + FsTreeNode.createDirectory("agentcore", [ + await FsTreeNode.fromAssetSource(config.assetSource, "cdk"), + FsTreeNode.createFile("agentcore.json", async () => + json({ + name: input.projectName, + version: 1, + managedBy: "CDK", + ...mergeSpecEntries(templates.map(({ spec }) => spec)), + }), + ), + FsTreeNode.createFile("aws-targets.json", async () => json([])), + FsTreeNode.createFile(".env.local", () => + config.assetSource.read("templates/shared/env.local.template"), + ), + ]), + FsTreeNode.createDirectory( + "app", + templates.map((t) => t.tree), + ), + ]); +} + +const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`; + +function mergeSpecEntries(entries: SpecEntries[]): SpecEntries { + const runtimes = entries.flatMap(({ runtimes }) => runtimes ?? []); + const credentials = entries.flatMap(({ credentials }) => credentials ?? []); + const memories = entries.flatMap(({ memories }) => memories ?? []); + const harnesses = entries.flatMap(({ harnesses }) => harnesses ?? []); + + return { + ...(runtimes.length > 0 && { runtimes }), + ...(credentials.length > 0 && { credentials }), + ...(memories.length > 0 && { memories }), + ...(harnesses.length > 0 && { harnesses }), + }; +} diff --git a/src/core/project/templates/renderer.ts b/src/core/project/templates/renderer.ts new file mode 100644 index 000000000..5a4412306 --- /dev/null +++ b/src/core/project/templates/renderer.ts @@ -0,0 +1,52 @@ +import Handlebars from "handlebars"; +import type { TemplateRenderer } from "./types"; + +/** An implementation of {@link TemplateRenderer} that leverages handlebars to substitute placeholders in the given string **/ +export class HandlebarsTemplateRenderer implements TemplateRenderer { + private readonly hbs: typeof Handlebars; + + constructor() { + this.hbs = Handlebars.create(); + // taken from https://github.com/aws/agentcore-cli/blob/cad94708aeaaa4c7d3e17ecac423453172f3fa86/src/cli/templates/render.ts#L5 + this.hbs.registerHelper("eq", (a: unknown, b: unknown) => a === b); + this.hbs.registerHelper( + "includes", + (arr: unknown[], val: unknown) => Array.isArray(arr) && arr.includes(val), + ); + this.hbs.registerHelper( + "some", + (arr: unknown[], key: string) => + Array.isArray(arr) && + arr.some( + (value) => + value !== null && + typeof value === "object" && + key in value && + Boolean((value as Record)[key]), + ), + ); + this.hbs.registerHelper("or", (...args: unknown[]) => { + for (let i = 0; i < args.length - 1; i++) if (args[i]) return true; + return false; + }); + this.hbs.registerHelper("snakeCase", (str: string) => + str.replace(/[^a-zA-Z0-9]/g, "_").toLowerCase(), + ); + this.hbs.registerHelper( + "safeJson", + (value: unknown) => new Handlebars.SafeString(JSON.stringify(value)), + ); + this.hbs.registerHelper( + "pyJsonStr", + (value: unknown) => new Handlebars.SafeString(JSON.stringify(JSON.stringify(value))), + ); + this.hbs.registerHelper("escapePyStr", (value: unknown) => { + const str = typeof value === "string" ? value : ""; + return new Handlebars.SafeString(str.replace(/\\/g, "\\\\").replace(/"""/g, '\\"\\"\\"')); + }); + } + + render(template: string, context: Record): string { + return this.hbs.compile(template, { noEscape: true })(context); + } +} diff --git a/src/core/project/templates/runtime.ts b/src/core/project/templates/runtime.ts new file mode 100644 index 000000000..275d63580 --- /dev/null +++ b/src/core/project/templates/runtime.ts @@ -0,0 +1,136 @@ +import { FsTreeNode } from "./fsTree"; +import type { AssetSource } from "../source"; +import type { RuntimeResourceConfig } from "../../../handlers/project/add/runtime/types"; +import type { ProjectRuntime } from "../../../projectSchemas/runtime"; +import type { TemplateRenderer, TemplateResolver } from "./types"; +import type { ScaffoldRuntimeInput } from "../../../handlers/project/types"; +import { InputValidationError } from "../../../errors"; + +function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { + const { scaffoldRuntimeInput, name, ...infra } = input; + return { + name, + build: scaffoldRuntimeInput.build, + entrypoint: scaffoldRuntimeInput.entrypoint, + codeLocation: `app/${name}` as ProjectRuntime["codeLocation"], + ...(scaffoldRuntimeInput.runtimeVersion && { + runtimeVersion: scaffoldRuntimeInput.runtimeVersion, + }), + ...(scaffoldRuntimeInput.build === "Container" && { dockerfile: "Dockerfile" }), + ...(infra.description && { description: infra.description }), + ...(infra.executionRoleArn && { executionRoleArn: infra.executionRoleArn }), + ...(infra.additionalPolicies && { additionalPolicies: infra.additionalPolicies }), + ...(infra.envVars && { envVars: infra.envVars }), + ...(infra.networkMode && { networkMode: infra.networkMode }), + ...(infra.networkConfig && { networkConfig: infra.networkConfig }), + ...(infra.authorizerType && { authorizerType: infra.authorizerType }), + ...(infra.authorizerConfiguration && { + authorizerConfiguration: infra.authorizerConfiguration, + }), + ...(infra.protocol && { protocol: infra.protocol }), + ...(infra.requestHeaderAllowlist && { requestHeaderAllowlist: infra.requestHeaderAllowlist }), + ...(infra.lifecycleConfiguration && { lifecycleConfiguration: infra.lifecycleConfiguration }), + ...(infra.filesystemConfigurations && { + filesystemConfigurations: infra.filesystemConfigurations, + }), + ...(infra.tags && { tags: infra.tags }), + }; +} + +/** + * Normalize a name for use as a Python package name per PEP 508. + * Valid names consist only of ASCII letters, numbers, period, underscore, and + * hyphen, and must start and end with a letter or number. + */ +function toPythonPackageName(name: string): string { + return name + .replace(/[^a-zA-Z0-9._-]/g, "-") + .replace(/^[^a-zA-Z0-9]+/, "") + .replace(/[^a-zA-Z0-9]+$/, ""); +} + +function buildResolverKey( + framework: ScaffoldRuntimeInput["framework"], + language: ScaffoldRuntimeInput["language"], +): `${ScaffoldRuntimeInput["framework"]}/${ScaffoldRuntimeInput["language"]}` { + return `${framework}/${language}`; +} + +const getTemplateResolvers = (assetSource: AssetSource, templateRenderer: TemplateRenderer) => ({ + [buildResolverKey("none", "Python")]: async (input: RuntimeResourceConfig) => { + if (input.protocol !== undefined && input.protocol !== "HTTP") + throw new InputValidationError(`hello-world-python only supports HTTP protocol`); + const tree = await FsTreeNode.fromAssetSource( + assetSource, + input.scaffoldRuntimeInput.build === "Container" + ? "templates/hello-world-python-container" + : "templates/hello-world-python", + input.name, + ); + return { tree, spec: { runtimes: [buildRuntimeSpec(input)] } }; + }, + [buildResolverKey("strands", "Python")]: async (input: RuntimeResourceConfig) => { + if (input.protocol !== undefined && input.protocol !== "HTTP") + throw new InputValidationError("the strands-python template only supports HTTP"); + + const filesystemConfigurations = input.filesystemConfigurations ?? []; + const sessionStorageMountPath = filesystemConfigurations.flatMap((configuration) => + "sessionStorage" in configuration ? [configuration.sessionStorage.mountPath] : [], + )[0]; + const efsMounts = filesystemConfigurations.flatMap((configuration) => + "efsAccessPoint" in configuration + ? [{ mountPath: configuration.efsAccessPoint.mountPath }] + : [], + ); + const s3Mounts = filesystemConfigurations.flatMap((configuration) => + "s3FilesAccessPoint" in configuration + ? [{ mountPath: configuration.s3FilesAccessPoint.mountPath }] + : [], + ); + const context = { + name: toPythonPackageName(input.name), + modelProvider: input.scaffoldRuntimeInput.modelProvider, + hasMemory: input.scaffoldRuntimeInput.memory !== "none", + hasIdentity: false, + hasGateway: false, + hasPayment: false, + isVpc: input.networkMode === "VPC", + identityProviders: [], + gatewayProviders: [], + gatewayAuthTypes: [], + sessionStorageMountPath, + efsMounts, + s3Mounts, + needsOs: filesystemConfigurations.length > 0, + hasConfigBundle: false, + }; + const tree = await FsTreeNode.fromAssetSource( + assetSource, + "templates/strands-http-python", + input.name, + (raw) => templateRenderer.render(raw, context), + ); + return { + tree, + spec: { runtimes: [{ ...buildRuntimeSpec(input), protocol: "HTTP" as const }] }, + }; + }, +}); + +type GetRuntimeTemplateResolverConfig = { + assetSource: AssetSource; + templateRenderer: TemplateRenderer; +}; + +/** Given the parameters for rendering, load the {@link TemplateResolver} that resolves to the correct template **/ +export function getRuntimeTemplateResolver( + config: GetRuntimeTemplateResolverConfig, + input: RuntimeResourceConfig, +): TemplateResolver | undefined { + const { framework, language } = input.scaffoldRuntimeInput; + const key = buildResolverKey(framework, language); + + const resolve = getTemplateResolvers(config.assetSource, config.templateRenderer)[key]; + if (!resolve) return undefined; + return { resolve }; +} diff --git a/src/core/project/templates/types.ts b/src/core/project/templates/types.ts new file mode 100644 index 000000000..f472be503 --- /dev/null +++ b/src/core/project/templates/types.ts @@ -0,0 +1,30 @@ +import type { FsTreeNode } from "./fsTree"; +import type { ProjectRuntime } from "../../../projectSchemas/runtime"; +import type { MemorySchema } from "../../../projectSchemas/memory"; +import type { CredentialSchema } from "../../../projectSchemas/credential"; +import type { HarnessRegistryEntry } from "../../../projectSchemas/harness"; +import type z from "zod"; + +/** AgentCore Project Spec Entries that rendered as part of a {@link Template} **/ +export type SpecEntries = { + runtimes?: ProjectRuntime[]; + credentials?: z.infer[]; + memories?: z.infer[]; + harnesses?: HarnessRegistryEntry[]; +}; + +/** A group of files and resources that can be rendered into a project **/ +export type Template = { + tree: FsTreeNode; + spec: SpecEntries; +}; + +/** A standard interface for resolving templates from a given input of paramters **/ +export interface TemplateResolver { + resolve(input: T): Promise