feat(eval): ondemand simulate — replay a dataset, evaluate synchronously

src/core/eval.tsx

@@ -216,6 +216,7 @@ export class EvalClient implements CoreEvalClient {
     // logger for batch-evaluation result-log diagnostics
     private readonly logger: Logger = noopLogger,
     private readonly newSessionId: () => string = randomUUID,
+    private readonly now: () => number = () => Date.now(),
   ) {}
 
   async createEvaluator(
@@ -582,7 +583,7 @@ export class EvalClient implements CoreEvalClient {
     const logGroupName = runtimeLogGroup(runtimeId, qualifier);
     const serviceName = runtimeServiceName(runtimeName, qualifier);
 
-    const endMs = input.window ? +input.window.endTime : Date.now();
+    const endMs = input.window ? +input.window.endTime : this.now();
     const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS;
     const startSec = Math.floor(startMs / 1000);
     const endSec = Math.floor(endMs / 1000);

src/core/index.tsx

@@ -39,6 +39,7 @@ type CoreClientConfig = {
   logger: Logger;
   fetch?: CoreFetch;
   newSessionId?: () => string;
+  now?: () => number;
 };
 
 // CoreClient is the single entry point to the Bedrock AgentCore APIs. It owns the
@@ -84,6 +85,7 @@ export class CoreClient implements AwsClients {
       fetch,
       this.logger.child({ module: "eval" }),
       config.newSessionId,
+      config.now,
     );
 
     this.projectManager = new FsProjectManager({

src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json

@@ -0,0 +1,114 @@
+{
+  "evaluationResults": [
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "00000000-0000-4000-8000-000000000001",
+          "traceId": "6a8f5b461983df572e2aca3264d32d4d"
+        }
+      },
+      "explanation": "The user simply said 'hi', which is a greeting with no specific goal expressed. The assistant responded with a polite greeting and an open invitation to help. This is appropriate conversational behavior that maintains the flow of interaction and invites the user to share their actual needs. Since the user hasn't expressed a specific goal yet, the assistant's response is a standard, appropriate reply that keeps the conversation open. It doesn't advance any specific goal (since none exists yet), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.",
+      "value": 0.5,
+      "label": "Neutral/Mixed",
+      "tokenUsage": {
+        "inputTokens": 815,
+        "outputTokens": 155,
+        "totalTokens": 970
+      },
+      "ignoredReferenceInputFields": [
+        "assertions"
+      ]
+    },
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "00000000-0000-4000-8000-000000000001",
+          "traceId": "6a8f5bbc1882c05547b64a9246f17ca4"
+        }
+      },
+      "explanation": "The user has said 'hi' twice without providing any specific goal or request. The assistant's response 'Hi there! How can I assist you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. Since the user hasn't expressed a specific goal yet, the assistant can't do much more than respond to the greeting and prompt the user to share what they need. This response is appropriate for the conversational context - it's a polite acknowledgment that maintains the conversation flow and opens the door for the user to state their actual needs. It doesn't advance any specific goal (since none has been stated), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.",
+      "value": 0.5,
+      "label": "Neutral/Mixed",
+      "tokenUsage": {
+        "inputTokens": 887,
+        "outputTokens": 194,
+        "totalTokens": 1081
+      },
+      "ignoredReferenceInputFields": [
+        "assertions"
+      ]
+    },
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "00000000-0000-4000-8000-000000000001",
+          "traceId": "6a8f639540ebe6cb6f30d5d80762c2e2"
+        }
+      },
+      "explanation": "The user has sent 'hi' three times in a row without providing any specific request or goal. The assistant's response 'Hello! How can I help you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. This is the third identical exchange, and the assistant is simply repeating the same greeting. While the response is appropriate and doesn't obstruct any goal, it also doesn't advance any specific goal since the user hasn't expressed one yet. The response is essentially neutral - it's appropriate chit-chat that maintains conversation flow without moving toward any particular goal (since no goal has been stated). The assistant could potentially note that the user has greeted multiple times and ask if they need help with something specific, which would be slightly more proactive, but the current response is still a reasonable reply to a simple greeting.",
+      "value": 0.5,
+      "label": "Neutral/Mixed",
+      "tokenUsage": {
+        "inputTokens": 958,
+        "outputTokens": 207,
+        "totalTokens": 1165
+      },
+      "ignoredReferenceInputFields": [
+        "assertions"
+      ]
+    },
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "00000000-0000-4000-8000-000000000001",
+          "traceId": "6a90a5987755c6e457ada5616a16dddb"
+        }
+      },
+      "explanation": "The user's final request was 'Say hello and offer help.' The assistant's response directly fulfills this request by:\n1. Saying hello with a greeting and wave emoji\n2. Offering help by listing specific capabilities it can assist with\n\nThe response is friendly, clear, and directly addresses what the user asked for. It goes slightly beyond the minimal requirement by providing specific examples of what it can help with (web search, fetching webpages, calculations), which gives the user actionable information about how to proceed.\n\nHowever, the listed capabilities (web search, fetch webpages, perform calculations) seem oddly specific and somewhat limiting - a general AI assistant can help with many more things like writing, analysis, coding, answering questions, etc. This specificity might actually mislead the user about the assistant's full range of capabilities.\n\nDespite this minor issue, the response does exactly what was requested - says hello and offers help - and does so in a clear, organized manner. The user's goal was simple and the assistant met it directly.",
+      "value": 0.83,
+      "label": "Very Helpful",
+      "tokenUsage": {
+        "inputTokens": 1112,
+        "outputTokens": 258,
+        "totalTokens": 1370
+      },
+      "ignoredReferenceInputFields": [
+        "assertions"
+      ]
+    },
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "00000000-0000-4000-8000-000000000001",
+          "traceId": "6a90a74750f33f567076943f7bce7966"
+        }
+      },
+      "explanation": "The user's request is simple and explicit: 'Say hello and offer help.' The assistant's response directly fulfills this request by greeting the user with 'Hello! 👋 Welcome!' and then offering help with a clear, organized list of capabilities (web searches, reading webpages, and calculations). The response is well-formatted, friendly, and ends with an open invitation for the user to specify what they need. This is essentially the same response as the previous turn (with minor variations like adding 'Welcome!'), which is appropriate since the user repeated the same request. The response fully satisfies the user's stated goal of having the assistant say hello and offer help. It's comprehensive and actionable, clearly communicating what the assistant can do. There's nothing missing or problematic about this response given the user's simple, direct request.",
+      "value": 0.83,
+      "label": "Very Helpful",
+      "tokenUsage": {
+        "inputTokens": 1267,
+        "outputTokens": 203,
+        "totalTokens": 1470
+      },
+      "ignoredReferenceInputFields": [
+        "assertions"
+      ]
+    }
+  ]
+}