feat: add new command to create scorer spec

command-snapshot.json

@@ -174,6 +174,14 @@
     ],
     "plugin": "@salesforce/plugin-agent"
   },
+  {
+    "alias": [],
+    "command": "agent:generate:scorer-spec",
+    "flagAliases": [],
+    "flagChars": ["f"],
+    "flags": ["agent-api-name", "data-type", "flags-dir", "force-overwrite", "json", "name", "output-file"],
+    "plugin": "@salesforce/plugin-agent"
+  },
   {
     "alias": [],
     "command": "agent:generate:template",

messages/agent.generate.scorer-spec.md

@@ -0,0 +1,49 @@
+# summary
+
+Generate a custom scorer spec YAML file from a template.
+
+# description
+
+Generate a template-based YAML scorer spec file for a custom Agentforce scorer (AiAgentScorerDefinition). Unlike the test spec command, this command does not use an interactive interview — it writes a ready-to-edit starter template to disk.
+
+Use the --data-type flag to choose between a numeric measurement scorer (Number) or a text classification scorer (Text). Edit the generated file to configure your scorer's prompt template, output values, and agent association.
+
+When your scorer spec is ready, run the "agent scorer spec create" command to convert it to metadata XML and deploy it to your org.
+
+# flags.output-file.summary
+
+Name of the generated scorer spec YAML file. Default value is "specs/<SCORER_NAME>-scorerSpec.yaml".
+
+# flags.force-overwrite.summary
+
+Don't prompt for confirmation when overwriting an existing scorer spec YAML file.
+
+# flags.name.summary
+
+API name for the scorer. Sets the "name" and "label" fields in the generated YAML template.
+
+# flags.agent-api-name.summary
+
+API name of the agent to associate with the scorer. Sets the "agentApiName" field in the generated YAML template.
+
+# flags.data-type.summary
+
+Data type for the scorer: Number (measurement) or Text (multilabel classifier). Determines the starter template used.
+
+# examples
+
+- Generate a numeric scorer spec YAML file using the default template:
+
+  <%= config.bin %> <%= command.id %>
+
+- Generate a text classifier scorer spec and write it to a specific file:
+
+  <%= config.bin %> <%= command.id %> --data-type Text --output-file specs/language_classifier-scorerSpec.yaml
+
+- Overwrite an existing scorer spec without confirmation:
+
+  <%= config.bin %> <%= command.id %> --output-file specs/my_scorer-scorerSpec.yaml --force-overwrite
+
+# info.cancel
+
+Operation canceled.

schemas/agent-generate-scorer__spec.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentGenerateScorerSpecResult",
+  "definitions": {
+    "AgentGenerateScorerSpecResult": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string"
+        }
+      },
+      "required": ["path"],
+      "additionalProperties": false
+    }
+  }
+}

src/commands/agent/generate/scorer-spec.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { existsSync } from 'node:fs';
+import { join, parse } from 'node:path';
+import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
+import { Messages } from '@salesforce/core';
+import { AgentScorer } from '@salesforce/agents';
+import { warn } from '@oclif/core/errors';
+import { theme } from '../../../inquirer-theme.js';
+import yesNoOrCancel from '../../../yes-no-cancel.js';
+
+Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
+const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.scorer-spec');
+
+export function ensureYamlExtension(filePath: string): string {
+  const parsedPath = parse(filePath);
+  if (parsedPath.ext === '.yaml' || parsedPath.ext === '.yml') return filePath;
+  const normalized = `${join(parsedPath.dir, parsedPath.name)}.yaml`;
+  warn(`Provided file path does not have a .yaml or .yml extension. Normalizing to ${normalized}`);
+  return normalized;
+}
+
+async function promptUntilUniqueFile(filePath?: string): Promise<string | undefined> {
+  const { input } = await import('@inquirer/prompts');
+
+  const outputFile =
+    filePath ??
+    (await input({
+      message: 'Enter a filepath for the scorer spec file',
+      validate(d: string): boolean | string {
+        if (!d.length) {
+          return 'Path cannot be empty';
+        }
+        return true;
+      },
+      theme,
+    }));
+
+  const normalized = ensureYamlExtension(outputFile);
+
+  if (!existsSync(normalized)) {
+    return normalized;
+  }
+
+  const confirmation = await yesNoOrCancel({
+    message: `File ${normalized} already exists. Overwrite?`,
+    default: false,
+  });
+
+  if (confirmation === 'cancel') {
+    return;
+  }
+
+  if (!confirmation) {
+    return promptUntilUniqueFile();
+  }
+
+  return normalized;
+}
+
+async function determineFilePath(
+  outputFile: string | undefined,
+  forceOverwrite: boolean,
+  name?: string
+): Promise<string | undefined> {
+  const defaultFile = ensureYamlExtension(outputFile ?? AgentScorer.defaultSpecPath(name ?? 'My_Custom_Scorer'));
+  return forceOverwrite ? defaultFile : promptUntilUniqueFile(defaultFile);
+}
+
+export type AgentGenerateScorerSpecResult = {
+  path: string;
+};
+
+export default class AgentGenerateScorerSpec extends SfCommand<AgentGenerateScorerSpecResult> {
+  public static state = 'beta';
+  public static readonly summary = messages.getMessage('summary');
+  public static readonly description = messages.getMessage('description');
+  public static readonly examples = messages.getMessages('examples');
+
+  public static readonly flags = {
+    'output-file': Flags.file({
+      char: 'f',
+      summary: messages.getMessage('flags.output-file.summary'),
+      parse: async (raw): Promise<string> => Promise.resolve(ensureYamlExtension(raw)),
+    }),
+    'force-overwrite': Flags.boolean({
+      summary: messages.getMessage('flags.force-overwrite.summary'),
+    }),
+    'data-type': Flags.option({
+      summary: messages.getMessage('flags.data-type.summary'),
+      options: ['Number', 'Text'] as const,
+      default: 'Number' as const,
+    })(),
+    name: Flags.string({
+      summary: messages.getMessage('flags.name.summary'),
+    }),
+    'agent-api-name': Flags.string({
+      summary: messages.getMessage('flags.agent-api-name.summary'),
+    }),
+  };
+
+  public async run(): Promise<AgentGenerateScorerSpecResult> {
+    const { flags } = await this.parse(AgentGenerateScorerSpec);
+
+    const outputFile = await determineFilePath(flags['output-file'], flags['force-overwrite'], flags['name']);
+    if (!outputFile) {
+      this.log(messages.getMessage('info.cancel'));
+      return { path: '' };
+    }
+
+    await AgentScorer.writeScorerSpecTemplate(outputFile, flags['data-type'], {
+      name: flags['name'],
+      agentApiName: flags['agent-api-name'],
+    });
+    this.log(`Created ${outputFile}`);
+    return { path: outputFile };
+  }
+}

test/commands/agent/generate/scorer-spec.test.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2026, Salesforce, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */
+
+import { join } from 'node:path';
+import { expect } from 'chai';
+import sinon from 'sinon';
+import esmock from 'esmock';
+import { TestContext } from '@salesforce/core/testSetup';
+import { ensureYamlExtension } from '../../../../src/commands/agent/generate/scorer-spec.js';
+
+describe('AgentGenerateScorerSpec', () => {
+  describe('ensureYamlExtension', () => {
+    it('leaves a .yaml path unchanged', () => {
+      expect(ensureYamlExtension('specs/my_scorer.yaml')).to.equal('specs/my_scorer.yaml');
+    });
+
+    it('leaves a .yml path unchanged', () => {
+      expect(ensureYamlExtension('specs/my_scorer.yml')).to.equal('specs/my_scorer.yml');
+    });
+
+    it('appends .yaml when extension is missing', () => {
+      expect(ensureYamlExtension('specs/my_scorer')).to.equal(join('specs', 'my_scorer.yaml'));
+    });
+
+    it('replaces a non-yaml extension with .yaml', () => {
+      expect(ensureYamlExtension('specs/my_scorer.txt')).to.equal(join('specs', 'my_scorer.yaml'));
+    });
+  });
+
+  describe('run', () => {
+    const $$ = new TestContext();
+    let writeScorerSpecTemplateStub: sinon.SinonStub;
+    let AgentGenerateScorerSpec: any;
+
+    beforeEach(async () => {
+      writeScorerSpecTemplateStub = $$.SANDBOX.stub().resolves();
+
+      const mod = await esmock('../../../../src/commands/agent/generate/scorer-spec.js', {
+        '@salesforce/agents': {
+          AgentScorer: {
+            writeScorerSpecTemplate: writeScorerSpecTemplateStub,
+            defaultSpecPath: (name: string) => join('specs', `${name}-scorerSpec.yaml`),
+          },
+        },
+      });
+      AgentGenerateScorerSpec = mod.default;
+    });
+
+    afterEach(() => {
+      $$.restore();
+    });
+
+    it('writes Number template to default path when no flags provided', async () => {
+      const result = await AgentGenerateScorerSpec.run(['--force-overwrite']);
+
+      expect(writeScorerSpecTemplateStub.calledOnce).to.be.true;
+      const [outputFile, dataType, overrides] = writeScorerSpecTemplateStub.firstCall.args;
+      expect(outputFile).to.equal(join('specs', 'My_Custom_Scorer-scorerSpec.yaml'));
+      expect(dataType).to.equal('Number');
+      expect(overrides).to.deep.equal({ name: undefined, agentApiName: undefined });
+      expect(result).to.deep.equal({ path: join('specs', 'My_Custom_Scorer-scorerSpec.yaml') });
+    });
+
+    it('passes --name to writeScorerSpecTemplate and uses it in default path', async () => {
+      await AgentGenerateScorerSpec.run(['--force-overwrite', '--name', 'Sentiment_Scorer']);
+
+      const [outputFile, , overrides] = writeScorerSpecTemplateStub.firstCall.args;
+      expect(outputFile).to.equal(join('specs', 'Sentiment_Scorer-scorerSpec.yaml'));
+      expect(overrides.name).to.equal('Sentiment_Scorer');
+    });
+
+    it('passes --agent-api-name to writeScorerSpecTemplate', async () => {
+      await AgentGenerateScorerSpec.run(['--force-overwrite', '--agent-api-name', 'Resort_Agent']);
+
+      const [, , overrides] = writeScorerSpecTemplateStub.firstCall.args;
+      expect(overrides.agentApiName).to.equal('Resort_Agent');
+    });
+
+    it('passes --data-type Text to writeScorerSpecTemplate', async () => {
+      await AgentGenerateScorerSpec.run(['--force-overwrite', '--data-type', 'Text']);
+
+      const [, dataType] = writeScorerSpecTemplateStub.firstCall.args;
+      expect(dataType).to.equal('Text');
+    });
+
+    it('uses --output-file when provided instead of default path', async () => {
+      await AgentGenerateScorerSpec.run(['--force-overwrite', '--output-file', 'custom/path.yaml']);
+
+      const [outputFile] = writeScorerSpecTemplateStub.firstCall.args;
+      expect(outputFile).to.equal('custom/path.yaml');
+    });
+
+    it('passes all flags together correctly', async () => {
+      await AgentGenerateScorerSpec.run([
+        '--force-overwrite',
+        '--name',
+        'Language_Classifier',
+        '--agent-api-name',
+        'My_Agent',
+        '--data-type',
+        'Text',
+      ]);
+
+      const [outputFile, dataType, overrides] = writeScorerSpecTemplateStub.firstCall.args;
+      expect(outputFile).to.equal(join('specs', 'Language_Classifier-scorerSpec.yaml'));
+      expect(dataType).to.equal('Text');
+      expect(overrides).to.deep.equal({ name: 'Language_Classifier', agentApiName: 'My_Agent' });
+    });
+  });
+});

test/commands/agent/trace/list.test.ts

@@ -153,7 +153,6 @@ describe('agent trace list', () => {
 
   describe('--since filter', () => {
     it('returns only traces at or after the given date (date-only)', async () => {
-      // RECENT_MTIME is 2026-04-07, OLD_MTIME is 2026-03-01
       const dateString = MIDDLE_MTIME.toISOString().slice(0, 10).toString();
       const result = await AgentTraceList.run(['--since', dateString]);
       const planIds = result.map((r: any) => r.planId);
@@ -163,19 +162,21 @@ describe('agent trace list', () => {
     });
 
     it('returns only traces at or after the given datetime', async () => {
-      const result = await AgentTraceList.run(['--since', '2026-04-07T17:00:00.000Z']);
+      const result = await AgentTraceList.run(['--since', RECENT_MTIME.toISOString()]);
       const planIds = result.map((r: any) => r.planId);
       expect(planIds).to.include('plan-1'); // exactly equal — mtime >= since
       expect(planIds).to.not.include('plan-2');
     });
 
     it('returns all traces when since is before all mtimes', async () => {
-      const result = await AgentTraceList.run(['--since', '2026-01-01']);
+      const beforeAll = new Date(OLD_MTIME.getTime() - 86_400_000).toISOString().slice(0, 10);
+      const result = await AgentTraceList.run(['--since', beforeAll]);
       expect(result).to.have.length(3);
     });
 
     it('returns empty when since is after all mtimes', async () => {
-      const result = await AgentTraceList.run(['--since', '2027-01-01']);
+      const afterAll = new Date(RECENT_MTIME.getTime() + 86_400_000).toISOString();
+      const result = await AgentTraceList.run(['--since', afterAll]);
       expect(result).to.deep.equal([]);
     });