From 9d8da649fdde325a558804d5e3e00875a88a0705 Mon Sep 17 00:00:00 2001 From: Esteban Romero Date: Mon, 8 Jun 2026 10:47:58 -0300 Subject: [PATCH 1/4] feat: new command to generate scorer spec --- messages/agent.generate.scorer-spec.md | 49 +++++++ src/commands/agent/generate/scorer-spec.ts | 127 ++++++++++++++++++ .../agent/generate/scorer-spec.test.ts | 124 +++++++++++++++++ 3 files changed, 300 insertions(+) create mode 100644 messages/agent.generate.scorer-spec.md create mode 100644 src/commands/agent/generate/scorer-spec.ts create mode 100644 test/commands/agent/generate/scorer-spec.test.ts diff --git a/messages/agent.generate.scorer-spec.md b/messages/agent.generate.scorer-spec.md new file mode 100644 index 00000000..5885e538 --- /dev/null +++ b/messages/agent.generate.scorer-spec.md @@ -0,0 +1,49 @@ +# summary + +Generate a custom scorer spec YAML file from a template. + +# description + +Generate a template-based YAML scorer spec file for a custom Agentforce scorer (AiAgentScorerDefinition). Unlike the test spec command, this command does not use an interactive interview — it writes a ready-to-edit starter template to disk. + +Use the --data-type flag to choose between a numeric measurement scorer (Number) or a text classification scorer (Text). Edit the generated file to configure your scorer's prompt template, output values, and agent association. + +When your scorer spec is ready, run the "agent scorer spec create" command to convert it to metadata XML and deploy it to your org. + +# flags.output-file.summary + +Name of the generated scorer spec YAML file. Default value is "specs/-scorerSpec.yaml". + +# flags.force-overwrite.summary + +Don't prompt for confirmation when overwriting an existing scorer spec YAML file. + +# flags.name.summary + +API name for the scorer. Sets the "name" and "label" fields in the generated YAML template. + +# flags.agent-api-name.summary + +API name of the agent to associate with the scorer. Sets the "agentApiName" field in the generated YAML template. + +# flags.data-type.summary + +Data type for the scorer: Number (measurement) or Text (multilabel classifier). Determines the starter template used. + +# examples + +- Generate a numeric scorer spec YAML file using the default template: + + <%= config.bin %> <%= command.id %> + +- Generate a text classifier scorer spec and write it to a specific file: + + <%= config.bin %> <%= command.id %> --data-type Text --output-file specs/language_classifier-scorerSpec.yaml + +- Overwrite an existing scorer spec without confirmation: + + <%= config.bin %> <%= command.id %> --output-file specs/my_scorer-scorerSpec.yaml --force-overwrite + +# info.cancel + +Operation canceled. diff --git a/src/commands/agent/generate/scorer-spec.ts b/src/commands/agent/generate/scorer-spec.ts new file mode 100644 index 00000000..e2b3f1fe --- /dev/null +++ b/src/commands/agent/generate/scorer-spec.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { existsSync } from 'node:fs'; +import { join, parse } from 'node:path'; +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { AgentScorer } from '@salesforce/agents'; +import { warn } from '@oclif/core/errors'; +import { theme } from '../../../inquirer-theme.js'; +import yesNoOrCancel from '../../../yes-no-cancel.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.scorer-spec'); + +export function ensureYamlExtension(filePath: string): string { + const parsedPath = parse(filePath); + if (parsedPath.ext === '.yaml' || parsedPath.ext === '.yml') return filePath; + const normalized = `${join(parsedPath.dir, parsedPath.name)}.yaml`; + warn(`Provided file path does not have a .yaml or .yml extension. Normalizing to ${normalized}`); + return normalized; +} + +async function promptUntilUniqueFile(filePath?: string): Promise { + const { input } = await import('@inquirer/prompts'); + + const outputFile = + filePath ?? + (await input({ + message: 'Enter a filepath for the scorer spec file', + validate(d: string): boolean | string { + if (!d.length) { + return 'Path cannot be empty'; + } + return true; + }, + theme, + })); + + const normalized = ensureYamlExtension(outputFile); + + if (!existsSync(normalized)) { + return normalized; + } + + const confirmation = await yesNoOrCancel({ + message: `File ${normalized} already exists. Overwrite?`, + default: false, + }); + + if (confirmation === 'cancel') { + return; + } + + if (!confirmation) { + return promptUntilUniqueFile(); + } + + return normalized; +} + +async function determineFilePath( + outputFile: string | undefined, + forceOverwrite: boolean, + name?: string +): Promise { + const defaultFile = ensureYamlExtension(outputFile ?? AgentScorer.defaultSpecPath(name ?? 'My_Custom_Scorer')); + return forceOverwrite ? defaultFile : promptUntilUniqueFile(defaultFile); +} + +export default class AgentGenerateScorerSpec extends SfCommand { + public static state = 'beta'; + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly enableJsonFlag = false; + + public static readonly flags = { + 'output-file': Flags.file({ + char: 'f', + summary: messages.getMessage('flags.output-file.summary'), + parse: async (raw): Promise => Promise.resolve(ensureYamlExtension(raw)), + }), + 'force-overwrite': Flags.boolean({ + summary: messages.getMessage('flags.force-overwrite.summary'), + }), + 'data-type': Flags.option({ + summary: messages.getMessage('flags.data-type.summary'), + options: ['Number', 'Text'] as const, + default: 'Number' as const, + })(), + name: Flags.string({ + summary: messages.getMessage('flags.name.summary'), + }), + 'agent-api-name': Flags.string({ + summary: messages.getMessage('flags.agent-api-name.summary'), + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentGenerateScorerSpec); + + const outputFile = await determineFilePath(flags['output-file'], flags['force-overwrite'], flags['name']); + if (!outputFile) { + this.log(messages.getMessage('info.cancel')); + return; + } + + await AgentScorer.writeScorerSpecTemplate(outputFile, flags['data-type'], { + name: flags['name'], + agentApiName: flags['agent-api-name'], + }); + this.log(`Created ${outputFile}`); + } +} diff --git a/test/commands/agent/generate/scorer-spec.test.ts b/test/commands/agent/generate/scorer-spec.test.ts new file mode 100644 index 00000000..1983a75a --- /dev/null +++ b/test/commands/agent/generate/scorer-spec.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ + +import { join } from 'node:path'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { TestContext } from '@salesforce/core/testSetup'; +import { ensureYamlExtension } from '../../../../src/commands/agent/generate/scorer-spec.js'; + +describe('AgentGenerateScorerSpec', () => { + describe('ensureYamlExtension', () => { + it('leaves a .yaml path unchanged', () => { + expect(ensureYamlExtension('specs/my_scorer.yaml')).to.equal('specs/my_scorer.yaml'); + }); + + it('leaves a .yml path unchanged', () => { + expect(ensureYamlExtension('specs/my_scorer.yml')).to.equal('specs/my_scorer.yml'); + }); + + it('appends .yaml when extension is missing', () => { + expect(ensureYamlExtension('specs/my_scorer')).to.equal(join('specs', 'my_scorer.yaml')); + }); + + it('replaces a non-yaml extension with .yaml', () => { + expect(ensureYamlExtension('specs/my_scorer.txt')).to.equal(join('specs', 'my_scorer.yaml')); + }); + }); + + describe('run', () => { + const $$ = new TestContext(); + let writeScorerSpecTemplateStub: sinon.SinonStub; + let AgentGenerateScorerSpec: any; + + beforeEach(async () => { + writeScorerSpecTemplateStub = $$.SANDBOX.stub().resolves(); + + const mod = await esmock('../../../../src/commands/agent/generate/scorer-spec.js', { + '@salesforce/agents': { + AgentScorer: { + writeScorerSpecTemplate: writeScorerSpecTemplateStub, + defaultSpecPath: (name: string) => join('specs', `${name}-scorerSpec.yaml`), + }, + }, + }); + AgentGenerateScorerSpec = mod.default; + }); + + afterEach(() => { + $$.restore(); + }); + + it('writes Number template to default path when no flags provided', async () => { + await AgentGenerateScorerSpec.run(['--force-overwrite']); + + expect(writeScorerSpecTemplateStub.calledOnce).to.be.true; + const [outputFile, dataType, overrides] = writeScorerSpecTemplateStub.firstCall.args; + expect(outputFile).to.equal(join('specs', 'My_Custom_Scorer-scorerSpec.yaml')); + expect(dataType).to.equal('Number'); + expect(overrides).to.deep.equal({ name: undefined, agentApiName: undefined }); + }); + + it('passes --name to writeScorerSpecTemplate and uses it in default path', async () => { + await AgentGenerateScorerSpec.run(['--force-overwrite', '--name', 'Sentiment_Scorer']); + + const [outputFile, , overrides] = writeScorerSpecTemplateStub.firstCall.args; + expect(outputFile).to.equal(join('specs', 'Sentiment_Scorer-scorerSpec.yaml')); + expect(overrides.name).to.equal('Sentiment_Scorer'); + }); + + it('passes --agent-api-name to writeScorerSpecTemplate', async () => { + await AgentGenerateScorerSpec.run(['--force-overwrite', '--agent-api-name', 'Resort_Agent']); + + const [, , overrides] = writeScorerSpecTemplateStub.firstCall.args; + expect(overrides.agentApiName).to.equal('Resort_Agent'); + }); + + it('passes --data-type Text to writeScorerSpecTemplate', async () => { + await AgentGenerateScorerSpec.run(['--force-overwrite', '--data-type', 'Text']); + + const [, dataType] = writeScorerSpecTemplateStub.firstCall.args; + expect(dataType).to.equal('Text'); + }); + + it('uses --output-file when provided instead of default path', async () => { + await AgentGenerateScorerSpec.run(['--force-overwrite', '--output-file', 'custom/path.yaml']); + + const [outputFile] = writeScorerSpecTemplateStub.firstCall.args; + expect(outputFile).to.equal('custom/path.yaml'); + }); + + it('passes all flags together correctly', async () => { + await AgentGenerateScorerSpec.run([ + '--force-overwrite', + '--name', + 'Language_Classifier', + '--agent-api-name', + 'My_Agent', + '--data-type', + 'Text', + ]); + + const [outputFile, dataType, overrides] = writeScorerSpecTemplateStub.firstCall.args; + expect(outputFile).to.equal(join('specs', 'Language_Classifier-scorerSpec.yaml')); + expect(dataType).to.equal('Text'); + expect(overrides).to.deep.equal({ name: 'Language_Classifier', agentApiName: 'My_Agent' }); + }); + }); +}); From 9dd89e7ab3b28286db59da93ef41f30de3286cf5 Mon Sep 17 00:00:00 2001 From: Esteban Romero Date: Mon, 8 Jun 2026 11:18:29 -0300 Subject: [PATCH 2/4] chore: add snapshot --- command-snapshot.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/command-snapshot.json b/command-snapshot.json index 5db34d1c..e906f374 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -174,6 +174,14 @@ ], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:generate:scorer-spec", + "flagAliases": [], + "flagChars": ["f"], + "flags": ["agent-api-name", "data-type", "flags-dir", "force-overwrite", "name", "output-file"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:generate:template", From ae10c6b325adcb8043a972dc942672509104fd46 Mon Sep 17 00:00:00 2001 From: Esteban Romero Date: Mon, 8 Jun 2026 11:40:05 -0300 Subject: [PATCH 3/4] chore: add return value to command --- command-snapshot.json | 2 +- schemas/agent-generate-scorer__spec.json | 16 ++++++++++++++++ src/commands/agent/generate/scorer-spec.ts | 12 ++++++++---- test/commands/agent/generate/scorer-spec.test.ts | 3 ++- 4 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 schemas/agent-generate-scorer__spec.json diff --git a/command-snapshot.json b/command-snapshot.json index e906f374..ac5616bd 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -179,7 +179,7 @@ "command": "agent:generate:scorer-spec", "flagAliases": [], "flagChars": ["f"], - "flags": ["agent-api-name", "data-type", "flags-dir", "force-overwrite", "name", "output-file"], + "flags": ["agent-api-name", "data-type", "flags-dir", "force-overwrite", "json", "name", "output-file"], "plugin": "@salesforce/plugin-agent" }, { diff --git a/schemas/agent-generate-scorer__spec.json b/schemas/agent-generate-scorer__spec.json new file mode 100644 index 00000000..cdb3bbc8 --- /dev/null +++ b/schemas/agent-generate-scorer__spec.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentGenerateScorerSpecResult", + "definitions": { + "AgentGenerateScorerSpecResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "required": ["path"], + "additionalProperties": false + } + } +} diff --git a/src/commands/agent/generate/scorer-spec.ts b/src/commands/agent/generate/scorer-spec.ts index e2b3f1fe..fa768ef9 100644 --- a/src/commands/agent/generate/scorer-spec.ts +++ b/src/commands/agent/generate/scorer-spec.ts @@ -80,12 +80,15 @@ async function determineFilePath( return forceOverwrite ? defaultFile : promptUntilUniqueFile(defaultFile); } -export default class AgentGenerateScorerSpec extends SfCommand { +export type AgentGenerateScorerSpecResult = { + path: string; +}; + +export default class AgentGenerateScorerSpec extends SfCommand { public static state = 'beta'; public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); - public static readonly enableJsonFlag = false; public static readonly flags = { 'output-file': Flags.file({ @@ -109,13 +112,13 @@ export default class AgentGenerateScorerSpec extends SfCommand { }), }; - public async run(): Promise { + public async run(): Promise { const { flags } = await this.parse(AgentGenerateScorerSpec); const outputFile = await determineFilePath(flags['output-file'], flags['force-overwrite'], flags['name']); if (!outputFile) { this.log(messages.getMessage('info.cancel')); - return; + return { path: '' }; } await AgentScorer.writeScorerSpecTemplate(outputFile, flags['data-type'], { @@ -123,5 +126,6 @@ export default class AgentGenerateScorerSpec extends SfCommand { agentApiName: flags['agent-api-name'], }); this.log(`Created ${outputFile}`); + return { path: outputFile }; } } diff --git a/test/commands/agent/generate/scorer-spec.test.ts b/test/commands/agent/generate/scorer-spec.test.ts index 1983a75a..c56c8c25 100644 --- a/test/commands/agent/generate/scorer-spec.test.ts +++ b/test/commands/agent/generate/scorer-spec.test.ts @@ -66,13 +66,14 @@ describe('AgentGenerateScorerSpec', () => { }); it('writes Number template to default path when no flags provided', async () => { - await AgentGenerateScorerSpec.run(['--force-overwrite']); + const result = await AgentGenerateScorerSpec.run(['--force-overwrite']); expect(writeScorerSpecTemplateStub.calledOnce).to.be.true; const [outputFile, dataType, overrides] = writeScorerSpecTemplateStub.firstCall.args; expect(outputFile).to.equal(join('specs', 'My_Custom_Scorer-scorerSpec.yaml')); expect(dataType).to.equal('Number'); expect(overrides).to.deep.equal({ name: undefined, agentApiName: undefined }); + expect(result).to.deep.equal({ path: join('specs', 'My_Custom_Scorer-scorerSpec.yaml') }); }); it('passes --name to writeScorerSpecTemplate and uses it in default path', async () => { From c7bfe9db56a92333fd094382dbe4f5896cb629ad Mon Sep 17 00:00:00 2001 From: Esteban Romero Date: Mon, 8 Jun 2026 11:44:40 -0300 Subject: [PATCH 4/4] test: fix failing test --- test/commands/agent/trace/list.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/commands/agent/trace/list.test.ts b/test/commands/agent/trace/list.test.ts index 7038d5b6..870c5e64 100644 --- a/test/commands/agent/trace/list.test.ts +++ b/test/commands/agent/trace/list.test.ts @@ -153,7 +153,6 @@ describe('agent trace list', () => { describe('--since filter', () => { it('returns only traces at or after the given date (date-only)', async () => { - // RECENT_MTIME is 2026-04-07, OLD_MTIME is 2026-03-01 const dateString = MIDDLE_MTIME.toISOString().slice(0, 10).toString(); const result = await AgentTraceList.run(['--since', dateString]); const planIds = result.map((r: any) => r.planId); @@ -163,19 +162,21 @@ describe('agent trace list', () => { }); it('returns only traces at or after the given datetime', async () => { - const result = await AgentTraceList.run(['--since', '2026-04-07T17:00:00.000Z']); + const result = await AgentTraceList.run(['--since', RECENT_MTIME.toISOString()]); const planIds = result.map((r: any) => r.planId); expect(planIds).to.include('plan-1'); // exactly equal — mtime >= since expect(planIds).to.not.include('plan-2'); }); it('returns all traces when since is before all mtimes', async () => { - const result = await AgentTraceList.run(['--since', '2026-01-01']); + const beforeAll = new Date(OLD_MTIME.getTime() - 86_400_000).toISOString().slice(0, 10); + const result = await AgentTraceList.run(['--since', beforeAll]); expect(result).to.have.length(3); }); it('returns empty when since is after all mtimes', async () => { - const result = await AgentTraceList.run(['--since', '2027-01-01']); + const afterAll = new Date(RECENT_MTIME.getTime() + 86_400_000).toISOString(); + const result = await AgentTraceList.run(['--since', afterAll]); expect(result).to.deep.equal([]); });