diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 2055a174..37383578 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -18,6 +18,7 @@ import { stripVTControlCharacters } from 'node:util'; import { writeFile, mkdir } from 'node:fs/promises'; import { AgentTestResultsResponse, + AgentforceStudioTestCaseResult, AgentforceStudioTestResultsResponse, convertTestResultsToFormat, humanFriendlyName, @@ -102,17 +103,90 @@ function parseScorerResponse(raw: string): ParsedScorerResponse { } } -function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { +type TestCaseInput = { name: string; value: string }; + +function getTestCaseInputs(testCase: AgentforceStudioTestCaseResult): TestCaseInput[] | undefined { + const inputs = (testCase as unknown as { inputs?: unknown }).inputs; + if (!Array.isArray(inputs)) { + return undefined; + } + const valid = inputs.filter( + (i): i is TestCaseInput => + typeof i === 'object' && + i !== null && + typeof (i as TestCaseInput).name === 'string' && + typeof (i as TestCaseInput).value === 'string' + ); + return valid.length > 0 ? valid : undefined; +} + +function formatInputsLine(inputs: TestCaseInput[]): string { + const shown = inputs.slice(0, 3); + const remaining = inputs.length - shown.length; + const pairs = shown.map((i) => `${i.name} = "${i.value}"`).join(', '); + return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; +} + +type ParsedSubjectResponseMetrics = { + performance?: { latency?: { duration?: number } }; + tokenUsage?: { completion?: number; prompt?: { total?: number }; total?: number }; +}; + +function parseSubjectResponseMetrics(raw: string): ParsedSubjectResponseMetrics { + try { + return JSON.parse(raw) as ParsedSubjectResponseMetrics; + } catch { + return {}; + } +} + +function formatMetricsLine(parsed: ParsedSubjectResponseMetrics): string | undefined { + const parts: string[] = []; + const latencyMs = parsed.performance?.latency?.duration; + if (typeof latencyMs === 'number') { + parts.push(`${ansis.dim('Latency')}: ${latencyMs}ms`); + } + const tokenUsage = parsed.tokenUsage; + const hasTokens = + tokenUsage !== undefined && + (typeof tokenUsage.completion === 'number' || + typeof tokenUsage.prompt?.total === 'number' || + typeof tokenUsage.total === 'number'); + if (hasTokens) { + const tokensIn = tokenUsage?.prompt?.total ?? 0; + const tokensOut = tokenUsage?.completion ?? 0; + const tokensTotal = tokenUsage?.total ?? 0; + parts.push(`${ansis.dim('Tokens')}: ${tokensIn} in / ${tokensOut} out / ${tokensTotal} total`); + } + return parts.length > 0 ? parts.join(' | ') : undefined; +} + +export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { const ux = new Ux(); const tables: string[] = []; for (const testCase of results.testCases) { - let userInput = ''; - try { - const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; - userInput = parsed.userInput ?? ''; - } catch { - // ignore + const inputs = getTestCaseInputs(testCase); + + const titleLines = [ansis.bold(`Test Case #${testCase.testNumber}`)]; + if (inputs) { + titleLines.push(`${ansis.dim('Inputs')}: ${formatInputsLine(inputs)}`); + } else { + let userInput = ''; + try { + const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; + userInput = parsed.userInput ?? ''; + } catch { + // ignore + } + if (userInput) { + titleLines.push(`${ansis.dim('User Input')}: ${userInput}`); + } + } + + const metricsLine = formatMetricsLine(parseSubjectResponseMetrics(testCase.subjectResponse)); + if (metricsLine) { + titleLines.push(metricsLine); } const scorerRows = testCase.testScorerResults.map((scorer) => { @@ -128,7 +202,7 @@ function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsRespons tables.push( ux.makeTable({ - title: `${ansis.bold(`Test Case #${testCase.testNumber}`)}\n${ansis.dim('User Input')}: ${userInput}`, + title: titleLines.join('\n'), overflow: 'wrap', columns: [ { key: 'scorer', name: 'Scorer' }, @@ -217,7 +291,10 @@ function tapFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse) return `TAP version 13\n1..${expectationCount}\n${lines.join('\n')}`; } -function convertAgentforceStudioTestResultsToFormat(results: AgentforceStudioTestResultsResponse, format: 'json' | 'junit' | 'tap'): string { +function convertAgentforceStudioTestResultsToFormat( + results: AgentforceStudioTestResultsResponse, + format: 'json' | 'junit' | 'tap' +): string { switch (format) { case 'json': return JSON.stringify(results, null, 2); @@ -392,9 +469,24 @@ export async function handleTestResults({ if (!isLegacyResponse(results)) { const ngtFormatConfig = { human: { ext: 'txt', label: 'human-readable', get: () => humanFormatAgentforceStudio(results), strip: true }, - json: { ext: 'json', label: 'JSON', get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), strip: false }, - junit: { ext: 'xml', label: 'JUnit', get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), strip: false }, - tap: { ext: 'txt', label: 'TAP', get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), strip: false }, + json: { + ext: 'json', + label: 'JSON', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), + strip: false, + }, + junit: { + ext: 'xml', + label: 'JUnit', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), + strip: false, + }, + tap: { + ext: 'txt', + label: 'TAP', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), + strip: false, + }, } as const; const cfg = ngtFormatConfig[format]; const formatted = cfg.get(); diff --git a/test/handleTestResults.test.ts b/test/handleTestResults.test.ts index 795269d7..fcdd8625 100644 --- a/test/handleTestResults.test.ts +++ b/test/handleTestResults.test.ts @@ -14,9 +14,10 @@ * limitations under the License. */ import { readFile } from 'node:fs/promises'; +import { stripVTControlCharacters } from 'node:util'; import { expect, config } from 'chai'; -import { AgentTestResultsResponse } from '@salesforce/agents'; -import { humanFormat, readableTime, truncate } from '../src/handleTestResults.js'; +import { AgentTestResultsResponse, AgentforceStudioTestResultsResponse } from '@salesforce/agents'; +import { humanFormat, humanFormatAgentforceStudio, readableTime, truncate } from '../src/handleTestResults.js'; config.truncateThreshold = 0; @@ -112,3 +113,68 @@ describe('metric calculations', () => { expect(output).to.include('Metric Pass % 0.00%'); }); }); + +describe('humanFormatAgentforceStudio - inputs line', () => { + it('renders Inputs line from testCase.inputs, using the raw field name as-is', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: account = "Acme", notes = "what is kafka"'); + }); + + it('falls back to User Input when testCase.inputs is absent but subjectResponse.userInput exists', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/legacy-user-input-fallback.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('User Input: What is the account status?'); + expect(output).to.not.include('Inputs:'); + }); + + it('omits the inputs line entirely when neither inputs nor userInput is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Inputs:'); + expect(output).to.not.include('User Input:'); + }); + + it('truncates to the first 3 inputs and appends a "+N more" suffix', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/many-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: account = "Acme", region = "ANZ", tier = "Gold" (+2 more)'); + }); +}); + +describe('humanFormatAgentforceStudio - latency/tokens line', () => { + it('renders combined Latency and Tokens line', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 842ms | Tokens: 156 in / 89 out / 245 total'); + }); + + it('renders Latency alone when tokenUsage is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/latency-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 500ms'); + expect(output).to.not.include('Tokens:'); + }); + + it('renders Tokens alone when performance is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/tokens-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Tokens: 30 in / 20 out / 50 total'); + expect(output).to.not.include('Latency:'); + }); + + it('omits the metrics line entirely when neither performance nor tokenUsage is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Latency:'); + expect(output).to.not.include('Tokens:'); + }); +}); diff --git a/test/mocks/agentforce-studio-results/latency-only.json b/test/mocks/agentforce-studio-results/latency-only.json new file mode 100644 index 00000000..fe58e5e9 --- /dev/null +++ b/test/mocks/agentforce-studio-results/latency-only.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with latency only.\",\"performance\":{\"latency\":{\"duration\":500}}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.1,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json b/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json new file mode 100644 index 00000000..92bc5ddf --- /dev/null +++ b/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"userInput\":\"What is the account status?\",\"text\":\"The account is active.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.5,\"reasoning\":\"Clear.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/many-inputs.json b/test/mocks/agentforce-studio-results/many-inputs.json new file mode 100644 index 00000000..5a18ebd9 --- /dev/null +++ b/test/mocks/agentforce-studio-results/many-inputs.json @@ -0,0 +1,22 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 2, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "region", "value": "ANZ" }, + { "name": "tier", "value": "Gold" }, + { "name": "segment", "value": "Enterprise" }, + { "name": "priority", "value": "High" } + ], + "subjectResponse": "{\"text\":\"Multi-input response.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.2,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json b/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json new file mode 100644 index 00000000..6e5b7200 --- /dev/null +++ b/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Some response with no metadata.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/tokens-only.json b/test/mocks/agentforce-studio-results/tokens-only.json new file mode 100644 index 00000000..12b2afc7 --- /dev/null +++ b/test/mocks/agentforce-studio-results/tokens-only.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with tokens only.\",\"tokenUsage\":{\"completion\":20,\"prompt\":{\"total\":30},\"total\":50}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.3,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/with-inputs.json b/test/mocks/agentforce-studio-results/with-inputs.json new file mode 100644 index 00000000..13219175 --- /dev/null +++ b/test/mocks/agentforce-studio-results/with-inputs.json @@ -0,0 +1,19 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "notes", "value": "what is kafka" } + ], + "subjectResponse": "{\"text\":\"Acme is a manufacturing prospect.\",\"performance\":{\"latency\":{\"duration\":842}},\"tokenUsage\":{\"completion\":89,\"prompt\":{\"total\":156},\"total\":245}}", + "testScorerResults": [ + { + "scorerName": "Conciseness Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.7,\"reasoning\":\"Good.\"}" + } + ] + } + ] +}