From 70c33e3f56fb77404718df408234250c95baa051 Mon Sep 17 00:00:00 2001 From: niharpatel Date: Wed, 26 Aug 2026 15:56:02 +0530 Subject: [PATCH 1/6] docs: add design spec for W-23524159 results rendering Scopes the CLI-only rendering of PROMPT test-case inputs, latency, and token usage in `sf agent test results` human format. --- ...26-prompt-test-results-rendering-design.md | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 specs/2026-08-26-prompt-test-results-rendering-design.md diff --git a/specs/2026-08-26-prompt-test-results-rendering-design.md b/specs/2026-08-26-prompt-test-results-rendering-design.md new file mode 100644 index 00000000..a130f401 --- /dev/null +++ b/specs/2026-08-26-prompt-test-results-rendering-design.md @@ -0,0 +1,69 @@ +# Design: Render inputs + latency/tokens in `sf agent test results` + +**Ticket:** W-23524159 — "[Headless PT] - Impl: CLI rendering of latency + token metrics" +**Depends on:** W-23524158 (server-side; "Ready for Review") — populates `subjectResponse` for PROMPT test cases. +**Repo:** `plugin-agent` (this repo). No changes to the `agents` package/repo — the ticket is explicitly scoped as a CLI-only change, with no Connect API / results-contract change. + +## Context + +`sf agent test results` renders results from two distinct APIs: + +1. **Legacy** (`AgentTestResultsResponse`, detected via `subjectName` field) — the original Bot Testing API. Out of scope for this change entirely. +2. **AgentforceStudio / NGT** (`AgentforceStudioTestResultsResponse`) — used by both AGENT and PROMPT subject types under the `agentforce-studio` test runner (PROMPT support added in forcedotcom/agents#353, not yet released). This is the path we're changing. + +Today, `humanFormatAgentforceStudio()` in `src/handleTestResults.ts` tries to read `userInput` off `testCase.subjectResponse` — but that field doesn't exist there. A real PROMPT-subject sample response shows: + +- Each test case has a top-level `inputs: [{name, value}, ...]` array. This field exists on the wire (confirmed by tracing `AgentforceStudioTester.results()` → `normalizeAgentforceStudioResults()`, which spreads `...tc` and so preserves it) but is **not declared** on the `AgentforceStudioTestCaseResult` TS type in `@salesforce/agents`. +- `subjectResponse` (a JSON string, already HTML-decoded by the SDK) contains `performance.latency.duration` (ms) and `tokenUsage.{completion, prompt.total, total}` for PROMPT test cases. + +The results payload has no `subjectType` field, so there's no way to positively distinguish AGENT vs. PROMPT test cases at render time. + +## Decisions + +- **Presence-based rendering, not subject-type-gated.** Render the new lines whenever the data exists, for any subject type. This is simpler (no detection needed) and, as a side effect, fixes AGENT's currently-broken "User Input" line for free if AGENT test cases populate the same `inputs` field. +- **Human format only.** JUnit and TAP are out of scope for this pass — revisit once human format has landed and been validated against a real org. +- **No changes to the `agents` package.** The `inputs` field is read via a local type extension in `plugin-agent`, not by modifying `AgentforceStudioTestCaseResult` upstream. +- **Fallback preserved.** If `testCase.inputs` is absent/empty, fall back to the existing `subjectResponse.userInput` parse (today's behavior, unchanged) rather than removing it outright. + +## Behavior + +For each test case in `humanFormatAgentforceStudio()`, after the `Test Case #N` title and before the scorer table: + +``` +Test Case #1 + Inputs: Account = "Acme", Notes = "what is kafka" + Latency: 842ms | Tokens: 156 in / 89 out / 245 total +``` + +### Inputs line + +- Source: `testCase.inputs: Array<{name, value}>` (locally typed, since not on the upstream `AgentforceStudioTestCaseResult` type). +- Format: `Inputs: Name1 = "value1", Name2 = "value2"`, values always double-quoted. +- Label: capitalize just the first letter of the raw `name` (`account` → `Account`). No snake_case/camelCase splitting — no evidence any real input names need it; simplest option that matches the sample data. +- Truncation: show the first 3 inputs in original array order; if there are more, append ` (+N more)` where N = total − 3. +- Fallback: if `inputs` is missing or empty, fall back to today's `User Input: {value}` line derived from `subjectResponse.userInput`, unchanged. If neither exists, omit the line (previously this rendered `User Input: ` with an empty value — omitting is strictly better). + +### Latency/Tokens line + +- Source: `JSON.parse(testCase.subjectResponse)`, reading `performance.latency.duration` and `tokenUsage.{completion, prompt.total, total}`. Reuses the existing try/catch-and-default-to-`{}` pattern already used for scorer responses (`parseScorerResponse`) — never throws on malformed/missing data. +- Format: `Latency: {duration}ms | Tokens: {prompt.total} in / {completion} out / {total} total`. +- Partial data: show only the parts that exist (e.g. `Latency: 842ms` alone if `tokenUsage` is missing). Omit the whole line if neither `performance.latency.duration` nor `tokenUsage` is present. + +## Out of scope (this ticket) + +- JUnit and TAP formats for AgentforceStudio results — revisit in a follow-up once human format is validated. +- Any change to `@salesforce/agents` types or the Connect API contract. +- Live-org / scratch-org test setup — the user will build (`yarn build`) and verify against a real connected org via their own already-linked local `sf` CLI. + +## Testing + +`humanFormatAgentforceStudio()` currently has zero unit test coverage. Add: + +- A fixture derived from the real sample response (trimmed `subjectResponse` prose for readability) covering: `inputs` present, `performance`/`tokenUsage` present. +- A fixture with no `inputs` and no `performance`/`tokenUsage`, to verify graceful omission (no crash, no blank/broken lines). +- A fixture with 4+ inputs on one test case, to verify the `(+N more)` truncation. +- Assertions cover the exact rendered lines for each case above. + +## Verification + +Unit tests only, in this session. The user has already `yarn link`ed their local `sf` CLI to this repo's build and will do the live-org pass themselves after `yarn build`. From 16cf58fc4c5173138f174216801f3aabdf0ad716 Mon Sep 17 00:00:00 2001 From: niharpatel Date: Wed, 26 Aug 2026 16:04:51 +0530 Subject: [PATCH 2/6] docs: add implementation plan for W-23524159 results rendering Two-task plan: inputs line, then latency/tokens line, both in humanFormatAgentforceStudio human-readable output only. --- ...8-26-prompt-test-results-rendering-plan.md | 496 ++++++++++++++++++ 1 file changed, 496 insertions(+) create mode 100644 specs/2026-08-26-prompt-test-results-rendering-plan.md diff --git a/specs/2026-08-26-prompt-test-results-rendering-plan.md b/specs/2026-08-26-prompt-test-results-rendering-plan.md new file mode 100644 index 00000000..a22ad0a7 --- /dev/null +++ b/specs/2026-08-26-prompt-test-results-rendering-plan.md @@ -0,0 +1,496 @@ +# PROMPT Test Results Rendering — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Render per-test-case `Inputs` and `Latency`/`Tokens` lines in the human-readable output of `sf agent test results`, for AgentforceStudio (NGT) results. + +**Architecture:** Two small, presence-based helpers extend `humanFormatAgentforceStudio()` in `src/handleTestResults.ts`. Each test case's table title grows from a fixed 2-line string into a variable-length array of lines (`titleLines`), populated only with the data that actually exists on that test case. No changes to `@salesforce/agents`. + +**Tech Stack:** TypeScript, Mocha/Chai, `@salesforce/sf-plugins-core` `Ux.makeTable`, `ansis`. + +## Global Constraints + +- CLI-only change — no modifications to the `@salesforce/agents` package/repo, no Connect API or results-contract changes (per W-23524159). +- Human format only. JUnit and TAP are explicitly out of scope for this pass. +- Rendering is presence-based, not gated by subject type (the payload has no `subjectType` field) — render whenever the data exists, for any subject type. +- Only `src/handleTestResults.ts` and its test file/fixtures change. + +--- + +## File Structure + +- Modify: `src/handleTestResults.ts` — add `TestCaseInput` type, `getTestCaseInputs`, `capitalizeInputName`, `formatInputsLine` (Task 1); add `ParsedSubjectResponseMetrics` type, `parseSubjectResponseMetrics`, `formatMetricsLine` (Task 2); export `humanFormatAgentforceStudio` (currently module-private). +- Modify: `test/handleTestResults.test.ts` — add two new `describe` blocks (one per task), following the file's existing pattern of loading a fixture and asserting on `humanFormatAgentforceStudio(...)` output. +- Create: `test/mocks/agentforce-studio-results/with-inputs.json`, `legacy-user-input-fallback.json`, `no-inputs-no-user-input.json`, `many-inputs.json`, `latency-only.json`, `tokens-only.json` — new fixtures modeling `AgentforceStudioTestResultsResponse`. + +--- + +### Task 1: Inputs line + +**Files:** + +- Modify: `src/handleTestResults.ts:105-163` (the `humanFormatAgentforceStudio` function and its imports) +- Test: `test/handleTestResults.test.ts` +- Create: `test/mocks/agentforce-studio-results/with-inputs.json` +- Create: `test/mocks/agentforce-studio-results/legacy-user-input-fallback.json` +- Create: `test/mocks/agentforce-studio-results/no-inputs-no-user-input.json` +- Create: `test/mocks/agentforce-studio-results/many-inputs.json` + +**Interfaces:** + +- Consumes: `AgentforceStudioTestResultsResponse`, `AgentforceStudioTestCaseResult` (import from `@salesforce/agents`, the latter is new to this file's imports). +- Produces (for Task 2 to build on): + + - `export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string` (newly exported; was module-private) + - Inside that function, a local `const titleLines: string[]` array per test case, built up before the `ux.makeTable({ title: titleLines.join('\n'), ... })` call — Task 2 appends one more line to this same array. + +- [ ] **Step 1: Create the four fixture files** + +`test/mocks/agentforce-studio-results/with-inputs.json`: + +```json +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "notes", "value": "what is kafka" } + ], + "subjectResponse": "{\"text\":\"Acme is a manufacturing prospect.\",\"performance\":{\"latency\":{\"duration\":842}},\"tokenUsage\":{\"completion\":89,\"prompt\":{\"total\":156},\"total\":245}}", + "testScorerResults": [ + { + "scorerName": "Conciseness Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.7,\"reasoning\":\"Good.\"}" + } + ] + } + ] +} +``` + +`test/mocks/agentforce-studio-results/legacy-user-input-fallback.json`: + +```json +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"userInput\":\"What is the account status?\",\"text\":\"The account is active.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.5,\"reasoning\":\"Clear.\"}" + } + ] + } + ] +} +``` + +`test/mocks/agentforce-studio-results/no-inputs-no-user-input.json`: + +```json +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Some response with no metadata.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} +``` + +`test/mocks/agentforce-studio-results/many-inputs.json`: + +```json +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 2, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "region", "value": "ANZ" }, + { "name": "tier", "value": "Gold" }, + { "name": "segment", "value": "Enterprise" }, + { "name": "priority", "value": "High" } + ], + "subjectResponse": "{\"text\":\"Multi-input response.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.2,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} +``` + +- [ ] **Step 2: Write the failing tests** + +Add to `test/handleTestResults.test.ts` (add `stripVTControlCharacters` and `AgentforceStudioTestResultsResponse` imports, and `humanFormatAgentforceStudio` to the existing `handleTestResults.js` import): + +```ts +import { stripVTControlCharacters } from 'node:util'; +import { AgentforceStudioTestResultsResponse } from '@salesforce/agents'; +import { humanFormat, humanFormatAgentforceStudio, readableTime, truncate } from '../src/handleTestResults.js'; +``` + +```ts +describe('humanFormatAgentforceStudio - inputs line', () => { + it('renders Inputs line from testCase.inputs, capitalizing each name', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: Account = "Acme", Notes = "what is kafka"'); + }); + + it('falls back to User Input when testCase.inputs is absent but subjectResponse.userInput exists', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/legacy-user-input-fallback.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('User Input: What is the account status?'); + expect(output).to.not.include('Inputs:'); + }); + + it('omits the inputs line entirely when neither inputs nor userInput is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Inputs:'); + expect(output).to.not.include('User Input:'); + }); + + it('truncates to the first 3 inputs and appends a "+N more" suffix', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/many-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: Account = "Acme", Region = "ANZ", Tier = "Gold" (+2 more)'); + }); +}); +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` +Expected: FAIL — `humanFormatAgentforceStudio` is not exported from `src/handleTestResults.ts` (TypeScript compile error via ts-node: `has no exported member 'humanFormatAgentforceStudio'`). + +- [ ] **Step 4: Implement** + +In `src/handleTestResults.ts`, update the `@salesforce/agents` import (around line 19-25) to add `AgentforceStudioTestCaseResult`: + +```ts +import { + AgentTestResultsResponse, + AgentforceStudioTestCaseResult, + AgentforceStudioTestResultsResponse, + convertTestResultsToFormat, + humanFriendlyName, + metric, +} from '@salesforce/agents'; +``` + +Add these three helpers directly above `function humanFormatAgentforceStudio` (i.e. right after the existing `parseScorerResponse` function, around line 103): + +```ts +type TestCaseInput = { name: string; value: string }; + +function getTestCaseInputs(testCase: AgentforceStudioTestCaseResult): TestCaseInput[] | undefined { + const inputs = (testCase as unknown as { inputs?: unknown }).inputs; + if (!Array.isArray(inputs)) { + return undefined; + } + const valid = inputs.filter( + (i): i is TestCaseInput => + typeof i === 'object' && + i !== null && + typeof (i as TestCaseInput).name === 'string' && + typeof (i as TestCaseInput).value === 'string' + ); + return valid.length > 0 ? valid : undefined; +} + +function capitalizeInputName(name: string): string { + return name.length > 0 ? `${name[0].toUpperCase()}${name.slice(1)}` : name; +} + +function formatInputsLine(inputs: TestCaseInput[]): string { + const shown = inputs.slice(0, 3); + const remaining = inputs.length - shown.length; + const pairs = shown.map((i) => `${capitalizeInputName(i.name)} = "${i.value}"`).join(', '); + return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; +} +``` + +Replace the body of `function humanFormatAgentforceStudio` (currently starting `function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string {`) — change the declaration to `export function humanFormatAgentforceStudio(...)`, and replace the per-test-case loop's title construction: + +```ts +export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { + const ux = new Ux(); + const tables: string[] = []; + + for (const testCase of results.testCases) { + const inputs = getTestCaseInputs(testCase); + + const titleLines = [ansis.bold(`Test Case #${testCase.testNumber}`)]; + if (inputs) { + titleLines.push(`${ansis.dim('Inputs')}: ${formatInputsLine(inputs)}`); + } else { + let userInput = ''; + try { + const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; + userInput = parsed.userInput ?? ''; + } catch { + // ignore + } + if (userInput) { + titleLines.push(`${ansis.dim('User Input')}: ${userInput}`); + } + } + + const scorerRows = testCase.testScorerResults.map((scorer) => { + const parsed = parseScorerResponse(scorer.scorerResponse); + return { + scorer: scorer.scorerName, + result: parsed.status === 'PASS' ? ansis.green('Pass') : ansis.red('Fail'), + expected: parsed.expectedValue ?? '', + actual: parsed.actualValue ?? '', + reasoning: parsed.reasoning ?? '', + }; + }); + + tables.push( + ux.makeTable({ + title: titleLines.join('\n'), + overflow: 'wrap', + columns: [ + { key: 'scorer', name: 'Scorer' }, + { key: 'result', name: 'Result' }, + { key: 'expected', name: 'Expected', width: '25%' }, + { key: 'actual', name: 'Actual', width: '25%' }, + { key: 'reasoning', name: 'Reasoning', width: '35%' }, + ], + data: scorerRows, + width: '100%', + }) + ); + tables.push('\n'); + } + + const totalCases = results.testCases.length; + const passCases = results.testCases.filter((tc) => + tc.testScorerResults.every((s) => parseScorerResponse(s.scorerResponse).status === 'PASS') + ).length; + + const summary = makeSimpleTable( + { + Status: results.status, + 'Total Test Cases': String(totalCases), + 'Passing Test Cases': String(passCases), + 'Failing Test Cases': String(totalCases - passCases), + }, + ansis.bold.blue('Test Results') + ); + + return tables.join('') + `\n${summary}\n`; +} +``` + +(Everything from `const totalCases = ...` to the end is unchanged from today — shown here only so the full function reads correctly.) + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` +Expected: PASS — all 4 new tests plus the existing 12 (16 total). + +- [ ] **Step 6: Commit** + +```bash +git add src/handleTestResults.ts test/handleTestResults.test.ts test/mocks/agentforce-studio-results/with-inputs.json test/mocks/agentforce-studio-results/legacy-user-input-fallback.json test/mocks/agentforce-studio-results/no-inputs-no-user-input.json test/mocks/agentforce-studio-results/many-inputs.json +git commit -m "feat: render test-case inputs in agent test results human format" +``` + +--- + +### Task 2: Latency/Tokens line + +**Files:** + +- Modify: `src/handleTestResults.ts` (the `humanFormatAgentforceStudio` function from Task 1) +- Test: `test/handleTestResults.test.ts` +- Create: `test/mocks/agentforce-studio-results/latency-only.json` +- Create: `test/mocks/agentforce-studio-results/tokens-only.json` + +**Interfaces:** + +- Consumes: `export function humanFormatAgentforceStudio(...)` and the `titleLines: string[]` array from Task 1 — this task appends one more line to that same array, after the `Inputs`/`User Input` line and before `scorerRows` is built. +- Produces: `formatMetricsLine(parsed: ParsedSubjectResponseMetrics): string | undefined`, `parseSubjectResponseMetrics(raw: string): ParsedSubjectResponseMetrics` — used only within this file; nothing downstream depends on them. + +- [ ] **Step 1: Create the two new fixture files** + +`test/mocks/agentforce-studio-results/latency-only.json`: + +```json +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with latency only.\",\"performance\":{\"latency\":{\"duration\":500}}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.1,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} +``` + +`test/mocks/agentforce-studio-results/tokens-only.json`: + +```json +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with tokens only.\",\"tokenUsage\":{\"completion\":20,\"prompt\":{\"total\":30},\"total\":50}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.3,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} +``` + +- [ ] **Step 2: Write the failing tests** + +Add to `test/handleTestResults.test.ts` (reuses `with-inputs.json` from Task 1 — it already has both `performance` and `tokenUsage` matching the numbers in the ticket's example, and `no-inputs-no-user-input.json`, which has neither): + +```ts +describe('humanFormatAgentforceStudio - latency/tokens line', () => { + it('renders combined Latency and Tokens line', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 842ms | Tokens: 156 in / 89 out / 245 total'); + }); + + it('renders Latency alone when tokenUsage is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/latency-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 500ms'); + expect(output).to.not.include('Tokens:'); + }); + + it('renders Tokens alone when performance is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/tokens-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Tokens: 30 in / 20 out / 50 total'); + expect(output).to.not.include('Latency:'); + }); + + it('omits the metrics line entirely when neither performance nor tokenUsage is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Latency:'); + expect(output).to.not.include('Tokens:'); + }); +}); +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` +Expected: FAIL — the 4 new tests fail because no `Latency:`/`Tokens:` line is rendered yet (`formatMetricsLine`/`parseSubjectResponseMetrics` don't exist yet). + +- [ ] **Step 4: Implement** + +Add these two helpers to `src/handleTestResults.ts`, directly below the `formatInputsLine` function added in Task 1: + +```ts +type ParsedSubjectResponseMetrics = { + performance?: { latency?: { duration?: number } }; + tokenUsage?: { completion?: number; prompt?: { total?: number }; total?: number }; +}; + +function parseSubjectResponseMetrics(raw: string): ParsedSubjectResponseMetrics { + try { + return JSON.parse(raw) as ParsedSubjectResponseMetrics; + } catch { + return {}; + } +} + +function formatMetricsLine(parsed: ParsedSubjectResponseMetrics): string | undefined { + const parts: string[] = []; + const latencyMs = parsed.performance?.latency?.duration; + if (typeof latencyMs === 'number') { + parts.push(`${ansis.dim('Latency')}: ${latencyMs}ms`); + } + const tokenUsage = parsed.tokenUsage; + const hasTokens = + tokenUsage !== undefined && + (typeof tokenUsage.completion === 'number' || + typeof tokenUsage.prompt?.total === 'number' || + typeof tokenUsage.total === 'number'); + if (hasTokens) { + const tokensIn = tokenUsage?.prompt?.total ?? 0; + const tokensOut = tokenUsage?.completion ?? 0; + const tokensTotal = tokenUsage?.total ?? 0; + parts.push(`${ansis.dim('Tokens')}: ${tokensIn} in / ${tokensOut} out / ${tokensTotal} total`); + } + return parts.length > 0 ? parts.join(' | ') : undefined; +} +``` + +In `humanFormatAgentforceStudio`, inside the per-test-case loop, add one block right after the `Inputs`/`User Input` `if`/`else` from Task 1 and before `const scorerRows = ...`: + +```ts +const metricsLine = formatMetricsLine(parseSubjectResponseMetrics(testCase.subjectResponse)); +if (metricsLine) { + titleLines.push(metricsLine); +} +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` +Expected: PASS — all 8 new tests (4 from Task 1, 4 from this task) plus the existing 12 (20 total). + +- [ ] **Step 6: Run the full test suite for a regression check** + +Run: `yarn test` +Expected: PASS — no regressions elsewhere (this only exercises `src/handleTestResults.ts`, an isolated, previously-under-tested file). + +- [ ] **Step 7: Commit** + +```bash +git add src/handleTestResults.ts test/handleTestResults.test.ts test/mocks/agentforce-studio-results/latency-only.json test/mocks/agentforce-studio-results/tokens-only.json +git commit -m "feat: render latency and token usage in agent test results human format" +``` + +--- + +## After both tasks + +Both tasks are on branch `feat/w-23524159-prompt-results-rendering`. Per your earlier direction: no PR yet. Next step is your own `yarn build` + live-org verification via your already-linked local `sf` CLI. JUnit/TAP enrichment is explicitly deferred to a follow-up ticket/plan. From db84f7301b40336000e7b48f79e2541e842ad136 Mon Sep 17 00:00:00 2001 From: niharpatel Date: Wed, 26 Aug 2026 16:16:03 +0530 Subject: [PATCH 3/6] feat: render test-case inputs in agent test results human format --- src/handleTestResults.ts | 81 ++++++++++++++++--- test/handleTestResults.test.ts | 37 ++++++++- .../legacy-user-input-fallback.json | 15 ++++ .../many-inputs.json | 22 +++++ .../no-inputs-no-user-input.json | 15 ++++ .../with-inputs.json | 19 +++++ 6 files changed, 175 insertions(+), 14 deletions(-) create mode 100644 test/mocks/agentforce-studio-results/legacy-user-input-fallback.json create mode 100644 test/mocks/agentforce-studio-results/many-inputs.json create mode 100644 test/mocks/agentforce-studio-results/no-inputs-no-user-input.json create mode 100644 test/mocks/agentforce-studio-results/with-inputs.json diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 2055a174..5533714e 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -18,6 +18,7 @@ import { stripVTControlCharacters } from 'node:util'; import { writeFile, mkdir } from 'node:fs/promises'; import { AgentTestResultsResponse, + AgentforceStudioTestCaseResult, AgentforceStudioTestResultsResponse, convertTestResultsToFormat, humanFriendlyName, @@ -102,17 +103,55 @@ function parseScorerResponse(raw: string): ParsedScorerResponse { } } -function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { +type TestCaseInput = { name: string; value: string }; + +function getTestCaseInputs(testCase: AgentforceStudioTestCaseResult): TestCaseInput[] | undefined { + const inputs = (testCase as unknown as { inputs?: unknown }).inputs; + if (!Array.isArray(inputs)) { + return undefined; + } + const valid = inputs.filter( + (i): i is TestCaseInput => + typeof i === 'object' && + i !== null && + typeof (i as TestCaseInput).name === 'string' && + typeof (i as TestCaseInput).value === 'string' + ); + return valid.length > 0 ? valid : undefined; +} + +function capitalizeInputName(name: string): string { + return name.length > 0 ? `${name[0].toUpperCase()}${name.slice(1)}` : name; +} + +function formatInputsLine(inputs: TestCaseInput[]): string { + const shown = inputs.slice(0, 3); + const remaining = inputs.length - shown.length; + const pairs = shown.map((i) => `${capitalizeInputName(i.name)} = "${i.value}"`).join(', '); + return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; +} + +export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { const ux = new Ux(); const tables: string[] = []; for (const testCase of results.testCases) { - let userInput = ''; - try { - const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; - userInput = parsed.userInput ?? ''; - } catch { - // ignore + const inputs = getTestCaseInputs(testCase); + + const titleLines = [ansis.bold(`Test Case #${testCase.testNumber}`)]; + if (inputs) { + titleLines.push(`${ansis.dim('Inputs')}: ${formatInputsLine(inputs)}`); + } else { + let userInput = ''; + try { + const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; + userInput = parsed.userInput ?? ''; + } catch { + // ignore + } + if (userInput) { + titleLines.push(`${ansis.dim('User Input')}: ${userInput}`); + } } const scorerRows = testCase.testScorerResults.map((scorer) => { @@ -128,7 +167,7 @@ function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsRespons tables.push( ux.makeTable({ - title: `${ansis.bold(`Test Case #${testCase.testNumber}`)}\n${ansis.dim('User Input')}: ${userInput}`, + title: titleLines.join('\n'), overflow: 'wrap', columns: [ { key: 'scorer', name: 'Scorer' }, @@ -217,7 +256,10 @@ function tapFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse) return `TAP version 13\n1..${expectationCount}\n${lines.join('\n')}`; } -function convertAgentforceStudioTestResultsToFormat(results: AgentforceStudioTestResultsResponse, format: 'json' | 'junit' | 'tap'): string { +function convertAgentforceStudioTestResultsToFormat( + results: AgentforceStudioTestResultsResponse, + format: 'json' | 'junit' | 'tap' +): string { switch (format) { case 'json': return JSON.stringify(results, null, 2); @@ -392,9 +434,24 @@ export async function handleTestResults({ if (!isLegacyResponse(results)) { const ngtFormatConfig = { human: { ext: 'txt', label: 'human-readable', get: () => humanFormatAgentforceStudio(results), strip: true }, - json: { ext: 'json', label: 'JSON', get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), strip: false }, - junit: { ext: 'xml', label: 'JUnit', get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), strip: false }, - tap: { ext: 'txt', label: 'TAP', get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), strip: false }, + json: { + ext: 'json', + label: 'JSON', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'json'), + strip: false, + }, + junit: { + ext: 'xml', + label: 'JUnit', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'junit'), + strip: false, + }, + tap: { + ext: 'txt', + label: 'TAP', + get: () => convertAgentforceStudioTestResultsToFormat(results, 'tap'), + strip: false, + }, } as const; const cfg = ngtFormatConfig[format]; const formatted = cfg.get(); diff --git a/test/handleTestResults.test.ts b/test/handleTestResults.test.ts index 795269d7..3e0ee3e4 100644 --- a/test/handleTestResults.test.ts +++ b/test/handleTestResults.test.ts @@ -14,9 +14,10 @@ * limitations under the License. */ import { readFile } from 'node:fs/promises'; +import { stripVTControlCharacters } from 'node:util'; import { expect, config } from 'chai'; -import { AgentTestResultsResponse } from '@salesforce/agents'; -import { humanFormat, readableTime, truncate } from '../src/handleTestResults.js'; +import { AgentTestResultsResponse, AgentforceStudioTestResultsResponse } from '@salesforce/agents'; +import { humanFormat, humanFormatAgentforceStudio, readableTime, truncate } from '../src/handleTestResults.js'; config.truncateThreshold = 0; @@ -112,3 +113,35 @@ describe('metric calculations', () => { expect(output).to.include('Metric Pass % 0.00%'); }); }); + +describe('humanFormatAgentforceStudio - inputs line', () => { + it('renders Inputs line from testCase.inputs, capitalizing each name', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: Account = "Acme", Notes = "what is kafka"'); + }); + + it('falls back to User Input when testCase.inputs is absent but subjectResponse.userInput exists', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/legacy-user-input-fallback.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('User Input: What is the account status?'); + expect(output).to.not.include('Inputs:'); + }); + + it('omits the inputs line entirely when neither inputs nor userInput is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Inputs:'); + expect(output).to.not.include('User Input:'); + }); + + it('truncates to the first 3 inputs and appends a "+N more" suffix', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/many-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Inputs: Account = "Acme", Region = "ANZ", Tier = "Gold" (+2 more)'); + }); +}); diff --git a/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json b/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json new file mode 100644 index 00000000..92bc5ddf --- /dev/null +++ b/test/mocks/agentforce-studio-results/legacy-user-input-fallback.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"userInput\":\"What is the account status?\",\"text\":\"The account is active.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.5,\"reasoning\":\"Clear.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/many-inputs.json b/test/mocks/agentforce-studio-results/many-inputs.json new file mode 100644 index 00000000..5a18ebd9 --- /dev/null +++ b/test/mocks/agentforce-studio-results/many-inputs.json @@ -0,0 +1,22 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 2, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "region", "value": "ANZ" }, + { "name": "tier", "value": "Gold" }, + { "name": "segment", "value": "Enterprise" }, + { "name": "priority", "value": "High" } + ], + "subjectResponse": "{\"text\":\"Multi-input response.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.2,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json b/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json new file mode 100644 index 00000000..6e5b7200 --- /dev/null +++ b/test/mocks/agentforce-studio-results/no-inputs-no-user-input.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Some response with no metadata.\"}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/with-inputs.json b/test/mocks/agentforce-studio-results/with-inputs.json new file mode 100644 index 00000000..13219175 --- /dev/null +++ b/test/mocks/agentforce-studio-results/with-inputs.json @@ -0,0 +1,19 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "inputs": [ + { "name": "account", "value": "Acme" }, + { "name": "notes", "value": "what is kafka" } + ], + "subjectResponse": "{\"text\":\"Acme is a manufacturing prospect.\",\"performance\":{\"latency\":{\"duration\":842}},\"tokenUsage\":{\"completion\":89,\"prompt\":{\"total\":156},\"total\":245}}", + "testScorerResults": [ + { + "scorerName": "Conciseness Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.7,\"reasoning\":\"Good.\"}" + } + ] + } + ] +} From 13a72a1786f83ceff184b82b215a30a700a690fb Mon Sep 17 00:00:00 2001 From: niharpatel Date: Wed, 26 Aug 2026 20:03:39 +0530 Subject: [PATCH 4/6] feat: render latency and token usage in agent test results human format --- src/handleTestResults.ts | 39 +++++++++++++++++++ test/handleTestResults.test.ts | 33 ++++++++++++++++ .../latency-only.json | 15 +++++++ .../tokens-only.json | 15 +++++++ 4 files changed, 102 insertions(+) create mode 100644 test/mocks/agentforce-studio-results/latency-only.json create mode 100644 test/mocks/agentforce-studio-results/tokens-only.json diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 5533714e..6ee458c4 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -131,6 +131,40 @@ function formatInputsLine(inputs: TestCaseInput[]): string { return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; } +type ParsedSubjectResponseMetrics = { + performance?: { latency?: { duration?: number } }; + tokenUsage?: { completion?: number; prompt?: { total?: number }; total?: number }; +}; + +function parseSubjectResponseMetrics(raw: string): ParsedSubjectResponseMetrics { + try { + return JSON.parse(raw) as ParsedSubjectResponseMetrics; + } catch { + return {}; + } +} + +function formatMetricsLine(parsed: ParsedSubjectResponseMetrics): string | undefined { + const parts: string[] = []; + const latencyMs = parsed.performance?.latency?.duration; + if (typeof latencyMs === 'number') { + parts.push(`${ansis.dim('Latency')}: ${latencyMs}ms`); + } + const tokenUsage = parsed.tokenUsage; + const hasTokens = + tokenUsage !== undefined && + (typeof tokenUsage.completion === 'number' || + typeof tokenUsage.prompt?.total === 'number' || + typeof tokenUsage.total === 'number'); + if (hasTokens) { + const tokensIn = tokenUsage?.prompt?.total ?? 0; + const tokensOut = tokenUsage?.completion ?? 0; + const tokensTotal = tokenUsage?.total ?? 0; + parts.push(`${ansis.dim('Tokens')}: ${tokensIn} in / ${tokensOut} out / ${tokensTotal} total`); + } + return parts.length > 0 ? parts.join(' | ') : undefined; +} + export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { const ux = new Ux(); const tables: string[] = []; @@ -154,6 +188,11 @@ export function humanFormatAgentforceStudio(results: AgentforceStudioTestResults } } + const metricsLine = formatMetricsLine(parseSubjectResponseMetrics(testCase.subjectResponse)); + if (metricsLine) { + titleLines.push(metricsLine); + } + const scorerRows = testCase.testScorerResults.map((scorer) => { const parsed = parseScorerResponse(scorer.scorerResponse); return { diff --git a/test/handleTestResults.test.ts b/test/handleTestResults.test.ts index 3e0ee3e4..bec45b0c 100644 --- a/test/handleTestResults.test.ts +++ b/test/handleTestResults.test.ts @@ -145,3 +145,36 @@ describe('humanFormatAgentforceStudio - inputs line', () => { expect(output).to.include('Inputs: Account = "Acme", Region = "ANZ", Tier = "Gold" (+2 more)'); }); }); + +describe('humanFormatAgentforceStudio - latency/tokens line', () => { + it('renders combined Latency and Tokens line', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 842ms | Tokens: 156 in / 89 out / 245 total'); + }); + + it('renders Latency alone when tokenUsage is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/latency-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Latency: 500ms'); + expect(output).to.not.include('Tokens:'); + }); + + it('renders Tokens alone when performance is missing', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/tokens-only.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.include('Tokens: 30 in / 20 out / 50 total'); + expect(output).to.not.include('Latency:'); + }); + + it('omits the metrics line entirely when neither performance nor tokenUsage is present', async () => { + const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); + const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; + const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); + expect(output).to.not.include('Latency:'); + expect(output).to.not.include('Tokens:'); + }); +}); diff --git a/test/mocks/agentforce-studio-results/latency-only.json b/test/mocks/agentforce-studio-results/latency-only.json new file mode 100644 index 00000000..fe58e5e9 --- /dev/null +++ b/test/mocks/agentforce-studio-results/latency-only.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with latency only.\",\"performance\":{\"latency\":{\"duration\":500}}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.1,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} diff --git a/test/mocks/agentforce-studio-results/tokens-only.json b/test/mocks/agentforce-studio-results/tokens-only.json new file mode 100644 index 00000000..12b2afc7 --- /dev/null +++ b/test/mocks/agentforce-studio-results/tokens-only.json @@ -0,0 +1,15 @@ +{ + "status": "SUCCESS", + "testCases": [ + { + "testNumber": 1, + "subjectResponse": "{\"text\":\"Response with tokens only.\",\"tokenUsage\":{\"completion\":20,\"prompt\":{\"total\":30},\"total\":50}}", + "testScorerResults": [ + { + "scorerName": "Coherence Evaluation", + "scorerResponse": "{\"status\":\"PASS\",\"score\":4.3,\"reasoning\":\"OK.\"}" + } + ] + } + ] +} From 908728e0dc97826bba96410013aace5c63464761 Mon Sep 17 00:00:00 2001 From: niharpatel Date: Thu, 27 Aug 2026 12:55:47 +0530 Subject: [PATCH 5/6] chore: drop specs/ planning docs from PR These design/plan markdown files were an artifact of my own tooling defaults, not a repo convention (no prior PR has anything like them) - removing them so the PR only carries the actual code change. The design/plan content lives in the PR description instead. --- ...26-prompt-test-results-rendering-design.md | 69 --- ...8-26-prompt-test-results-rendering-plan.md | 496 ------------------ 2 files changed, 565 deletions(-) delete mode 100644 specs/2026-08-26-prompt-test-results-rendering-design.md delete mode 100644 specs/2026-08-26-prompt-test-results-rendering-plan.md diff --git a/specs/2026-08-26-prompt-test-results-rendering-design.md b/specs/2026-08-26-prompt-test-results-rendering-design.md deleted file mode 100644 index a130f401..00000000 --- a/specs/2026-08-26-prompt-test-results-rendering-design.md +++ /dev/null @@ -1,69 +0,0 @@ -# Design: Render inputs + latency/tokens in `sf agent test results` - -**Ticket:** W-23524159 — "[Headless PT] - Impl: CLI rendering of latency + token metrics" -**Depends on:** W-23524158 (server-side; "Ready for Review") — populates `subjectResponse` for PROMPT test cases. -**Repo:** `plugin-agent` (this repo). No changes to the `agents` package/repo — the ticket is explicitly scoped as a CLI-only change, with no Connect API / results-contract change. - -## Context - -`sf agent test results` renders results from two distinct APIs: - -1. **Legacy** (`AgentTestResultsResponse`, detected via `subjectName` field) — the original Bot Testing API. Out of scope for this change entirely. -2. **AgentforceStudio / NGT** (`AgentforceStudioTestResultsResponse`) — used by both AGENT and PROMPT subject types under the `agentforce-studio` test runner (PROMPT support added in forcedotcom/agents#353, not yet released). This is the path we're changing. - -Today, `humanFormatAgentforceStudio()` in `src/handleTestResults.ts` tries to read `userInput` off `testCase.subjectResponse` — but that field doesn't exist there. A real PROMPT-subject sample response shows: - -- Each test case has a top-level `inputs: [{name, value}, ...]` array. This field exists on the wire (confirmed by tracing `AgentforceStudioTester.results()` → `normalizeAgentforceStudioResults()`, which spreads `...tc` and so preserves it) but is **not declared** on the `AgentforceStudioTestCaseResult` TS type in `@salesforce/agents`. -- `subjectResponse` (a JSON string, already HTML-decoded by the SDK) contains `performance.latency.duration` (ms) and `tokenUsage.{completion, prompt.total, total}` for PROMPT test cases. - -The results payload has no `subjectType` field, so there's no way to positively distinguish AGENT vs. PROMPT test cases at render time. - -## Decisions - -- **Presence-based rendering, not subject-type-gated.** Render the new lines whenever the data exists, for any subject type. This is simpler (no detection needed) and, as a side effect, fixes AGENT's currently-broken "User Input" line for free if AGENT test cases populate the same `inputs` field. -- **Human format only.** JUnit and TAP are out of scope for this pass — revisit once human format has landed and been validated against a real org. -- **No changes to the `agents` package.** The `inputs` field is read via a local type extension in `plugin-agent`, not by modifying `AgentforceStudioTestCaseResult` upstream. -- **Fallback preserved.** If `testCase.inputs` is absent/empty, fall back to the existing `subjectResponse.userInput` parse (today's behavior, unchanged) rather than removing it outright. - -## Behavior - -For each test case in `humanFormatAgentforceStudio()`, after the `Test Case #N` title and before the scorer table: - -``` -Test Case #1 - Inputs: Account = "Acme", Notes = "what is kafka" - Latency: 842ms | Tokens: 156 in / 89 out / 245 total -``` - -### Inputs line - -- Source: `testCase.inputs: Array<{name, value}>` (locally typed, since not on the upstream `AgentforceStudioTestCaseResult` type). -- Format: `Inputs: Name1 = "value1", Name2 = "value2"`, values always double-quoted. -- Label: capitalize just the first letter of the raw `name` (`account` → `Account`). No snake_case/camelCase splitting — no evidence any real input names need it; simplest option that matches the sample data. -- Truncation: show the first 3 inputs in original array order; if there are more, append ` (+N more)` where N = total − 3. -- Fallback: if `inputs` is missing or empty, fall back to today's `User Input: {value}` line derived from `subjectResponse.userInput`, unchanged. If neither exists, omit the line (previously this rendered `User Input: ` with an empty value — omitting is strictly better). - -### Latency/Tokens line - -- Source: `JSON.parse(testCase.subjectResponse)`, reading `performance.latency.duration` and `tokenUsage.{completion, prompt.total, total}`. Reuses the existing try/catch-and-default-to-`{}` pattern already used for scorer responses (`parseScorerResponse`) — never throws on malformed/missing data. -- Format: `Latency: {duration}ms | Tokens: {prompt.total} in / {completion} out / {total} total`. -- Partial data: show only the parts that exist (e.g. `Latency: 842ms` alone if `tokenUsage` is missing). Omit the whole line if neither `performance.latency.duration` nor `tokenUsage` is present. - -## Out of scope (this ticket) - -- JUnit and TAP formats for AgentforceStudio results — revisit in a follow-up once human format is validated. -- Any change to `@salesforce/agents` types or the Connect API contract. -- Live-org / scratch-org test setup — the user will build (`yarn build`) and verify against a real connected org via their own already-linked local `sf` CLI. - -## Testing - -`humanFormatAgentforceStudio()` currently has zero unit test coverage. Add: - -- A fixture derived from the real sample response (trimmed `subjectResponse` prose for readability) covering: `inputs` present, `performance`/`tokenUsage` present. -- A fixture with no `inputs` and no `performance`/`tokenUsage`, to verify graceful omission (no crash, no blank/broken lines). -- A fixture with 4+ inputs on one test case, to verify the `(+N more)` truncation. -- Assertions cover the exact rendered lines for each case above. - -## Verification - -Unit tests only, in this session. The user has already `yarn link`ed their local `sf` CLI to this repo's build and will do the live-org pass themselves after `yarn build`. diff --git a/specs/2026-08-26-prompt-test-results-rendering-plan.md b/specs/2026-08-26-prompt-test-results-rendering-plan.md deleted file mode 100644 index a22ad0a7..00000000 --- a/specs/2026-08-26-prompt-test-results-rendering-plan.md +++ /dev/null @@ -1,496 +0,0 @@ -# PROMPT Test Results Rendering — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Render per-test-case `Inputs` and `Latency`/`Tokens` lines in the human-readable output of `sf agent test results`, for AgentforceStudio (NGT) results. - -**Architecture:** Two small, presence-based helpers extend `humanFormatAgentforceStudio()` in `src/handleTestResults.ts`. Each test case's table title grows from a fixed 2-line string into a variable-length array of lines (`titleLines`), populated only with the data that actually exists on that test case. No changes to `@salesforce/agents`. - -**Tech Stack:** TypeScript, Mocha/Chai, `@salesforce/sf-plugins-core` `Ux.makeTable`, `ansis`. - -## Global Constraints - -- CLI-only change — no modifications to the `@salesforce/agents` package/repo, no Connect API or results-contract changes (per W-23524159). -- Human format only. JUnit and TAP are explicitly out of scope for this pass. -- Rendering is presence-based, not gated by subject type (the payload has no `subjectType` field) — render whenever the data exists, for any subject type. -- Only `src/handleTestResults.ts` and its test file/fixtures change. - ---- - -## File Structure - -- Modify: `src/handleTestResults.ts` — add `TestCaseInput` type, `getTestCaseInputs`, `capitalizeInputName`, `formatInputsLine` (Task 1); add `ParsedSubjectResponseMetrics` type, `parseSubjectResponseMetrics`, `formatMetricsLine` (Task 2); export `humanFormatAgentforceStudio` (currently module-private). -- Modify: `test/handleTestResults.test.ts` — add two new `describe` blocks (one per task), following the file's existing pattern of loading a fixture and asserting on `humanFormatAgentforceStudio(...)` output. -- Create: `test/mocks/agentforce-studio-results/with-inputs.json`, `legacy-user-input-fallback.json`, `no-inputs-no-user-input.json`, `many-inputs.json`, `latency-only.json`, `tokens-only.json` — new fixtures modeling `AgentforceStudioTestResultsResponse`. - ---- - -### Task 1: Inputs line - -**Files:** - -- Modify: `src/handleTestResults.ts:105-163` (the `humanFormatAgentforceStudio` function and its imports) -- Test: `test/handleTestResults.test.ts` -- Create: `test/mocks/agentforce-studio-results/with-inputs.json` -- Create: `test/mocks/agentforce-studio-results/legacy-user-input-fallback.json` -- Create: `test/mocks/agentforce-studio-results/no-inputs-no-user-input.json` -- Create: `test/mocks/agentforce-studio-results/many-inputs.json` - -**Interfaces:** - -- Consumes: `AgentforceStudioTestResultsResponse`, `AgentforceStudioTestCaseResult` (import from `@salesforce/agents`, the latter is new to this file's imports). -- Produces (for Task 2 to build on): - - - `export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string` (newly exported; was module-private) - - Inside that function, a local `const titleLines: string[]` array per test case, built up before the `ux.makeTable({ title: titleLines.join('\n'), ... })` call — Task 2 appends one more line to this same array. - -- [ ] **Step 1: Create the four fixture files** - -`test/mocks/agentforce-studio-results/with-inputs.json`: - -```json -{ - "status": "SUCCESS", - "testCases": [ - { - "testNumber": 1, - "inputs": [ - { "name": "account", "value": "Acme" }, - { "name": "notes", "value": "what is kafka" } - ], - "subjectResponse": "{\"text\":\"Acme is a manufacturing prospect.\",\"performance\":{\"latency\":{\"duration\":842}},\"tokenUsage\":{\"completion\":89,\"prompt\":{\"total\":156},\"total\":245}}", - "testScorerResults": [ - { - "scorerName": "Conciseness Evaluation", - "scorerResponse": "{\"status\":\"PASS\",\"score\":4.7,\"reasoning\":\"Good.\"}" - } - ] - } - ] -} -``` - -`test/mocks/agentforce-studio-results/legacy-user-input-fallback.json`: - -```json -{ - "status": "SUCCESS", - "testCases": [ - { - "testNumber": 1, - "subjectResponse": "{\"userInput\":\"What is the account status?\",\"text\":\"The account is active.\"}", - "testScorerResults": [ - { - "scorerName": "Coherence Evaluation", - "scorerResponse": "{\"status\":\"PASS\",\"score\":4.5,\"reasoning\":\"Clear.\"}" - } - ] - } - ] -} -``` - -`test/mocks/agentforce-studio-results/no-inputs-no-user-input.json`: - -```json -{ - "status": "SUCCESS", - "testCases": [ - { - "testNumber": 1, - "subjectResponse": "{\"text\":\"Some response with no metadata.\"}", - "testScorerResults": [ - { - "scorerName": "Coherence Evaluation", - "scorerResponse": "{\"status\":\"PASS\",\"score\":4.0,\"reasoning\":\"OK.\"}" - } - ] - } - ] -} -``` - -`test/mocks/agentforce-studio-results/many-inputs.json`: - -```json -{ - "status": "SUCCESS", - "testCases": [ - { - "testNumber": 2, - "inputs": [ - { "name": "account", "value": "Acme" }, - { "name": "region", "value": "ANZ" }, - { "name": "tier", "value": "Gold" }, - { "name": "segment", "value": "Enterprise" }, - { "name": "priority", "value": "High" } - ], - "subjectResponse": "{\"text\":\"Multi-input response.\"}", - "testScorerResults": [ - { - "scorerName": "Coherence Evaluation", - "scorerResponse": "{\"status\":\"PASS\",\"score\":4.2,\"reasoning\":\"OK.\"}" - } - ] - } - ] -} -``` - -- [ ] **Step 2: Write the failing tests** - -Add to `test/handleTestResults.test.ts` (add `stripVTControlCharacters` and `AgentforceStudioTestResultsResponse` imports, and `humanFormatAgentforceStudio` to the existing `handleTestResults.js` import): - -```ts -import { stripVTControlCharacters } from 'node:util'; -import { AgentforceStudioTestResultsResponse } from '@salesforce/agents'; -import { humanFormat, humanFormatAgentforceStudio, readableTime, truncate } from '../src/handleTestResults.js'; -``` - -```ts -describe('humanFormatAgentforceStudio - inputs line', () => { - it('renders Inputs line from testCase.inputs, capitalizing each name', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Inputs: Account = "Acme", Notes = "what is kafka"'); - }); - - it('falls back to User Input when testCase.inputs is absent but subjectResponse.userInput exists', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/legacy-user-input-fallback.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('User Input: What is the account status?'); - expect(output).to.not.include('Inputs:'); - }); - - it('omits the inputs line entirely when neither inputs nor userInput is present', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.not.include('Inputs:'); - expect(output).to.not.include('User Input:'); - }); - - it('truncates to the first 3 inputs and appends a "+N more" suffix', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/many-inputs.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Inputs: Account = "Acme", Region = "ANZ", Tier = "Gold" (+2 more)'); - }); -}); -``` - -- [ ] **Step 3: Run tests to verify they fail** - -Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` -Expected: FAIL — `humanFormatAgentforceStudio` is not exported from `src/handleTestResults.ts` (TypeScript compile error via ts-node: `has no exported member 'humanFormatAgentforceStudio'`). - -- [ ] **Step 4: Implement** - -In `src/handleTestResults.ts`, update the `@salesforce/agents` import (around line 19-25) to add `AgentforceStudioTestCaseResult`: - -```ts -import { - AgentTestResultsResponse, - AgentforceStudioTestCaseResult, - AgentforceStudioTestResultsResponse, - convertTestResultsToFormat, - humanFriendlyName, - metric, -} from '@salesforce/agents'; -``` - -Add these three helpers directly above `function humanFormatAgentforceStudio` (i.e. right after the existing `parseScorerResponse` function, around line 103): - -```ts -type TestCaseInput = { name: string; value: string }; - -function getTestCaseInputs(testCase: AgentforceStudioTestCaseResult): TestCaseInput[] | undefined { - const inputs = (testCase as unknown as { inputs?: unknown }).inputs; - if (!Array.isArray(inputs)) { - return undefined; - } - const valid = inputs.filter( - (i): i is TestCaseInput => - typeof i === 'object' && - i !== null && - typeof (i as TestCaseInput).name === 'string' && - typeof (i as TestCaseInput).value === 'string' - ); - return valid.length > 0 ? valid : undefined; -} - -function capitalizeInputName(name: string): string { - return name.length > 0 ? `${name[0].toUpperCase()}${name.slice(1)}` : name; -} - -function formatInputsLine(inputs: TestCaseInput[]): string { - const shown = inputs.slice(0, 3); - const remaining = inputs.length - shown.length; - const pairs = shown.map((i) => `${capitalizeInputName(i.name)} = "${i.value}"`).join(', '); - return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; -} -``` - -Replace the body of `function humanFormatAgentforceStudio` (currently starting `function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string {`) — change the declaration to `export function humanFormatAgentforceStudio(...)`, and replace the per-test-case loop's title construction: - -```ts -export function humanFormatAgentforceStudio(results: AgentforceStudioTestResultsResponse): string { - const ux = new Ux(); - const tables: string[] = []; - - for (const testCase of results.testCases) { - const inputs = getTestCaseInputs(testCase); - - const titleLines = [ansis.bold(`Test Case #${testCase.testNumber}`)]; - if (inputs) { - titleLines.push(`${ansis.dim('Inputs')}: ${formatInputsLine(inputs)}`); - } else { - let userInput = ''; - try { - const parsed = JSON.parse(testCase.subjectResponse) as { userInput?: string }; - userInput = parsed.userInput ?? ''; - } catch { - // ignore - } - if (userInput) { - titleLines.push(`${ansis.dim('User Input')}: ${userInput}`); - } - } - - const scorerRows = testCase.testScorerResults.map((scorer) => { - const parsed = parseScorerResponse(scorer.scorerResponse); - return { - scorer: scorer.scorerName, - result: parsed.status === 'PASS' ? ansis.green('Pass') : ansis.red('Fail'), - expected: parsed.expectedValue ?? '', - actual: parsed.actualValue ?? '', - reasoning: parsed.reasoning ?? '', - }; - }); - - tables.push( - ux.makeTable({ - title: titleLines.join('\n'), - overflow: 'wrap', - columns: [ - { key: 'scorer', name: 'Scorer' }, - { key: 'result', name: 'Result' }, - { key: 'expected', name: 'Expected', width: '25%' }, - { key: 'actual', name: 'Actual', width: '25%' }, - { key: 'reasoning', name: 'Reasoning', width: '35%' }, - ], - data: scorerRows, - width: '100%', - }) - ); - tables.push('\n'); - } - - const totalCases = results.testCases.length; - const passCases = results.testCases.filter((tc) => - tc.testScorerResults.every((s) => parseScorerResponse(s.scorerResponse).status === 'PASS') - ).length; - - const summary = makeSimpleTable( - { - Status: results.status, - 'Total Test Cases': String(totalCases), - 'Passing Test Cases': String(passCases), - 'Failing Test Cases': String(totalCases - passCases), - }, - ansis.bold.blue('Test Results') - ); - - return tables.join('') + `\n${summary}\n`; -} -``` - -(Everything from `const totalCases = ...` to the end is unchanged from today — shown here only so the full function reads correctly.) - -- [ ] **Step 5: Run tests to verify they pass** - -Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` -Expected: PASS — all 4 new tests plus the existing 12 (16 total). - -- [ ] **Step 6: Commit** - -```bash -git add src/handleTestResults.ts test/handleTestResults.test.ts test/mocks/agentforce-studio-results/with-inputs.json test/mocks/agentforce-studio-results/legacy-user-input-fallback.json test/mocks/agentforce-studio-results/no-inputs-no-user-input.json test/mocks/agentforce-studio-results/many-inputs.json -git commit -m "feat: render test-case inputs in agent test results human format" -``` - ---- - -### Task 2: Latency/Tokens line - -**Files:** - -- Modify: `src/handleTestResults.ts` (the `humanFormatAgentforceStudio` function from Task 1) -- Test: `test/handleTestResults.test.ts` -- Create: `test/mocks/agentforce-studio-results/latency-only.json` -- Create: `test/mocks/agentforce-studio-results/tokens-only.json` - -**Interfaces:** - -- Consumes: `export function humanFormatAgentforceStudio(...)` and the `titleLines: string[]` array from Task 1 — this task appends one more line to that same array, after the `Inputs`/`User Input` line and before `scorerRows` is built. -- Produces: `formatMetricsLine(parsed: ParsedSubjectResponseMetrics): string | undefined`, `parseSubjectResponseMetrics(raw: string): ParsedSubjectResponseMetrics` — used only within this file; nothing downstream depends on them. - -- [ ] **Step 1: Create the two new fixture files** - -`test/mocks/agentforce-studio-results/latency-only.json`: - -```json -{ - "status": "SUCCESS", - "testCases": [ - { - "testNumber": 1, - "subjectResponse": "{\"text\":\"Response with latency only.\",\"performance\":{\"latency\":{\"duration\":500}}}", - "testScorerResults": [ - { - "scorerName": "Coherence Evaluation", - "scorerResponse": "{\"status\":\"PASS\",\"score\":4.1,\"reasoning\":\"OK.\"}" - } - ] - } - ] -} -``` - -`test/mocks/agentforce-studio-results/tokens-only.json`: - -```json -{ - "status": "SUCCESS", - "testCases": [ - { - "testNumber": 1, - "subjectResponse": "{\"text\":\"Response with tokens only.\",\"tokenUsage\":{\"completion\":20,\"prompt\":{\"total\":30},\"total\":50}}", - "testScorerResults": [ - { - "scorerName": "Coherence Evaluation", - "scorerResponse": "{\"status\":\"PASS\",\"score\":4.3,\"reasoning\":\"OK.\"}" - } - ] - } - ] -} -``` - -- [ ] **Step 2: Write the failing tests** - -Add to `test/handleTestResults.test.ts` (reuses `with-inputs.json` from Task 1 — it already has both `performance` and `tokenUsage` matching the numbers in the ticket's example, and `no-inputs-no-user-input.json`, which has neither): - -```ts -describe('humanFormatAgentforceStudio - latency/tokens line', () => { - it('renders combined Latency and Tokens line', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Latency: 842ms | Tokens: 156 in / 89 out / 245 total'); - }); - - it('renders Latency alone when tokenUsage is missing', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/latency-only.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Latency: 500ms'); - expect(output).to.not.include('Tokens:'); - }); - - it('renders Tokens alone when performance is missing', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/tokens-only.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Tokens: 30 in / 20 out / 50 total'); - expect(output).to.not.include('Latency:'); - }); - - it('omits the metrics line entirely when neither performance nor tokenUsage is present', async () => { - const raw = await readFile('./test/mocks/agentforce-studio-results/no-inputs-no-user-input.json', 'utf8'); - const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; - const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.not.include('Latency:'); - expect(output).to.not.include('Tokens:'); - }); -}); -``` - -- [ ] **Step 3: Run tests to verify they fail** - -Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` -Expected: FAIL — the 4 new tests fail because no `Latency:`/`Tokens:` line is rendered yet (`formatMetricsLine`/`parseSubjectResponseMetrics` don't exist yet). - -- [ ] **Step 4: Implement** - -Add these two helpers to `src/handleTestResults.ts`, directly below the `formatInputsLine` function added in Task 1: - -```ts -type ParsedSubjectResponseMetrics = { - performance?: { latency?: { duration?: number } }; - tokenUsage?: { completion?: number; prompt?: { total?: number }; total?: number }; -}; - -function parseSubjectResponseMetrics(raw: string): ParsedSubjectResponseMetrics { - try { - return JSON.parse(raw) as ParsedSubjectResponseMetrics; - } catch { - return {}; - } -} - -function formatMetricsLine(parsed: ParsedSubjectResponseMetrics): string | undefined { - const parts: string[] = []; - const latencyMs = parsed.performance?.latency?.duration; - if (typeof latencyMs === 'number') { - parts.push(`${ansis.dim('Latency')}: ${latencyMs}ms`); - } - const tokenUsage = parsed.tokenUsage; - const hasTokens = - tokenUsage !== undefined && - (typeof tokenUsage.completion === 'number' || - typeof tokenUsage.prompt?.total === 'number' || - typeof tokenUsage.total === 'number'); - if (hasTokens) { - const tokensIn = tokenUsage?.prompt?.total ?? 0; - const tokensOut = tokenUsage?.completion ?? 0; - const tokensTotal = tokenUsage?.total ?? 0; - parts.push(`${ansis.dim('Tokens')}: ${tokensIn} in / ${tokensOut} out / ${tokensTotal} total`); - } - return parts.length > 0 ? parts.join(' | ') : undefined; -} -``` - -In `humanFormatAgentforceStudio`, inside the per-test-case loop, add one block right after the `Inputs`/`User Input` `if`/`else` from Task 1 and before `const scorerRows = ...`: - -```ts -const metricsLine = formatMetricsLine(parseSubjectResponseMetrics(testCase.subjectResponse)); -if (metricsLine) { - titleLines.push(metricsLine); -} -``` - -- [ ] **Step 5: Run tests to verify they pass** - -Run: `node_modules/.bin/mocha test/handleTestResults.test.ts` -Expected: PASS — all 8 new tests (4 from Task 1, 4 from this task) plus the existing 12 (20 total). - -- [ ] **Step 6: Run the full test suite for a regression check** - -Run: `yarn test` -Expected: PASS — no regressions elsewhere (this only exercises `src/handleTestResults.ts`, an isolated, previously-under-tested file). - -- [ ] **Step 7: Commit** - -```bash -git add src/handleTestResults.ts test/handleTestResults.test.ts test/mocks/agentforce-studio-results/latency-only.json test/mocks/agentforce-studio-results/tokens-only.json -git commit -m "feat: render latency and token usage in agent test results human format" -``` - ---- - -## After both tasks - -Both tasks are on branch `feat/w-23524159-prompt-results-rendering`. Per your earlier direction: no PR yet. Next step is your own `yarn build` + live-org verification via your already-linked local `sf` CLI. JUnit/TAP enrichment is explicitly deferred to a follow-up ticket/plan. From 9481dbfdd78492b1b2a31489df0d5ca29c893617 Mon Sep 17 00:00:00 2001 From: niharpatel Date: Thu, 27 Aug 2026 17:43:51 +0530 Subject: [PATCH 6/6] fix: show raw input field name in Inputs line, not a capitalized label inputs[].name is an API field name, not a display label - show it verbatim instead of title-casing it. --- src/handleTestResults.ts | 6 +----- test/handleTestResults.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 6ee458c4..37383578 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -120,14 +120,10 @@ function getTestCaseInputs(testCase: AgentforceStudioTestCaseResult): TestCaseIn return valid.length > 0 ? valid : undefined; } -function capitalizeInputName(name: string): string { - return name.length > 0 ? `${name[0].toUpperCase()}${name.slice(1)}` : name; -} - function formatInputsLine(inputs: TestCaseInput[]): string { const shown = inputs.slice(0, 3); const remaining = inputs.length - shown.length; - const pairs = shown.map((i) => `${capitalizeInputName(i.name)} = "${i.value}"`).join(', '); + const pairs = shown.map((i) => `${i.name} = "${i.value}"`).join(', '); return remaining > 0 ? `${pairs} (+${remaining} more)` : pairs; } diff --git a/test/handleTestResults.test.ts b/test/handleTestResults.test.ts index bec45b0c..fcdd8625 100644 --- a/test/handleTestResults.test.ts +++ b/test/handleTestResults.test.ts @@ -115,11 +115,11 @@ describe('metric calculations', () => { }); describe('humanFormatAgentforceStudio - inputs line', () => { - it('renders Inputs line from testCase.inputs, capitalizing each name', async () => { + it('renders Inputs line from testCase.inputs, using the raw field name as-is', async () => { const raw = await readFile('./test/mocks/agentforce-studio-results/with-inputs.json', 'utf8'); const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Inputs: Account = "Acme", Notes = "what is kafka"'); + expect(output).to.include('Inputs: account = "Acme", notes = "what is kafka"'); }); it('falls back to User Input when testCase.inputs is absent but subjectResponse.userInput exists', async () => { @@ -142,7 +142,7 @@ describe('humanFormatAgentforceStudio - inputs line', () => { const raw = await readFile('./test/mocks/agentforce-studio-results/many-inputs.json', 'utf8'); const input = JSON.parse(raw) as AgentforceStudioTestResultsResponse; const output = stripVTControlCharacters(humanFormatAgentforceStudio(input)); - expect(output).to.include('Inputs: Account = "Acme", Region = "ANZ", Tier = "Gold" (+2 more)'); + expect(output).to.include('Inputs: account = "Acme", region = "ANZ", tier = "Gold" (+2 more)'); }); });