-
Notifications
You must be signed in to change notification settings - Fork 244
test(webview): e2e tests + API task controls (branch 3/3) #1140
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
ee0cd01
62989b7
1477389
c24b7ba
a59e01d
2dab0de
1becf87
42ffba7
8727a47
d06d70f
797c819
c77de69
7d07e53
cf8fd95
ef0f0d3
601e6bc
e55f596
7fe9f6d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| import type { ChatCompletionRequest, ChatMessage, LLMock } from "@copilotkit/aimock" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🔴 Critical | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: Repository: Zoo-Code-Org/Zoo-Code Length of output: 221 🏁 Script executed: Repository: Zoo-Code-Org/Zoo-Code Length of output: 4968 Use
Proposed fix🤖 Prompt for AI Agents |
||
|
|
||
| const TASKS = ["A", "B", "C"] as const | ||
| const ROUNDS = 10 | ||
|
|
||
| const MODE_SEQUENCES: Record<(typeof TASKS)[number], string[]> = { | ||
| A: ["ask", "debug", "architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code"], | ||
| B: ["debug", "architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code", "ask"], | ||
| C: ["architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code", "ask", "debug"], | ||
| } | ||
|
|
||
| const markerFor = (taskName: (typeof TASKS)[number]) => `FOLLOWUP_MODE_ISOLATION_${taskName}` | ||
| const answerFor = (taskName: (typeof TASKS)[number], round: number) => `${taskName} follow-up round ${round}` | ||
| const callIdFor = (taskName: (typeof TASKS)[number], round: number) => | ||
| `call_followup_mode_${taskName.toLowerCase()}_${String(round).padStart(2, "0")}` | ||
|
|
||
| const lastToolResultContains = (req: ChatCompletionRequest, toolCallId: string, expected: string[]) => { | ||
| const messages = Array.isArray(req?.messages) ? req.messages : [] | ||
| const toolMessage = messages.filter((message: ChatMessage) => message?.role === "tool").at(-1) | ||
| const content = toolMessage?.content | ||
|
|
||
| return ( | ||
| toolMessage?.tool_call_id === toolCallId && | ||
| typeof content === "string" && | ||
| expected.every((text) => content.includes(text)) | ||
| ) | ||
| } | ||
|
|
||
| const followupToolCall = (taskName: (typeof TASKS)[number], round: number) => ({ | ||
| name: "ask_followup_question", | ||
| arguments: JSON.stringify({ | ||
| question: `Task ${taskName}: choose mode for round ${round}`, | ||
| follow_up: [ | ||
| { | ||
| text: answerFor(taskName, round), | ||
| mode: MODE_SEQUENCES[taskName][round - 1], | ||
| }, | ||
| ], | ||
| }), | ||
| id: callIdFor(taskName, round), | ||
| }) | ||
|
|
||
| export const getFollowupModeIsolationPlan = () => | ||
| TASKS.map((taskName) => ({ | ||
| taskName, | ||
| marker: markerFor(taskName), | ||
| rounds: MODE_SEQUENCES[taskName].map((mode, index) => ({ | ||
| round: index + 1, | ||
| answer: answerFor(taskName, index + 1), | ||
| mode, | ||
| })), | ||
| })) | ||
|
|
||
| export function addViewStateFixtures(mock: InstanceType<typeof LLMock>) { | ||
| for (const taskName of TASKS) { | ||
| mock.addFixture({ | ||
| match: { | ||
| userMessage: markerFor(taskName), | ||
| }, | ||
| response: { | ||
| toolCalls: [followupToolCall(taskName, 1)], | ||
| }, | ||
| }) | ||
|
|
||
| for (let round = 1; round < ROUNDS; round++) { | ||
| mock.addFixture({ | ||
| match: { | ||
| predicate: (req) => | ||
| lastToolResultContains(req, callIdFor(taskName, round), [answerFor(taskName, round)]), | ||
| }, | ||
| response: { | ||
| toolCalls: [followupToolCall(taskName, round + 1)], | ||
| }, | ||
| }) | ||
| } | ||
|
|
||
| mock.addFixture({ | ||
| match: { | ||
| predicate: (req) => | ||
| lastToolResultContains(req, callIdFor(taskName, ROUNDS), [answerFor(taskName, ROUNDS)]), | ||
| }, | ||
| response: { | ||
| toolCalls: [ | ||
| { | ||
| name: "attempt_completion", | ||
| arguments: JSON.stringify({ | ||
| result: `Task ${taskName} completed ${ROUNDS} follow-up mode switches.`, | ||
| }), | ||
| id: `call_followup_mode_${taskName.toLowerCase()}_complete`, | ||
| }, | ||
| ], | ||
| }, | ||
| }) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win
🧩 Analysis chain
🏁 Script executed:
Repository: Zoo-Code-Org/Zoo-Code
Length of output: 10770
Mark the first debug tool response as turn zero.
The JSON fixture does not include
sequenceIndex, while later fixtures consumecall_modes_switch_002. Add"sequenceIndex": 0to thematchobject for this turn so replay associates the follow-up response with the same request sequence.Proposed fix
📝 Committable suggestion
🤖 Prompt for AI Agents
Source: Coding guidelines