diff --git a/CITATION.cff b/CITATION.cff index 374a9768..51bc85c8 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,6 +4,6 @@ authors: - family-names: 'Leymet' given-names: 'Arnaud' title: 'DEVS' -version: 0.8.0 -date-released: 2026-05-01 +version: 0.8.1 +date-released: 2026-08-01 url: 'https://github.com/codename-co/devs' diff --git a/package.json b/package.json index b31c35f7..db3ed5f5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "devs", "private": true, - "version": "0.8.0", + "version": "0.8.1", "license": "MIT", "type": "module", "sideEffects": false, diff --git a/src/lib/llm/ai-sdk/adapter.ts b/src/lib/llm/ai-sdk/adapter.ts index 7ec5aad3..0e6cc9dc 100644 --- a/src/lib/llm/ai-sdk/adapter.ts +++ b/src/lib/llm/ai-sdk/adapter.ts @@ -19,6 +19,7 @@ import type { ToolDefinition, ToolChoice, FinishReason, + GroundingMetadata, } from '../types' import { stripModelPrefix } from '../types' @@ -58,6 +59,14 @@ export interface AiSdkBinding { providerOptions?( config: FullConfig, ): Record> | undefined + /** + * Provider-executed (hosted) tools to merge into the request's tool set — + * e.g. Anthropic's native `web_search` tool when `enableWebSearch` is set. + * These run server-side on the provider; DEVS never executes them itself. + */ + providerTools?( + config: FullConfig, + ): Promise | ToolSet | undefined /** Validate an API key / endpoint (thin GET). Omitted ⇒ always valid. */ validateApiKey?(apiKey: string, baseUrl?: string): Promise /** List models live from the provider (thin GET). Omitted ⇒ `[]`. */ @@ -108,21 +117,47 @@ function toAiToolChoice( return undefined } -/** Map an AI SDK tool-call → the canonical OpenAI-style ToolCall. */ -function toCanonicalToolCall(tc: { - toolCallId: string - toolName: string - input: unknown -}): ToolCall { - return { +/** Map an AI SDK tool-call → the canonical OpenAI-style ToolCall. Filters + * out provider-executed calls (e.g. Anthropic's native web_search): those + * run server-side and must not be surfaced to DEVS' own tool-execution + * loop. */ +function toCanonicalToolCalls( + calls: + | Array<{ + toolCallId: string + toolName: string + input: unknown + providerExecuted?: boolean + }> + | undefined, +): ToolCall[] | undefined { + const clientCalls = calls?.filter((tc) => !tc.providerExecuted) + if (!clientCalls || clientCalls.length === 0) return undefined + return clientCalls.map((tc) => ({ id: tc.toolCallId, type: 'function', function: { name: tc.toolName, arguments: - typeof tc.input === 'string' ? tc.input : JSON.stringify(tc.input ?? {}), + typeof tc.input === 'string' + ? tc.input + : JSON.stringify(tc.input ?? {}), }, - } + })) +} + +/** Map AI SDK `sources` (unified across providers' web search / grounding) → + * the canonical {@link GroundingMetadata}. */ +function toGroundingMetadata( + sources: + | Array<{ sourceType: string; url?: string; title?: string }> + | undefined, +): GroundingMetadata | undefined { + const webResults = (sources ?? []) + .filter((s) => s.sourceType === 'url' && s.url) + .map((s) => ({ title: s.title || s.url!, url: s.url! })) + if (webResults.length === 0) return undefined + return { isGrounded: true, webResults } } /** @@ -184,12 +219,25 @@ export class AiSdkProvider implements LLMProviderInterface { return { ai, binding, model } } + /** Merge DEVS' client tool definitions with the binding's provider-executed + * (hosted) tools, e.g. Anthropic's native `web_search`. */ + private async tools( + ai: typeof import('ai'), + binding: AiSdkBinding, + config?: FullConfig, + ): Promise { + const definedTools = toAiTools(config?.tools, ai.tool, ai.jsonSchema) + const providerTools = await binding.providerTools?.(config ?? {}) + if (!definedTools && !providerTools) return undefined + return { ...definedTools, ...providerTools } + } + async chat( messages: LLMMessage[], config?: FullConfig, ): Promise { const { ai, binding, model } = await this.build(config) - const tools = toAiTools(config?.tools, ai.tool, ai.jsonSchema) + const tools = await this.tools(ai, binding, config) const providerOptions = binding.providerOptions?.(config ?? {}) const result = await ai.generateText({ @@ -198,19 +246,27 @@ export class AiSdkProvider implements LLMProviderInterface { allowSystemInMessages: true, temperature: config?.temperature ?? 0.7, ...(config?.maxTokens ? { maxOutputTokens: config.maxTokens } : {}), - ...(tools ? { tools, toolChoice: toAiToolChoice(config?.tool_choice) } : {}), + ...(tools + ? { tools, toolChoice: toAiToolChoice(config?.tool_choice) } + : {}), ...(providerOptions ? { providerOptions: providerOptions as ProviderOptions } : {}), abortSignal: config?.signal, }) - const toolCalls = result.toolCalls?.map(toCanonicalToolCall) + const toolCalls = toCanonicalToolCalls(result.toolCalls) + const groundingMetadata = toGroundingMetadata(result.sources) + // A provider-executed web search doesn't need a client round-trip: if + // every tool call was filtered out as provider-executed, the turn is done. + const finishReason = mapFinishReason(result.finishReason) return { content: result.text ?? '', ...(result.reasoningText ? { thinking: result.reasoningText } : {}), - tool_calls: toolCalls && toolCalls.length > 0 ? toolCalls : undefined, - finish_reason: mapFinishReason(result.finishReason), + tool_calls: toolCalls, + ...(groundingMetadata ? { groundingMetadata } : {}), + finish_reason: + finishReason === 'tool_calls' && !toolCalls ? 'stop' : finishReason, usage: result.usage ? { promptTokens: result.usage.inputTokens ?? 0, @@ -226,7 +282,7 @@ export class AiSdkProvider implements LLMProviderInterface { config?: FullConfig, ): AsyncIterableIterator { const { ai, binding, model } = await this.build(config) - const tools = toAiTools(config?.tools, ai.tool, ai.jsonSchema) + const tools = await this.tools(ai, binding, config) const providerOptions = binding.providerOptions?.(config ?? {}) const result = ai.streamText({ @@ -235,7 +291,9 @@ export class AiSdkProvider implements LLMProviderInterface { allowSystemInMessages: true, temperature: config?.temperature ?? 0.7, ...(config?.maxTokens ? { maxOutputTokens: config.maxTokens } : {}), - ...(tools ? { tools, toolChoice: toAiToolChoice(config?.tool_choice) } : {}), + ...(tools + ? { tools, toolChoice: toAiToolChoice(config?.tool_choice) } + : {}), ...(providerOptions ? { providerOptions: providerOptions as ProviderOptions } : {}), @@ -246,9 +304,17 @@ export class AiSdkProvider implements LLMProviderInterface { if (delta) yield delta } + // Grounding metadata (web search sources) is flushed before tool calls — + // mirrors the legacy Google provider's stream ordering, which + // `parseToolCallsFromStream` relies on. + const groundingMetadata = toGroundingMetadata(await result.sources) + if (groundingMetadata) { + yield `\n__GROUNDING_METADATA__${JSON.stringify(groundingMetadata)}` + } + // Preserve the legacy streaming protocol: tool calls are flushed at the end // as a `__TOOL_CALLS__`-prefixed JSON marker (parsed by the agent loop). - const toolCalls = (await result.toolCalls)?.map(toCanonicalToolCall) + const toolCalls = toCanonicalToolCalls(await result.toolCalls) if (toolCalls && toolCalls.length > 0) { yield `\n__TOOL_CALLS__${JSON.stringify(toolCalls)}` } diff --git a/src/lib/llm/ai-sdk/bindings.ts b/src/lib/llm/ai-sdk/bindings.ts index ce26ce90..fd6b86e7 100644 --- a/src/lib/llm/ai-sdk/bindings.ts +++ b/src/lib/llm/ai-sdk/bindings.ts @@ -21,7 +21,10 @@ function bearer(apiKey?: string): Record { } /** Validate a key against an OpenAI-style `GET {base}/models`. */ -async function openAiStyleValidate(base: string, apiKey?: string): Promise { +async function openAiStyleValidate( + base: string, + apiKey?: string, +): Promise { try { const res = await fetch(`${base}/models`, { headers: bearer(apiKey) }) return res.ok @@ -31,7 +34,10 @@ async function openAiStyleValidate(base: string, apiKey?: string): Promise { +async function openAiStyleList( + base: string, + apiKey?: string, +): Promise { try { const res = await fetch(`${base}/models`, { headers: bearer(apiKey) }) if (!res.ok) return [] @@ -126,6 +132,14 @@ export const anthropicBinding: AiSdkBinding = { if (config.effort) anthropic.effort = config.effort return Object.keys(anthropic).length ? { anthropic } : undefined }, + async providerTools(config: FullConfig) { + if (!config.enableWebSearch) return undefined + // Native `web_search_20250305` server tool: Anthropic runs the search + // itself and folds the results back into the same turn, so this never + // needs DEVS' own tool-execution loop (filtered out via `providerExecuted`). + const { anthropic } = await import('@ai-sdk/anthropic') + return { web_search: anthropic.tools.webSearch_20250305({ maxUses: 5 }) } + }, async validateApiKey(apiKey) { // Validate on the endpoint the key is actually used with (`/v1/messages`, // which honours the browser-access header). What matters is whether *auth* @@ -224,7 +238,9 @@ export function makeCompatBinding(opts: CompatOptions): AiSdkBinding { defaultModel: opts.defaultModel, async createModel(config: AiSdkModelConfig) { const baseURL = resolveNormalized(config) - const { createOpenAICompatible } = await import('@ai-sdk/openai-compatible') + const { createOpenAICompatible } = await import( + '@ai-sdk/openai-compatible' + ) const provider = createOpenAICompatible({ name: opts.name, baseURL, @@ -271,7 +287,9 @@ export const ollamaBinding = makeCompatBinding({ resolveBase: (c) => c.baseUrl || OLLAMA_DEFAULT_HOST, // Ollama lists installed models via its native /api/tags endpoint. list: async (config) => { - const host = trimTrailingSlash(ensureAbsolute(config?.baseUrl || OLLAMA_DEFAULT_HOST)) + const host = trimTrailingSlash( + ensureAbsolute(config?.baseUrl || OLLAMA_DEFAULT_HOST), + ) try { const res = await fetch(`${host}/api/tags`) if (!res.ok) return [] @@ -284,7 +302,8 @@ export const ollamaBinding = makeCompatBinding({ validate: async (apiKey, baseUrl) => { const host = trimTrailingSlash( ensureAbsolute( - baseUrl || (apiKey && apiKey !== 'ollama-no-key' ? apiKey : OLLAMA_DEFAULT_HOST), + baseUrl || + (apiKey && apiKey !== 'ollama-no-key' ? apiKey : OLLAMA_DEFAULT_HOST), ), ) try { diff --git a/src/test/lib/llm/ai-sdk/web-search.test.ts b/src/test/lib/llm/ai-sdk/web-search.test.ts new file mode 100644 index 00000000..ec3d1191 --- /dev/null +++ b/src/test/lib/llm/ai-sdk/web-search.test.ts @@ -0,0 +1,156 @@ +/** + * Regression: provider-native web search grounding must actually reach the + * model and its results must reach the caller. + * + * Two things regressed silently during the AI SDK migration (REPORT §4 + * Phase 3): + * 1. Only the Google binding ever looked at `config.enableWebSearch` — the + * Anthropic binding documented `web_search` support (see + * `lib/llm/types.ts`) but never registered the tool, so Claude had no way + * to search the web at all. + * 2. Even when a provider *did* search (Google), the AI SDK adapter never + * turned `result.sources` into `GroundingMetadata` / the + * `__GROUNDING_METADATA__` stream marker `chat.ts` expects, so citations + * were silently dropped. + * + * These tests drive `AiSdkProvider` against a fake `LanguageModelV4` (same + * technique as `chatjimmy-parity.test.ts`) so they exercise the real + * `adapter.ts` tool-merging / source-extraction logic without depending on + * the exact Anthropic wire format. + */ +import { describe, it, expect } from 'vitest' +import type { + LanguageModelV4, + LanguageModelV4CallOptions, + LanguageModelV4Usage, +} from '@ai-sdk/provider' +import { AiSdkProvider } from '@/lib/llm/ai-sdk/adapter' +import { anthropicBinding } from '@/lib/llm/ai-sdk/bindings' +import type { AiSdkBinding, FullConfig } from '@/lib/llm/ai-sdk/adapter' +import type { LLMMessage } from '@/lib/llm' + +const USAGE: LanguageModelV4Usage = { + inputTokens: { + total: 10, + noCache: 10, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, +} + +const messages: LLMMessage[] = [ + { role: 'user', content: 'quelle sera la météo demain à Palaiseau ?' }, +] + +describe('anthropicBinding.providerTools', () => { + it('registers the native web_search tool only when enableWebSearch is set', async () => { + const withSearch = await anthropicBinding.providerTools!({ + enableWebSearch: true, + } as FullConfig) + expect(withSearch).toBeDefined() + expect(Object.keys(withSearch!)).toEqual(['web_search']) + + const withoutSearch = await anthropicBinding.providerTools!( + {} as FullConfig, + ) + expect(withoutSearch).toBeUndefined() + }) +}) + +describe('AiSdkProvider — provider-executed web search (chat)', () => { + /** A fake model whose `doGenerate` returns a server-executed web_search + * tool call + its source, mirroring what @ai-sdk/anthropic normalizes + * Claude's `server_tool_use` / `web_search_tool_result` blocks into. */ + function fakeSearchModel(): LanguageModelV4 { + return { + specificationVersion: 'v4', + provider: 'fake-anthropic', + modelId: 'fake-claude', + supportedUrls: {}, + async doGenerate(_options: LanguageModelV4CallOptions) { + return { + content: [ + { + type: 'tool-call' as const, + toolCallId: 'srvtoolu_1', + toolName: 'web_search', + input: JSON.stringify({ query: 'météo demain Palaiseau' }), + providerExecuted: true, + }, + { + type: 'source' as const, + sourceType: 'url' as const, + id: 'src_1', + url: 'https://meteofrance.com/meteo-france/palaiseau-91120', + title: 'Météo Palaiseau demain - Météo France', + }, + { + type: 'text' as const, + text: 'Demain à Palaiseau : ciel nuageux, 14°C.', + }, + ], + finishReason: { unified: 'stop' as const, raw: 'end_turn' }, + usage: USAGE, + warnings: [], + } + }, + async doStream() { + throw new Error('not used in this test') + }, + } + } + + function fakeBinding(): AiSdkBinding { + return { + defaultModel: 'fake-claude', + createModel: () => Promise.resolve(fakeSearchModel()), + // The fake model ignores tool defs and always "searches" — using the + // real Anthropic factory here just proves a valid provider-defined tool + // survives `generateText`'s tool validation end to end. + providerTools: async (config: FullConfig) => { + if (!config.enableWebSearch) return undefined + const { anthropic } = await import('@ai-sdk/anthropic') + return { web_search: anthropic.tools.webSearch_20250305() } + }, + } + } + + it('surfaces grounding metadata and drops the provider-executed tool call', async () => { + const provider = new AiSdkProvider(() => Promise.resolve(fakeBinding())) + const result = await provider.chat(messages, { enableWebSearch: true }) + + expect(result.content).toContain('Palaiseau') + // The web_search call ran server-side — DEVS must not see it as a + // pending tool call requiring client execution. + expect(result.tool_calls).toBeUndefined() + expect(result.finish_reason).toBe('stop') + expect(result.groundingMetadata).toEqual({ + isGrounded: true, + webResults: [ + { + title: 'Météo Palaiseau demain - Météo France', + url: 'https://meteofrance.com/meteo-france/palaiseau-91120', + }, + ], + }) + }) + + it('does not request the tool when enableWebSearch is unset', async () => { + let seenConfig: FullConfig | undefined + const binding: AiSdkBinding = { + defaultModel: 'fake-claude', + createModel: () => Promise.resolve(fakeSearchModel()), + providerTools: async (config: FullConfig) => { + seenConfig = config + if (!config.enableWebSearch) return undefined + const { anthropic } = await import('@ai-sdk/anthropic') + return { web_search: anthropic.tools.webSearch_20250305() } + }, + } + const provider = new AiSdkProvider(() => Promise.resolve(binding)) + await provider.chat(messages, {}) + + expect(seenConfig?.enableWebSearch).toBeFalsy() + }) +})