diff --git a/.changeset/langchain-js-token-total-metrics.md b/.changeset/langchain-js-token-total-metrics.md new file mode 100644 index 000000000..548b13b62 --- /dev/null +++ b/.changeset/langchain-js-token-total-metrics.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix: Emit canonical LangChain JS total token metrics diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json index d649a4b4a..159e7c4ae 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.log-payloads.json @@ -37,11 +37,7 @@ "span_id": "" }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": {}, @@ -85,11 +81,7 @@ ] }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": { @@ -187,11 +179,7 @@ ] }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": [ @@ -283,7 +271,7 @@ "prompt_cached_tokens": "", "prompt_tokens": "", "start": 0, - "total_tokens": "" + "tokens": "" }, "output": { "generations": [ @@ -430,11 +418,7 @@ ] }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": { diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json index ed1dba52c..801834fae 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.json @@ -116,7 +116,7 @@ "completion_tokens", "prompt_cached_tokens", "prompt_tokens", - "total_tokens" + "tokens" ] } ], diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt index 2e3c03b9b..1c06fb02c 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1-latest.span-tree.txt @@ -116,7 +116,7 @@ span_tree: │ "completion_tokens", │ "prompt_cached_tokens", │ "prompt_tokens", - │ "total_tokens" + │ "tokens" │ ] └── sayBye [task] input: { diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json index 258a602c8..2b4b57071 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.log-payloads.json @@ -37,11 +37,7 @@ "span_id": "" }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": {}, @@ -85,11 +81,7 @@ ] }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": { @@ -186,11 +178,7 @@ ] }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": [ @@ -282,7 +270,7 @@ "prompt_cached_tokens": "", "prompt_tokens": "", "start": 0, - "total_tokens": "" + "tokens": "" }, "output": { "generations": [ @@ -429,11 +417,7 @@ ] }, { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, + "context": {}, "created": "", "id": "", "input": { diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json index ed1dba52c..801834fae 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.json @@ -116,7 +116,7 @@ "completion_tokens", "prompt_cached_tokens", "prompt_tokens", - "total_tokens" + "tokens" ] } ], diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt index 2e3c03b9b..1c06fb02c 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/langgraph-v1.span-tree.txt @@ -116,7 +116,7 @@ span_tree: │ "completion_tokens", │ "prompt_cached_tokens", │ "prompt_tokens", - │ "total_tokens" + │ "tokens" │ ] └── sayBye [task] input: { diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json index d649a4b4a..fc593404d 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json +++ b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/log-payloads.json @@ -283,7 +283,7 @@ "prompt_cached_tokens": "", "prompt_tokens": "", "start": 0, - "total_tokens": "" + "tokens": "" }, "output": { "generations": [ diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/span-tree.json b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/span-tree.json deleted file mode 100644 index ed1dba52c..000000000 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/span-tree.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "span_tree": [ - { - "name": "langgraph-auto-instrumentation-root", - "type": "task", - "children": [ - { - "name": "LangGraph", - "type": "task", - "children": [ - { - "name": "sayHello", - "type": "task", - "children": [ - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with exactly: hello from langgraph", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "hello from langgraph", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 4, - "promptTokens": 15, - "totalTokens": 19 - }, - "usage": { - "completion_tokens": 4, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 15, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 19 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 15, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 4, - "total_tokens": 19 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "hello from langgraph" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 4, - "promptTokens": 15, - "totalTokens": 19 - } - } - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "total_tokens" - ] - } - ], - "input": { - "message": "" - }, - "output": { - "message": "hello from langgraph" - } - }, - { - "name": "sayBye", - "type": "task", - "children": [], - "input": { - "message": "hello from langgraph" - }, - "output": "" - } - ], - "input": "", - "output": { - "message": "hello from langgraph" - } - } - ], - "metadata": { - "scenario": "langgraph-auto-instrumentation" - } - } - ] -} diff --git a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/span-tree.txt b/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/span-tree.txt deleted file mode 100644 index 2e3c03b9b..000000000 --- a/e2e/scenarios/langgraph-auto-instrumentation/__snapshots__/span-tree.txt +++ /dev/null @@ -1,125 +0,0 @@ -span_tree: -└── langgraph-auto-instrumentation-root [task] - metadata: { - "scenario": "langgraph-auto-instrumentation" - } - └── LangGraph [task] - input: "" - output: { - "message": "hello from langgraph" - } - ├── sayHello [task] - │ input: { - │ "message": "" - │ } - │ output: { - │ "message": "hello from langgraph" - │ } - │ └── ChatOpenAI [llm] - │ input: [ - │ [ - │ { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "HumanMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "Reply with exactly: hello from langgraph", - │ "response_metadata": {} - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ ] - │ output: { - │ "generations": [ - │ [ - │ { - │ "generationInfo": { - │ "finish_reason": "stop" - │ }, - │ "message": { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "hello from langgraph", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "finish_reason": "stop", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "system_fingerprint": "", - │ "tokenUsage": { - │ "completionTokens": 4, - │ "promptTokens": 15, - │ "totalTokens": 19 - │ }, - │ "usage": { - │ "completion_tokens": 4, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 15, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 19 - │ } - │ }, - │ "tool_calls": [], - │ "type": "ai", - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 15, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 4, - │ "total_tokens": 19 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "text": "hello from langgraph" - │ } - │ ] - │ ], - │ "llmOutput": { - │ "tokenUsage": { - │ "completionTokens": 4, - │ "promptTokens": 15, - │ "totalTokens": 19 - │ } - │ } - │ } - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18" - │ } - │ metric_keys: [ - │ "completion_tokens", - │ "prompt_cached_tokens", - │ "prompt_tokens", - │ "total_tokens" - │ ] - └── sayBye [task] - input: { - "message": "hello from langgraph" - } - output: "" diff --git a/e2e/scenarios/langgraph-auto-instrumentation/assertions.ts b/e2e/scenarios/langgraph-auto-instrumentation/assertions.ts index 5b71b2e3a..52e22a7c8 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/assertions.ts +++ b/e2e/scenarios/langgraph-auto-instrumentation/assertions.ts @@ -55,10 +55,27 @@ function normalizeLangGraphPayloadRows(rows: unknown[]): unknown[] { normalizeTokenMetrics(normalized.metrics); normalizeLLMOutput(normalized.output); normalizeLangchainMetadata(normalized); + normalizeInternalCallerContext(normalized); return normalized; }); } +function normalizeInternalCallerContext(row: Record): void { + const context = row.context; + if (!context || typeof context !== "object" || Array.isArray(context)) { + return; + } + + const record = context as Record; + const callerFilename = record.caller_filename; + if ( + typeof callerFilename === "string" && + (callerFilename.startsWith("node:") || callerFilename === "") + ) { + row.context = {}; + } +} + const LANGCHAIN_LS_VOLATILE_KEYS = new Set([ "max_tokens", "model", @@ -209,14 +226,14 @@ export function assertLangGraphAutoInstrumentation(options: { (event) => typeof event.metrics?.completion_tokens === "number" && typeof event.metrics?.prompt_tokens === "number" && - typeof event.metrics?.total_tokens === "number", + typeof event.metrics?.tokens === "number", ); expect(llmSpan).toBeDefined(); expect(llmSpan?.span.type).toBe("llm"); expect(llmSpan?.metrics).toMatchObject({ completion_tokens: expect.any(Number), prompt_tokens: expect.any(Number), - total_tokens: expect.any(Number), + tokens: expect.any(Number), }); const spanTree = [root, graphSpan, sayHelloSpan, llmSpan, sayByeSpan].map( diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.json deleted file mode 100644 index 1bfa61b77..000000000 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.json +++ /dev/null @@ -1,1552 +0,0 @@ -{ - "span_tree": [ - { - "name": "langchain-wrapper-root", - "type": "task", - "children": [ - { - "name": "langchain-invoke-operation", - "children": [ - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with exactly OK.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "OK.", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 2, - "promptTokens": 12, - "totalTokens": 14 - }, - "usage": { - "completion_tokens": 2, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 12, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 14 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 12, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 2, - "total_tokens": 14 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "OK." - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 2, - "promptTokens": 12, - "totalTokens": 14 - } - } - }, - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 24, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0 - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 24, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 24, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": {}, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 24, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": 2, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "total_tokens": 14 - } - } - ], - "metadata": { - "operation": "invoke", - "testRunId": "" - } - }, - { - "name": "langchain-chain-operation", - "children": [ - { - "name": "RunnableSequence", - "type": "task", - "children": [ - { - "name": "ChatPromptTemplate", - "type": "task", - "children": [], - "input": { - "word": "PARIS" - }, - "output": { - "id": [ - "langchain_core", - "prompt_values", - "ChatPromptValue" - ], - "kwargs": { - "messages": [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with the single word PARIS and nothing else.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - "metadata": { - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "metadata": {}, - "name": "ChatPromptTemplate", - "parent_run_id": "", - "run_id": "", - "run_type": "prompt", - "serialized": { - "id": [ - "langchain_core", - "prompts", - "chat", - "ChatPromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "messages": [ - { - "id": [ - "langchain_core", - "prompts", - "chat", - "HumanMessagePromptTemplate" - ], - "kwargs": { - "prompt": { - "id": [ - "langchain_core", - "prompts", - "prompt", - "PromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "template": "Reply with the single word {word} and nothing else.", - "template_format": "f-string" - }, - "lc": 1, - "type": "constructor" - } - }, - "lc": 1, - "type": "constructor" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - "tags": [ - "seq:step:1" - ] - } - }, - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with the single word PARIS and nothing else.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "PARIS", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 2, - "promptTokens": 18, - "totalTokens": 20 - }, - "usage": { - "completion_tokens": 2, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 18, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 20 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 18, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 2, - "total_tokens": 20 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "PARIS" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 2, - "promptTokens": 18, - "totalTokens": 20 - } - } - }, - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0 - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 32, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": {}, - "parent_run_id": "", - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [ - "seq:step:2" - ] - }, - "metrics": { - "completion_tokens": 2, - "prompt_cached_tokens": 0, - "prompt_tokens": 18, - "total_tokens": 20 - } - } - ], - "input": { - "word": "PARIS" - }, - "output": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "PARIS", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 2, - "promptTokens": 18, - "totalTokens": 20 - }, - "usage": { - "completion_tokens": 2, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 18, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 20 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 18, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 2, - "total_tokens": 20 - } - }, - "lc": 1, - "type": "constructor" - }, - "metadata": { - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "metadata": {}, - "run_id": "", - "serialized": { - "id": [ - "langchain_core", - "runnables", - "RunnableSequence" - ], - "kwargs": { - "first": { - "id": [ - "langchain_core", - "prompts", - "chat", - "ChatPromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "messages": [ - { - "id": [ - "langchain_core", - "prompts", - "chat", - "HumanMessagePromptTemplate" - ], - "kwargs": { - "prompt": { - "id": [ - "langchain_core", - "prompts", - "prompt", - "PromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "template": "Reply with the single word {word} and nothing else.", - "template_format": "f-string" - }, - "lc": 1, - "type": "constructor" - } - }, - "lc": 1, - "type": "constructor" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - "last": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - } - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - } - } - ], - "metadata": { - "operation": "chain", - "testRunId": "" - } - }, - { - "name": "langchain-stream-operation", - "children": [ - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Count from 1 to 3 and include the words one two three.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "completion": 0, - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "prompt": 0, - "service_tier": "default", - "system_fingerprint": "" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessageChunk" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "One, two, three.", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "completion": 0, - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "prompt": 0, - "service_tier": "default", - "system_fingerprint": "", - "usage": { - "completion_tokens": 6, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 22, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 28 - } - }, - "tool_call_chunks": [], - "tool_calls": [], - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 22, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 6, - "total_tokens": 28 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "One, two, three." - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 6, - "promptTokens": 22, - "totalTokens": 28 - } - } - }, - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "stream": true, - "stream_options": { - "include_usage": true - }, - "temperature": 0 - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 32, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "stream": true, - "stream_options": { - "include_usage": true - }, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": {}, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "streaming": true, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": 6, - "prompt_cached_tokens": 0, - "prompt_tokens": 22, - "time_to_first_token": 0, - "total_tokens": 28 - } - } - ], - "metadata": { - "operation": "stream", - "testRunId": "" - } - }, - { - "name": "langchain-tool-operation", - "children": [ - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "tool_calls" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": { - "tool_calls": [ - { - "function": { - "arguments": "{\"location\":\"Paris, France\"}", - "name": "get_weather" - }, - "id": "", - "type": "function" - } - ] - }, - "content": "", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "tool_calls", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 16, - "promptTokens": 72, - "totalTokens": 88 - }, - "usage": { - "completion_tokens": 16, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 72, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 88 - } - }, - "tool_calls": [ - { - "args": { - "location": "Paris, France" - }, - "id": "", - "name": "get_weather", - "type": "tool_call" - } - ], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 72, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 16, - "total_tokens": 88 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 16, - "promptTokens": 72, - "totalTokens": 88 - } - } - }, - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Get the current weather in a given location", - "name": "get_weather", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Get the current weather in a given location", - "name": "get_weather", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": 16, - "prompt_cached_tokens": 0, - "prompt_tokens": 72, - "total_tokens": 88 - } - } - ], - "metadata": { - "operation": "tool", - "testRunId": "" - } - }, - { - "name": "langchain-tool-result-operation", - "children": [ - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "What is 127 multiplied by 49? Use the calculate tool.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "tool_calls" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": { - "tool_calls": [ - { - "function": { - "arguments": "{\"operation\":\"multiply\",\"a\":127,\"b\":49}", - "name": "calculate" - }, - "id": "", - "type": "function" - } - ] - }, - "content": "", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "tool_calls", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 21, - "promptTokens": 76, - "totalTokens": 97 - }, - "usage": { - "completion_tokens": 21, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 76, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 97 - } - }, - "tool_calls": [ - { - "args": { - "a": 127, - "b": 49, - "operation": "multiply" - }, - "id": "", - "name": "calculate", - "type": "tool_call" - } - ], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 76, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 21, - "total_tokens": 97 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 21, - "promptTokens": 76, - "totalTokens": 97 - } - } - }, - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": 21, - "prompt_cached_tokens": 0, - "prompt_tokens": 76, - "total_tokens": 97 - } - }, - { - "name": "ChatOpenAI", - "type": "llm", - "children": [], - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "What is 127 multiplied by 49? Use the calculate tool.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - }, - { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "invalid_tool_calls": [], - "response_metadata": {}, - "tool_calls": [ - { - "args": { - "a": 127, - "b": 49, - "operation": "multiply" - }, - "id": "", - "name": "calculate", - "type": "tool_call" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - { - "id": [ - "langchain_core", - "messages", - "ToolMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "6223", - "response_metadata": {}, - "tool_call_id": "call_V61pHdchBYqDmROrrAZ2d664" - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "127 multiplied by 49 is 6223.", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 11, - "promptTokens": 106, - "totalTokens": 117 - }, - "usage": { - "completion_tokens": 11, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 106, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 117 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 106, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 11, - "total_tokens": 117 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "127 multiplied by 49 is 6223." - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 11, - "promptTokens": 106, - "totalTokens": 117 - } - } - }, - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": 11, - "prompt_cached_tokens": 0, - "prompt_tokens": 106, - "total_tokens": 117 - } - } - ], - "metadata": { - "operation": "tool-result", - "testRunId": "" - } - } - ], - "metadata": { - "scenario": "wrap-langchain-js-traces", - "testRunId": "" - } - } - ] -} diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.txt b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.txt deleted file mode 100644 index 8558e7123..000000000 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.txt +++ /dev/null @@ -1,1491 +0,0 @@ -span_tree: -└── langchain-wrapper-root [task] - metadata: { - "scenario": "wrap-langchain-js-traces", - "testRunId": "" - } - ├── langchain-invoke-operation - │ metadata: { - │ "operation": "invoke", - │ "testRunId": "" - │ } - │ └── ChatOpenAI [llm] - │ input: [ - │ [ - │ { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "HumanMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "Reply with exactly OK.", - │ "response_metadata": {} - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ ] - │ output: { - │ "generations": [ - │ [ - │ { - │ "generationInfo": { - │ "finish_reason": "stop" - │ }, - │ "message": { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "OK.", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "finish_reason": "stop", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "system_fingerprint": "", - │ "tokenUsage": { - │ "completionTokens": 2, - │ "promptTokens": 12, - │ "totalTokens": 14 - │ }, - │ "usage": { - │ "completion_tokens": 2, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 12, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 14 - │ } - │ }, - │ "tool_calls": [], - │ "type": "ai", - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 12, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 2, - │ "total_tokens": 14 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "text": "OK." - │ } - │ ] - │ ], - │ "llmOutput": { - │ "tokenUsage": { - │ "completionTokens": 2, - │ "promptTokens": 12, - │ "totalTokens": 14 - │ } - │ } - │ } - │ metadata: { - │ "batch_size": 1, - │ "braintrust": { - │ "integration_name": "langchain-js", - │ "sdk_language": "javascript" - │ }, - │ "invocation_params": { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0 - │ }, - │ "metadata": { - │ "ls_integration": "langchain_chat_model", - │ "ls_max_tokens": 24, - │ "ls_model_name": "gpt-4o-mini-2024-07-18", - │ "ls_model_type": "chat", - │ "ls_provider": "openai", - │ "ls_temperature": 0, - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0, - │ "versions": { - │ "@langchain/core": "1.1.35", - │ "@langchain/openai": "1.3.0" - │ } - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": {}, - │ "run_id": "", - │ "serialized": { - │ "id": [ - │ "langchain", - │ "chat_models", - │ "openai", - │ "ChatOpenAI" - │ ], - │ "kwargs": { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "openai_api_key": { - │ "id": [ - │ "OPENAI_API_KEY" - │ ], - │ "lc": 1, - │ "type": "secret" - │ }, - │ "temperature": 0 - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "tags": [] - │ } - │ metrics: { - │ "completion_tokens": 2, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "total_tokens": 14 - │ } - ├── langchain-chain-operation - │ metadata: { - │ "operation": "chain", - │ "testRunId": "" - │ } - │ └── RunnableSequence [task] - │ input: { - │ "word": "PARIS" - │ } - │ output: { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "PARIS", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "finish_reason": "stop", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "system_fingerprint": "", - │ "tokenUsage": { - │ "completionTokens": 2, - │ "promptTokens": 18, - │ "totalTokens": 20 - │ }, - │ "usage": { - │ "completion_tokens": 2, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 18, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 20 - │ } - │ }, - │ "tool_calls": [], - │ "type": "ai", - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 18, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 2, - │ "total_tokens": 20 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "langchain-js", - │ "sdk_language": "javascript" - │ }, - │ "metadata": {}, - │ "run_id": "", - │ "serialized": { - │ "id": [ - │ "langchain_core", - │ "runnables", - │ "RunnableSequence" - │ ], - │ "kwargs": { - │ "first": { - │ "id": [ - │ "langchain_core", - │ "prompts", - │ "chat", - │ "ChatPromptTemplate" - │ ], - │ "kwargs": { - │ "input_variables": [ - │ "word" - │ ], - │ "messages": [ - │ { - │ "id": [ - │ "langchain_core", - │ "prompts", - │ "chat", - │ "HumanMessagePromptTemplate" - │ ], - │ "kwargs": { - │ "prompt": { - │ "id": [ - │ "langchain_core", - │ "prompts", - │ "prompt", - │ "PromptTemplate" - │ ], - │ "kwargs": { - │ "input_variables": [ - │ "word" - │ ], - │ "template": "Reply with the single word {word} and nothing else.", - │ "template_format": "f-string" - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "last": { - │ "id": [ - │ "langchain", - │ "chat_models", - │ "openai", - │ "ChatOpenAI" - │ ], - │ "kwargs": { - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "openai_api_key": { - │ "id": [ - │ "OPENAI_API_KEY" - │ ], - │ "lc": 1, - │ "type": "secret" - │ }, - │ "temperature": 0 - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "tags": [] - │ } - │ ├── ChatPromptTemplate [task] - │ │ input: { - │ │ "word": "PARIS" - │ │ } - │ │ output: { - │ │ "id": [ - │ │ "langchain_core", - │ │ "prompt_values", - │ │ "ChatPromptValue" - │ │ ], - │ │ "kwargs": { - │ │ "messages": [ - │ │ { - │ │ "id": [ - │ │ "langchain_core", - │ │ "messages", - │ │ "HumanMessage" - │ │ ], - │ │ "kwargs": { - │ │ "additional_kwargs": {}, - │ │ "content": "Reply with the single word PARIS and nothing else.", - │ │ "response_metadata": {} - │ │ }, - │ │ "lc": 1, - │ │ "type": "constructor" - │ │ } - │ │ ] - │ │ }, - │ │ "lc": 1, - │ │ "type": "constructor" - │ │ } - │ │ metadata: { - │ │ "braintrust": { - │ │ "integration_name": "langchain-js", - │ │ "sdk_language": "javascript" - │ │ }, - │ │ "metadata": {}, - │ │ "name": "ChatPromptTemplate", - │ │ "parent_run_id": "", - │ │ "run_id": "", - │ │ "run_type": "prompt", - │ │ "serialized": { - │ │ "id": [ - │ │ "langchain_core", - │ │ "prompts", - │ │ "chat", - │ │ "ChatPromptTemplate" - │ │ ], - │ │ "kwargs": { - │ │ "input_variables": [ - │ │ "word" - │ │ ], - │ │ "messages": [ - │ │ { - │ │ "id": [ - │ │ "langchain_core", - │ │ "prompts", - │ │ "chat", - │ │ "HumanMessagePromptTemplate" - │ │ ], - │ │ "kwargs": { - │ │ "prompt": { - │ │ "id": [ - │ │ "langchain_core", - │ │ "prompts", - │ │ "prompt", - │ │ "PromptTemplate" - │ │ ], - │ │ "kwargs": { - │ │ "input_variables": [ - │ │ "word" - │ │ ], - │ │ "template": "Reply with the single word {word} and nothing else.", - │ │ "template_format": "f-string" - │ │ }, - │ │ "lc": 1, - │ │ "type": "constructor" - │ │ } - │ │ }, - │ │ "lc": 1, - │ │ "type": "constructor" - │ │ } - │ │ ] - │ │ }, - │ │ "lc": 1, - │ │ "type": "constructor" - │ │ }, - │ │ "tags": [ - │ │ "seq:step:1" - │ │ ] - │ │ } - │ └── ChatOpenAI [llm] - │ input: [ - │ [ - │ { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "HumanMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "Reply with the single word PARIS and nothing else.", - │ "response_metadata": {} - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ ] - │ output: { - │ "generations": [ - │ [ - │ { - │ "generationInfo": { - │ "finish_reason": "stop" - │ }, - │ "message": { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "PARIS", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "finish_reason": "stop", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "system_fingerprint": "", - │ "tokenUsage": { - │ "completionTokens": 2, - │ "promptTokens": 18, - │ "totalTokens": 20 - │ }, - │ "usage": { - │ "completion_tokens": 2, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 18, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 20 - │ } - │ }, - │ "tool_calls": [], - │ "type": "ai", - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 18, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 2, - │ "total_tokens": 20 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "text": "PARIS" - │ } - │ ] - │ ], - │ "llmOutput": { - │ "tokenUsage": { - │ "completionTokens": 2, - │ "promptTokens": 18, - │ "totalTokens": 20 - │ } - │ } - │ } - │ metadata: { - │ "batch_size": 1, - │ "braintrust": { - │ "integration_name": "langchain-js", - │ "sdk_language": "javascript" - │ }, - │ "invocation_params": { - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0 - │ }, - │ "metadata": { - │ "ls_integration": "langchain_chat_model", - │ "ls_max_tokens": 32, - │ "ls_model_name": "gpt-4o-mini-2024-07-18", - │ "ls_model_type": "chat", - │ "ls_provider": "openai", - │ "ls_temperature": 0, - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0, - │ "versions": { - │ "@langchain/core": "1.1.35", - │ "@langchain/openai": "1.3.0" - │ } - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": {}, - │ "parent_run_id": "", - │ "run_id": "", - │ "serialized": { - │ "id": [ - │ "langchain", - │ "chat_models", - │ "openai", - │ "ChatOpenAI" - │ ], - │ "kwargs": { - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "openai_api_key": { - │ "id": [ - │ "OPENAI_API_KEY" - │ ], - │ "lc": 1, - │ "type": "secret" - │ }, - │ "temperature": 0 - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "tags": [ - │ "seq:step:2" - │ ] - │ } - │ metrics: { - │ "completion_tokens": 2, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 18, - │ "total_tokens": 20 - │ } - ├── langchain-stream-operation - │ metadata: { - │ "operation": "stream", - │ "testRunId": "" - │ } - │ └── ChatOpenAI [llm] - │ input: [ - │ [ - │ { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "HumanMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "Count from 1 to 3 and include the words one two three.", - │ "response_metadata": {} - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ ] - │ output: { - │ "generations": [ - │ [ - │ { - │ "generationInfo": { - │ "completion": 0, - │ "finish_reason": "stop", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "prompt": 0, - │ "service_tier": "default", - │ "system_fingerprint": "" - │ }, - │ "message": { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessageChunk" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "One, two, three.", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "completion": 0, - │ "finish_reason": "stop", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "prompt": 0, - │ "service_tier": "default", - │ "system_fingerprint": "", - │ "usage": { - │ "completion_tokens": 6, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 22, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 28 - │ } - │ }, - │ "tool_call_chunks": [], - │ "tool_calls": [], - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 22, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 6, - │ "total_tokens": 28 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "text": "One, two, three." - │ } - │ ] - │ ], - │ "llmOutput": { - │ "tokenUsage": { - │ "completionTokens": 6, - │ "promptTokens": 22, - │ "totalTokens": 28 - │ } - │ } - │ } - │ metadata: { - │ "batch_size": 1, - │ "braintrust": { - │ "integration_name": "langchain-js", - │ "sdk_language": "javascript" - │ }, - │ "invocation_params": { - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ }, - │ "metadata": { - │ "ls_integration": "langchain_chat_model", - │ "ls_max_tokens": 32, - │ "ls_model_name": "gpt-4o-mini-2024-07-18", - │ "ls_model_type": "chat", - │ "ls_provider": "openai", - │ "ls_temperature": 0, - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "versions": { - │ "@langchain/core": "1.1.35", - │ "@langchain/openai": "1.3.0" - │ } - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": {}, - │ "run_id": "", - │ "serialized": { - │ "id": [ - │ "langchain", - │ "chat_models", - │ "openai", - │ "ChatOpenAI" - │ ], - │ "kwargs": { - │ "max_tokens": 32, - │ "model": "gpt-4o-mini-2024-07-18", - │ "openai_api_key": { - │ "id": [ - │ "OPENAI_API_KEY" - │ ], - │ "lc": 1, - │ "type": "secret" - │ }, - │ "streaming": true, - │ "temperature": 0 - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "tags": [] - │ } - │ metrics: { - │ "completion_tokens": 6, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 22, - │ "time_to_first_token": 0, - │ "total_tokens": 28 - │ } - ├── langchain-tool-operation - │ metadata: { - │ "operation": "tool", - │ "testRunId": "" - │ } - │ └── ChatOpenAI [llm] - │ input: [ - │ [ - │ { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "HumanMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "response_metadata": {} - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ ] - │ output: { - │ "generations": [ - │ [ - │ { - │ "generationInfo": { - │ "finish_reason": "tool_calls" - │ }, - │ "message": { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": { - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ }, - │ "content": "", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "finish_reason": "tool_calls", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "system_fingerprint": "", - │ "tokenUsage": { - │ "completionTokens": 16, - │ "promptTokens": 72, - │ "totalTokens": 88 - │ }, - │ "usage": { - │ "completion_tokens": 16, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 72, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 88 - │ } - │ }, - │ "tool_calls": [ - │ { - │ "args": { - │ "location": "Paris, France" - │ }, - │ "id": "", - │ "name": "get_weather", - │ "type": "tool_call" - │ } - │ ], - │ "type": "ai", - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 72, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 16, - │ "total_tokens": 88 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "text": "" - │ } - │ ] - │ ], - │ "llmOutput": { - │ "tokenUsage": { - │ "completionTokens": 16, - │ "promptTokens": 72, - │ "totalTokens": 88 - │ } - │ } - │ } - │ metadata: { - │ "batch_size": 1, - │ "braintrust": { - │ "integration_name": "langchain-js", - │ "sdk_language": "javascript" - │ }, - │ "invocation_params": { - │ "max_tokens": 128, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the current weather in a given location", - │ "name": "get_weather", - │ "parameters": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and state or city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ }, - │ "metadata": { - │ "ls_integration": "langchain_chat_model", - │ "ls_max_tokens": 128, - │ "ls_model_name": "gpt-4o-mini-2024-07-18", - │ "ls_model_type": "chat", - │ "ls_provider": "openai", - │ "ls_temperature": 0, - │ "max_tokens": 128, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0, - │ "versions": { - │ "@langchain/core": "1.1.35", - │ "@langchain/openai": "1.3.0" - │ } - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the current weather in a given location", - │ "name": "get_weather", - │ "parameters": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and state or city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ }, - │ "run_id": "", - │ "serialized": { - │ "id": [ - │ "langchain", - │ "chat_models", - │ "openai", - │ "ChatOpenAI" - │ ], - │ "kwargs": { - │ "max_tokens": 128, - │ "model": "gpt-4o-mini-2024-07-18", - │ "openai_api_key": { - │ "id": [ - │ "OPENAI_API_KEY" - │ ], - │ "lc": 1, - │ "type": "secret" - │ }, - │ "temperature": 0 - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "tags": [] - │ } - │ metrics: { - │ "completion_tokens": 16, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 72, - │ "total_tokens": 88 - │ } - └── langchain-tool-result-operation - metadata: { - "operation": "tool-result", - "testRunId": "" - } - ├── ChatOpenAI [llm] - │ input: [ - │ [ - │ { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "HumanMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": {}, - │ "content": "What is 127 multiplied by 49? Use the calculate tool.", - │ "response_metadata": {} - │ }, - │ "lc": 1, - │ "type": "constructor" - │ } - │ ] - │ ] - │ output: { - │ "generations": [ - │ [ - │ { - │ "generationInfo": { - │ "finish_reason": "tool_calls" - │ }, - │ "message": { - │ "id": [ - │ "langchain_core", - │ "messages", - │ "AIMessage" - │ ], - │ "kwargs": { - │ "additional_kwargs": { - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"operation\":\"multiply\",\"a\":127,\"b\":49}", - │ "name": "calculate" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ }, - │ "content": "", - │ "id": "", - │ "invalid_tool_calls": [], - │ "response_metadata": { - │ "finish_reason": "tool_calls", - │ "model_name": "gpt-4o-mini-2024-07-18", - │ "model_provider": "openai", - │ "system_fingerprint": "", - │ "tokenUsage": { - │ "completionTokens": 21, - │ "promptTokens": 76, - │ "totalTokens": 97 - │ }, - │ "usage": { - │ "completion_tokens": 21, - │ "completion_tokens_details": { - │ "accepted_prediction_tokens": 0, - │ "audio_tokens": 0, - │ "reasoning_tokens": 0, - │ "rejected_prediction_tokens": 0 - │ }, - │ "prompt_tokens": 76, - │ "prompt_tokens_details": { - │ "audio_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "total_tokens": 97 - │ } - │ }, - │ "tool_calls": [ - │ { - │ "args": { - │ "a": 127, - │ "b": 49, - │ "operation": "multiply" - │ }, - │ "id": "", - │ "name": "calculate", - │ "type": "tool_call" - │ } - │ ], - │ "type": "ai", - │ "usage_metadata": { - │ "input_token_details": { - │ "audio": 0, - │ "cache_read": 0 - │ }, - │ "input_tokens": 76, - │ "output_token_details": { - │ "audio": 0, - │ "reasoning": 0 - │ }, - │ "output_tokens": 21, - │ "total_tokens": 97 - │ } - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "text": "" - │ } - │ ] - │ ], - │ "llmOutput": { - │ "tokenUsage": { - │ "completionTokens": 21, - │ "promptTokens": 76, - │ "totalTokens": 97 - │ } - │ } - │ } - │ metadata: { - │ "batch_size": 1, - │ "braintrust": { - │ "integration_name": "langchain-js", - │ "sdk_language": "javascript" - │ }, - │ "invocation_params": { - │ "max_tokens": 128, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0, - │ "tools": [ - │ { - │ "function": { - │ "description": "Perform a mathematical calculation", - │ "name": "calculate", - │ "parameters": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "a": { - │ "type": "number" - │ }, - │ "b": { - │ "type": "number" - │ }, - │ "operation": { - │ "enum": [ - │ "add", - │ "subtract", - │ "multiply", - │ "divide" - │ ], - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "operation", - │ "a", - │ "b" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ }, - │ "metadata": { - │ "ls_integration": "langchain_chat_model", - │ "ls_max_tokens": 128, - │ "ls_model_name": "gpt-4o-mini-2024-07-18", - │ "ls_model_type": "chat", - │ "ls_provider": "openai", - │ "ls_temperature": 0, - │ "max_tokens": 128, - │ "model": "gpt-4o-mini-2024-07-18", - │ "stream": false, - │ "temperature": 0, - │ "versions": { - │ "@langchain/core": "1.1.35", - │ "@langchain/openai": "1.3.0" - │ } - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "tools": [ - │ { - │ "function": { - │ "description": "Perform a mathematical calculation", - │ "name": "calculate", - │ "parameters": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "a": { - │ "type": "number" - │ }, - │ "b": { - │ "type": "number" - │ }, - │ "operation": { - │ "enum": [ - │ "add", - │ "subtract", - │ "multiply", - │ "divide" - │ ], - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "operation", - │ "a", - │ "b" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ }, - │ "run_id": "", - │ "serialized": { - │ "id": [ - │ "langchain", - │ "chat_models", - │ "openai", - │ "ChatOpenAI" - │ ], - │ "kwargs": { - │ "max_tokens": 128, - │ "model": "gpt-4o-mini-2024-07-18", - │ "openai_api_key": { - │ "id": [ - │ "OPENAI_API_KEY" - │ ], - │ "lc": 1, - │ "type": "secret" - │ }, - │ "temperature": 0 - │ }, - │ "lc": 1, - │ "type": "constructor" - │ }, - │ "tags": [] - │ } - │ metrics: { - │ "completion_tokens": 21, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 76, - │ "total_tokens": 97 - │ } - └── ChatOpenAI [llm] - input: [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "What is 127 multiplied by 49? Use the calculate tool.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - }, - { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "invalid_tool_calls": [], - "response_metadata": {}, - "tool_calls": [ - { - "args": { - "a": 127, - "b": 49, - "operation": "multiply" - }, - "id": "", - "name": "calculate", - "type": "tool_call" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - { - "id": [ - "langchain_core", - "messages", - "ToolMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "6223", - "response_metadata": {}, - "tool_call_id": "call_V61pHdchBYqDmROrrAZ2d664" - }, - "lc": 1, - "type": "constructor" - } - ] - ] - output: { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "127 multiplied by 49 is 6223.", - "id": "", - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 11, - "promptTokens": 106, - "totalTokens": 117 - }, - "usage": { - "completion_tokens": 11, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 106, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 117 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 106, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 11, - "total_tokens": 117 - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "127 multiplied by 49 is 6223." - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": 11, - "promptTokens": 106, - "totalTokens": 117 - } - } - } - metadata: { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "versions": { - "@langchain/core": "1.1.35", - "@langchain/openai": "1.3.0" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - } - metrics: { - "completion_tokens": 11, - "prompt_cached_tokens": 0, - "prompt_tokens": 106, - "total_tokens": 117 - } diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json index b52b2d5f9..ad63c46a4 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.json @@ -159,7 +159,7 @@ "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 12, - "total_tokens": 14 + "tokens": 14 } } ], @@ -425,7 +425,7 @@ "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 18, - "total_tokens": 20 + "tokens": 20 } } ], @@ -746,7 +746,7 @@ "prompt_cached_tokens": 0, "prompt_tokens": 22, "time_to_first_token": 0, - "total_tokens": 28 + "tokens": 28 } } ], @@ -977,7 +977,7 @@ "completion_tokens": 16, "prompt_cached_tokens": 0, "prompt_tokens": 72, - "total_tokens": 88 + "tokens": 88 } } ], @@ -1236,7 +1236,7 @@ "completion_tokens": 21, "prompt_cached_tokens": 0, "prompt_tokens": 76, - "total_tokens": 97 + "tokens": 97 } }, { @@ -1506,7 +1506,7 @@ "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, - "total_tokens": 117 + "tokens": 117 } } ], diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt index 63bfc1c3c..da37b45bb 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1-latest.span-tree.txt @@ -158,7 +158,7 @@ span_tree: │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, - │ "total_tokens": 14 + │ "tokens": 14 │ } ├── langchain-chain-operation │ metadata: { @@ -556,7 +556,7 @@ span_tree: │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 18, - │ "total_tokens": 20 + │ "tokens": 20 │ } ├── langchain-stream-operation │ metadata: { @@ -720,7 +720,7 @@ span_tree: │ "prompt_cached_tokens": 0, │ "prompt_tokens": 22, │ "time_to_first_token": 0, - │ "total_tokens": 28 + │ "tokens": 28 │ } ├── langchain-tool-operation │ metadata: { @@ -943,7 +943,7 @@ span_tree: │ "completion_tokens": 16, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 72, - │ "total_tokens": 88 + │ "tokens": 88 │ } └── langchain-tool-result-operation metadata: { @@ -1194,7 +1194,7 @@ span_tree: │ "completion_tokens": 21, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 76, - │ "total_tokens": 97 + │ "tokens": 97 │ } └── ChatOpenAI [llm] input: [ @@ -1460,5 +1460,5 @@ span_tree: "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, - "total_tokens": 117 + "tokens": 117 } diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json index d316480b9..74a535155 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.json @@ -159,7 +159,7 @@ "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 12, - "total_tokens": 14 + "tokens": 14 } } ], @@ -425,7 +425,7 @@ "completion_tokens": 2, "prompt_cached_tokens": 0, "prompt_tokens": 18, - "total_tokens": 20 + "tokens": 20 } } ], @@ -746,7 +746,7 @@ "prompt_cached_tokens": 0, "prompt_tokens": 22, "time_to_first_token": 0, - "total_tokens": 28 + "tokens": 28 } } ], @@ -977,7 +977,7 @@ "completion_tokens": 16, "prompt_cached_tokens": 0, "prompt_tokens": 72, - "total_tokens": 88 + "tokens": 88 } } ], @@ -1236,7 +1236,7 @@ "completion_tokens": 21, "prompt_cached_tokens": 0, "prompt_tokens": 76, - "total_tokens": 97 + "tokens": 97 } }, { @@ -1506,7 +1506,7 @@ "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, - "total_tokens": 117 + "tokens": 117 } } ], diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt index 4c1b75e45..cb09946d7 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/wrap-langchain-js-traces-v1.span-tree.txt @@ -158,7 +158,7 @@ span_tree: │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 12, - │ "total_tokens": 14 + │ "tokens": 14 │ } ├── langchain-chain-operation │ metadata: { @@ -556,7 +556,7 @@ span_tree: │ "completion_tokens": 2, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 18, - │ "total_tokens": 20 + │ "tokens": 20 │ } ├── langchain-stream-operation │ metadata: { @@ -720,7 +720,7 @@ span_tree: │ "prompt_cached_tokens": 0, │ "prompt_tokens": 22, │ "time_to_first_token": 0, - │ "total_tokens": 28 + │ "tokens": 28 │ } ├── langchain-tool-operation │ metadata: { @@ -943,7 +943,7 @@ span_tree: │ "completion_tokens": 16, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 72, - │ "total_tokens": 88 + │ "tokens": 88 │ } └── langchain-tool-result-operation metadata: { @@ -1194,7 +1194,7 @@ span_tree: │ "completion_tokens": 21, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 76, - │ "total_tokens": 97 + │ "tokens": 97 │ } └── ChatOpenAI [llm] input: [ @@ -1460,5 +1460,5 @@ span_tree: "completion_tokens": 11, "prompt_cached_tokens": 0, "prompt_tokens": 106, - "total_tokens": 117 + "tokens": 117 } diff --git a/integrations/langchain-js/src/BraintrustCallbackHandler.test.ts b/integrations/langchain-js/src/BraintrustCallbackHandler.test.ts index 9b2af8ca1..e45afe2e6 100644 --- a/integrations/langchain-js/src/BraintrustCallbackHandler.test.ts +++ b/integrations/langchain-js/src/BraintrustCallbackHandler.test.ts @@ -134,7 +134,7 @@ describe("BraintrustCallbackHandler", () => { ], metrics: { start: expect.any(Number), - total_tokens: expect.any(Number), + tokens: expect.any(Number), prompt_tokens: expect.any(Number), completion_tokens: expect.any(Number), end: expect.any(Number), @@ -295,7 +295,7 @@ it("should handle streaming LLM calls", async () => { end: expect.any(Number), prompt_tokens: 16, start: expect.any(Number), - total_tokens: 22, + tokens: 22, }, metadata: { tags: ["seq:step:2"], @@ -373,7 +373,7 @@ it("should track time-to-first-token in streaming calls", async () => { time_to_first_token: expect.any(Number), prompt_tokens: expect.any(Number), completion_tokens: expect.any(Number), - total_tokens: expect.any(Number), + tokens: expect.any(Number), }); // Verify TTFT is a reasonable value (positive and less than total time) @@ -482,7 +482,7 @@ it("should handle multi-step chains with memory", async () => { end: expect.any(Number), prompt_tokens: expect.any(Number), start: expect.any(Number), - total_tokens: expect.any(Number), + tokens: expect.any(Number), }, metadata: { tags: ["seq:step:2", "test"], @@ -597,7 +597,7 @@ it("should handle tool/agent usage", async () => { end: expect.any(Number), prompt_tokens: expect.any(Number), start: expect.any(Number), - total_tokens: expect.any(Number), + tokens: expect.any(Number), }, metadata: { model: "gpt-4o-mini-2024-07-18", diff --git a/integrations/langchain-js/src/setGlobalHandler.test.ts b/integrations/langchain-js/src/setGlobalHandler.test.ts index f5a16c2c3..42cdffb96 100644 --- a/integrations/langchain-js/src/setGlobalHandler.test.ts +++ b/integrations/langchain-js/src/setGlobalHandler.test.ts @@ -122,7 +122,7 @@ describe("setGlobalHandler", () => { end: expect.any(Number), prompt_tokens: 15, start: expect.any(Number), - total_tokens: 23, + tokens: 23, }, metadata: { tags: ["seq:step:2"], diff --git a/js/src/wrappers/langchain/callback-handler.test.ts b/js/src/wrappers/langchain/callback-handler.test.ts new file mode 100644 index 000000000..e441fa1ad --- /dev/null +++ b/js/src/wrappers/langchain/callback-handler.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it, vi } from "vitest"; +import { BraintrustLangChainCallbackHandler } from "./callback-handler"; + +function createHarness() { + const logs: unknown[] = []; + const span = { + log: vi.fn((event: unknown) => { + logs.push(event); + }), + end: vi.fn(), + }; + const parent = { + startSpan: vi.fn(() => span), + }; + const handler = new BraintrustLangChainCallbackHandler({ + parent: parent as never, + }); + + return { handler, logs, parent, span }; +} + +async function finishChatModelRun(output: unknown) { + const harness = createHarness(); + + await harness.handler.handleChatModelStart( + { name: "ChatOpenAI" }, + [[{ role: "user", content: "hello" }]], + "run-1", + ); + await harness.handler.handleLLMEnd(output as never, "run-1"); + + const endLog = harness.logs.at(-1) as { + metrics?: Record; + }; + return { ...harness, endLog }; +} + +describe("BraintrustLangChainCallbackHandler metrics", () => { + it("synthesizes tokens from message usage metadata prompt and completion counts", async () => { + const { endLog } = await finishChatModelRun({ + generations: [ + [ + { + message: { + usage_metadata: { + input_tokens: 10, + output_tokens: 2, + }, + }, + }, + ], + ], + }); + + expect(endLog.metrics).toEqual({ + prompt_tokens: 10, + completion_tokens: 2, + tokens: 12, + }); + }); + + it("prefers explicit total tokens from message usage metadata", async () => { + const { endLog } = await finishChatModelRun({ + generations: [ + [ + { + message: { + usage_metadata: { + input_tokens: 10, + output_tokens: 2, + total_tokens: 99, + }, + }, + }, + ], + ], + }); + + expect(endLog.metrics).toEqual({ + prompt_tokens: 10, + completion_tokens: 2, + tokens: 99, + }); + }); + + it("synthesizes tokens from llmOutput token usage prompt and completion counts", async () => { + const { endLog } = await finishChatModelRun({ + llmOutput: { + tokenUsage: { + promptTokens: 10, + completionTokens: 2, + }, + }, + }); + + expect(endLog.metrics).toEqual({ + prompt_tokens: 10, + completion_tokens: 2, + tokens: 12, + }); + }); + + it("preserves cache metrics from message usage metadata", async () => { + const { endLog } = await finishChatModelRun({ + generations: [ + [ + { + message: { + usage_metadata: { + input_tokens: 10, + output_tokens: 2, + input_token_details: { + cache_creation: 4, + cache_read: 3, + }, + }, + }, + }, + ], + ], + }); + + expect(endLog.metrics).toEqual({ + prompt_tokens: 10, + completion_tokens: 2, + prompt_cache_creation_tokens: 4, + prompt_cached_tokens: 3, + tokens: 12, + }); + }); +}); diff --git a/js/src/wrappers/langchain/callback-handler.ts b/js/src/wrappers/langchain/callback-handler.ts index e5612735d..29292ed1e 100644 --- a/js/src/wrappers/langchain/callback-handler.ts +++ b/js/src/wrappers/langchain/callback-handler.ts @@ -438,6 +438,24 @@ function cleanObject(obj: Record): Record { ) as Record; } +function normalizeTokenMetrics( + obj: Record, +): Record { + const metrics = cleanObject(obj); + + if (metrics.total_tokens !== undefined) { + metrics.tokens = metrics.total_tokens; + } else if ( + metrics.prompt_tokens !== undefined && + metrics.completion_tokens !== undefined + ) { + metrics.tokens = metrics.prompt_tokens + metrics.completion_tokens; + } + + delete metrics.total_tokens; + return metrics; +} + function walkGenerations( response: LangChainLLMResult, ): Record[] { @@ -497,7 +515,7 @@ function getMetricsFromResponse( } const inputTokenDetails = usageMetadata.input_token_details; - return cleanObject({ + return normalizeTokenMetrics({ total_tokens: usageMetadata.total_tokens, prompt_tokens: usageMetadata.input_tokens, completion_tokens: usageMetadata.output_tokens, @@ -517,7 +535,7 @@ function getMetricsFromResponse( ? llmOutput.estimatedTokens : {}; - return cleanObject({ + return normalizeTokenMetrics({ total_tokens: tokenUsage.totalTokens, prompt_tokens: tokenUsage.promptTokens, completion_tokens: tokenUsage.completionTokens,