Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .agents/skills/sdk-integrations/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ The integration that directly owns the model/provider API response owns token ac
- Do not over-serialize. Braintrust serializes at send/log time. Integration tracing only needs readable Python dicts/lists/scalars and materialized attachments.
- Keep wrapper bodies thin: prepare traced input, open span, call provider, normalize result, log.
- Do not wrap `span.log(...)` / `span.set_attributes(...)` in broad try/except. Braintrust span methods are boundary-safe.
- **Rerank exception to "no silent caps":** rerank result lists can be huge and are noisy in the span UI; cap the `output` list at a fixed max (e.g. 100 entries) via a named constant. The cap is compact-span hygiene, not coverage bounding — the model already ranked everything.

### Span origin

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,6 @@
spans = memory_logger.pop()
assert len(spans) == 1, f"Expected 1 span, got {len(spans)}"
span = spans[0]
assert span["metadata"]["provider"] == "litellm"
assert span["metadata"]["provider"] == "openai"

print("SUCCESS")
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ async def main():
for key, value in span["metrics"].items():
assert isinstance(value, (int, float)) and not isinstance(value, bool)
assert span["metadata"]["model"] == "gpt-4o-mini"
assert span["metadata"]["provider"] == "litellm"
assert span["metadata"]["provider"] == "openai"
assert "What's 12 + 12?" in str(span["input"])


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
for key, value in span["metrics"].items():
assert isinstance(value, (int, float)) and not isinstance(value, bool)
assert span["metadata"]["model"] == "gpt-4o-mini"
assert span["metadata"]["provider"] == "litellm"
assert span["metadata"]["provider"] == "openai"
assert "What's 12 + 12?" in str(span["input"])

print("SUCCESS")
64 changes: 4 additions & 60 deletions py/src/braintrust/integrations/litellm/__init__.py
Original file line number Diff line number Diff line change
@@ -1,74 +1,18 @@
"""Braintrust LiteLLM integration."""

import importlib

from .integration import LiteLLMIntegration
from .patchers import LiteLLMAcompletionPatcher, LiteLLMCompletionPatcher, wrap_litellm
from .patchers import wrap_litellm


def patch_litellm() -> bool:
"""Patch LiteLLM to add Braintrust tracing.

This wraps litellm.completion, litellm.acompletion, litellm.responses,
litellm.aresponses, litellm.image_generation, litellm.aimage_generation,
litellm.embedding, litellm.aembedding, litellm.moderation,
litellm.speech, litellm.aspeech, litellm.transcription,
litellm.atranscription, litellm.rerank, and litellm.arerank to automatically
create Braintrust spans with detailed token metrics, timing, and costs.

Returns:
True if LiteLLM was patched (or already patched), False if LiteLLM is not installed.
"""Patch LiteLLM top-level callables to emit Braintrust spans.

Example:
```python
import braintrust
braintrust.integrations.litellm.patch_litellm()

import litellm
from braintrust import init_logger

logger = init_logger(project="my-project")
response = litellm.completion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Hello"}]
)
```
Returns ``True`` if LiteLLM was patched (or already patched), ``False``
if LiteLLM is not installed.
"""
return LiteLLMIntegration.setup()


def _is_litellm_patched() -> bool:
"""Return ``True`` when the LiteLLM completion entry points are patched.

Internal helper used by other Braintrust integrations (e.g. CrewAI,
which delegates its LLM calls to LiteLLM) to decide whether to emit
token metrics on their own LLM spans. When LiteLLM is already
patched, the Braintrust ``Completion`` span it produces is the leaf
span and must own token accounting; non-leaf wrappers should emit
timing-only metrics to avoid double-counting during trace-tree
rollup.

Not part of the public API — cross-integration callers import this
directly by its underscore-prefixed name.

Returns ``False`` if ``litellm`` is not importable or neither of the
``completion`` / ``acompletion`` entry points has been patched.
"""
try:
litellm = importlib.import_module("litellm")
except ImportError:
return False

completion = getattr(litellm, "completion", None)
acompletion = getattr(litellm, "acompletion", None)
return bool(
LiteLLMCompletionPatcher.has_patch_marker(completion)
or LiteLLMCompletionPatcher.has_patch_marker(litellm)
or LiteLLMAcompletionPatcher.has_patch_marker(acompletion)
or LiteLLMAcompletionPatcher.has_patch_marker(litellm)
)


__all__ = [
"LiteLLMIntegration",
"patch_litellm",
Expand Down
32 changes: 2 additions & 30 deletions py/src/braintrust/integrations/litellm/patchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,37 +171,9 @@ class LiteLLMArerankPatcher(FunctionWrapperPatcher):


def wrap_litellm(litellm: Any) -> Any:
"""Wrap a LiteLLM module to add Braintrust tracing.
"""Wrap a LiteLLM module-like object in-place with Braintrust tracing.

Unlike :func:`patch_litellm`, which patches the globally-imported ``litellm``
module, this function instruments a specific module object (or any object
that exposes the same top-level callables such as ``completion``,
``acompletion``, ``responses``, ``aresponses``, ``image_generation``,
``text_completion``, ``atext_completion``, ``aimage_generation``,
``embedding``, ``aembedding``, ``moderation``, ``amoderation``, ``speech``,
``aspeech``, ``transcription``, ``atranscription``, ``rerank``, and
``arerank``). Each patcher is applied idempotently — calling
``wrap_litellm`` twice on the same object is safe.

Args:
litellm: The ``litellm`` module or a module-like object that exposes
the standard LiteLLM top-level functions.

Returns:
The same *litellm* object, with tracing wrappers applied in-place.

Example::

import litellm
from braintrust.integrations.litellm import wrap_litellm

wrap_litellm(litellm)

# All subsequent calls are automatically traced.
response = litellm.completion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Hello"}],
)
Idempotent; safe to call twice on the same object.
"""
for patcher in _ALL_LITELLM_PATCHERS:
patcher.wrap_target(litellm)
Expand Down
Loading