Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions py/src/braintrust/integrations/instructor/test_instructor.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ def test_instructor_openai_single_success(self, setup_logger, memory_logger):
assert parent["context"]["span_origin"]["instrumentation"]["name"] == "instructor-auto"
assert parent["span_attributes"]["name"] == "instructor.create"
meta = parent.get("metadata", {})
assert meta.get("model") == "gpt-4o-mini"
assert meta.get("provider") == "openai"
assert meta.get("response_model") == "Person"
assert meta.get("mode") == "TOOLS"
assert meta.get("max_retries") == 3
Expand Down Expand Up @@ -172,6 +174,8 @@ def test_instructor_openai_retries_then_succeeds(self, setup_logger, memory_logg

parent = task_spans[0]
meta = parent.get("metadata", {})
assert meta.get("model") == "gpt-4o-mini"
assert meta.get("provider") == "openai"
assert meta.get("response_model") == "Person"
assert meta.get("mode") == "TOOLS"
assert meta.get("max_retries") == 3
Expand Down
55 changes: 48 additions & 7 deletions py/src/braintrust/integrations/instructor/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,21 @@
- ``output``: the extracted Pydantic model (or list of models for
iterable/partial helpers). This is Instructor's product and is not
present on any provider child span.
- ``metadata``: ``response_model`` (class name), ``mode``, ``max_retries``,
``retry_count``, ``validation_errors``.
- ``metadata``: ``model``, ``provider``, ``response_model`` (class name),
``mode``, ``max_retries``, ``retry_count``, ``validation_errors``. The
``model`` / ``provider`` pair carries attribution for the framework span
per the SKILL "framework delegates transport" rule; the provider child
span still owns token accounting.
- ``metrics``: empty. Token usage stays on the provider child span.
"""

from __future__ import annotations

import inspect
import logging
from collections.abc import AsyncIterator, Iterator
from typing import Any

from braintrust.logger import start_span as _bt_start_span
from braintrust.span_types import SpanTypeAttribute


_INSTRUMENTATION = "instructor-auto"
Expand All @@ -41,9 +43,6 @@ def start_span(*args, **kwargs):
return _bt_start_span(*args, **kwargs)


from braintrust.span_types import SpanTypeAttribute


log = logging.getLogger(__name__)


Expand Down Expand Up @@ -81,6 +80,37 @@ def _mode_name(instance: Any) -> str | None:
return getattr(mode, "name", None) or str(mode)


def _provider_name(instance: Any) -> str | None:
"""Read the underlying provider ("openai", "anthropic", ...) from the client.

Instructor exposes a ``Provider`` enum on every ``Instructor`` /
``AsyncInstructor`` instance whose ``value`` is exactly the short provider
string we want for ``metadata.provider``. Fall back to the enum ``name``
lowercased for older builds that only expose the name.
"""
provider = getattr(instance, "provider", None)
if provider is None:
return None
value = getattr(provider, "value", None)
if isinstance(value, str) and value:
return value
name = getattr(provider, "name", None)
if isinstance(name, str) and name:
return name.lower()
return None


def _model_name(kwargs: Any, instance: Any) -> str | None:
"""Pull the underlying model name from call kwargs (or the client default)."""
model = kwargs.get("model") if isinstance(kwargs, dict) else None
if isinstance(model, str) and model:
return model
default = getattr(instance, "default_model", None)
if isinstance(default, str) and default:
return default
return None


def _max_retries_value(max_retries: Any) -> Any:
"""Normalize Instructor's ``max_retries`` argument for metadata logging.

Expand Down Expand Up @@ -124,11 +154,14 @@ def _build_metadata(
*,
response_model: Any,
instance: Any,
kwargs: Any,
max_retries: Any,
retry_count: int,
validation_errors: list[str],
) -> dict[str, Any]:
return {
"model": _model_name(kwargs, instance),
"provider": _provider_name(instance),
"response_model": _response_model_name(response_model),
"mode": _mode_name(instance),
"max_retries": _max_retries_value(max_retries),
Expand Down Expand Up @@ -211,6 +244,7 @@ def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand All @@ -222,6 +256,7 @@ def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand Down Expand Up @@ -252,6 +287,7 @@ async def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand All @@ -263,6 +299,7 @@ async def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand Down Expand Up @@ -299,6 +336,7 @@ def _iterate() -> Iterator[Any]:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand All @@ -310,6 +348,7 @@ def _iterate() -> Iterator[Any]:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand Down Expand Up @@ -349,6 +388,7 @@ async def _iterate() -> AsyncIterator[Any]:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand All @@ -360,6 +400,7 @@ async def _iterate() -> AsyncIterator[Any]:
metadata=_build_metadata(
response_model=response_model,
instance=instance,
kwargs=kwargs,
max_retries=max_retries,
retry_count=tracker.retry_count,
validation_errors=tracker.validation_errors,
Expand Down