fix(crewai): add provider metadata, route tools to metadata, drop eager serialization

Align the CrewAI integration with .agents/skills/sdk-integrations/SKILL.md.

- Add metadata.provider on crewai.llm spans, read from LLM.provider on the
  event source. Every llm span must carry both metadata.model AND
  metadata.provider per the instrumentation spec.
- Route tool definitions into metadata.tools (both LLMCallStartedEvent
  tools and AgentExecutionStartedEvent tools). Tools no longer leak into
  input.
- Drop eager _try_to_dict / _normalize_output / _normalize_tools passes
  in kickoff/task/agent/llm/tool output paths. bt_json at log time
  already handles Pydantic v2/v1 and dataclasses, so the eager pass is
  wasted work.
- Narrow _agent_metadata's agent_llm read: previously ``str(llm)`` was a
  fallback, which dumps the pydantic repr including api_key / api_base /
  client_params. Now the allowlist reads only the model name.
- Extend the existing direct-event tests: assert metadata.provider on
  the kickoff/llm tree and add a positive+negative check that tools
  route to metadata (not input). No new cassettes — the file docstring
  documents why VCR is impractical for CrewAI.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: starfolkbot

py/src/braintrust/integrations/crewai/test_crewai.py

@@ -250,8 +250,23 @@ def test_kickoff_llm_event_tree_parents_and_shape(memory_logger):
 
     kickoff = _build_kickoff_started()
     _emit(kickoff)
+
+    # Emit with a real CrewAI LLM as source so ``metadata.provider`` gets
+    # populated from ``LLM.provider`` — the spec requires every llm span
+    # to carry provider metadata.
+    from crewai import LLM
+    from crewai.events.event_bus import crewai_event_bus
+
+    llm_source = LLM(model="gpt-4o-mini")
     llm_started = _build_llm_started(parent_event_id=kickoff.event_id)
-    _emit(llm_started)
+    future = crewai_event_bus.emit(llm_source, llm_started)
+    if future is not None:
+        try:
+            future.result(timeout=5.0)
+        except Exception:
+            pass
+    _flush_event_bus(crewai_event_bus, timeout=5.0)
+
     _emit(_build_llm_completed(llm_started, usage={"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}))
     _emit(_build_kickoff_completed(kickoff))
 
@@ -277,12 +292,34 @@ def test_kickoff_llm_event_tree_parents_and_shape(memory_logger):
     # Shape assertions.
     assert llm_span["input"]["messages"] == llm_started.messages
     assert llm_span["metadata"]["model"] == "gpt-4o-mini"
+    # Every llm span must carry ``metadata.provider`` per the instrumentation spec.
+    assert llm_span["metadata"]["provider"] == "openai"
     assert llm_span["metadata"]["call_id"] == "call-1"
     assert llm_span["output"] == "4"
     assert kickoff_span["output"] == "final answer"
     assert kickoff_span["input"] == kickoff.inputs
 
 
+def test_llm_tools_route_to_metadata_not_input(memory_logger):
+    """Tool definitions belong in ``metadata.tools`` per the spec, not in ``input``."""
+    patch_crewai()
+
+    tools = [
+        {"type": "function", "function": {"name": "search", "description": "search the web"}},
+        {"type": "function", "function": {"name": "sum", "description": "add two numbers"}},
+    ]
+    started = _build_llm_started(tools=tools)
+    _emit(started)
+    _emit(_build_llm_completed(started))
+
+    span = memory_logger.pop()[0]
+    assert span["span_attributes"]["name"] == "crewai.llm"
+    # Positive: tools land in metadata untouched (no eager serialization).
+    assert span["metadata"]["tools"] == tools
+    # Negative: tools MUST NOT leak into input.
+    assert "tools" not in span["input"], span["input"]
+
+
 def test_llm_tokens_skipped_when_litellm_patched(memory_logger, monkeypatch):
     """Leaf-only rule: LiteLLM patched -> no token metrics on crewai.llm."""
     # Pretend LiteLLM is patched without actually wrapping the module, so

py/src/braintrust/integrations/crewai/tracing.py

@@ -45,7 +45,6 @@
 from braintrust.integrations.utils import (
     _normalize_chat_messages,
     _parse_openai_usage_metrics,
-    _try_to_dict,
 )
 from braintrust.logger import NOOP_SPAN, Span, current_span
 from braintrust.logger import start_span as _bt_start_span
@@ -127,7 +126,12 @@ def _litellm_owns_leaf_span() -> bool:
 
 
 def _agent_metadata(agent: Any) -> dict[str, Any]:
-    """Extract identity + configuration metadata from a CrewAI agent object."""
+    """Extract identity + configuration metadata from a CrewAI agent object.
+
+    Only attributes on the explicit allowlist below are read, so an SDK-level
+    change that adds new fields (potentially containing API keys or other
+    secrets) does not silently leak into span metadata.
+    """
     if agent is None:
         return {}
     meta: dict[str, Any] = {}
@@ -137,7 +141,12 @@ def _agent_metadata(agent: Any) -> dict[str, Any]:
             meta[f"agent_{attr}"] = str(value) if attr == "id" else value
     llm = getattr(agent, "llm", None)
     if llm is not None:
-        meta["agent_llm"] = getattr(llm, "model", None) or str(llm)
+        # Only read the model name off the LLM object. Falling back to
+        # ``str(llm)`` would dump the full pydantic repr, which on CrewAI's
+        # ``LLM`` includes ``api_key`` / ``api_base`` / ``client_params``.
+        model_name = getattr(llm, "model", None)
+        if model_name:
+            meta["agent_llm"] = model_name
     return meta
 
 
@@ -198,32 +207,20 @@ def _llm_config_metadata(source: Any) -> dict[str, Any]:
     return meta
 
 
-def _normalize_tools(tools: Any) -> Any:
-    """Normalize a list of CrewAI tool descriptors for logging."""
-    if tools is None:
-        return None
-    if isinstance(tools, (list, tuple)):
-        out = []
-        for tool in tools:
-            if isinstance(tool, dict):
-                out.append(tool)
-            else:
-                out.append(_try_to_dict(tool))
-        return out
-    return _try_to_dict(tools)
-
-
-def _normalize_output(value: Any) -> Any:
-    """Coerce provider-owned output objects into plain dicts/strings for logging."""
-    if value is None:
+def _provider_from_source(source: Any) -> str | None:
+    """Best-effort extraction of the underlying provider for a CrewAI LLM call.
+
+    CrewAI's :class:`LLM` object exposes a ``provider`` string
+    (``"openai"``, ``"anthropic"``, ``"bedrock"``, ...). Every ``llm`` span
+    must carry ``metadata.provider`` per the instrumentation spec, so we
+    pull it directly from the emitting source when possible.
+    """
+    if source is None:
         return None
-    if isinstance(value, (str, int, float, bool, dict, list)):
-        return value
-    coerced = _try_to_dict(value)
-    if coerced is value and not isinstance(value, (str, int, float, bool, dict, list)):
-        # Last-resort: render as a string rather than log a raw SDK object.
-        return str(value)
-    return coerced
+    provider = getattr(source, "provider", None)
+    if isinstance(provider, str) and provider:
+        return provider
+    return None
 
 
 # ---------------------------------------------------------------------------
@@ -366,7 +363,10 @@ def on_crew_kickoff_started(source: Any, event: CrewKickoffStartedEvent) -> None
 
     @crewai_event_bus.on(CrewKickoffCompletedEvent)
     def on_crew_kickoff_completed(_source: Any, event: CrewKickoffCompletedEvent) -> None:
-        self._end_span(event, output=_normalize_output(getattr(event, "output", None)))
+        # Pass provider objects through untouched — ``bt_json`` at log time
+        # handles Pydantic models, dataclasses, and stringly fallbacks, so
+        # eagerly ``model_dump``-ing here would just be wasted work.
+        self._end_span(event, output=getattr(event, "output", None))
         # Kickoff is the outermost scope; drop any orphan entries left over
         # when an inner end-event was never delivered so state does not grow
         # unbounded in long-running services.
@@ -399,7 +399,7 @@ def on_task_started(source: Any, event: TaskStartedEvent) -> None:
 
     @crewai_event_bus.on(TaskCompletedEvent)
     def on_task_completed(_source: Any, event: TaskCompletedEvent) -> None:
-        self._end_span(event, output=_normalize_output(getattr(event, "output", None)))
+        self._end_span(event, output=getattr(event, "output", None))
 
     @crewai_event_bus.on(TaskFailedEvent)
     def on_task_failed(_source: Any, event: TaskFailedEvent) -> None:
@@ -418,7 +418,9 @@ def on_agent_started(_source: Any, event: AgentExecutionStartedEvent) -> None:
             **_task_metadata(task),
             **_causal_metadata(event),
         }
-        metadata["tools"] = _normalize_tools(getattr(event, "tools", None))
+        agent_tools = getattr(event, "tools", None)
+        if agent_tools:
+            metadata["tools"] = agent_tools
         self._open_span(
             event,
             name="crewai.agent",
@@ -447,18 +449,27 @@ def on_llm_started(source: Any, event: LLMCallStartedEvent) -> None:
         }
         if getattr(event, "model", None):
             metadata["model"] = event.model
+        # Every ``llm`` span must carry ``metadata.provider`` per the
+        # instrumentation spec. CrewAI ``LLM`` objects expose ``.provider``
+        # directly (``openai`` / ``anthropic`` / ``bedrock`` / ...).
+        provider = _provider_from_source(source)
+        if provider:
+            metadata["provider"] = provider
         if getattr(event, "call_id", None):
             metadata["call_id"] = event.call_id
         available_functions = getattr(event, "available_functions", None)
         if available_functions:
+            # Only the callable *names* — never the callables themselves.
             metadata["available_functions"] = list(available_functions)
-        span_input: dict[str, Any] = {
-            "messages": getattr(event, "messages", None),
-        }
+        # Tool definitions belong in ``metadata.tools``, not ``input``,
+        # per the spec. ``bt_json`` at log time handles Pydantic models
+        # and dataclasses, so pass through without eager serialization.
         tools = getattr(event, "tools", None)
         if tools:
-            span_input["tools"] = _normalize_tools(tools)
-        span_input["messages"] = _normalize_chat_messages(span_input["messages"])
+            metadata["tools"] = tools
+        span_input: dict[str, Any] = {
+            "messages": _normalize_chat_messages(getattr(event, "messages", None)),
+        }
         self._open_span(
             event,
             name="crewai.llm",
@@ -497,7 +508,7 @@ def on_llm_completed(_source: Any, event: LLMCallCompletedEvent) -> None:
 
         self._end_span(
             event,
-            output=_normalize_output(getattr(event, "response", None)),
+            output=getattr(event, "response", None),
             metadata=metadata or None,
             extra_metrics=extra_metrics,
         )
@@ -540,7 +551,7 @@ def on_tool_finished(_source: Any, event: ToolUsageFinishedEvent) -> None:
             extra_metadata["run_attempts"] = run_attempts
         self._end_span(
             event,
-            output=_normalize_output(getattr(event, "output", None)),
+            output=getattr(event, "output", None),
             metadata=extra_metadata or None,
         )