Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions sentry-agents/Gemfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

source "https://rubygems.org"
git_source(:github) { |name| "https://github.com/#{name}.git" }

Expand Down
4 changes: 4 additions & 0 deletions sentry-agents/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ Add the gem to your bundle:
gem "sentry-agents"
```

## Agent runs

`RubyLLM::Agent` runs create `gen_ai.invoke_agent` spans. Nested agent runs are preserved, while calls on the same agent chat are grouped into one run.

## Workflows

`RubyLLM.workflow` and its steps create pipeline grouping spans. Chat and tool descendants carry `gen_ai.pipeline.name` without being labeled as agent runs.
Expand Down
2 changes: 2 additions & 0 deletions sentry-agents/lib/sentry-agents.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
require "sentry/agents/version"
require "sentry/agents/ruby_llm/instrumenter"
require "sentry/agents/ruby_llm/tool_concurrency"
require "sentry/agents/ruby_llm/agent_runs"

module Sentry
module Agents
Expand All @@ -16,6 +17,7 @@ module Agents
module RubyLLM
def self.install(ruby_llm_config = ::RubyLLM.config)
config = ruby_llm_config.respond_to?(:config) ? ruby_llm_config.config : ruby_llm_config
AgentRuns.install
ToolConcurrency.install
Instrumenter.install(config)
end
Expand Down
194 changes: 194 additions & 0 deletions sentry-agents/lib/sentry/agents/ruby_llm/agent_runs.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
# frozen_string_literal: true

require "ruby_llm/agent"
require "ruby_llm/chat"

module Sentry
module Agents
module RubyLLM
module AgentRuns
AGENT_NAME_IVAR = :@sentry_agents_agent_name
DEPTH_IVAR = :@sentry_agents_agent_run_depth
OPERATION = "gen_ai.invoke_agent"
ORIGIN = "auto.ai.ruby_llm"

module AgentIdentityPatch
def apply_configuration(chat, input_values:, persist_instructions:, **options)
result = super(chat, input_values: input_values, persist_instructions: persist_instructions, **options)
chat.instance_variable_set(AGENT_NAME_IVAR, name.to_s)
result
end
end

module ChatPatch
def complete(*args, **kwargs, &block)
AgentRuns.with(self) { super(*args, **kwargs, &block) }
end

def generate(*args, **kwargs, &block)
AgentRuns.with(self) { super(*args, **kwargs, &block) }
end

def run_tools(*args, **kwargs, &block)
AgentRuns.with(self) { super(*args, **kwargs, &block) }
end

def step(*args, **kwargs, &block)
AgentRuns.with(self) { super(*args, **kwargs, &block) }
end
end

class << self
def install
install_agent_identity
install_chat_patch
end

def with(chat)
agent_name = chat.instance_variable_get(AGENT_NAME_IVAR)
return yield if agent_name.nil? || agent_name.empty?

depth = chat.instance_variable_get(DEPTH_IVAR).to_i
return yield if depth.positive?

chat.instance_variable_set(DEPTH_IVAR, depth + 1)
with_span(chat, agent_name) { yield }
ensure
if depth&.positive?
depth.zero? ? chat.remove_instance_variable(DEPTH_IVAR) : chat.instance_variable_set(DEPTH_IVAR, depth)
end

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agent run depth never resets

High Severity

The ensure in with only restores depth when depth is already positive, so the outer wrapper never clears @sentry_agents_agent_run_depth. After the first run, later calls on the same chat skip gen_ai.invoke_agent spans, including follow-up turns on a reused agent chat.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 18541b7. Configure here.

end

private

def install_agent_identity
target = ::RubyLLM::Agent.singleton_class
target.prepend(AgentIdentityPatch) unless target.ancestors.include?(AgentIdentityPatch)
end

def install_chat_patch
target = ::RubyLLM::Chat
target.prepend(ChatPatch) unless target.ancestors.include?(ChatPatch)
end

def with_span(chat, agent_name)
return yield unless Sentry.initialized?

scope = Sentry.get_current_scope
current_span = scope&.get_span
if current_span
with_child_span(scope, current_span, chat, agent_name) { yield }
else
with_root_transaction(chat, agent_name) { yield }
end
end

def with_child_span(scope, current_span, chat, agent_name)
span = nil
begin
span = current_span.start_child(op: OPERATION, description: description(agent_name), origin: ORIGIN)
scope.set_span(span)
rescue StandardError => error
safe_finish(span)
report(error)
return yield
end

execute_span(span, scope: scope, previous_span: current_span, chat: chat, agent_name: agent_name) { yield }
end

def with_root_transaction(chat, agent_name)
transaction = begin
Sentry.start_transaction(name: description(agent_name), op: OPERATION, origin: ORIGIN)
rescue StandardError => error
report(error)
nil
end
return yield unless transaction

application_called = false
begin
Sentry.with_scope do |scope|
scope.set_span(transaction)
execute_span(transaction, chat: chat, agent_name: agent_name) do
application_called = true
yield
end
end
rescue StandardError => error
raise if application_called

safe_finish(transaction)
report(error)
yield
end
end

def execute_span(span, chat:, agent_name:, scope: nil, previous_span: nil)
result = nil
begin
set_data(span, chat, agent_name)
result = yield
rescue StandardError => error
set_error(span, error)
raise
ensure
safe_restore_scope(scope, previous_span) if scope && previous_span
safe_finish(span)
end
result
end

def set_data(span, chat, agent_name)
span.set_data("gen_ai.operation.name", "invoke_agent")
span.set_data("gen_ai.agent.name", agent_name)
workflow = ::RubyLLM::Support::Instrumentation.current_workflow
pipeline = workflow && workflow[:workflow_name]
span.set_data("gen_ai.pipeline.name", pipeline) if pipeline
rescue StandardError => error
report(error)
end

def set_error(span, error)
safe_set_data(span, "error.type", error.class.name)
safe_set_status(span, "internal_error")
end

def safe_set_data(span, key, value)
span.set_data(key, value)
rescue StandardError => error
report(error)
end

def safe_set_status(span, status)
span.set_status(status)
rescue StandardError => error
report(error)
end

def safe_restore_scope(scope, span)
scope.set_span(span)
rescue StandardError => error
report(error)
end

def safe_finish(span)
span&.finish
rescue StandardError => error
report(error)
end

def report(error)
Sentry.configuration.sdk_logger.debug("sentry-agents instrumentation failed: #{error.class}: #{error.message}")
rescue StandardError
nil
end

def description(agent_name)
"invoke_agent #{agent_name}"
end
end
end
end
end
end
8 changes: 8 additions & 0 deletions sentry-agents/lib/sentry/agents/ruby_llm/instrumenter.rb
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ def set_request_data(span, payload, operation)
def set_chat_request_data(span, payload)
safe_set_data(span, "gen_ai.operation.name", "chat")
set_pipeline_data(span, payload)
set_agent_data(span, payload)
set_data_if_present(span, "gen_ai.provider.name", string_value(payload[:provider]))
set_data_if_present(span, "gen_ai.request.model", string_value(payload[:model]))
set_data_if_present(span, "gen_ai.request.temperature", payload[:temperature])
Expand All @@ -171,6 +172,7 @@ def set_chat_request_data(span, payload)
def set_tool_request_data(span, payload)
safe_set_data(span, "gen_ai.operation.name", "execute_tool")
set_pipeline_data(span, payload)
set_agent_data(span, payload)
set_data_if_present(span, "gen_ai.tool.name", string_value(payload[:tool_name] || Serializer.value(payload[:tool], :name)))
return unless collect_inputs?

Expand All @@ -189,6 +191,12 @@ def set_pipeline_data(span, payload)
set_data_if_present(span, "gen_ai.pipeline.name", string_value(payload[:workflow_name]))
end

def set_agent_data(span, payload)
chat = payload[:chat]
agent_name = chat&.instance_variable_get(:@sentry_agents_agent_name)
set_data_if_present(span, "gen_ai.agent.name", string_value(agent_name))
end

def set_response_data(span, payload, operation)
case operation
when OPERATION
Expand Down
107 changes: 107 additions & 0 deletions sentry-agents/spec/sentry/agents/ruby_llm/agent_runs_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# frozen_string_literal: true

require "spec_helper"

RSpec.describe Sentry::Agents::RubyLLM::AgentRuns do
before do
RubyLLM.config.instrumenter = nil
RubyLLM.configure { |config| config.openai_api_key = "test" }
Sentry.init do |config|
config.dsn = Sentry::TestHelper::DUMMY_DSN
config.sdk_logger = Logger.new(nil)
config.background_worker_threads = 0
config.transport.transport_class = Sentry::DummyTransport
config.traces_sample_rate = 1.0
config.data_collection.gen_ai.inputs = true
config.data_collection.gen_ai.outputs = true
end
end

after do
RubyLLM.config.instrumenter = nil
end

it "creates one invoke_agent span with model and tool descendants" do
agent_class = agent_class("SupportAgent")
chat = agent_class.chat
adapter = Sentry::Agents::RubyLLM::Instrumenter.new(nil)

result = described_class.with(chat) do
adapter.instrument("chat.ruby_llm", chat_payload(chat)) { :model_result }
adapter.instrument("tool_call.ruby_llm", tool_payload(chat)) { :tool_result }
:agent_result
end

expect(result).to eq(:agent_result)
event = transaction_event
root_trace = event.contexts[:trace]
spans = event.spans

expect(root_trace[:op]).to eq("gen_ai.invoke_agent")
expect(root_trace[:data]["gen_ai.agent.name"]).to eq("SupportAgent")
expect(spans.map { |span| span[:op] }).to contain_exactly("gen_ai.chat", "gen_ai.execute_tool")
expect(spans.map { |span| span[:data]["gen_ai.agent.name"] }).to all(eq("SupportAgent"))
expect(spans.map { |span| span[:parent_span_id] }).to all(eq(root_trace[:span_id]))
end

it "suppresses duplicate wrappers for the same chat and keeps nested agents" do
outer_class = agent_class("OuterAgent")
inner_class = agent_class("InnerAgent")
outer_chat = outer_class.chat
inner_chat = inner_class.chat

described_class.with(outer_chat) do
described_class.with(outer_chat) { :same_chat }
described_class.with(inner_chat) { :nested_chat }
end

event = transaction_event
root_trace = event.contexts[:trace]
invoke_spans = event.spans.select { |span| span[:op] == "gen_ai.invoke_agent" }

expect(invoke_spans.size).to eq(1)
expect(invoke_spans.first[:data]["gen_ai.agent.name"]).to eq("InnerAgent")
expect(invoke_spans.first[:parent_span_id]).to eq(root_trace[:span_id])
end

it "finishes the invoke_agent span when the operation raises" do
chat = agent_class("FailingAgent").chat

expect do
described_class.with(chat) { raise "agent failure" }
end.to raise_error(RuntimeError, "agent failure")

event = transaction_event
trace = event.contexts[:trace]
expect(trace[:status]).to eq("internal_error")
expect(trace[:data]["error.type"]).to eq("RuntimeError")
end

def agent_class(name)
stub_const(name, Class.new(RubyLLM::Agent)).tap do |klass|
klass.model "gpt-4o"
end
end

def chat_payload(chat)
{
chat: chat,
provider: "openai",
model: "gpt-4o",
input_messages: []
}
end

def tool_payload(chat)
{
chat: chat,
tool_name: "weather",
tool_arguments: { city: "Paris" }
}
end

def transaction_event
Sentry.get_current_client.flush
sentry_events.find { |event| event.is_a?(Sentry::TransactionEvent) }
end
end
Loading