Skip to content

Commit cbb73ec

Browse files
sl0thentr0pycodex
andcommitted
feat(agents): Instrument RubyLLM agent runs
Co-Authored-By: OpenAI <noreply@openai.com>
1 parent 15100b6 commit cbb73ec

6 files changed

Lines changed: 317 additions & 0 deletions

File tree

‎sentry-agents/Gemfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# frozen_string_literal: true
2+
13
source "https://rubygems.org"
24
git_source(:github) { |name| "https://github.com/#{name}.git" }
35

‎sentry-agents/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ Add the gem to your bundle:
2424
gem "sentry-agents"
2525
```
2626

27+
## Agent runs
28+
29+
`RubyLLM::Agent` runs create `gen_ai.invoke_agent` spans. Nested agent runs are preserved, while calls on the same agent chat are grouped into one run.
30+
2731
## Workflows
2832

2933
`RubyLLM.workflow` and its steps create pipeline grouping spans. Chat and tool descendants carry `gen_ai.pipeline.name` without being labeled as agent runs.

‎sentry-agents/lib/sentry-agents.rb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
require "sentry/agents/version"
77
require "sentry/agents/ruby_llm/instrumenter"
88
require "sentry/agents/ruby_llm/tool_concurrency"
9+
require "sentry/agents/ruby_llm/agent_runs"
910

1011
module Sentry
1112
module Agents
@@ -16,6 +17,7 @@ module Agents
1617
module RubyLLM
1718
def self.install(ruby_llm_config = ::RubyLLM.config)
1819
config = ruby_llm_config.respond_to?(:config) ? ruby_llm_config.config : ruby_llm_config
20+
AgentRuns.install
1921
ToolConcurrency.install
2022
Instrumenter.install(config)
2123
end
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
# frozen_string_literal: true
2+
3+
require "ruby_llm/agent"
4+
require "ruby_llm/chat"
5+
6+
module Sentry
7+
module Agents
8+
module RubyLLM
9+
module AgentRuns
10+
AGENT_NAME_IVAR = :@sentry_agents_agent_name
11+
DEPTH_IVAR = :@sentry_agents_agent_run_depth
12+
OPERATION = "gen_ai.invoke_agent"
13+
ORIGIN = "auto.ai.ruby_llm"
14+
15+
module AgentIdentityPatch
16+
def apply_configuration(chat, input_values:, persist_instructions:, **options)
17+
result = super(chat, input_values: input_values, persist_instructions: persist_instructions, **options)
18+
chat.instance_variable_set(AGENT_NAME_IVAR, name.to_s)
19+
result
20+
end
21+
end
22+
23+
module ChatPatch
24+
def complete(*args, **kwargs, &block)
25+
AgentRuns.with(self) { super(*args, **kwargs, &block) }
26+
end
27+
28+
def generate(*args, **kwargs, &block)
29+
AgentRuns.with(self) { super(*args, **kwargs, &block) }
30+
end
31+
32+
def run_tools(*args, **kwargs, &block)
33+
AgentRuns.with(self) { super(*args, **kwargs, &block) }
34+
end
35+
36+
def step(*args, **kwargs, &block)
37+
AgentRuns.with(self) { super(*args, **kwargs, &block) }
38+
end
39+
end
40+
41+
class << self
42+
def install
43+
install_agent_identity
44+
install_chat_patch
45+
end
46+
47+
def with(chat)
48+
agent_name = chat.instance_variable_get(AGENT_NAME_IVAR)
49+
return yield if agent_name.nil? || agent_name.empty?
50+
51+
depth = chat.instance_variable_get(DEPTH_IVAR).to_i
52+
return yield if depth.positive?
53+
54+
chat.instance_variable_set(DEPTH_IVAR, depth + 1)
55+
with_span(chat, agent_name) { yield }
56+
ensure
57+
if depth&.positive?
58+
depth.zero? ? chat.remove_instance_variable(DEPTH_IVAR) : chat.instance_variable_set(DEPTH_IVAR, depth)
59+
end
60+
end
61+
62+
private
63+
64+
def install_agent_identity
65+
target = ::RubyLLM::Agent.singleton_class
66+
target.prepend(AgentIdentityPatch) unless target.ancestors.include?(AgentIdentityPatch)
67+
end
68+
69+
def install_chat_patch
70+
target = ::RubyLLM::Chat
71+
target.prepend(ChatPatch) unless target.ancestors.include?(ChatPatch)
72+
end
73+
74+
def with_span(chat, agent_name)
75+
return yield unless Sentry.initialized?
76+
77+
scope = Sentry.get_current_scope
78+
current_span = scope&.get_span
79+
if current_span
80+
with_child_span(scope, current_span, chat, agent_name) { yield }
81+
else
82+
with_root_transaction(chat, agent_name) { yield }
83+
end
84+
end
85+
86+
def with_child_span(scope, current_span, chat, agent_name)
87+
span = nil
88+
begin
89+
span = current_span.start_child(op: OPERATION, description: description(agent_name), origin: ORIGIN)
90+
scope.set_span(span)
91+
rescue StandardError => error
92+
safe_finish(span)
93+
report(error)
94+
return yield
95+
end
96+
97+
execute_span(span, scope: scope, previous_span: current_span, chat: chat, agent_name: agent_name) { yield }
98+
end
99+
100+
def with_root_transaction(chat, agent_name)
101+
transaction = begin
102+
Sentry.start_transaction(name: description(agent_name), op: OPERATION, origin: ORIGIN)
103+
rescue StandardError => error
104+
report(error)
105+
nil
106+
end
107+
return yield unless transaction
108+
109+
application_called = false
110+
begin
111+
Sentry.with_scope do |scope|
112+
scope.set_span(transaction)
113+
execute_span(transaction, chat: chat, agent_name: agent_name) do
114+
application_called = true
115+
yield
116+
end
117+
end
118+
rescue StandardError => error
119+
raise if application_called
120+
121+
safe_finish(transaction)
122+
report(error)
123+
yield
124+
end
125+
end
126+
127+
def execute_span(span, chat:, agent_name:, scope: nil, previous_span: nil)
128+
result = nil
129+
begin
130+
set_data(span, chat, agent_name)
131+
result = yield
132+
rescue StandardError => error
133+
set_error(span, error)
134+
raise
135+
ensure
136+
safe_restore_scope(scope, previous_span) if scope && previous_span
137+
safe_finish(span)
138+
end
139+
result
140+
end
141+
142+
def set_data(span, chat, agent_name)
143+
span.set_data("gen_ai.operation.name", "invoke_agent")
144+
span.set_data("gen_ai.agent.name", agent_name)
145+
workflow = ::RubyLLM::Support::Instrumentation.current_workflow
146+
pipeline = workflow && workflow[:workflow_name]
147+
span.set_data("gen_ai.pipeline.name", pipeline) if pipeline
148+
rescue StandardError => error
149+
report(error)
150+
end
151+
152+
def set_error(span, error)
153+
safe_set_data(span, "error.type", error.class.name)
154+
safe_set_status(span, "internal_error")
155+
end
156+
157+
def safe_set_data(span, key, value)
158+
span.set_data(key, value)
159+
rescue StandardError => error
160+
report(error)
161+
end
162+
163+
def safe_set_status(span, status)
164+
span.set_status(status)
165+
rescue StandardError => error
166+
report(error)
167+
end
168+
169+
def safe_restore_scope(scope, span)
170+
scope.set_span(span)
171+
rescue StandardError => error
172+
report(error)
173+
end
174+
175+
def safe_finish(span)
176+
span&.finish
177+
rescue StandardError => error
178+
report(error)
179+
end
180+
181+
def report(error)
182+
Sentry.configuration.sdk_logger.debug("sentry-agents instrumentation failed: #{error.class}: #{error.message}")
183+
rescue StandardError
184+
nil
185+
end
186+
187+
def description(agent_name)
188+
"invoke_agent #{agent_name}"
189+
end
190+
end
191+
end
192+
end
193+
end
194+
end

‎sentry-agents/lib/sentry/agents/ruby_llm/instrumenter.rb‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ def set_request_data(span, payload, operation)
154154
def set_chat_request_data(span, payload)
155155
safe_set_data(span, "gen_ai.operation.name", "chat")
156156
set_pipeline_data(span, payload)
157+
set_agent_data(span, payload)
157158
set_data_if_present(span, "gen_ai.provider.name", string_value(payload[:provider]))
158159
set_data_if_present(span, "gen_ai.request.model", string_value(payload[:model]))
159160
set_data_if_present(span, "gen_ai.request.temperature", payload[:temperature])
@@ -171,6 +172,7 @@ def set_chat_request_data(span, payload)
171172
def set_tool_request_data(span, payload)
172173
safe_set_data(span, "gen_ai.operation.name", "execute_tool")
173174
set_pipeline_data(span, payload)
175+
set_agent_data(span, payload)
174176
set_data_if_present(span, "gen_ai.tool.name", string_value(payload[:tool_name] || Serializer.value(payload[:tool], :name)))
175177
return unless collect_inputs?
176178

@@ -189,6 +191,12 @@ def set_pipeline_data(span, payload)
189191
set_data_if_present(span, "gen_ai.pipeline.name", string_value(payload[:workflow_name]))
190192
end
191193

194+
def set_agent_data(span, payload)
195+
chat = payload[:chat]
196+
agent_name = chat&.instance_variable_get(:@sentry_agents_agent_name)
197+
set_data_if_present(span, "gen_ai.agent.name", string_value(agent_name))
198+
end
199+
192200
def set_response_data(span, payload, operation)
193201
case operation
194202
when OPERATION
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# frozen_string_literal: true
2+
3+
require "spec_helper"
4+
5+
RSpec.describe Sentry::Agents::RubyLLM::AgentRuns do
6+
before do
7+
RubyLLM.config.instrumenter = nil
8+
RubyLLM.configure { |config| config.openai_api_key = "test" }
9+
Sentry.init do |config|
10+
config.dsn = Sentry::TestHelper::DUMMY_DSN
11+
config.sdk_logger = Logger.new(nil)
12+
config.background_worker_threads = 0
13+
config.transport.transport_class = Sentry::DummyTransport
14+
config.traces_sample_rate = 1.0
15+
config.data_collection.gen_ai.inputs = true
16+
config.data_collection.gen_ai.outputs = true
17+
end
18+
end
19+
20+
after do
21+
RubyLLM.config.instrumenter = nil
22+
end
23+
24+
it "creates one invoke_agent span with model and tool descendants" do
25+
agent_class = agent_class("SupportAgent")
26+
chat = agent_class.chat
27+
adapter = Sentry::Agents::RubyLLM::Instrumenter.new(nil)
28+
29+
result = described_class.with(chat) do
30+
adapter.instrument("chat.ruby_llm", chat_payload(chat)) { :model_result }
31+
adapter.instrument("tool_call.ruby_llm", tool_payload(chat)) { :tool_result }
32+
:agent_result
33+
end
34+
35+
expect(result).to eq(:agent_result)
36+
event = transaction_event
37+
root_trace = event.contexts[:trace]
38+
spans = event.spans
39+
40+
expect(root_trace[:op]).to eq("gen_ai.invoke_agent")
41+
expect(root_trace[:data]["gen_ai.agent.name"]).to eq("SupportAgent")
42+
expect(spans.map { |span| span[:op] }).to contain_exactly("gen_ai.chat", "gen_ai.execute_tool")
43+
expect(spans.map { |span| span[:data]["gen_ai.agent.name"] }).to all(eq("SupportAgent"))
44+
expect(spans.map { |span| span[:parent_span_id] }).to all(eq(root_trace[:span_id]))
45+
end
46+
47+
it "suppresses duplicate wrappers for the same chat and keeps nested agents" do
48+
outer_class = agent_class("OuterAgent")
49+
inner_class = agent_class("InnerAgent")
50+
outer_chat = outer_class.chat
51+
inner_chat = inner_class.chat
52+
53+
described_class.with(outer_chat) do
54+
described_class.with(outer_chat) { :same_chat }
55+
described_class.with(inner_chat) { :nested_chat }
56+
end
57+
58+
event = transaction_event
59+
root_trace = event.contexts[:trace]
60+
invoke_spans = event.spans.select { |span| span[:op] == "gen_ai.invoke_agent" }
61+
62+
expect(invoke_spans.size).to eq(1)
63+
expect(invoke_spans.first[:data]["gen_ai.agent.name"]).to eq("InnerAgent")
64+
expect(invoke_spans.first[:parent_span_id]).to eq(root_trace[:span_id])
65+
end
66+
67+
it "finishes the invoke_agent span when the operation raises" do
68+
chat = agent_class("FailingAgent").chat
69+
70+
expect do
71+
described_class.with(chat) { raise "agent failure" }
72+
end.to raise_error(RuntimeError, "agent failure")
73+
74+
event = transaction_event
75+
trace = event.contexts[:trace]
76+
expect(trace[:status]).to eq("internal_error")
77+
expect(trace[:data]["error.type"]).to eq("RuntimeError")
78+
end
79+
80+
def agent_class(name)
81+
stub_const(name, Class.new(RubyLLM::Agent)).tap do |klass|
82+
klass.model "gpt-4o"
83+
end
84+
end
85+
86+
def chat_payload(chat)
87+
{
88+
chat: chat,
89+
provider: "openai",
90+
model: "gpt-4o",
91+
input_messages: []
92+
}
93+
end
94+
95+
def tool_payload(chat)
96+
{
97+
chat: chat,
98+
tool_name: "weather",
99+
tool_arguments: { city: "Paris" }
100+
}
101+
end
102+
103+
def transaction_event
104+
Sentry.get_current_client.flush
105+
sentry_events.find { |event| event.is_a?(Sentry::TransactionEvent) }
106+
end
107+
end

0 commit comments

Comments
 (0)