Skip to content

Commit 8734db7

Browse files
sl0thentr0pycodex
andcommitted
feat(agents): Instrument RubyLLM chats
Co-Authored-By: OpenAI <noreply@openai.com>
1 parent d3bc613 commit 8734db7

4 files changed

Lines changed: 547 additions & 1 deletion

File tree

‎sentry-agents/README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22

33
[![Build Status](https://github.com/getsentry/sentry-ruby/actions/workflows/tests.yml/badge.svg)](https://github.com/getsentry/sentry-ruby/actions/workflows/tests.yml)
44

5-
`sentry-agents` integrates [RubyLLM](https://github.com/crmne/ruby_llm) with Sentry.
5+
`sentry-agents` integrates [RubyLLM](https://github.com/crmne/ruby_llm) with Sentry. Loading the gem instruments the global RubyLLM configuration when Sentry is initialized.
6+
7+
For an isolated RubyLLM context, install the adapter explicitly:
8+
9+
```ruby
10+
context = RubyLLM.context
11+
Sentry::Agents::RubyLLM.install(context)
12+
```
613

714
## Requirements
815

‎sentry-agents/lib/sentry-agents.rb‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,21 @@
44
require "sentry-ruby"
55
require "sentry/integrable"
66
require "sentry/agents/version"
7+
require "sentry/agents/ruby_llm/instrumenter"
78

89
module Sentry
910
module Agents
1011
extend Sentry::Integrable
1112

1213
register_integration name: "agents", version: Sentry::Agents::VERSION
14+
15+
module RubyLLM
16+
def self.install(ruby_llm_config = ::RubyLLM.config)
17+
config = ruby_llm_config.respond_to?(:config) ? ruby_llm_config.config : ruby_llm_config
18+
Instrumenter.install(config)
19+
end
20+
end
1321
end
1422
end
23+
24+
Sentry::Configuration.after(:configured) { Sentry::Agents::RubyLLM.install }
Lines changed: 346 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,346 @@
1+
# frozen_string_literal: true
2+
3+
require "json"
4+
5+
module Sentry
6+
module Agents
7+
module RubyLLM
8+
class Instrumenter
9+
CHAT_EVENT = "chat.ruby_llm"
10+
USAGE_EVENT = "usage.ruby_llm"
11+
OPERATION = "gen_ai.chat"
12+
ORIGIN = "auto.ai.ruby_llm"
13+
14+
attr_reader :delegate
15+
16+
def self.install(config)
17+
return config.instrumenter if config.instrumenter.is_a?(self)
18+
19+
config.instrumenter = new(config.instrumenter)
20+
end
21+
22+
def initialize(delegate)
23+
@delegate = delegate
24+
end
25+
26+
def instrument(name, payload = {}, &block)
27+
return delegate_instrument(name, payload, &block) unless name.to_s == CHAT_EVENT || name.to_s == USAGE_EVENT
28+
29+
if name.to_s == USAGE_EVENT
30+
update_usage(payload)
31+
return delegate_instrument(name, payload, &block)
32+
end
33+
34+
return delegate_instrument(name, payload, &block) unless block
35+
36+
delegate_instrument(name, payload) do |event|
37+
with_chat_span(event || payload) { yield(event || payload) }
38+
end
39+
end
40+
41+
private
42+
43+
def delegate_instrument(name, payload, &block)
44+
return delegate.instrument(name, payload, &block) if delegate&.respond_to?(:instrument)
45+
block.call(payload) if block
46+
end
47+
48+
def with_chat_span(payload)
49+
return yield unless Sentry.initialized?
50+
51+
scope = Sentry.get_current_scope
52+
current_span = scope&.get_span
53+
return with_child_span(scope, current_span, payload) { yield } if current_span
54+
55+
with_root_transaction(payload) { yield }
56+
end
57+
58+
def with_child_span(scope, current_span, payload)
59+
span = nil
60+
begin
61+
span = current_span.start_child(
62+
op: OPERATION,
63+
description: span_description(payload),
64+
origin: ORIGIN
65+
)
66+
scope.set_span(span)
67+
rescue StandardError => error
68+
safe_finish(span)
69+
report_instrumentation_error(error)
70+
return yield
71+
end
72+
73+
execute_span(span, payload, scope: scope, previous_span: current_span) { yield }
74+
end
75+
76+
def with_root_transaction(payload)
77+
transaction = begin
78+
Sentry.start_transaction(
79+
name: span_description(payload),
80+
op: OPERATION,
81+
origin: ORIGIN
82+
)
83+
rescue StandardError => error
84+
report_instrumentation_error(error)
85+
nil
86+
end
87+
return yield unless transaction
88+
89+
application_called = false
90+
begin
91+
Sentry.with_scope do |scope|
92+
scope.set_span(transaction)
93+
execute_span(transaction, payload) do
94+
application_called = true
95+
yield
96+
end
97+
end
98+
rescue StandardError => error
99+
raise if application_called
100+
101+
safe_finish(transaction)
102+
report_instrumentation_error(error)
103+
yield
104+
end
105+
end
106+
107+
def execute_span(span, payload, scope: nil, previous_span: nil)
108+
result = nil
109+
begin
110+
set_request_data(span, payload)
111+
result = yield
112+
rescue StandardError => error
113+
set_error(span, error)
114+
raise
115+
ensure
116+
set_response_data(span, payload)
117+
safe_restore_scope(scope, previous_span) if scope && previous_span
118+
safe_finish(span)
119+
end
120+
result
121+
end
122+
123+
def set_request_data(span, payload)
124+
safe_set_data(span, "gen_ai.operation.name", "chat")
125+
set_data_if_present(span, "gen_ai.provider.name", string_value(payload[:provider]))
126+
set_data_if_present(span, "gen_ai.request.model", string_value(payload[:model]))
127+
set_data_if_present(span, "gen_ai.request.temperature", payload[:temperature])
128+
set_data_if_present(span, "gen_ai.request.max_tokens", payload[:max_output_tokens])
129+
return unless collect_inputs?
130+
131+
messages, system_instructions = Serializer.messages(payload[:input_messages])
132+
set_json_data(span, "gen_ai.input.messages", messages) unless messages.empty?
133+
set_json_data(span, "gen_ai.system_instructions", system_instructions) unless system_instructions.empty?
134+
135+
definitions = Serializer.tool_definitions(payload)
136+
set_json_data(span, "gen_ai.tool.definitions", definitions) unless definitions.empty?
137+
rescue StandardError => error
138+
report_instrumentation_error(error)
139+
end
140+
141+
def set_response_data(span, payload)
142+
response = payload[:response] || Array(payload[:messages_after]).last
143+
response_model = payload[:response_model] || Serializer.value(response, :model)
144+
finish_reason = Serializer.value(response, :finish_reason)
145+
146+
set_data_if_present(span, "gen_ai.response.model", string_value(response_model))
147+
set_data_if_present(span, "gen_ai.response.finish_reasons", finish_reason&.to_s)
148+
return unless collect_outputs? && response
149+
150+
messages = Serializer.messages([response]).first
151+
set_json_data(span, "gen_ai.output.messages", messages) unless messages.empty?
152+
rescue StandardError => error
153+
report_instrumentation_error(error)
154+
end
155+
156+
def update_usage(payload)
157+
return unless Sentry.initialized?
158+
159+
span = Sentry.get_current_scope&.get_span
160+
return unless span&.op == OPERATION
161+
162+
tokens = payload[:tokens]
163+
return unless tokens
164+
165+
input_tokens = sum_tokens(tokens, :input, :cache_read, :cache_write)
166+
add_usage(span, "gen_ai.usage.input_tokens", input_tokens) unless input_tokens.nil?
167+
add_usage(span, "gen_ai.usage.output_tokens", token_value(tokens, :output))
168+
add_usage(span, "gen_ai.usage.cache_read.input_tokens", token_value(tokens, :cache_read))
169+
add_usage(span, "gen_ai.usage.cache_creation.input_tokens", token_value(tokens, :cache_write))
170+
add_usage(span, "gen_ai.usage.reasoning.output_tokens", token_value(tokens, :thinking))
171+
rescue StandardError => error
172+
report_instrumentation_error(error)
173+
end
174+
175+
def sum_tokens(tokens, *names)
176+
values = names.map { |name| token_value(tokens, name) }
177+
return if values.all?(&:nil?)
178+
179+
values.compact.sum
180+
end
181+
182+
def token_value(tokens, name)
183+
if tokens.respond_to?(name)
184+
tokens.public_send(name)
185+
elsif tokens.respond_to?(:key?) && tokens.key?(name)
186+
tokens[name]
187+
elsif tokens.respond_to?(:key?) && tokens.key?(name.to_s)
188+
tokens[name.to_s]
189+
end
190+
end
191+
192+
def add_usage(span, key, value)
193+
return if value.nil?
194+
195+
previous = span.data[key]
196+
safe_set_data(span, key, previous.nil? ? value : previous + value)
197+
end
198+
199+
def span_description(payload)
200+
model = payload[:model]
201+
model ? "chat #{model}" : "chat"
202+
end
203+
204+
def collect_inputs?
205+
Sentry.configuration.data_collection.gen_ai.inputs
206+
end
207+
208+
def collect_outputs?
209+
Sentry.configuration.data_collection.gen_ai.outputs
210+
end
211+
212+
def set_error(span, error)
213+
safe_set_data(span, "error.type", error.class.name)
214+
safe_set_status(span, "internal_error")
215+
end
216+
217+
def set_data_if_present(span, key, value)
218+
safe_set_data(span, key, value) unless value.nil?
219+
end
220+
221+
def string_value(value)
222+
value&.to_s
223+
end
224+
225+
def set_json_data(span, key, value)
226+
set_data_if_present(span, key, JSON.generate(value))
227+
rescue StandardError => error
228+
report_instrumentation_error(error)
229+
end
230+
231+
def safe_set_data(span, key, value)
232+
span.set_data(key, value)
233+
rescue StandardError => error
234+
report_instrumentation_error(error)
235+
end
236+
237+
def safe_set_status(span, status)
238+
span.set_status(status)
239+
rescue StandardError => error
240+
report_instrumentation_error(error)
241+
end
242+
243+
def safe_restore_scope(scope, span)
244+
scope.set_span(span)
245+
rescue StandardError => error
246+
report_instrumentation_error(error)
247+
end
248+
249+
def safe_finish(span)
250+
span&.finish
251+
rescue StandardError => error
252+
report_instrumentation_error(error)
253+
end
254+
255+
def report_instrumentation_error(error)
256+
Sentry.configuration.sdk_logger.debug("sentry-agents instrumentation failed: #{error.class}: #{error.message}")
257+
rescue StandardError
258+
nil
259+
end
260+
261+
module Serializer
262+
module_function
263+
264+
def messages(messages)
265+
messages = Array(messages)
266+
system = messages.select { |message| value(message, :role).to_s == "system" }
267+
non_system = messages.reject { |message| value(message, :role).to_s == "system" }
268+
269+
[non_system.filter_map { |message| normalize_message(message) }, system.flat_map { |message| parts(message) }]
270+
end
271+
272+
def tool_definitions(payload)
273+
chat = payload[:chat]
274+
tools =
275+
if chat&.respond_to?(:tools)
276+
chat.tools.values
277+
else
278+
Array(payload[:tools])
279+
end
280+
281+
definitions = tools.filter_map do |tool|
282+
if tool.respond_to?(:name)
283+
{
284+
name: tool.name,
285+
description: tool.description,
286+
parameters: tool.parameters_schema
287+
}.compact
288+
elsif tool.is_a?(Hash)
289+
tool
290+
elsif tool
291+
{ name: tool.to_s }
292+
end
293+
end
294+
295+
definitions + Array(payload[:provider_tools]).select { |tool| tool.is_a?(Hash) }
296+
end
297+
298+
def normalize_message(message)
299+
role = normalize_role(value(message, :role))
300+
{ role: role, parts: parts(message) }
301+
end
302+
303+
def parts(message)
304+
role = normalize_role(value(message, :role))
305+
return [{ type: "tool_call_response", id: value(message, :tool_call_id), response: value(message, :content) }.compact] if role == "tool"
306+
307+
result = []
308+
content = value(message, :content)
309+
result << { type: "text", content: content } unless content.nil?
310+
311+
tool_calls = value(message, :tool_calls)
312+
tool_calls = tool_calls.values if tool_calls.is_a?(Hash)
313+
Array(tool_calls).each do |tool_call|
314+
result << {
315+
type: "tool_call",
316+
id: value(tool_call, :id),
317+
name: value(tool_call, :name),
318+
arguments: value(tool_call, :arguments)
319+
}.compact
320+
end
321+
result
322+
end
323+
324+
def normalize_role(role)
325+
case role.to_s
326+
when "human" then "user"
327+
when "ai" then "assistant"
328+
when "function" then "tool"
329+
else role.to_s
330+
end
331+
end
332+
333+
def value(object, key)
334+
if object.respond_to?(key)
335+
object.public_send(key)
336+
elsif object.respond_to?(:key?) && object.key?(key)
337+
object[key]
338+
elsif object.respond_to?(:key?) && object.key?(key.to_s)
339+
object[key.to_s]
340+
end
341+
end
342+
end
343+
end
344+
end
345+
end
346+
end

0 commit comments

Comments
 (0)