Skip to content

Commit f1a41f0

Browse files
sl0thentr0pycodex
andcommitted
feat(agents): Instrument RubyLLM chats
Co-Authored-By: OpenAI <noreply@openai.com>
1 parent d3bc613 commit f1a41f0

4 files changed

Lines changed: 543 additions & 1 deletion

File tree

‎sentry-agents/README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22

33
[![Build Status](https://github.com/getsentry/sentry-ruby/actions/workflows/tests.yml/badge.svg)](https://github.com/getsentry/sentry-ruby/actions/workflows/tests.yml)
44

5-
`sentry-agents` integrates [RubyLLM](https://github.com/crmne/ruby_llm) with Sentry.
5+
`sentry-agents` integrates [RubyLLM](https://github.com/crmne/ruby_llm) with Sentry. Loading the gem instruments the global RubyLLM configuration when Sentry is initialized.
6+
7+
For an isolated RubyLLM context, install the adapter explicitly:
8+
9+
```ruby
10+
context = RubyLLM.context
11+
Sentry::Agents.install(context)
12+
```
613

714
## Requirements
815

‎sentry-agents/lib/sentry-agents.rb‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,19 @@
44
require "sentry-ruby"
55
require "sentry/integrable"
66
require "sentry/agents/version"
7+
require "sentry/agents/instrumenter"
78

89
module Sentry
910
module Agents
1011
extend Sentry::Integrable
1112

1213
register_integration name: "agents", version: Sentry::Agents::VERSION
14+
15+
def self.install(ruby_llm_config = RubyLLM.config)
16+
config = ruby_llm_config.respond_to?(:config) ? ruby_llm_config.config : ruby_llm_config
17+
Instrumenter.install(config)
18+
end
1319
end
1420
end
21+
22+
Sentry::Configuration.after(:configured) { Sentry::Agents.install }
Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
# frozen_string_literal: true
2+
3+
require "json"
4+
5+
module Sentry
6+
module Agents
7+
class Instrumenter
8+
CHAT_EVENT = "chat.ruby_llm"
9+
USAGE_EVENT = "usage.ruby_llm"
10+
OPERATION = "gen_ai.chat"
11+
ORIGIN = "auto.ai.ruby_llm"
12+
13+
attr_reader :delegate
14+
15+
def self.install(config)
16+
return config.instrumenter if config.instrumenter.is_a?(self)
17+
18+
config.instrumenter = new(config.instrumenter)
19+
end
20+
21+
def initialize(delegate)
22+
@delegate = delegate
23+
end
24+
25+
def instrument(name, payload = {}, &block)
26+
return delegate_instrument(name, payload, &block) unless name.to_s == CHAT_EVENT || name.to_s == USAGE_EVENT
27+
28+
if name.to_s == USAGE_EVENT
29+
update_usage(payload)
30+
return delegate_instrument(name, payload, &block)
31+
end
32+
33+
return delegate_instrument(name, payload, &block) unless block
34+
35+
delegate_instrument(name, payload) do |event|
36+
with_chat_span(event || payload) { yield(event || payload) }
37+
end
38+
end
39+
40+
private
41+
42+
def delegate_instrument(name, payload, &block)
43+
return delegate.instrument(name, payload, &block) if delegate&.respond_to?(:instrument)
44+
block.call(payload) if block
45+
end
46+
47+
def with_chat_span(payload)
48+
return yield unless Sentry.initialized?
49+
50+
scope = Sentry.get_current_scope
51+
current_span = scope&.get_span
52+
return with_child_span(scope, current_span, payload) { yield } if current_span
53+
54+
with_root_transaction(payload) { yield }
55+
end
56+
57+
def with_child_span(scope, current_span, payload)
58+
span = nil
59+
begin
60+
span = current_span.start_child(
61+
op: OPERATION,
62+
description: span_description(payload),
63+
origin: ORIGIN
64+
)
65+
scope.set_span(span)
66+
rescue StandardError => error
67+
safe_finish(span)
68+
report_instrumentation_error(error)
69+
return yield
70+
end
71+
72+
execute_span(span, payload, scope: scope, previous_span: current_span) { yield }
73+
end
74+
75+
def with_root_transaction(payload)
76+
transaction = begin
77+
Sentry.start_transaction(
78+
name: span_description(payload),
79+
op: OPERATION,
80+
origin: ORIGIN
81+
)
82+
rescue StandardError => error
83+
report_instrumentation_error(error)
84+
nil
85+
end
86+
return yield unless transaction
87+
88+
application_called = false
89+
begin
90+
Sentry.with_scope do |scope|
91+
scope.set_span(transaction)
92+
execute_span(transaction, payload) do
93+
application_called = true
94+
yield
95+
end
96+
end
97+
rescue StandardError => error
98+
raise if application_called
99+
100+
safe_finish(transaction)
101+
report_instrumentation_error(error)
102+
yield
103+
end
104+
end
105+
106+
def execute_span(span, payload, scope: nil, previous_span: nil)
107+
result = nil
108+
begin
109+
set_request_data(span, payload)
110+
result = yield
111+
rescue StandardError => error
112+
set_error(span, error)
113+
raise
114+
ensure
115+
set_response_data(span, payload)
116+
safe_restore_scope(scope, previous_span) if scope && previous_span
117+
safe_finish(span)
118+
end
119+
result
120+
end
121+
122+
def set_request_data(span, payload)
123+
safe_set_data(span, "gen_ai.operation.name", "chat")
124+
set_data_if_present(span, "gen_ai.provider.name", string_value(payload[:provider]))
125+
set_data_if_present(span, "gen_ai.request.model", string_value(payload[:model]))
126+
set_data_if_present(span, "gen_ai.request.temperature", payload[:temperature])
127+
set_data_if_present(span, "gen_ai.request.max_tokens", payload[:max_output_tokens])
128+
return unless collect_inputs?
129+
130+
messages, system_instructions = Serializer.messages(payload[:input_messages])
131+
set_json_data(span, "gen_ai.input.messages", messages) unless messages.empty?
132+
set_json_data(span, "gen_ai.system_instructions", system_instructions) unless system_instructions.empty?
133+
134+
definitions = Serializer.tool_definitions(payload)
135+
set_json_data(span, "gen_ai.tool.definitions", definitions) unless definitions.empty?
136+
rescue StandardError => error
137+
report_instrumentation_error(error)
138+
end
139+
140+
def set_response_data(span, payload)
141+
response = payload[:response] || Array(payload[:messages_after]).last
142+
response_model = payload[:response_model] || Serializer.value(response, :model)
143+
finish_reason = Serializer.value(response, :finish_reason)
144+
145+
set_data_if_present(span, "gen_ai.response.model", string_value(response_model))
146+
set_data_if_present(span, "gen_ai.response.finish_reasons", finish_reason&.to_s)
147+
return unless collect_outputs? && response
148+
149+
messages = Serializer.messages([response]).first
150+
set_json_data(span, "gen_ai.output.messages", messages) unless messages.empty?
151+
rescue StandardError => error
152+
report_instrumentation_error(error)
153+
end
154+
155+
def update_usage(payload)
156+
return unless Sentry.initialized?
157+
158+
span = Sentry.get_current_scope&.get_span
159+
return unless span&.op == OPERATION
160+
161+
tokens = payload[:tokens]
162+
return unless tokens
163+
164+
input_tokens = sum_tokens(tokens, :input, :cache_read, :cache_write)
165+
add_usage(span, "gen_ai.usage.input_tokens", input_tokens) unless input_tokens.nil?
166+
add_usage(span, "gen_ai.usage.output_tokens", token_value(tokens, :output))
167+
add_usage(span, "gen_ai.usage.cache_read.input_tokens", token_value(tokens, :cache_read))
168+
add_usage(span, "gen_ai.usage.cache_creation.input_tokens", token_value(tokens, :cache_write))
169+
add_usage(span, "gen_ai.usage.reasoning.output_tokens", token_value(tokens, :thinking))
170+
rescue StandardError => error
171+
report_instrumentation_error(error)
172+
end
173+
174+
def sum_tokens(tokens, *names)
175+
values = names.map { |name| token_value(tokens, name) }
176+
return if values.all?(&:nil?)
177+
178+
values.compact.sum
179+
end
180+
181+
def token_value(tokens, name)
182+
if tokens.respond_to?(name)
183+
tokens.public_send(name)
184+
elsif tokens.respond_to?(:key?) && tokens.key?(name)
185+
tokens[name]
186+
elsif tokens.respond_to?(:key?) && tokens.key?(name.to_s)
187+
tokens[name.to_s]
188+
end
189+
end
190+
191+
def add_usage(span, key, value)
192+
return if value.nil?
193+
194+
previous = span.data[key]
195+
safe_set_data(span, key, previous.nil? ? value : previous + value)
196+
end
197+
198+
def span_description(payload)
199+
model = payload[:model]
200+
model ? "chat #{model}" : "chat"
201+
end
202+
203+
def collect_inputs?
204+
Sentry.configuration.data_collection.gen_ai.inputs
205+
end
206+
207+
def collect_outputs?
208+
Sentry.configuration.data_collection.gen_ai.outputs
209+
end
210+
211+
def set_error(span, error)
212+
safe_set_data(span, "error.type", error.class.name)
213+
safe_set_status(span, "internal_error")
214+
end
215+
216+
def set_data_if_present(span, key, value)
217+
safe_set_data(span, key, value) unless value.nil?
218+
end
219+
220+
def string_value(value)
221+
value&.to_s
222+
end
223+
224+
def set_json_data(span, key, value)
225+
set_data_if_present(span, key, JSON.generate(value))
226+
rescue StandardError => error
227+
report_instrumentation_error(error)
228+
end
229+
230+
def safe_set_data(span, key, value)
231+
span.set_data(key, value)
232+
rescue StandardError => error
233+
report_instrumentation_error(error)
234+
end
235+
236+
def safe_set_status(span, status)
237+
span.set_status(status)
238+
rescue StandardError => error
239+
report_instrumentation_error(error)
240+
end
241+
242+
def safe_restore_scope(scope, span)
243+
scope.set_span(span)
244+
rescue StandardError => error
245+
report_instrumentation_error(error)
246+
end
247+
248+
def safe_finish(span)
249+
span&.finish
250+
rescue StandardError => error
251+
report_instrumentation_error(error)
252+
end
253+
254+
def report_instrumentation_error(error)
255+
Sentry.configuration.sdk_logger.debug("sentry-agents instrumentation failed: #{error.class}: #{error.message}")
256+
rescue StandardError
257+
nil
258+
end
259+
260+
module Serializer
261+
module_function
262+
263+
def messages(messages)
264+
messages = Array(messages)
265+
system = messages.select { |message| value(message, :role).to_s == "system" }
266+
non_system = messages.reject { |message| value(message, :role).to_s == "system" }
267+
268+
[non_system.filter_map { |message| normalize_message(message) }, system.flat_map { |message| parts(message) }]
269+
end
270+
271+
def tool_definitions(payload)
272+
chat = payload[:chat]
273+
tools =
274+
if chat&.respond_to?(:tools)
275+
chat.tools.values
276+
else
277+
Array(payload[:tools])
278+
end
279+
280+
definitions = tools.filter_map do |tool|
281+
if tool.respond_to?(:name)
282+
{
283+
name: tool.name,
284+
description: tool.description,
285+
parameters: tool.parameters_schema
286+
}.compact
287+
elsif tool.is_a?(Hash)
288+
tool
289+
elsif tool
290+
{ name: tool.to_s }
291+
end
292+
end
293+
294+
definitions + Array(payload[:provider_tools]).select { |tool| tool.is_a?(Hash) }
295+
end
296+
297+
def normalize_message(message)
298+
role = normalize_role(value(message, :role))
299+
{ role: role, parts: parts(message) }
300+
end
301+
302+
def parts(message)
303+
role = normalize_role(value(message, :role))
304+
return [{ type: "tool_call_response", id: value(message, :tool_call_id), response: value(message, :content) }.compact] if role == "tool"
305+
306+
result = []
307+
content = value(message, :content)
308+
result << { type: "text", content: content } unless content.nil?
309+
310+
tool_calls = value(message, :tool_calls)
311+
tool_calls = tool_calls.values if tool_calls.is_a?(Hash)
312+
Array(tool_calls).each do |tool_call|
313+
result << {
314+
type: "tool_call",
315+
id: value(tool_call, :id),
316+
name: value(tool_call, :name),
317+
arguments: value(tool_call, :arguments)
318+
}.compact
319+
end
320+
result
321+
end
322+
323+
def normalize_role(role)
324+
case role.to_s
325+
when "human" then "user"
326+
when "ai" then "assistant"
327+
when "function" then "tool"
328+
else role.to_s
329+
end
330+
end
331+
332+
def value(object, key)
333+
if object.respond_to?(key)
334+
object.public_send(key)
335+
elsif object.respond_to?(:key?) && object.key?(key)
336+
object[key]
337+
elsif object.respond_to?(:key?) && object.key?(key.to_s)
338+
object[key.to_s]
339+
end
340+
end
341+
end
342+
end
343+
end
344+
end

0 commit comments

Comments
 (0)