From cbb73ec15994a2bf07bec5618b69a3547d5fdbfd Mon Sep 17 00:00:00 2001 From: Neel Shah Date: Wed, 4 Nov 2026 17:30:47 +0100 Subject: [PATCH] feat(agents): Instrument RubyLLM agent runs Co-Authored-By: OpenAI --- sentry-agents/Gemfile | 2 + sentry-agents/README.md | 4 + sentry-agents/lib/sentry-agents.rb | 2 + .../lib/sentry/agents/ruby_llm/agent_runs.rb | 194 ++++++++++++++++++ .../sentry/agents/ruby_llm/instrumenter.rb | 8 + .../sentry/agents/ruby_llm/agent_runs_spec.rb | 107 ++++++++++ 6 files changed, 317 insertions(+) create mode 100644 sentry-agents/lib/sentry/agents/ruby_llm/agent_runs.rb create mode 100644 sentry-agents/spec/sentry/agents/ruby_llm/agent_runs_spec.rb diff --git a/sentry-agents/Gemfile b/sentry-agents/Gemfile index ae888581a..65a0fd599 100644 --- a/sentry-agents/Gemfile +++ b/sentry-agents/Gemfile @@ -1,3 +1,5 @@ +# frozen_string_literal: true + source "https://rubygems.org" git_source(:github) { |name| "https://github.com/#{name}.git" } diff --git a/sentry-agents/README.md b/sentry-agents/README.md index 5b101e6f7..c16011f8b 100644 --- a/sentry-agents/README.md +++ b/sentry-agents/README.md @@ -24,6 +24,10 @@ Add the gem to your bundle: gem "sentry-agents" ``` +## Agent runs + +`RubyLLM::Agent` runs create `gen_ai.invoke_agent` spans. Nested agent runs are preserved, while calls on the same agent chat are grouped into one run. + ## Workflows `RubyLLM.workflow` and its steps create pipeline grouping spans. Chat and tool descendants carry `gen_ai.pipeline.name` without being labeled as agent runs. diff --git a/sentry-agents/lib/sentry-agents.rb b/sentry-agents/lib/sentry-agents.rb index 09874a279..3902b51ac 100644 --- a/sentry-agents/lib/sentry-agents.rb +++ b/sentry-agents/lib/sentry-agents.rb @@ -6,6 +6,7 @@ require "sentry/agents/version" require "sentry/agents/ruby_llm/instrumenter" require "sentry/agents/ruby_llm/tool_concurrency" +require "sentry/agents/ruby_llm/agent_runs" module Sentry module Agents @@ -16,6 +17,7 @@ module Agents module RubyLLM def self.install(ruby_llm_config = ::RubyLLM.config) config = ruby_llm_config.respond_to?(:config) ? ruby_llm_config.config : ruby_llm_config + AgentRuns.install ToolConcurrency.install Instrumenter.install(config) end diff --git a/sentry-agents/lib/sentry/agents/ruby_llm/agent_runs.rb b/sentry-agents/lib/sentry/agents/ruby_llm/agent_runs.rb new file mode 100644 index 000000000..0e78b2d5b --- /dev/null +++ b/sentry-agents/lib/sentry/agents/ruby_llm/agent_runs.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +require "ruby_llm/agent" +require "ruby_llm/chat" + +module Sentry + module Agents + module RubyLLM + module AgentRuns + AGENT_NAME_IVAR = :@sentry_agents_agent_name + DEPTH_IVAR = :@sentry_agents_agent_run_depth + OPERATION = "gen_ai.invoke_agent" + ORIGIN = "auto.ai.ruby_llm" + + module AgentIdentityPatch + def apply_configuration(chat, input_values:, persist_instructions:, **options) + result = super(chat, input_values: input_values, persist_instructions: persist_instructions, **options) + chat.instance_variable_set(AGENT_NAME_IVAR, name.to_s) + result + end + end + + module ChatPatch + def complete(*args, **kwargs, &block) + AgentRuns.with(self) { super(*args, **kwargs, &block) } + end + + def generate(*args, **kwargs, &block) + AgentRuns.with(self) { super(*args, **kwargs, &block) } + end + + def run_tools(*args, **kwargs, &block) + AgentRuns.with(self) { super(*args, **kwargs, &block) } + end + + def step(*args, **kwargs, &block) + AgentRuns.with(self) { super(*args, **kwargs, &block) } + end + end + + class << self + def install + install_agent_identity + install_chat_patch + end + + def with(chat) + agent_name = chat.instance_variable_get(AGENT_NAME_IVAR) + return yield if agent_name.nil? || agent_name.empty? + + depth = chat.instance_variable_get(DEPTH_IVAR).to_i + return yield if depth.positive? + + chat.instance_variable_set(DEPTH_IVAR, depth + 1) + with_span(chat, agent_name) { yield } + ensure + if depth&.positive? + depth.zero? ? chat.remove_instance_variable(DEPTH_IVAR) : chat.instance_variable_set(DEPTH_IVAR, depth) + end + end + + private + + def install_agent_identity + target = ::RubyLLM::Agent.singleton_class + target.prepend(AgentIdentityPatch) unless target.ancestors.include?(AgentIdentityPatch) + end + + def install_chat_patch + target = ::RubyLLM::Chat + target.prepend(ChatPatch) unless target.ancestors.include?(ChatPatch) + end + + def with_span(chat, agent_name) + return yield unless Sentry.initialized? + + scope = Sentry.get_current_scope + current_span = scope&.get_span + if current_span + with_child_span(scope, current_span, chat, agent_name) { yield } + else + with_root_transaction(chat, agent_name) { yield } + end + end + + def with_child_span(scope, current_span, chat, agent_name) + span = nil + begin + span = current_span.start_child(op: OPERATION, description: description(agent_name), origin: ORIGIN) + scope.set_span(span) + rescue StandardError => error + safe_finish(span) + report(error) + return yield + end + + execute_span(span, scope: scope, previous_span: current_span, chat: chat, agent_name: agent_name) { yield } + end + + def with_root_transaction(chat, agent_name) + transaction = begin + Sentry.start_transaction(name: description(agent_name), op: OPERATION, origin: ORIGIN) + rescue StandardError => error + report(error) + nil + end + return yield unless transaction + + application_called = false + begin + Sentry.with_scope do |scope| + scope.set_span(transaction) + execute_span(transaction, chat: chat, agent_name: agent_name) do + application_called = true + yield + end + end + rescue StandardError => error + raise if application_called + + safe_finish(transaction) + report(error) + yield + end + end + + def execute_span(span, chat:, agent_name:, scope: nil, previous_span: nil) + result = nil + begin + set_data(span, chat, agent_name) + result = yield + rescue StandardError => error + set_error(span, error) + raise + ensure + safe_restore_scope(scope, previous_span) if scope && previous_span + safe_finish(span) + end + result + end + + def set_data(span, chat, agent_name) + span.set_data("gen_ai.operation.name", "invoke_agent") + span.set_data("gen_ai.agent.name", agent_name) + workflow = ::RubyLLM::Support::Instrumentation.current_workflow + pipeline = workflow && workflow[:workflow_name] + span.set_data("gen_ai.pipeline.name", pipeline) if pipeline + rescue StandardError => error + report(error) + end + + def set_error(span, error) + safe_set_data(span, "error.type", error.class.name) + safe_set_status(span, "internal_error") + end + + def safe_set_data(span, key, value) + span.set_data(key, value) + rescue StandardError => error + report(error) + end + + def safe_set_status(span, status) + span.set_status(status) + rescue StandardError => error + report(error) + end + + def safe_restore_scope(scope, span) + scope.set_span(span) + rescue StandardError => error + report(error) + end + + def safe_finish(span) + span&.finish + rescue StandardError => error + report(error) + end + + def report(error) + Sentry.configuration.sdk_logger.debug("sentry-agents instrumentation failed: #{error.class}: #{error.message}") + rescue StandardError + nil + end + + def description(agent_name) + "invoke_agent #{agent_name}" + end + end + end + end + end +end diff --git a/sentry-agents/lib/sentry/agents/ruby_llm/instrumenter.rb b/sentry-agents/lib/sentry/agents/ruby_llm/instrumenter.rb index cec927525..0fc50c449 100644 --- a/sentry-agents/lib/sentry/agents/ruby_llm/instrumenter.rb +++ b/sentry-agents/lib/sentry/agents/ruby_llm/instrumenter.rb @@ -154,6 +154,7 @@ def set_request_data(span, payload, operation) def set_chat_request_data(span, payload) safe_set_data(span, "gen_ai.operation.name", "chat") set_pipeline_data(span, payload) + set_agent_data(span, payload) set_data_if_present(span, "gen_ai.provider.name", string_value(payload[:provider])) set_data_if_present(span, "gen_ai.request.model", string_value(payload[:model])) set_data_if_present(span, "gen_ai.request.temperature", payload[:temperature]) @@ -171,6 +172,7 @@ def set_chat_request_data(span, payload) def set_tool_request_data(span, payload) safe_set_data(span, "gen_ai.operation.name", "execute_tool") set_pipeline_data(span, payload) + set_agent_data(span, payload) set_data_if_present(span, "gen_ai.tool.name", string_value(payload[:tool_name] || Serializer.value(payload[:tool], :name))) return unless collect_inputs? @@ -189,6 +191,12 @@ def set_pipeline_data(span, payload) set_data_if_present(span, "gen_ai.pipeline.name", string_value(payload[:workflow_name])) end + def set_agent_data(span, payload) + chat = payload[:chat] + agent_name = chat&.instance_variable_get(:@sentry_agents_agent_name) + set_data_if_present(span, "gen_ai.agent.name", string_value(agent_name)) + end + def set_response_data(span, payload, operation) case operation when OPERATION diff --git a/sentry-agents/spec/sentry/agents/ruby_llm/agent_runs_spec.rb b/sentry-agents/spec/sentry/agents/ruby_llm/agent_runs_spec.rb new file mode 100644 index 000000000..a87ffb328 --- /dev/null +++ b/sentry-agents/spec/sentry/agents/ruby_llm/agent_runs_spec.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Sentry::Agents::RubyLLM::AgentRuns do + before do + RubyLLM.config.instrumenter = nil + RubyLLM.configure { |config| config.openai_api_key = "test" } + Sentry.init do |config| + config.dsn = Sentry::TestHelper::DUMMY_DSN + config.sdk_logger = Logger.new(nil) + config.background_worker_threads = 0 + config.transport.transport_class = Sentry::DummyTransport + config.traces_sample_rate = 1.0 + config.data_collection.gen_ai.inputs = true + config.data_collection.gen_ai.outputs = true + end + end + + after do + RubyLLM.config.instrumenter = nil + end + + it "creates one invoke_agent span with model and tool descendants" do + agent_class = agent_class("SupportAgent") + chat = agent_class.chat + adapter = Sentry::Agents::RubyLLM::Instrumenter.new(nil) + + result = described_class.with(chat) do + adapter.instrument("chat.ruby_llm", chat_payload(chat)) { :model_result } + adapter.instrument("tool_call.ruby_llm", tool_payload(chat)) { :tool_result } + :agent_result + end + + expect(result).to eq(:agent_result) + event = transaction_event + root_trace = event.contexts[:trace] + spans = event.spans + + expect(root_trace[:op]).to eq("gen_ai.invoke_agent") + expect(root_trace[:data]["gen_ai.agent.name"]).to eq("SupportAgent") + expect(spans.map { |span| span[:op] }).to contain_exactly("gen_ai.chat", "gen_ai.execute_tool") + expect(spans.map { |span| span[:data]["gen_ai.agent.name"] }).to all(eq("SupportAgent")) + expect(spans.map { |span| span[:parent_span_id] }).to all(eq(root_trace[:span_id])) + end + + it "suppresses duplicate wrappers for the same chat and keeps nested agents" do + outer_class = agent_class("OuterAgent") + inner_class = agent_class("InnerAgent") + outer_chat = outer_class.chat + inner_chat = inner_class.chat + + described_class.with(outer_chat) do + described_class.with(outer_chat) { :same_chat } + described_class.with(inner_chat) { :nested_chat } + end + + event = transaction_event + root_trace = event.contexts[:trace] + invoke_spans = event.spans.select { |span| span[:op] == "gen_ai.invoke_agent" } + + expect(invoke_spans.size).to eq(1) + expect(invoke_spans.first[:data]["gen_ai.agent.name"]).to eq("InnerAgent") + expect(invoke_spans.first[:parent_span_id]).to eq(root_trace[:span_id]) + end + + it "finishes the invoke_agent span when the operation raises" do + chat = agent_class("FailingAgent").chat + + expect do + described_class.with(chat) { raise "agent failure" } + end.to raise_error(RuntimeError, "agent failure") + + event = transaction_event + trace = event.contexts[:trace] + expect(trace[:status]).to eq("internal_error") + expect(trace[:data]["error.type"]).to eq("RuntimeError") + end + + def agent_class(name) + stub_const(name, Class.new(RubyLLM::Agent)).tap do |klass| + klass.model "gpt-4o" + end + end + + def chat_payload(chat) + { + chat: chat, + provider: "openai", + model: "gpt-4o", + input_messages: [] + } + end + + def tool_payload(chat) + { + chat: chat, + tool_name: "weather", + tool_arguments: { city: "Paris" } + } + end + + def transaction_event + Sentry.get_current_client.flush + sentry_events.find { |event| event.is_a?(Sentry::TransactionEvent) } + end +end