Skip to content

Commit dba55b3

Browse files
AbhiPrasadAbhiPrasad
committed
feat: Add agentscope eval instrumentation
1 parent fd6b14b commit dba55b3

7 files changed

Lines changed: 1053 additions & 5 deletions

File tree

‎py/src/braintrust/integrations/agentscope/__init__.py‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,59 @@
11
"""Braintrust integration for AgentScope."""
22

3+
from typing import Any
4+
35
from braintrust.logger import NOOP_SPAN, current_span, init_logger
46

57
from .integration import AgentScopeIntegration
8+
from .patchers import (
9+
GeneralEvaluatorPatcher,
10+
MetricCallPatcher,
11+
RayEvaluatorRunPatcher,
12+
TaskEvaluatePatcher,
13+
)
614

715

8-
__all__ = ["AgentScopeIntegration", "setup_agentscope"]
16+
__all__ = ["AgentScopeIntegration", "setup_agentscope", "wrap_evaluator"]
917

1018

1119
def setup_agentscope(
1220
api_key: str | None = None,
1321
project_id: str | None = None,
1422
project_name: str | None = None,
23+
instrument_evals: bool = True,
1524
) -> bool:
1625
"""Setup Braintrust integration with AgentScope."""
1726
if current_span() == NOOP_SPAN:
1827
init_logger(project=project_name, api_key=api_key, project_id=project_id)
1928

20-
return AgentScopeIntegration.setup()
29+
return AgentScopeIntegration.setup(instrument_evals=instrument_evals)
30+
31+
32+
def wrap_evaluator(Evaluator: Any) -> Any:
33+
"""Manually patch an AgentScope evaluator class for tracing.
34+
35+
This helper patches the evaluator class itself and, when available, also
36+
enables task and metric tracing from the exported ``agentscope.evaluate``
37+
module so ``GeneralEvaluator`` produces nested evaluation spans even when
38+
global setup is not used.
39+
"""
40+
class_name = getattr(Evaluator, "__name__", "")
41+
if class_name == "RayEvaluator":
42+
RayEvaluatorRunPatcher.wrap_target(Evaluator)
43+
else:
44+
GeneralEvaluatorPatcher.wrap_target(Evaluator)
45+
46+
try:
47+
import agentscope.evaluate as agentscope_evaluate
48+
except ImportError:
49+
return Evaluator
50+
51+
task_cls = getattr(agentscope_evaluate, "Task", None)
52+
if task_cls is not None:
53+
TaskEvaluatePatcher.wrap_target(task_cls)
54+
55+
metric_cls = getattr(agentscope_evaluate, "MetricBase", None)
56+
if metric_cls is not None:
57+
MetricCallPatcher.wrap_target(metric_cls)
58+
59+
return Evaluator

‎py/src/braintrust/integrations/agentscope/cassettes/test_agentscope_general_evaluator_creates_eval_spans.yaml‎

Lines changed: 320 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,320 @@
1+
interactions:
2+
- request:
3+
body: '{"messages":[{"role":"system","name":"system","content":[{"type":"text","text":"You
4+
are a concise assistant. Answer in one sentence."}]},{"role":"user","name":"user","content":[{"type":"text","text":"Say
5+
hello in exactly two words."}]}],"model":"gpt-4o-mini","stream":false,"temperature":0}'
6+
headers:
7+
Accept:
8+
- application/json
9+
Accept-Encoding:
10+
- gzip, deflate
11+
Connection:
12+
- keep-alive
13+
Content-Length:
14+
- '290'
15+
Content-Type:
16+
- application/json
17+
Host:
18+
- api.openai.com
19+
User-Agent:
20+
- AsyncOpenAI/Python 2.29.0
21+
X-Stainless-Arch:
22+
- arm64
23+
X-Stainless-Async:
24+
- async:asyncio
25+
X-Stainless-Lang:
26+
- python
27+
X-Stainless-OS:
28+
- MacOS
29+
X-Stainless-Package-Version:
30+
- 2.29.0
31+
X-Stainless-Runtime:
32+
- CPython
33+
X-Stainless-Runtime-Version:
34+
- 3.13.3
35+
x-stainless-read-timeout:
36+
- '600'
37+
x-stainless-retry-count:
38+
- '0'
39+
method: POST
40+
uri: https://api.openai.com/v1/chat/completions
41+
response:
42+
body:
43+
string: !!binary |
44+
H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdfELyCzNtgAAAAD//4xSsU7DMBDd+xWV56Rq
45+
05TQGYbODCyoilz7khgc29hOFYT675ydtglQJBZLvnfv/N7zPRzNkb33u02/qXTPN48AT89bkgSG
46+
PrwC8xfWgmnkgRdaDTCzQD2EqauiyPMiK7Z3EWg1BxlotfFprtNWKJFmyyxPl0W6uj+zGy0YOGx7
47+
wet8/hnPoFNx6LG8TC6VFpyjNWDt0oRFq2WoEOqccJ4qT5IRZFp5UFH6DqTUc9+AhcW0xULVORpk
48+
qk7KCUCV0p4Gm1Hc/oycrnKkro3VB/eDSiq06ZoSU3EYET7tvDYkoic899F2980JwUGt8aXXbxCf
49+
W2fDODKGPQHPmEd9clLeJDeGlRw8FdJNUiOMsgb4yBwjph0XegLMJpZ/a7k1e7AtVP2f8SPAGBhc
50+
otJY4IJ99zu2WQib+FfbNeIomDiwR1yt0guw4Rs4VLSTw34Q9+E8tCX+VQ3WWDEsSWVKRtdQ8GJ1
51+
qMjsNPsCAAD//wMAQQCebTIDAAA=
52+
headers:
53+
CF-Cache-Status:
54+
- DYNAMIC
55+
CF-Ray:
56+
- 9e20e1608aaf6142-SJC
57+
Connection:
58+
- keep-alive
59+
Content-Encoding:
60+
- gzip
61+
Content-Type:
62+
- application/json
63+
Date:
64+
- Wed, 25 Mar 2026 21:06:37 GMT
65+
Server:
66+
- cloudflare
67+
Strict-Transport-Security:
68+
- max-age=31536000; includeSubDomains; preload
69+
Transfer-Encoding:
70+
- chunked
71+
X-Content-Type-Options:
72+
- nosniff
73+
access-control-expose-headers:
74+
- X-Request-ID
75+
alt-svc:
76+
- h3=":443"; ma=86400
77+
openai-organization:
78+
- braintrust-data
79+
openai-processing-ms:
80+
- '373'
81+
openai-project:
82+
- proj_vsCSXafhhByzWOThMrJcZiw9
83+
openai-version:
84+
- '2020-10-01'
85+
set-cookie:
86+
- __cf_bm=0r2bm90d_zmUe.y8EYZpgoGyTVS4QQSDoJVx.yDpJng-1774472796.2480545-1.0.1.1-onYisUL_Bju9EhfGXQnZBZkwk3gdjG7tHXVdr34BVePUh3JL0OqfVWApVIaF_KDBKfw4HIiGBvzONzv_AS91kbK7eL.FzFDwILNg8_F1h3hsPpZO.pIoeUN1dp_.acW6;
87+
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026
88+
21:36:37 GMT
89+
x-openai-proxy-wasm:
90+
- v0.1
91+
x-ratelimit-limit-requests:
92+
- '30000'
93+
x-ratelimit-limit-tokens:
94+
- '150000000'
95+
x-ratelimit-remaining-requests:
96+
- '29999'
97+
x-ratelimit-remaining-tokens:
98+
- '149999975'
99+
x-ratelimit-reset-requests:
100+
- 2ms
101+
x-ratelimit-reset-tokens:
102+
- 0s
103+
x-request-id:
104+
- req_0b44866bb8cd459db8712e04e4248889
105+
status:
106+
code: 200
107+
message: OK
108+
- request:
109+
body: '{"messages":[{"role":"system","name":"system","content":[{"type":"text","text":"You
110+
are a concise assistant. Answer in one sentence."}]},{"role":"user","name":"user","content":[{"type":"text","text":"Say
111+
hello in exactly two words."}]}],"model":"gpt-4o-mini","stream":false,"temperature":0}'
112+
headers:
113+
Accept:
114+
- application/json
115+
Accept-Encoding:
116+
- gzip, deflate
117+
Connection:
118+
- keep-alive
119+
Content-Length:
120+
- '290'
121+
Content-Type:
122+
- application/json
123+
Host:
124+
- api.openai.com
125+
User-Agent:
126+
- AsyncOpenAI/Python 2.29.0
127+
X-Stainless-Arch:
128+
- arm64
129+
X-Stainless-Async:
130+
- async:asyncio
131+
X-Stainless-Lang:
132+
- python
133+
X-Stainless-OS:
134+
- MacOS
135+
X-Stainless-Package-Version:
136+
- 2.29.0
137+
X-Stainless-Runtime:
138+
- CPython
139+
X-Stainless-Runtime-Version:
140+
- 3.13.3
141+
x-stainless-read-timeout:
142+
- '600'
143+
x-stainless-retry-count:
144+
- '0'
145+
method: POST
146+
uri: https://api.openai.com/v1/chat/completions
147+
response:
148+
body:
149+
string: !!binary |
150+
H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdfELyPJOswAAAAD//4xS0WrCMBR99yskz61o
151+
7aiv7mX6MJgwtsGQEpPbNjNNQpKOjeG/76ZVWzcHewnknntuzjm5t/dsebeB9Wq/qZ7WD4/L55eU
152+
piQKDL17A+ZPrAnTyAMvtOpgZoF6CFNnWZamWbJIkhaoNQcZaKXxcarjWigRJ9MkjadZPFsc2ZUW
153+
DBy2veJ1PP5qz6BTcfjA8jQ6VWpwjpaAtVMTFq2WoUKoc8J5qjyJepBp5UG10lcgpR77CixMhi0W
154+
isbRIFM1Ug4AqpT2NNhsxW2PyOEsR+rSWL1zP6ikQJuuyjEVhxHh085rQ1r0gOe2td1cOCE4qDY+
155+
93oP7XPzpBtH+rAH4BHzqE8OyjfRlWE5B0+FdIPUCKOsAt4z+4hpw4UeAKOB5d9ars3ubAtV/md8
156+
DzAGBpcoNxa4YJd++zYLYRP/ajtH3AomDuw7rlbuBdjwDRwK2shuP4j7dB7qHP+qBGus6JakMDmj
157+
c8h4NtsVZHQYfQMAAP//AwDajPLhMgMAAA==
158+
headers:
159+
CF-Cache-Status:
160+
- DYNAMIC
161+
CF-Ray:
162+
- 9e20e202698a251d-SJC
163+
Connection:
164+
- keep-alive
165+
Content-Encoding:
166+
- gzip
167+
Content-Type:
168+
- application/json
169+
Date:
170+
- Wed, 25 Mar 2026 21:07:03 GMT
171+
Server:
172+
- cloudflare
173+
Strict-Transport-Security:
174+
- max-age=31536000; includeSubDomains; preload
175+
Transfer-Encoding:
176+
- chunked
177+
X-Content-Type-Options:
178+
- nosniff
179+
access-control-expose-headers:
180+
- X-Request-ID
181+
alt-svc:
182+
- h3=":443"; ma=86400
183+
openai-organization:
184+
- braintrust-data
185+
openai-processing-ms:
186+
- '334'
187+
openai-project:
188+
- proj_vsCSXafhhByzWOThMrJcZiw9
189+
openai-version:
190+
- '2020-10-01'
191+
set-cookie:
192+
- __cf_bm=GJt8W0lyxqfgtKXEJ4M8twcG5pNqc4RmQiPqQ_IukiU-1774472822.1494222-1.0.1.1-Jv3zKpnCjFAAQQaXBEt3RElEP.QjtEFbqrvr8BASrk5X7XSiOj1UBc4tUR3t9QbKmOM0VrcVW6R3HYLaYxbTHQqz4Dpjl7Z.Sz9BslefycjBprbfLjQ1aoOYxrSO7lkO;
193+
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026
194+
21:37:03 GMT
195+
x-openai-proxy-wasm:
196+
- v0.1
197+
x-ratelimit-limit-requests:
198+
- '30000'
199+
x-ratelimit-limit-tokens:
200+
- '150000000'
201+
x-ratelimit-remaining-requests:
202+
- '29999'
203+
x-ratelimit-remaining-tokens:
204+
- '149999977'
205+
x-ratelimit-reset-requests:
206+
- 2ms
207+
x-ratelimit-reset-tokens:
208+
- 0s
209+
x-request-id:
210+
- req_8c118a37a5da44069de28fba911081e0
211+
status:
212+
code: 200
213+
message: OK
214+
- request:
215+
body: '{"messages":[{"role":"system","name":"system","content":[{"type":"text","text":"You
216+
are a concise assistant. Answer in one sentence."}]},{"role":"user","name":"user","content":[{"type":"text","text":"Say
217+
hello in exactly two words."}]}],"model":"gpt-4o-mini","stream":false,"temperature":0}'
218+
headers:
219+
Accept:
220+
- application/json
221+
Accept-Encoding:
222+
- gzip, deflate
223+
Connection:
224+
- keep-alive
225+
Content-Length:
226+
- '290'
227+
Content-Type:
228+
- application/json
229+
Host:
230+
- api.openai.com
231+
User-Agent:
232+
- AsyncOpenAI/Python 2.29.0
233+
X-Stainless-Arch:
234+
- arm64
235+
X-Stainless-Async:
236+
- async:asyncio
237+
X-Stainless-Lang:
238+
- python
239+
X-Stainless-OS:
240+
- MacOS
241+
X-Stainless-Package-Version:
242+
- 2.29.0
243+
X-Stainless-Runtime:
244+
- CPython
245+
X-Stainless-Runtime-Version:
246+
- 3.13.3
247+
x-stainless-read-timeout:
248+
- '600'
249+
x-stainless-retry-count:
250+
- '0'
251+
method: POST
252+
uri: https://api.openai.com/v1/chat/completions
253+
response:
254+
body:
255+
string: !!binary |
256+
H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdfELyDbKMgIAAAD//4xSQU7DMBC89xWVz0mV
257+
pKHhDALBDSFxQlXk2pvE4NiW7dAi1L+zTtomhSJxseSdnfXMeO+zuxuWbp9vt4/pU/WyonrHk45E
258+
gaE3b8D8kbVgGnnghVYDzCxQD2FqWhR5XmTXq1UPtJqDDLTa+DjXcSuUiLMky+OkiNPrA7vRgoHD
259+
tle8zudf/Rl0Kg47LCfRsdKCc7QGrB2bsGi1DBVCnRPOU+VJNIJMKw+ql/4AUuq5b8DCYtpioeoc
260+
DTJVJ+UEoEppT4PNXtz6gOxPcqSujdUb94NKKrTpmhJTcRgRPu28NqRH93iue9vdmROCg1rjS6/f
261+
oX9umQ3jyBj2BDxgHvXJSfkqujCs5OCpkG6SGmGUNcBH5hgx7bjQE2A2sfxby6XZg22h6v+MHwHG
262+
wOASlcYCF+zc79hmIWziX22niHvBxIH9wNUqvQAbvoFDRTs57Adxn85DW+Jf1WCNFcOSVKZkdAkF
263+
L9JNRWb72TcAAAD//wMAHqai9DIDAAA=
264+
headers:
265+
CF-Cache-Status:
266+
- DYNAMIC
267+
CF-Ray:
268+
- 9e20e3168f481d99-SJC
269+
Connection:
270+
- keep-alive
271+
Content-Encoding:
272+
- gzip
273+
Content-Type:
274+
- application/json
275+
Date:
276+
- Wed, 25 Mar 2026 21:07:46 GMT
277+
Server:
278+
- cloudflare
279+
Strict-Transport-Security:
280+
- max-age=31536000; includeSubDomains; preload
281+
Transfer-Encoding:
282+
- chunked
283+
X-Content-Type-Options:
284+
- nosniff
285+
access-control-expose-headers:
286+
- X-Request-ID
287+
alt-svc:
288+
- h3=":443"; ma=86400
289+
openai-organization:
290+
- braintrust-data
291+
openai-processing-ms:
292+
- '370'
293+
openai-project:
294+
- proj_vsCSXafhhByzWOThMrJcZiw9
295+
openai-version:
296+
- '2020-10-01'
297+
set-cookie:
298+
- __cf_bm=QmJ_mn8jkQXEiES_f8MtNfFrG.TdUhMi9F3CsYesW.Y-1774472866.3273165-1.0.1.1-9jM9NCSYxoeVuM4GWc_rXbAFSxA2USepp4rz5niKNzmDK.TtF1V7PSKWLir8akfbpuyXvva5QQRFgDCcMT3P0xa_1Tzm9mW9uidzQIoQBbq5O6t0XUQzY8bQC4ddKj1i;
299+
HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026
300+
21:37:46 GMT
301+
x-openai-proxy-wasm:
302+
- v0.1
303+
x-ratelimit-limit-requests:
304+
- '30000'
305+
x-ratelimit-limit-tokens:
306+
- '150000000'
307+
x-ratelimit-remaining-requests:
308+
- '29999'
309+
x-ratelimit-remaining-tokens:
310+
- '149999977'
311+
x-ratelimit-reset-requests:
312+
- 2ms
313+
x-ratelimit-reset-tokens:
314+
- 0s
315+
x-request-id:
316+
- req_2ac33c8b77a343ea881d0fe0fbeccc14
317+
status:
318+
code: 200
319+
message: OK
320+
version: 1

0 commit comments

Comments
 (0)