diff --git a/e2e/helpers/deno-test-helpers.ts b/e2e/helpers/deno-test-helpers.ts index f9a4e7093..57ceba901 100644 --- a/e2e/helpers/deno-test-helpers.ts +++ b/e2e/helpers/deno-test-helpers.ts @@ -20,6 +20,13 @@ export function getTestRunId(): string { } export function scopedName(base: string): string { + const configuredProjectName = Deno.env + .get("BRAINTRUST_E2E_PROJECT_NAME") + ?.trim(); + if (configuredProjectName) { + return configuredProjectName; + } + return `${base}-${getTestRunId() .toLowerCase() .replace(/[^a-z0-9-]/g, "-")}`; diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.json index e467f2a99..a037b2466 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.json @@ -186,20 +186,6 @@ "prompt_tokens": 14771, "tokens": 14883 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { @@ -534,20 +520,6 @@ "prompt_tokens": 22371, "tokens": 22567 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.txt index 3b9685df7..484f4b2a7 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-auto-hook.span-tree.txt @@ -266,62 +266,52 @@ span_tree: │ "model": "gpt-5.4-mini", │ "provider": "codex" │ } -│ ├── doGenerate [llm] -│ │ input: { -│ │ "messages": [ -│ │ { -│ │ "role": "user" -│ │ } -│ │ ], -│ │ "model": { -│ │ "modelId": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ } -│ │ output: { -│ │ "finishReason": { -│ │ "raw": "stop", -│ │ "unified": "stop" -│ │ }, -│ │ "response": { -│ │ "id": "" -│ │ }, -│ │ "usage": { -│ │ "inputTokens": { -│ │ "cacheRead": 7168, -│ │ "cacheWrite": 0, -│ │ "noCache": 7603, -│ │ "total": 14771 -│ │ }, -│ │ "outputTokens": { -│ │ "total": 112 -│ │ } -│ │ } -│ │ } -│ │ metadata: { -│ │ "braintrust": { -│ │ "integration_name": "ai-sdk", -│ │ "sdk_language": "typescript" -│ │ }, -│ │ "model": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 112, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 7168, -│ │ "prompt_tokens": 14771, -│ │ "tokens": 14883 -│ │ } -│ └── bash [tool] -│ input: "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}" +│ └── doGenerate [llm] +│ input: { +│ "messages": [ +│ { +│ "role": "user" +│ } +│ ], +│ "model": { +│ "modelId": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ } +│ output: { +│ "finishReason": { +│ "raw": "stop", +│ "unified": "stop" +│ }, +│ "response": { +│ "id": "" +│ }, +│ "usage": { +│ "inputTokens": { +│ "cacheRead": 7168, +│ "cacheWrite": 0, +│ "noCache": 7603, +│ "total": 14771 +│ }, +│ "outputTokens": { +│ "total": 112 +│ } +│ } +│ } │ metadata: { │ "braintrust": { │ "integration_name": "ai-sdk", │ "sdk_language": "typescript" │ }, -│ "toolCallId": "", -│ "toolName": "bash" +│ "model": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ metrics: { +│ "completion_tokens": 112, +│ "prompt_cache_creation_tokens": 0, +│ "prompt_cached_tokens": 7168, +│ "prompt_tokens": 14771, +│ "tokens": 14883 │ } ├── HarnessAgent.stream [task] │ input: { @@ -550,60 +540,50 @@ span_tree: "model": "gpt-5.4-mini", "provider": "codex" } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "role": "user" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ } - │ output: { - │ "finishReason": { - │ "raw": "stop", - │ "unified": "stop" - │ }, - │ "response": { - │ "id": "" - │ }, - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 14336, - │ "cacheWrite": 0, - │ "noCache": 8035, - │ "total": 22371 - │ }, - │ "outputTokens": { - │ "total": 196 - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 14336, - │ "prompt_tokens": 22371, - │ "tokens": 22567 - │ } - └── bash [tool] - input: "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}" + └── doGenerate [llm] + input: { + "messages": [ + { + "role": "user" + } + ], + "model": { + "modelId": "gpt-5.4-mini", + "provider": "codex" + } + } + output: { + "finishReason": { + "raw": "stop", + "unified": "stop" + }, + "response": { + "id": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 14336, + "cacheWrite": 0, + "noCache": 8035, + "total": 22371 + }, + "outputTokens": { + "total": 196 + } + } + } metadata: { "braintrust": { "integration_name": "ai-sdk", "sdk_language": "typescript" }, - "toolCallId": "", - "toolName": "bash" + "model": "gpt-5.4-mini", + "provider": "codex" + } + metrics: { + "completion_tokens": 196, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 14336, + "prompt_tokens": 22371, + "tokens": 22567 } diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.json index 129ea347d..b564ed9ea 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.json @@ -186,20 +186,6 @@ "prompt_tokens": 22357, "tokens": 22546 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { @@ -534,20 +520,6 @@ "prompt_tokens": 22309, "tokens": 22462 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.txt index 4cbb7c1a3..aa91d3bce 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-auto-hook.span-tree.txt @@ -266,62 +266,52 @@ span_tree: │ "model": "gpt-5.4-mini", │ "provider": "codex" │ } -│ ├── doGenerate [llm] -│ │ input: { -│ │ "messages": [ -│ │ { -│ │ "role": "user" -│ │ } -│ │ ], -│ │ "model": { -│ │ "modelId": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ } -│ │ output: { -│ │ "finishReason": { -│ │ "raw": "stop", -│ │ "unified": "stop" -│ │ }, -│ │ "response": { -│ │ "id": "" -│ │ }, -│ │ "usage": { -│ │ "inputTokens": { -│ │ "cacheRead": 14336, -│ │ "cacheWrite": 0, -│ │ "noCache": 8021, -│ │ "total": 22357 -│ │ }, -│ │ "outputTokens": { -│ │ "total": 189 -│ │ } -│ │ } -│ │ } -│ │ metadata: { -│ │ "braintrust": { -│ │ "integration_name": "ai-sdk", -│ │ "sdk_language": "typescript" -│ │ }, -│ │ "model": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 189, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 14336, -│ │ "prompt_tokens": 22357, -│ │ "tokens": 22546 -│ │ } -│ └── bash [tool] -│ input: "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}" +│ └── doGenerate [llm] +│ input: { +│ "messages": [ +│ { +│ "role": "user" +│ } +│ ], +│ "model": { +│ "modelId": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ } +│ output: { +│ "finishReason": { +│ "raw": "stop", +│ "unified": "stop" +│ }, +│ "response": { +│ "id": "" +│ }, +│ "usage": { +│ "inputTokens": { +│ "cacheRead": 14336, +│ "cacheWrite": 0, +│ "noCache": 8021, +│ "total": 22357 +│ }, +│ "outputTokens": { +│ "total": 189 +│ } +│ } +│ } │ metadata: { │ "braintrust": { │ "integration_name": "ai-sdk", │ "sdk_language": "typescript" │ }, -│ "toolCallId": "", -│ "toolName": "bash" +│ "model": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ metrics: { +│ "completion_tokens": 189, +│ "prompt_cache_creation_tokens": 0, +│ "prompt_cached_tokens": 14336, +│ "prompt_tokens": 22357, +│ "tokens": 22546 │ } ├── HarnessAgent.stream [task] │ input: { @@ -550,60 +540,50 @@ span_tree: "model": "gpt-5.4-mini", "provider": "codex" } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "role": "user" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ } - │ output: { - │ "finishReason": { - │ "raw": "stop", - │ "unified": "stop" - │ }, - │ "response": { - │ "id": "" - │ }, - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 14336, - │ "cacheWrite": 0, - │ "noCache": 7973, - │ "total": 22309 - │ }, - │ "outputTokens": { - │ "total": 153 - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ metrics: { - │ "completion_tokens": 153, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 14336, - │ "prompt_tokens": 22309, - │ "tokens": 22462 - │ } - └── bash [tool] - input: "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}" + └── doGenerate [llm] + input: { + "messages": [ + { + "role": "user" + } + ], + "model": { + "modelId": "gpt-5.4-mini", + "provider": "codex" + } + } + output: { + "finishReason": { + "raw": "stop", + "unified": "stop" + }, + "response": { + "id": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 14336, + "cacheWrite": 0, + "noCache": 7973, + "total": 22309 + }, + "outputTokens": { + "total": 153 + } + } + } metadata: { "braintrust": { "integration_name": "ai-sdk", "sdk_language": "typescript" }, - "toolCallId": "", - "toolName": "bash" + "model": "gpt-5.4-mini", + "provider": "codex" + } + metrics: { + "completion_tokens": 153, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 14336, + "prompt_tokens": 22309, + "tokens": 22462 } diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.json index 129ea347d..b564ed9ea 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.json +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.json @@ -186,20 +186,6 @@ "prompt_tokens": 22357, "tokens": 22546 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { @@ -534,20 +520,6 @@ "prompt_tokens": 22309, "tokens": 22462 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.txt index 4cbb7c1a3..aa91d3bce 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-latest-wrapped.span-tree.txt @@ -266,62 +266,52 @@ span_tree: │ "model": "gpt-5.4-mini", │ "provider": "codex" │ } -│ ├── doGenerate [llm] -│ │ input: { -│ │ "messages": [ -│ │ { -│ │ "role": "user" -│ │ } -│ │ ], -│ │ "model": { -│ │ "modelId": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ } -│ │ output: { -│ │ "finishReason": { -│ │ "raw": "stop", -│ │ "unified": "stop" -│ │ }, -│ │ "response": { -│ │ "id": "" -│ │ }, -│ │ "usage": { -│ │ "inputTokens": { -│ │ "cacheRead": 14336, -│ │ "cacheWrite": 0, -│ │ "noCache": 8021, -│ │ "total": 22357 -│ │ }, -│ │ "outputTokens": { -│ │ "total": 189 -│ │ } -│ │ } -│ │ } -│ │ metadata: { -│ │ "braintrust": { -│ │ "integration_name": "ai-sdk", -│ │ "sdk_language": "typescript" -│ │ }, -│ │ "model": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 189, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 14336, -│ │ "prompt_tokens": 22357, -│ │ "tokens": 22546 -│ │ } -│ └── bash [tool] -│ input: "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}" +│ └── doGenerate [llm] +│ input: { +│ "messages": [ +│ { +│ "role": "user" +│ } +│ ], +│ "model": { +│ "modelId": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ } +│ output: { +│ "finishReason": { +│ "raw": "stop", +│ "unified": "stop" +│ }, +│ "response": { +│ "id": "" +│ }, +│ "usage": { +│ "inputTokens": { +│ "cacheRead": 14336, +│ "cacheWrite": 0, +│ "noCache": 8021, +│ "total": 22357 +│ }, +│ "outputTokens": { +│ "total": 189 +│ } +│ } +│ } │ metadata: { │ "braintrust": { │ "integration_name": "ai-sdk", │ "sdk_language": "typescript" │ }, -│ "toolCallId": "", -│ "toolName": "bash" +│ "model": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ metrics: { +│ "completion_tokens": 189, +│ "prompt_cache_creation_tokens": 0, +│ "prompt_cached_tokens": 14336, +│ "prompt_tokens": 22357, +│ "tokens": 22546 │ } ├── HarnessAgent.stream [task] │ input: { @@ -550,60 +540,50 @@ span_tree: "model": "gpt-5.4-mini", "provider": "codex" } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "role": "user" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ } - │ output: { - │ "finishReason": { - │ "raw": "stop", - │ "unified": "stop" - │ }, - │ "response": { - │ "id": "" - │ }, - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 14336, - │ "cacheWrite": 0, - │ "noCache": 7973, - │ "total": 22309 - │ }, - │ "outputTokens": { - │ "total": 153 - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ metrics: { - │ "completion_tokens": 153, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 14336, - │ "prompt_tokens": 22309, - │ "tokens": 22462 - │ } - └── bash [tool] - input: "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}" + └── doGenerate [llm] + input: { + "messages": [ + { + "role": "user" + } + ], + "model": { + "modelId": "gpt-5.4-mini", + "provider": "codex" + } + } + output: { + "finishReason": { + "raw": "stop", + "unified": "stop" + }, + "response": { + "id": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 14336, + "cacheWrite": 0, + "noCache": 7973, + "total": 22309 + }, + "outputTokens": { + "total": 153 + } + } + } metadata: { "braintrust": { "integration_name": "ai-sdk", "sdk_language": "typescript" }, - "toolCallId": "", - "toolName": "bash" + "model": "gpt-5.4-mini", + "provider": "codex" + } + metrics: { + "completion_tokens": 153, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 14336, + "prompt_tokens": 22309, + "tokens": 22462 } diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.json index e467f2a99..a037b2466 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.json +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.json @@ -186,20 +186,6 @@ "prompt_tokens": 14771, "tokens": 14883 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { @@ -534,20 +520,6 @@ "prompt_tokens": 22371, "tokens": 22567 } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}", - "metadata": { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "toolCallId": "", - "toolName": "bash" - } } ], "input": { diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.txt index 3b9685df7..484f4b2a7 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.txt +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/__snapshots__/ai-sdk-harness-v1-wrapped.span-tree.txt @@ -266,62 +266,52 @@ span_tree: │ "model": "gpt-5.4-mini", │ "provider": "codex" │ } -│ ├── doGenerate [llm] -│ │ input: { -│ │ "messages": [ -│ │ { -│ │ "role": "user" -│ │ } -│ │ ], -│ │ "model": { -│ │ "modelId": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ } -│ │ output: { -│ │ "finishReason": { -│ │ "raw": "stop", -│ │ "unified": "stop" -│ │ }, -│ │ "response": { -│ │ "id": "" -│ │ }, -│ │ "usage": { -│ │ "inputTokens": { -│ │ "cacheRead": 7168, -│ │ "cacheWrite": 0, -│ │ "noCache": 7603, -│ │ "total": 14771 -│ │ }, -│ │ "outputTokens": { -│ │ "total": 112 -│ │ } -│ │ } -│ │ } -│ │ metadata: { -│ │ "braintrust": { -│ │ "integration_name": "ai-sdk", -│ │ "sdk_language": "typescript" -│ │ }, -│ │ "model": "gpt-5.4-mini", -│ │ "provider": "codex" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 112, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 7168, -│ │ "prompt_tokens": 14771, -│ │ "tokens": 14883 -│ │ } -│ └── bash [tool] -│ input: "{\"command\":\"/bin/bash -lc 'touch /workspace/generate-started; sleep 5; printf GENERATE_OK'\"}" +│ └── doGenerate [llm] +│ input: { +│ "messages": [ +│ { +│ "role": "user" +│ } +│ ], +│ "model": { +│ "modelId": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ } +│ output: { +│ "finishReason": { +│ "raw": "stop", +│ "unified": "stop" +│ }, +│ "response": { +│ "id": "" +│ }, +│ "usage": { +│ "inputTokens": { +│ "cacheRead": 7168, +│ "cacheWrite": 0, +│ "noCache": 7603, +│ "total": 14771 +│ }, +│ "outputTokens": { +│ "total": 112 +│ } +│ } +│ } │ metadata: { │ "braintrust": { │ "integration_name": "ai-sdk", │ "sdk_language": "typescript" │ }, -│ "toolCallId": "", -│ "toolName": "bash" +│ "model": "gpt-5.4-mini", +│ "provider": "codex" +│ } +│ metrics: { +│ "completion_tokens": 112, +│ "prompt_cache_creation_tokens": 0, +│ "prompt_cached_tokens": 7168, +│ "prompt_tokens": 14771, +│ "tokens": 14883 │ } ├── HarnessAgent.stream [task] │ input: { @@ -550,60 +540,50 @@ span_tree: "model": "gpt-5.4-mini", "provider": "codex" } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "role": "user" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ } - │ output: { - │ "finishReason": { - │ "raw": "stop", - │ "unified": "stop" - │ }, - │ "response": { - │ "id": "" - │ }, - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 14336, - │ "cacheWrite": 0, - │ "noCache": 8035, - │ "total": 22371 - │ }, - │ "outputTokens": { - │ "total": 196 - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-5.4-mini", - │ "provider": "codex" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 14336, - │ "prompt_tokens": 22371, - │ "tokens": 22567 - │ } - └── bash [tool] - input: "{\"command\":\"/bin/bash -lc 'touch /workspace/stream-started; sleep 5; printf STREAM_OK'\"}" + └── doGenerate [llm] + input: { + "messages": [ + { + "role": "user" + } + ], + "model": { + "modelId": "gpt-5.4-mini", + "provider": "codex" + } + } + output: { + "finishReason": { + "raw": "stop", + "unified": "stop" + }, + "response": { + "id": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 14336, + "cacheWrite": 0, + "noCache": 8035, + "total": 22371 + }, + "outputTokens": { + "total": 196 + } + } + } metadata: { "braintrust": { "integration_name": "ai-sdk", "sdk_language": "typescript" }, - "toolCallId": "", - "toolName": "bash" + "model": "gpt-5.4-mini", + "provider": "codex" + } + metrics: { + "completion_tokens": 196, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 14336, + "prompt_tokens": 22371, + "tokens": 22567 } diff --git a/e2e/scenarios/ai-sdk-harness-instrumentation/scenario.test.ts b/e2e/scenarios/ai-sdk-harness-instrumentation/scenario.test.ts index 094c88987..4b51cb2f6 100644 --- a/e2e/scenarios/ai-sdk-harness-instrumentation/scenario.test.ts +++ b/e2e/scenarios/ai-sdk-harness-instrumentation/scenario.test.ts @@ -129,8 +129,14 @@ describe.sequential("HarnessAgent instrumentation variants", () => { ); expect(findAllSpans(events, "bash").length).toBeGreaterThan(0); + // Suspension can attach the bash tool span to either side of the + // continued turn. Its presence is asserted above; omit its + // placement from the stable tree contract. + const snapshotEvents = events.filter( + (event) => event.span.name !== "bash", + ); await matchSpanTreeSnapshot( - events, + snapshotEvents, resolveFileSnapshotPath( import.meta.url, `${scenario.variantKey}-${mode}.span-tree.json`, diff --git a/e2e/scenarios/deno-browser/scenario.test.ts b/e2e/scenarios/deno-browser/scenario.test.ts index 291ffbf8b..46fe46f7d 100644 --- a/e2e/scenarios/deno-browser/scenario.test.ts +++ b/e2e/scenarios/deno-browser/scenario.test.ts @@ -19,6 +19,8 @@ const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const CONFIGURED_PROJECT_NAME = + process.env.BRAINTRUST_E2E_PROJECT_NAME || "sdk-e2e-tests"; function findEventByCase(events: CapturedLogEvent[], testCase: string) { return events.find((event) => { @@ -44,6 +46,7 @@ test( const cursor = requestCursor(); await runDenoScenarioDir({ + env: { BRAINTRUST_E2E_PROJECT_NAME: CONFIGURED_PROJECT_NAME }, scenarioDir, timeoutMs: TIMEOUT_MS, }); diff --git a/e2e/scenarios/deno-node/scenario.test.ts b/e2e/scenarios/deno-node/scenario.test.ts index e588b6c52..9fa8b34f2 100644 --- a/e2e/scenarios/deno-node/scenario.test.ts +++ b/e2e/scenarios/deno-node/scenario.test.ts @@ -19,6 +19,8 @@ const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const CONFIGURED_PROJECT_NAME = + process.env.BRAINTRUST_E2E_PROJECT_NAME || "sdk-e2e-tests"; function findEventByCase(events: CapturedLogEvent[], testCase: string) { return events.find((event) => { @@ -44,6 +46,7 @@ test( const cursor = requestCursor(); await runDenoScenarioDir({ + env: { BRAINTRUST_E2E_PROJECT_NAME: CONFIGURED_PROJECT_NAME }, scenarioDir, timeoutMs: TIMEOUT_MS, }); diff --git a/e2e/scenarios/test-framework-evals-jest/runner.case.cjs b/e2e/scenarios/test-framework-evals-jest/runner.case.cjs index 030bb75e6..cffa7756d 100644 --- a/e2e/scenarios/test-framework-evals-jest/runner.case.cjs +++ b/e2e/scenarios/test-framework-evals-jest/runner.case.cjs @@ -13,6 +13,11 @@ const testRunId = process.env.BRAINTRUST_E2E_RUN_ID; const scenario = "test-framework-evals-jest"; function scopedName(base) { + const configuredProjectName = process.env.BRAINTRUST_E2E_PROJECT_NAME?.trim(); + if (configuredProjectName) { + return configuredProjectName; + } + return `${base}-${testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-")}`; } diff --git a/e2e/scenarios/test-framework-evals-jest/scenario.test.ts b/e2e/scenarios/test-framework-evals-jest/scenario.test.ts index c1c397d10..6e0fd08e3 100644 --- a/e2e/scenarios/test-framework-evals-jest/scenario.test.ts +++ b/e2e/scenarios/test-framework-evals-jest/scenario.test.ts @@ -19,6 +19,8 @@ const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const CONFIGURED_PROJECT_NAME = + process.env.BRAINTRUST_E2E_PROJECT_NAME || "sdk-e2e-tests"; function findEventByCase(events: CapturedLogEvent[], testCase: string) { return events.find((event) => { @@ -44,6 +46,7 @@ test( const cursor = requestCursor(); await runNodeScenarioDir({ + env: { BRAINTRUST_E2E_PROJECT_NAME: CONFIGURED_PROJECT_NAME }, scenarioDir, timeoutMs: TIMEOUT_MS, }); diff --git a/e2e/scenarios/test-framework-evals-node/runner.case.mjs b/e2e/scenarios/test-framework-evals-node/runner.case.mjs index 42ef3c74d..a5345e5c4 100644 --- a/e2e/scenarios/test-framework-evals-node/runner.case.mjs +++ b/e2e/scenarios/test-framework-evals-node/runner.case.mjs @@ -3,8 +3,14 @@ import { currentSpan, initNodeTestSuite, login } from "braintrust/node"; const testRunId = process.env.BRAINTRUST_E2E_RUN_ID; const scenario = "test-framework-evals-node"; -const scopedName = (base) => - `${base}-${testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-")}`; +const scopedName = (base) => { + const configuredProjectName = process.env.BRAINTRUST_E2E_PROJECT_NAME?.trim(); + if (configuredProjectName) { + return configuredProjectName; + } + + return `${base}-${testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-")}`; +}; await login({ apiKey: process.env.BRAINTRUST_API_KEY, diff --git a/e2e/scenarios/test-framework-evals-node/scenario.test.ts b/e2e/scenarios/test-framework-evals-node/scenario.test.ts index 964f9076f..61f1ff244 100644 --- a/e2e/scenarios/test-framework-evals-node/scenario.test.ts +++ b/e2e/scenarios/test-framework-evals-node/scenario.test.ts @@ -12,6 +12,8 @@ const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const CONFIGURED_PROJECT_NAME = + process.env.BRAINTRUST_E2E_PROJECT_NAME || "sdk-e2e-tests"; const spanTreeSnapshotPath = resolveFileSnapshotPath( import.meta.url, "span-tree.json", @@ -25,7 +27,11 @@ test( async () => { await withScenarioHarness( async ({ runScenarioDir, testRunEvents, testRunId }) => { - await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS }); + await runScenarioDir({ + env: { BRAINTRUST_E2E_PROJECT_NAME: CONFIGURED_PROJECT_NAME }, + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); const capturedEvents = testRunEvents(); const basicEval = findLatestSpan(