diff --git a/typescript/openai-agents/src/__tests__/stubs/openai-agents-fakes.ts b/typescript/openai-agents/src/__tests__/stubs/openai-agents-fakes.ts index c72420d8..bf3d8e15 100644 --- a/typescript/openai-agents/src/__tests__/stubs/openai-agents-fakes.ts +++ b/typescript/openai-agents/src/__tests__/stubs/openai-agents-fakes.ts @@ -140,13 +140,17 @@ export function textResponse(text: string): ModelResponse { return { output, usage: fakeUsage(text.length) }; } -export function toolCallResponse(toolName: string, args: Record): ModelResponse { +export function toolCallResponse( + toolName: string, + args: Record, + callId = `call_fake_${toolName}` +): ModelResponse { const output: AgentOutputItem[] = [ { type: 'function_call', name: toolName, arguments: JSON.stringify(args), - callId: `call_fake_${toolName}`, + callId, status: 'completed', }, ]; @@ -157,12 +161,14 @@ export function handoffResponse(handoffToolName: string, args: Record }>): ModelResponse { +export function multiToolCallResponse( + calls: Array<{ name: string; args: Record; callId?: string }> +): ModelResponse { const output: AgentOutputItem[] = calls.map((c) => ({ type: 'function_call' as const, name: c.name, arguments: JSON.stringify(c.args), - callId: `call_fake_${c.name}`, + callId: c.callId ?? `call_fake_${c.name}`, status: 'completed' as const, })); return { output, usage: fakeUsage(20) }; diff --git a/typescript/openai-agents/src/__tests__/stubs/private-env-value.ts b/typescript/openai-agents/src/__tests__/stubs/private-env-value.ts new file mode 100644 index 00000000..14b9aae1 --- /dev/null +++ b/typescript/openai-agents/src/__tests__/stubs/private-env-value.ts @@ -0,0 +1,22 @@ +import { EnvValueReference, registerEnvValueReference } from '@openai/agents-core/sandbox'; + +/** A Manifest environment reference that keeps its value in a private field, which only `serialize` exposes. */ +export class PrivateEnvValue extends EnvValueReference { + static readonly type = 'test.private-env-value'; + readonly #value: string; + + constructor(value: string) { + super(); + this.#value = value; + } + + serialize(): Record { + return { secret: this.#value }; + } + + async resolve(): Promise { + return this.#value; + } +} + +registerEnvValueReference(PrivateEnvValue, (payload) => new PrivateEnvValue(String(payload.secret))); diff --git a/typescript/openai-agents/src/__tests__/stubs/sandbox-fakes.ts b/typescript/openai-agents/src/__tests__/stubs/sandbox-fakes.ts index 193d6636..5c889866 100644 --- a/typescript/openai-agents/src/__tests__/stubs/sandbox-fakes.ts +++ b/typescript/openai-agents/src/__tests__/stubs/sandbox-fakes.ts @@ -18,6 +18,9 @@ import { } from '@openai/agents-core/sandbox'; import { deserializeManifest, mergeManifestDelta, mergeManifestEntryDelta } from '@openai/agents-core/sandbox/internal'; +// Re-exported so the Worker can deserialize this reference in sandbox Activities. +export { PrivateEnvValue } from './private-env-value'; + export class FakeSandboxSession implements SandboxSession { state: SandboxSessionState; execCalls: ExecCommandArgs[] = []; diff --git a/typescript/openai-agents/src/__tests__/test-openai-agents.ts b/typescript/openai-agents/src/__tests__/test-openai-agents.ts index cf660dee..f299c508 100644 --- a/typescript/openai-agents/src/__tests__/test-openai-agents.ts +++ b/typescript/openai-agents/src/__tests__/test-openai-agents.ts @@ -13,6 +13,7 @@ import { StatelessMCPServerProvider, } from '..'; import { + approvalResumeContextWorkflow, localActivityAgentWorkflow, mcpFactoryArgWorkflow, summaryOverrideStringWorkflow, @@ -25,7 +26,13 @@ import { streamingNoTopicWorkflow, } from './workflows/openai-agents'; import { makeTestFunction } from './helpers/test-fn'; -import { FakeModelProvider, textResponse, toolCallResponse } from './stubs/openai-agents'; +import { + FakeModelProvider, + handoffResponse, + multiToolCallResponse, + textResponse, + toolCallResponse, +} from './stubs/openai-agents'; import { StreamingFakeModelProvider, streamingTextEvents } from './stubs/openai-agents-fakes'; import EventType = temporal.api.enums.v1.EventType; @@ -524,3 +531,186 @@ test('Streaming: useLocalActivity fails fast before scheduling an Activity', asy t.is(result, 'StreamingLocalActivityUnsupported'); }); }); + +test('Approval resume keeps the caller context object', async (t) => { + const { createWorker, executeWorkflow } = helpers(t); + const worker = await createWorker({ + plugins: [ + new OpenAIAgentsPlugin({ + modelProvider: new FakeModelProvider([ + toolCallResponse('set_after', { value: 'S-1' }, 'call_set_after_1'), + toolCallResponse('set_after', { value: 'S-2' }, 'call_set_after_2'), + toolCallResponse('set_after', { value: 'S-after' }, 'call_set_after_3'), + textResponse('done'), + toolCallResponse('set_after', { value: 'A' }, 'call_shared_a'), + toolCallResponse('set_after', { value: 'B' }, 'call_shared_b'), + toolCallResponse('set_after', { value: 'C' }, 'call_shared_c'), + textResponse('done A'), + textResponse('done B'), + textResponse('done C'), + toolCallResponse('flaky', { value: 'R' }, 'call_flaky'), + textResponse('done R'), + toolCallResponse('gated', { value: 'D' }, 'call_gated_d'), + toolCallResponse('gated', { value: 'E1' }, 'call_gated_e1'), + toolCallResponse('gated', { value: 'E2' }, 'call_gated_e2'), + textResponse('done E'), + toolCallResponse('set_after', { value: 'R-1' }, 'call_rehydrated_1'), + toolCallResponse('set_after', { value: 'R-2' }, 'call_rehydrated_2'), + toolCallResponse('set_after', { value: 'R-3' }, 'call_rehydrated_3'), + textResponse('done rehydrated'), + toolCallResponse('set_after', { value: 'H-1' }, 'call_route_1'), + handoffResponse('transfer_to_FirstTarget'), + textResponse('done routed'), + toolCallResponse('set_after', { value: 'H-2' }, 'call_route_2'), + handoffResponse('transfer_to_FirstTarget'), + toolCallResponse('extend', {}, 'call_extend'), + toolCallResponse('extra', { value: 'X' }, 'call_extra'), + textResponse('done extended'), + handoffResponse('transfer_to_FailChild'), + toolCallResponse('set_after', { value: 'F' }, 'call_fail_child'), + textResponse('done failed conversion'), + toolCallResponse('set_after', { value: 'V' }, 'call_versioned'), + textResponse('done versioned'), + handoffResponse('transfer_to_CustomChild'), + toolCallResponse('set_after', { value: 'C-1' }, 'call_custom_child'), + textResponse('done custom child'), + handoffResponse('transfer_to_ReplacedChild'), + toolCallResponse('set_after', { value: 'C-2' }, 'call_replaced_child'), + textResponse('done replaced child'), + handoffResponse('transfer_to_RestoreChild'), + toolCallResponse('set_after', { value: 'D-1' }, 'call_restore_child'), + textResponse('done restore child'), + toolCallResponse('set_after', { value: 'W' }, 'call_registered'), + textResponse('done registered'), + handoffResponse('transfer_to_RejectCustomChild'), + toolCallResponse('rejectable', { value: 'J-1' }, 'call_reject_custom_child'), + textResponse('done rejected custom child'), + handoffResponse('transfer_to_RejectChild'), + toolCallResponse('rejectable', { value: 'J-2' }, 'call_reject_child'), + textResponse('done rejected child'), + handoffResponse('transfer_to_AlwaysChild'), + toolCallResponse('rejectable', { value: 'J-3' }, 'call_always_child'), + textResponse('done always child'), + toolCallResponse('rejectable', { value: 'J-4' }, 'call_persisted'), + textResponse('done persisted'), + toolCallResponse('guarded', { value: 'G' }, 'call_guarded'), + textResponse('done guarded'), + toolCallResponse('vetoed', { value: 'V-1' }, 'call_veto_persisted'), + textResponse('done veto persisted'), + toolCallResponse('vetoed', { value: 'V-2' }, 'call_veto_persisted_always'), + textResponse('done veto persisted always'), + toolCallResponse('vetoed', { value: 'V-3' }, 'call_veto_direct'), + textResponse('done veto direct'), + multiToolCallResponse([ + { name: 'vetoed', args: { value: 'K-A' }, callId: 'call_batch_persisted_a' }, + { name: 'vetoed', args: { value: 'K-B' }, callId: 'call_batch_persisted_b' }, + ]), + textResponse('done batch persisted'), + multiToolCallResponse([ + { name: 'vetoed', args: { value: 'L-A' }, callId: 'call_batch_direct_a' }, + { name: 'vetoed', args: { value: 'L-B' }, callId: 'call_batch_direct_b' }, + ]), + textResponse('done batch direct'), + toolCallResponse('same_name', { value: 'N-0' }, 'call_same_direct_0'), + handoffResponse('transfer_to_Same'), + toolCallResponse('same_name', { value: 'N-1' }, 'call_same_direct_1'), + toolCallResponse('same_name', { value: 'N-2' }, 'call_same_direct_2'), + toolCallResponse('same_name', { value: 'N-3' }, 'call_same_direct_3'), + textResponse('done same name direct'), + toolCallResponse('same_name', { value: 'M-0' }, 'call_same_rehydrated_0'), + handoffResponse('transfer_to_Same'), + toolCallResponse('same_name', { value: 'M-1' }, 'call_same_rehydrated_1'), + toolCallResponse('same_name', { value: 'M-2' }, 'call_same_rehydrated_2'), + toolCallResponse('same_name', { value: 'M-3' }, 'call_same_rehydrated_3'), + textResponse('done same name rehydrated'), + toolCallResponse('long_history', { value: 'H' }, 'call_long_history'), + textResponse('done long history'), + ]), + }), + ], + }); + + const result = await worker.runUntil( + executeWorkflow(approvalResumeContextWorkflow, { workflowExecutionTimeout: '30 seconds' }) + ); + + // Each resume converts into the agent objects its state runs on, so the approvals keep one owner, and + // no resume drops approvals that another state on the context, or a retry of the same state, needs. + t.deepEqual(result, { + callerAfter: 'S-after', + resumedUsesCallerObject: true, + writes: ['S-1', 'S-2', 'S-after'], + approvalRounds: 3, + failedResumeThrew: true, + approvalOwners: 1, + approvedCallIds: 3, + sharedContextWrites: ['A', 'B', 'C'], + sharedContextInterruptions: 0, + retryThrew: true, + retryAttempts: 2, + retryWrites: ['R'], + retryInterruptions: 0, + concurrentFailedResumeThrew: true, + concurrentWrites: ['E1', 'E2'], + concurrentInterruptions: 0, + rehydratedWrites: ['R-1', 'R-2', 'R-3'], + rehydratedApprovalOwners: 2, + rehydratedApprovedCallIds: 6, + handoffTarget: 'SecondTarget', + handoffFilters: ['second'], + disabledHandoffThrew: true, + sharedFieldsWrites: ['X'], + sharedFieldsTemperatures: [0.1, 0.9, 0.9], + failedConversionThrew: true, + failedConversionWrites: ['F'], + failedConversionInterruptions: 0, + versions: ['before', 'after'], + lastAgentVersion: 'after', + customChildWrites: ['C-1'], + customChildInterruptions: 0, + customChildSavedApprovalOwners: 1, + replacedChildWrites: ['C-2'], + replacedChildInterruptions: 0, + failedRestoreThrew: true, + failedRestoreWrites: ['D-1'], + failedRestoreInterruptions: 0, + registeredVersions: ['before', 'after'], + lastAgentRegisteredVersion: 'after', + rejectedResumesThrew: [true, true, true, true], + rejectedExecutions: ['J-1', 'J-2', 'J-3', 'J-4'], + rejectedInterruptions: 0, + failedRestores: 3, + failedRestoreApprovalOwners: 2, + failedRestoreApprovedCallIds: 2, + recoveredWrites: ['G'], + recoveredApprovalOwners: 2, + recoveredApprovedCallIds: 2, + rejectionMessages: [ + 'VetoPersisted before', + 'VetoPersisted after', + 'VetoPersistedAlways before', + 'VetoPersistedAlways after', + 'VetoDirect before', + 'VetoDirect after', + ], + revisedRejectionWrites: [], + revisedRejectionInterruptions: 0, + batchMessages: [ + 'call_batch_persisted_a: ok', + 'call_batch_persisted_b: BatchPersisted before', + 'call_batch_persisted_a: BatchPersisted after', + 'call_batch_persisted_b: BatchPersisted after', + 'call_batch_direct_a: ok', + 'call_batch_direct_b: BatchDirect before', + 'call_batch_direct_a: BatchDirect after', + 'call_batch_direct_b: BatchDirect after', + ], + batchInterruptions: 0, + sameNameWrites: ['N-0', 'M-0'], + sameNameApprovalOwners: [2, 4], + longHistoryThrew: true, + longHistoryWrites: ['H'], + longHistoryInterruptions: 0, + longHistoryApprovedCallIds: 100_002, + }); +}); diff --git a/typescript/openai-agents/src/__tests__/test-openai-sandbox.ts b/typescript/openai-agents/src/__tests__/test-openai-sandbox.ts index 4dcab3fd..cd447f5d 100644 --- a/typescript/openai-agents/src/__tests__/test-openai-sandbox.ts +++ b/typescript/openai-agents/src/__tests__/test-openai-sandbox.ts @@ -11,6 +11,7 @@ import { sandboxApprovalResumeWorkflow, sandboxArchiveLimitsWorkflow, sandboxExecWorkflow, + sandboxHandoffConfigResumeWorkflow, sandboxManifestResumeWorkflow, sandboxValidationWorkflow, } from './workflows/openai-sandbox'; @@ -114,6 +115,45 @@ test.serial('a SandboxAgent whose manifest carries a workerEnvValue survives an t.true(client.resumeCalls >= 1, 'the preserved session is re-established through resume()'); }); +test('a SandboxAgent manifest or runAs change before the resume reaches the sandbox', async (t) => { + const { createWorker, executeWorkflow } = helpers(t); + + const client = new FakeSandboxClient(); + const responses = () => [ + toolCallResponse('confirm', {}), + toolCallResponse('transfer_to_sandbox_target', {}), + toolCallResponse('run_command', { cmd: 'echo hello' }), + textResponse('Done.'), + ]; + const worker = await createWorker({ + plugins: [ + new OpenAIAgentsPlugin({ + modelProvider: new FakeModelProvider([...responses(), ...responses(), ...responses()]), + modelParams: { startToCloseTimeout: '30s' }, + sandboxClientProviders: [new SandboxClientProvider('fake', client)], + }), + ], + }); + + const users = () => client.session.state.manifest.users.map((user) => user.name); + await worker.runUntil(async () => { + t.is(await executeWorkflow(sandboxHandoffConfigResumeWorkflow, { args: ['environment'] }), 'Done.'); + t.is(client.createCalls.length, 1); + t.is(client.session.state.environment?.VERSION, 'after'); + t.deepEqual(users(), ['before']); + + t.is(await executeWorkflow(sandboxHandoffConfigResumeWorkflow, { args: ['runAs'] }), 'Done.'); + t.is(client.createCalls.length, 2); + t.is(client.session.state.environment?.VERSION, 'before'); + t.deepEqual(users(), ['after']); + + t.is(await executeWorkflow(sandboxHandoffConfigResumeWorkflow, { args: ['reference'] }), 'Done.'); + t.is(client.createCalls.length, 3); + t.is(client.session.state.environment?.VERSION, 'after'); + t.deepEqual(users(), ['before']); + }); +}); + test('setArchiveLimits is forwarded to the persistWorkspace and hydrateWorkspace Activities', async (t) => { const { createWorker, executeWorkflow } = helpers(t); diff --git a/typescript/openai-agents/src/__tests__/workflows/openai-agents.ts b/typescript/openai-agents/src/__tests__/workflows/openai-agents.ts index b739c0d1..27cf35e2 100644 --- a/typescript/openai-agents/src/__tests__/workflows/openai-agents.ts +++ b/typescript/openai-agents/src/__tests__/workflows/openai-agents.ts @@ -1,5 +1,5 @@ -import { Agent } from '@openai/agents-core'; -import { ApplicationFailure } from '@temporalio/workflow'; +import { Agent, type AgentOptions, type CallModelInputFilter, RunState, handoff, tool } from '@openai/agents-core'; +import { ApplicationFailure, condition } from '@temporalio/workflow'; import { WorkflowStream } from '@temporalio/workflow-streams/workflow'; import { TemporalOpenAIRunner, statelessMcpServer, statefulMcpServer } from '../../workflow'; @@ -176,3 +176,828 @@ export async function statefulMcpSlowConnectHeartbeatWorkflow(): Promise await server.cleanup(); } } + +interface ResumeContext { + original: string; + after?: string; + writes: string[]; +} + +export interface ApprovalResumeContextResult { + callerAfter: string | undefined; + resumedUsesCallerObject: boolean; + writes: string[]; + approvalRounds: number; + failedResumeThrew: boolean; + approvalOwners: number; + approvedCallIds: number; + sharedContextWrites: string[]; + sharedContextInterruptions: number; + retryThrew: boolean; + retryAttempts: number; + retryWrites: string[]; + retryInterruptions: number; + concurrentFailedResumeThrew: boolean; + concurrentWrites: string[]; + concurrentInterruptions: number; + rehydratedWrites: string[]; + rehydratedApprovalOwners: number; + rehydratedApprovedCallIds: number; + handoffTarget: string | undefined; + handoffFilters: string[]; + disabledHandoffThrew: boolean; + sharedFieldsWrites: string[]; + sharedFieldsTemperatures: Array; + failedConversionThrew: boolean; + failedConversionWrites: string[]; + failedConversionInterruptions: number; + versions: string[]; + lastAgentVersion: string | undefined; + customChildWrites: string[]; + customChildInterruptions: number; + customChildSavedApprovalOwners: number; + replacedChildWrites: string[]; + replacedChildInterruptions: number; + failedRestoreThrew: boolean; + failedRestoreWrites: string[]; + failedRestoreInterruptions: number; + registeredVersions: Array; + lastAgentRegisteredVersion: string | undefined; + rejectedResumesThrew: boolean[]; + rejectedExecutions: string[]; + rejectedInterruptions: number; + failedRestores: number; + failedRestoreApprovalOwners: number; + failedRestoreApprovedCallIds: number; + recoveredWrites: string[]; + recoveredApprovalOwners: number; + recoveredApprovedCallIds: number; + rejectionMessages: string[]; + revisedRejectionWrites: string[]; + revisedRejectionInterruptions: number; + batchMessages: string[]; + batchInterruptions: number; + sameNameWrites: string[]; + sameNameApprovalOwners: number[]; + longHistoryThrew: boolean; + longHistoryWrites: string[]; + longHistoryInterruptions: number; + longHistoryApprovedCallIds: number; +} + +/** An agent whose `clone` carries a private field, which can't be copied onto another object. */ +class VersionedAgent extends Agent { + #version: string; + + constructor(config: AgentOptions, version: string) { + super(config); + this.#version = version; + } + + get version(): string { + return this.#version; + } + + setVersion(version: string): void { + this.#version = version; + } + + override clone(config: Partial>): VersionedAgent { + return new VersionedAgent({ ...this, ...config } as AgentOptions, this.#version); + } +} + +/** Versions that `RegisteredVersionAgent.clone` registers for each clone. */ +const registeredVersions = new WeakMap, string>(); + +/** An agent whose `clone` returns a plain `Agent` and registers its version outside the clone. */ +class RegisteredVersionAgent extends Agent { + override clone(config: Partial>): Agent { + const clone = super.clone(config); + registeredVersions.set(clone, registeredVersions.get(this)!); + return clone; + } +} + +/** Serialized approval record, as the `RunContext` internal below returns it. */ +interface ApprovalRecord { + approved: boolean | string[]; +} + +/** Counts the agents that own function approvals in `context`, and the call IDs they approve. */ +function countApprovalOwners(context: unknown): { approvalOwners: number; approvedCallIds: number } { + let approvalOwners = 0; + // _toJSONForRunState is internal to @openai/agents-core; it looks up each owner in the map it's given. + const json = ( + context as { + _toJSONForRunState(keys: { get(agent: unknown): string }): { + functionApprovals?: Array<{ approvals: Record }>; + }; + } + )._toJSONForRunState({ get: () => `owner-${++approvalOwners}` }); + let approvedCallIds = 0; + for (const { approvals } of json.functionApprovals ?? []) { + for (const record of Object.values(approvals)) { + approvedCallIds += Array.isArray(record.approved) ? record.approved.length : 0; + } + } + return { approvalOwners, approvedCallIds }; +} + +/** + * Approves a tool that writes the run context and resumes the same `RunState` until the run + * finishes, then reads the caller's context object. The first resume uses an agent the state + * doesn't match, which must fail without losing the approval. Then three paused states share one + * `RunContext`, all are approved, and resuming one must keep the others' approvals. Last, an approved + * tool throws once, and resuming the same state again must run it without asking again. Then a + * resume fails while another state on its context advances and is approved again. Then each resume + * rehydrates the saved state onto the original agent and context. Then a handoff changes between an + * interruption and its resume. Then an agent's tools and model settings are replaced with equal copies + * before a resume, and the approved tool changes them. Then a resume fails to convert a handoff target, + * and the saved state must still carry its approval. Then an agent class whose clone keeps a private + * field changes that field before the resume. Then a resume replaces a handoff target's object, because + * its class overrides `clone` or the target itself was replaced. Last, a `clone` that returns a plain + * `Agent` keeps state outside it, which changes before the resume. Then an approved call fails after + * the resume replaced its agent's object, and is rejected through the original state, whose retry must + * not run it. Then the restore of a saved state fails repeatedly after merging its approvals, which must + * not add a copy of them each time. Then a rejection message changes after a failed resume, and the retry + * must send the model the new one, also for a call that a later rejection for good covers. Last, a root and + * its handoff target share a name, and their resumes must not add a copy of the approvals each time. Last, + * the retry of a saved state with a long approval history must finish. + */ +export async function approvalResumeContextWorkflow(): Promise { + const setAfter = tool({ + name: 'set_after', + description: 'Set a value after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + execute: async (args, runContext) => { + const context = runContext!.context as ResumeContext; + context.after = (args as { value: string }).value; + context.writes.push(context.after); + return 'ok'; + }, + }); + const agent = new Agent({ + name: 'ResumeContextAgent', + instructions: 'Set the value.', + model: 'gpt-4o-mini', + tools: [setAfter], + }); + const otherAgent = new Agent({ name: 'OtherAgent', model: 'gpt-4o-mini', tools: [setAfter] }); + const runner = new TemporalOpenAIRunner(); + const ctx: ResumeContext = { original: 'orig', writes: [] }; + let result = await runner.run(agent, 'go', { context: ctx }); + let approvalRounds = 0; + let failedResumeThrew = false; + while (result.interruptions.length > 0 && approvalRounds < 10) { + result.state.approve(result.interruptions[0]!); + if (approvalRounds++ === 0) { + try { + await runner.run(otherAgent, result.state); + } catch { + failedResumeThrew = true; + } + } + result = await runner.run(agent, result.state); + } + const owners = countApprovalOwners(result.runContext); + + const sharedAgentA = new Agent({ name: 'SharedAgentA', model: 'gpt-4o-mini', tools: [setAfter] }); + const sharedAgentB = new Agent({ name: 'SharedAgentB', model: 'gpt-4o-mini', tools: [setAfter] }); + const sharedCtx: ResumeContext = { original: 'shared', writes: [] }; + const pausedA = await runner.run(sharedAgentA, 'go', { context: sharedCtx }); + const pausedB = await runner.run(sharedAgentB, 'go', { context: sharedCtx }); + const pausedC = await runner.run(sharedAgentA, 'go', { context: sharedCtx }); + // State A keeps the converted graph its run produced. B is rehydrated onto A's context, and C onto + // A's context and A's converted graph, so A and C share their approval owners. + const stateA = pausedA.state; + const stateB = await RunState.fromStringWithContext(sharedAgentB, pausedB.state.toString(), stateA._context); + const stateC = await RunState.fromStringWithContext(pausedA.lastAgent!, pausedC.state.toString(), stateA._context); + for (const state of [stateA, stateB, stateC]) state.approve(state.getInterruptions()[0]!); + const resumed = [ + await runner.run(sharedAgentA, stateA), + await runner.run(sharedAgentB, stateB), + await runner.run(sharedAgentA, stateC), + ]; + + const flaky = tool({ + name: 'flaky', + description: 'Fail once after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + errorFunction: null, + execute: async (args, runContext) => { + const context = runContext!.context as ResumeContext & { attempts: number }; + if (context.attempts++ === 0) throw new Error('flaky failed'); + context.writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const retryAgent = new Agent({ name: 'RetryAgent', model: 'gpt-4o-mini', tools: [flaky] }); + const retryCtx = { original: 'retry', writes: [] as string[], attempts: 0 }; + const pausedRetry = await runner.run(retryAgent, 'go', { context: retryCtx }); + pausedRetry.state.approve(pausedRetry.interruptions[0]!); + let retryThrew = false; + try { + await runner.run(retryAgent, pausedRetry.state); + } catch { + retryThrew = true; + } + const retried = await runner.run(retryAgent, pausedRetry.state); + + // State D's resume waits in a tool's isEnabled check, then throws. Meanwhile state E, which shares + // D's context, resumes, interrupts on its next call, and has that call approved. + const gate = { holdD: false, eApproved: false }; + const gated = tool({ + name: 'gated', + description: 'Record a value after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + isEnabled: async ({ agent }) => { + if (gate.holdD && agent.name === 'ConcurrentAgentD') { + await condition(() => gate.eApproved); + throw new Error('isEnabled failed'); + } + return true; + }, + execute: async (args, runContext) => { + (runContext!.context as ResumeContext).writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const agentD = new Agent({ name: 'ConcurrentAgentD', model: 'gpt-4o-mini', tools: [gated] }); + const agentE = new Agent({ name: 'ConcurrentAgentE', model: 'gpt-4o-mini', tools: [gated] }); + const concurrentCtx: ResumeContext = { original: 'concurrent', writes: [] }; + const pausedD = await runner.run(agentD, 'go', { context: concurrentCtx }); + const pausedE = await runner.run(agentE, 'go', { context: concurrentCtx }); + const stateD = pausedD.state; + const stateE = await RunState.fromStringWithContext(agentE, pausedE.state.toString(), stateD._context); + stateE.approve(stateE.getInterruptions()[0]!); + gate.holdD = true; + const failedD = runner.run(agentD, stateD).then( + () => false, + () => true + ); + const pausedAgainE = await runner.run(agentE, stateE); + pausedAgainE.state.approve(pausedAgainE.interruptions[0]!); + gate.eApproved = true; + const concurrentFailedResumeThrew = await failedD; + gate.holdD = false; + const finishedE = await runner.run(agentE, pausedAgainE.state); + + // Each round saves the state as a string and rehydrates it onto the original agent and the caller's + // context. The agent has a handoff, which stays unchanged here. + const handoffFilters: string[] = []; + const firstTarget = new Agent({ name: 'FirstTarget', model: 'gpt-4o-mini' }); + const secondTarget = new Agent({ name: 'SecondTarget', model: 'gpt-4o-mini' }); + const routing = handoff(firstTarget, { + inputFilter: (data) => { + handoffFilters.push('first'); + return data; + }, + }); + const routingAgent = new Agent({ + name: 'RoutingAgent', + model: 'gpt-4o-mini', + tools: [setAfter], + handoffs: [routing], + }); + const rehydratedCtx: ResumeContext = { original: 'rehydrated', writes: [] }; + let rehydrated = await runner.run(routingAgent, 'go', { context: rehydratedCtx }); + const rehydratedRunContext = rehydrated.runContext; + for (let round = 0; rehydrated.interruptions.length > 0 && round < 10; round++) { + const state = await RunState.fromStringWithContext(routingAgent, rehydrated.state.toString(), rehydratedRunContext); + state.approve(state.getInterruptions()[0]!); + rehydrated = await runner.run(routingAgent, state); + } + const rehydratedOwners = countApprovalOwners(rehydratedRunContext); + + // The handoff's target and input filter change between the interruption and the resume, and then + // the handoff is disabled before another resume. Each resume must use the handoff as it is now. + const pausedRoute = await runner.run(routingAgent, 'go', { context: { original: 'route', writes: [] } }); + pausedRoute.state.approve(pausedRoute.interruptions[0]!); + routing.agent = secondTarget; + routing.inputFilter = (data) => { + handoffFilters.push('second'); + return data; + }; + const routed = await runner.run(routingAgent, pausedRoute.state); + const pausedDisabled = await runner.run(routingAgent, 'go', { context: { original: 'disabled', writes: [] } }); + pausedDisabled.state.approve(pausedDisabled.interruptions[0]!); + routing.isEnabled = async () => false; + let disabledHandoffThrew = false; + try { + await runner.run(routingAgent, pausedDisabled.state); + } catch { + disabledHandoffThrew = true; + } + + // Conversion shares an agent's tools array and model settings with the converted agent. They're + // replaced with equal copies before the resume, and the approved tool then adds a tool and changes a + // setting, which the next model call must see. + const extra = tool({ + name: 'extra', + description: 'Record a value.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + execute: async (args, runContext) => { + (runContext!.context as ResumeContext).writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const sharedFieldsTemperatures: Array = []; + const sharedFieldsAgent: Agent = new Agent({ + name: 'SharedFieldsAgent', + model: 'gpt-4o-mini', + instructions: (_runContext, current) => { + sharedFieldsTemperatures.push(current.modelSettings.temperature); + return 'Extend yourself.'; + }, + modelSettings: { temperature: 0.1 }, + tools: [ + tool({ + name: 'extend', + description: 'Add a tool and change a setting after approval.', + parameters: { type: 'object', properties: {}, required: [], additionalProperties: false }, + needsApproval: true, + execute: async () => { + sharedFieldsAgent.tools.push(extra); + sharedFieldsAgent.modelSettings.temperature = 0.9; + return 'ok'; + }, + }), + ], + }); + const sharedFieldsCtx: ResumeContext = { original: 'shared-fields', writes: [] }; + const pausedSharedFields = await runner.run(sharedFieldsAgent, 'go', { context: sharedFieldsCtx }); + pausedSharedFields.state.approve(pausedSharedFields.interruptions[0]!); + sharedFieldsAgent.tools = [...sharedFieldsAgent.tools]; + sharedFieldsAgent.modelSettings = { ...sharedFieldsAgent.modelSettings }; + await runner.run(sharedFieldsAgent, pausedSharedFields.state); + + // The run pauses on the handoff target's call. A resume while the target has a raw function tool + // fails to convert it. Once the tool is removed, the state saved as a string must still carry the + // approval: a new context rehydrated from it must run the call without asking again. + const failChild = new Agent({ name: 'FailChild', model: 'gpt-4o-mini', tools: [setAfter] }); + const failRoot = new Agent({ name: 'FailRoot', model: 'gpt-4o-mini', handoffs: [failChild] }); + const failedConversionCtx: ResumeContext = { original: 'failed-conversion', writes: [] }; + const pausedFail = await runner.run(failRoot, 'go', { context: failedConversionCtx }); + pausedFail.state.approve(pausedFail.interruptions[0]!); + failChild.tools = [setAfter, (() => 'raw') as any]; + let failedConversionThrew = false; + try { + await runner.run(failRoot, pausedFail.state); + } catch { + failedConversionThrew = true; + } + failChild.tools = [setAfter]; + const failRehydrated = await RunState.fromString(failRoot, pausedFail.state.toString()); + const failedConversionResumed = await runner.run(failRoot, failRehydrated); + + // The version changes between the interruption and the resume, and the agent the resumed run uses must + // have the new one, as a fresh clone does. + const versions: string[] = []; + const versionedAgent = new VersionedAgent( + { + name: 'VersionedAgent', + model: 'gpt-4o-mini', + instructions: (_runContext, current) => { + versions.push((current as VersionedAgent).version); + return 'Set the value.'; + }, + tools: [setAfter], + }, + 'before' + ); + const pausedVersioned = await runner.run(versionedAgent, 'go', { context: { original: 'versioned', writes: [] } }); + pausedVersioned.state.approve(pausedVersioned.interruptions[0]!); + versionedAgent.setVersion('after'); + const versioned = await runner.run(versionedAgent, pausedVersioned.state); + + // The run pauses on a handoff target's call. The resume reuses the root, but gives the target a new + // object, because its class overrides clone. The approved call must run, and the state saved again + // afterwards must still carry its approval. + const customChild = new VersionedAgent({ name: 'CustomChild', model: 'gpt-4o-mini', tools: [setAfter] }, 'v'); + const customRoot = new Agent({ name: 'CustomRoot', model: 'gpt-4o-mini', handoffs: [customChild] }); + const customChildCtx: ResumeContext = { original: 'custom-child', writes: [] }; + const pausedCustomChild = await runner.run(customRoot, 'go', { context: customChildCtx }); + pausedCustomChild.state.approve(pausedCustomChild.interruptions[0]!); + const customChildResumed = await runner.run(customRoot, pausedCustomChild.state); + const customChildSaved = await RunState.fromString(customRoot, pausedCustomChild.state.toString()); + + // The handoff target is replaced by another agent with the same name before the resume. + const replacedChild = new Agent({ name: 'ReplacedChild', model: 'gpt-4o-mini', tools: [setAfter] }); + const replacedRoot = new Agent({ + name: 'ReplacedRoot', + model: 'gpt-4o-mini', + handoffs: [replacedChild], + }); + const replacedChildCtx: ResumeContext = { original: 'replaced-child', writes: [] }; + const pausedReplacedChild = await runner.run(replacedRoot, 'go', { context: replacedChildCtx }); + pausedReplacedChild.state.approve(pausedReplacedChild.interruptions[0]!); + replacedRoot.handoffs = [replacedChild.clone({})]; + const replacedChildResumed = await runner.run(replacedRoot, pausedReplacedChild.state); + + // The handoff to the paused target is removed, so the restore throws after the conversion. Once the + // handoff is back, the state saved as a string must still carry the approval. + const restoreChild = new Agent({ name: 'RestoreChild', model: 'gpt-4o-mini', tools: [setAfter] }); + const restoreRoot = new Agent({ name: 'RestoreRoot', model: 'gpt-4o-mini', handoffs: [restoreChild] }); + const pausedRestore = await runner.run(restoreRoot, 'go', { context: { original: 'restore', writes: [] } }); + pausedRestore.state.approve(pausedRestore.interruptions[0]!); + restoreRoot.handoffs = []; + let failedRestoreThrew = false; + try { + await runner.run(restoreRoot, pausedRestore.state); + } catch { + failedRestoreThrew = true; + } + restoreRoot.handoffs = [restoreChild]; + const restoreRehydrated = await RunState.fromString(restoreRoot, pausedRestore.state.toString()); + const failedRestoreResumed = await runner.run(restoreRoot, restoreRehydrated); + + // The registered version changes between the interruption and the resume, and the agent the resumed run + // uses must have the new one, as a fresh clone does. + const versionsSeen: Array = []; + const registeredAgent = new RegisteredVersionAgent({ + name: 'RegisteredVersionAgent', + model: 'gpt-4o-mini', + instructions: (_runContext, current) => { + versionsSeen.push(registeredVersions.get(current)); + return 'Set the value.'; + }, + tools: [setAfter], + }); + registeredVersions.set(registeredAgent, 'before'); + const pausedRegistered = await runner.run(registeredAgent, 'go', { context: { original: 'registered', writes: [] } }); + pausedRegistered.state.approve(pausedRegistered.interruptions[0]!); + registeredVersions.set(registeredAgent, 'after'); + const registered = await runner.run(registeredAgent, pausedRegistered.state); + + // The run pauses on a handoff target's call, and the approved call throws once its resume has given the + // target a new object, because its class overrides clone or the target was replaced. The call is then + // always rejected through the original state, and the retry of that state must not run it again, even + // if the call was approved for good. A state saved as a string and rehydrated onto the original agent + // must not run it again either. + const rejectedExecutions: string[] = []; + const rejectable = tool({ + name: 'rejectable', + description: 'Fail once after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + errorFunction: null, + execute: async (args) => { + const { value } = args as { value: string }; + const firstRun = !rejectedExecutions.includes(value); + rejectedExecutions.push(value); + if (firstRun) throw new Error('rejectable failed'); + return 'ok'; + }, + }); + const rejectAfterFailedResume = async ( + child: Agent, + replacement?: Agent, + alwaysApprove = false + ) => { + const root = new Agent({ name: `${child.name}Root`, model: 'gpt-4o-mini', handoffs: [child] }); + const paused = await runner.run(root, 'go', { context: { original: child.name, writes: [] } }); + const interruption = paused.interruptions[0]!; + paused.state.approve(interruption, { alwaysApprove }); + if (replacement !== undefined) root.handoffs = [replacement]; + let threw = false; + try { + await runner.run(root, paused.state); + } catch { + threw = true; + } + paused.state.reject(interruption, { alwaysReject: true }); + const retried = await runner.run(root, paused.state); + return { threw, interruptions: retried.interruptions.length }; + }; + const rejectChild = new Agent({ name: 'RejectChild', model: 'gpt-4o-mini', tools: [rejectable] }); + const alwaysChild = new Agent({ name: 'AlwaysChild', model: 'gpt-4o-mini', tools: [rejectable] }); + const rejected = [ + await rejectAfterFailedResume( + new VersionedAgent({ name: 'RejectCustomChild', model: 'gpt-4o-mini', tools: [rejectable] }, 'v') + ), + await rejectAfterFailedResume(rejectChild, rejectChild.clone({})), + await rejectAfterFailedResume(alwaysChild, alwaysChild.clone({}), true), + ]; + const persistedAgent = new Agent({ + name: 'PersistedAgent', + model: 'gpt-4o-mini', + tools: [rejectable], + }); + const pausedPersisted = await runner.run(persistedAgent, 'go', { context: { original: 'persisted', writes: [] } }); + const persistedState = await RunState.fromString(persistedAgent, pausedPersisted.state.toString()); + const persistedInterruption = persistedState.getInterruptions()[0]!; + persistedState.approve(persistedInterruption, { alwaysApprove: true }); + let persistedThrew = false; + try { + await runner.run(persistedAgent, persistedState); + } catch { + persistedThrew = true; + } + persistedState.reject(persistedInterruption, { alwaysReject: true }); + const persistedRetried = await runner.run(persistedAgent, persistedState); + rejected.push({ threw: persistedThrew, interruptions: persistedRetried.interruptions.length }); + + // A saved state is rehydrated onto the original agent. Its restore merges the approvals, then fails in a + // tool's isEnabled check, three times. Neither the failures nor the resume that then succeeds may add a + // copy of the approvals. + const guard = { fail: false }; + const guarded = tool({ + name: 'guarded', + description: 'Record a value after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + isEnabled: async () => { + if (guard.fail) throw new Error('isEnabled failed'); + return true; + }, + execute: async (args, runContext) => { + (runContext!.context as ResumeContext).writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const guardedAgent = new Agent({ name: 'GuardedAgent', model: 'gpt-4o-mini', tools: [guarded] }); + const pausedGuarded = await runner.run(guardedAgent, 'go', { context: { original: 'guarded', writes: [] } }); + pausedGuarded.state.approve(pausedGuarded.interruptions[0]!); + const guardedState = await RunState.fromString(guardedAgent, pausedGuarded.state.toString()); + guard.fail = true; + let failedRestores = 0; + for (let attempt = 0; attempt < 3; attempt++) { + try { + await runner.run(guardedAgent, guardedState); + } catch { + failedRestores++; + } + } + const failedRestoreOwners = countApprovalOwners(guardedState._context); + guard.fail = false; + const recovered = await runner.run(guardedAgent, guardedState); + const recoveredOwners = countApprovalOwners(guardedState._context); + + // A call is rejected with a message, and the next model call fails once after the resume. The call is + // then rejected again with a new message, and the model must receive the new one when the state is + // retried, whether the state was rehydrated onto the original agent or kept from the run. + const vetoed = tool({ + name: 'vetoed', + description: 'Record a value after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + execute: async (args, runContext) => { + (runContext!.context as ResumeContext).writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const rejectionMessages: string[] = []; + const vetoedWrites: string[] = []; + const reviseRejection = async (name: string, rehydrate: boolean, alwaysReject: boolean) => { + const vetoAgent = new Agent({ name, model: 'gpt-4o-mini', tools: [vetoed] }); + const vetoCtx: ResumeContext = { original: name, writes: vetoedWrites }; + const paused = await runner.run(vetoAgent, 'go', { context: vetoCtx }); + const state = rehydrate ? await RunState.fromString(vetoAgent, paused.state.toString()) : paused.state; + const interruption = state.getInterruptions()[0]!; + let failModel = true; + const callModelInputFilter: CallModelInputFilter = ({ modelData }) => { + for (const item of modelData.input) { + if (item.type === 'function_call_result') { + const { output } = item; + rejectionMessages.push( + typeof output === 'string' ? output : 'text' in output ? output.text : JSON.stringify(output) + ); + } + } + if (failModel) { + failModel = false; + throw new Error('model failed'); + } + return modelData; + }; + state.reject(interruption, { alwaysReject, message: `${name} before` }); + await runner.run(vetoAgent, state, { callModelInputFilter }).catch(() => undefined); + state.reject(interruption, { alwaysReject, message: `${name} after` }); + return (await runner.run(vetoAgent, state, { callModelInputFilter })).interruptions.length; + }; + const revisedInterruptions = [ + await reviseRejection('VetoPersisted', true, false), + await reviseRejection('VetoPersistedAlways', true, true), + await reviseRejection('VetoDirect', false, false), + ]; + + // Calls A and B pause together. A is approved and B rejected, and the next model call fails once after + // the resume. A is then rejected for good with a new message, which also covers B, and the model must + // receive the new message for both when the state is retried. + const batchMessages: string[] = []; + const rejectBatch = async (name: string, rehydrate: boolean) => { + const batchAgent = new Agent({ name, model: 'gpt-4o-mini', tools: [vetoed] }); + const paused = await runner.run(batchAgent, 'go', { context: { original: name, writes: [] } }); + const state = rehydrate ? await RunState.fromString(batchAgent, paused.state.toString()) : paused.state; + const [callA, callB] = state.getInterruptions(); + let failModel = true; + const callModelInputFilter: CallModelInputFilter = ({ modelData }) => { + for (const item of modelData.input) { + if (item.type === 'function_call_result') { + const { output } = item; + batchMessages.push( + `${item.callId}: ${typeof output === 'string' ? output : 'text' in output ? output.text : JSON.stringify(output)}` + ); + } + } + if (failModel) { + failModel = false; + throw new Error('model failed'); + } + return modelData; + }; + state.approve(callA!); + state.reject(callB!, { message: `${name} before` }); + await runner.run(batchAgent, state, { callModelInputFilter }).catch(() => undefined); + state.reject(callA!, { alwaysReject: true, message: `${name} after` }); + return (await runner.run(batchAgent, state, { callModelInputFilter })).interruptions.length; + }; + const batchInterruptions = [await rejectBatch('BatchPersisted', true), await rejectBatch('BatchDirect', false)]; + + // A root and its handoff target share a name. The root's call is approved for good, and the target's + // calls are then rejected one at a time. The resumes must keep converting into the same agents, whether + // each resumes the run's state or one rehydrated onto the original agent and the run's context. + const sameNameTool = tool({ + name: 'same_name', + description: 'Record a value after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + execute: async (args, runContext) => { + (runContext!.context as ResumeContext).writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const sameNameWrites: string[] = []; + const resumeSameName = async (rehydrate: boolean) => { + const target = new Agent({ name: 'Same', model: 'gpt-4o-mini', tools: [sameNameTool] }); + const root = new Agent({ + name: 'Same', + model: 'gpt-4o-mini', + tools: [sameNameTool], + handoffs: [target], + }); + let paused = await runner.run(root, 'go', { context: { original: 'same-name', writes: sameNameWrites } }); + const runContext = paused.runContext; + for (let round = 0; paused.interruptions.length > 0 && round < 10; round++) { + const state = rehydrate + ? await RunState.fromStringWithContext(root, paused.state.toString(), runContext) + : paused.state; + const interruption = state.getInterruptions()[0]!; + if (round === 0) state.approve(interruption, { alwaysApprove: true }); + else state.reject(interruption); + paused = await runner.run(root, state); + } + return countApprovalOwners(runContext).approvalOwners; + }; + const sameNameApprovalOwners = [await resumeSameName(false), await resumeSameName(true)]; + + // A saved state carries a long history of earlier approvals. Its approved call throws once after the + // resume, and the retry, which compares that history with the one the reused graph owns, must finish. + const longHistoryCtx = { original: 'long-history', writes: [] as string[], attempts: 0 }; + const longHistoryTool = tool({ + name: 'long_history', + description: 'Fail once after approval.', + parameters: { + type: 'object' as const, + properties: { value: { type: 'string' } }, + required: ['value'] as const, + additionalProperties: false as const, + }, + needsApproval: true, + errorFunction: null, + execute: async (args, runContext) => { + const context = runContext!.context as ResumeContext & { attempts: number }; + if (context.attempts++ === 0) throw new Error('long history failed'); + context.writes.push((args as { value: string }).value); + return 'ok'; + }, + }); + const longHistoryAgent = new Agent({ + name: 'LongHistoryAgent', + model: 'gpt-4o-mini', + tools: [longHistoryTool], + }); + const pausedLongHistory = await runner.run(longHistoryAgent, 'go', { context: longHistoryCtx }); + pausedLongHistory.state.approve(pausedLongHistory.interruptions[0]!); + const longHistoryJson = JSON.parse(pausedLongHistory.state.toString()); + for (const { approvals } of longHistoryJson.context.functionApprovals) { + for (const record of Object.values(approvals) as ApprovalRecord[]) { + if (Array.isArray(record.approved)) { + record.approved.push(...Array.from({ length: 50_000 }, (_, index) => `call_earlier_${index}`)); + } + } + } + const longHistoryState = await RunState.fromString(longHistoryAgent, JSON.stringify(longHistoryJson)); + let longHistoryThrew = false; + try { + await runner.run(longHistoryAgent, longHistoryState); + } catch { + longHistoryThrew = true; + } + const longHistoryRetried = await runner.run(longHistoryAgent, longHistoryState); + + return { + callerAfter: ctx.after, + resumedUsesCallerObject: result.runContext.context === ctx, + writes: ctx.writes, + approvalRounds, + failedResumeThrew, + ...owners, + sharedContextWrites: sharedCtx.writes, + sharedContextInterruptions: resumed.reduce((count, { interruptions }) => count + interruptions.length, 0), + retryThrew, + retryAttempts: retryCtx.attempts, + retryWrites: retryCtx.writes, + retryInterruptions: retried.interruptions.length, + concurrentFailedResumeThrew, + concurrentWrites: concurrentCtx.writes, + concurrentInterruptions: finishedE.interruptions.length, + rehydratedWrites: rehydratedCtx.writes, + rehydratedApprovalOwners: rehydratedOwners.approvalOwners, + rehydratedApprovedCallIds: rehydratedOwners.approvedCallIds, + handoffTarget: routed.lastAgent?.name, + handoffFilters, + disabledHandoffThrew, + sharedFieldsWrites: sharedFieldsCtx.writes, + sharedFieldsTemperatures, + failedConversionThrew, + failedConversionWrites: failedConversionResumed.runContext.context.writes, + failedConversionInterruptions: failedConversionResumed.interruptions.length, + versions, + lastAgentVersion: (versioned.lastAgent as VersionedAgent | undefined)?.version, + customChildWrites: customChildCtx.writes, + customChildInterruptions: customChildResumed.interruptions.length, + customChildSavedApprovalOwners: countApprovalOwners(customChildSaved._context).approvalOwners, + replacedChildWrites: replacedChildCtx.writes, + replacedChildInterruptions: replacedChildResumed.interruptions.length, + failedRestoreThrew, + failedRestoreWrites: failedRestoreResumed.runContext.context.writes, + failedRestoreInterruptions: failedRestoreResumed.interruptions.length, + registeredVersions: versionsSeen, + lastAgentRegisteredVersion: registered.lastAgent && registeredVersions.get(registered.lastAgent), + rejectedResumesThrew: rejected.map(({ threw }) => threw), + rejectedExecutions, + rejectedInterruptions: rejected.reduce((count, { interruptions }) => count + interruptions, 0), + failedRestores, + failedRestoreApprovalOwners: failedRestoreOwners.approvalOwners, + failedRestoreApprovedCallIds: failedRestoreOwners.approvedCallIds, + recoveredWrites: recovered.runContext.context.writes, + recoveredApprovalOwners: recoveredOwners.approvalOwners, + recoveredApprovedCallIds: recoveredOwners.approvedCallIds, + rejectionMessages, + revisedRejectionWrites: vetoedWrites, + revisedRejectionInterruptions: revisedInterruptions.reduce((count, interruptions) => count + interruptions, 0), + batchMessages, + batchInterruptions: batchInterruptions.reduce((count, interruptions) => count + interruptions, 0), + sameNameWrites, + sameNameApprovalOwners, + longHistoryThrew, + longHistoryWrites: longHistoryRetried.runContext.context.writes, + longHistoryInterruptions: longHistoryRetried.interruptions.length, + longHistoryApprovedCallIds: countApprovalOwners(longHistoryState._context).approvedCallIds, + }; +} diff --git a/typescript/openai-agents/src/__tests__/workflows/openai-sandbox.ts b/typescript/openai-agents/src/__tests__/workflows/openai-sandbox.ts index f259645d..577bba03 100644 --- a/typescript/openai-agents/src/__tests__/workflows/openai-sandbox.ts +++ b/typescript/openai-agents/src/__tests__/workflows/openai-sandbox.ts @@ -1,12 +1,14 @@ -import { Agent, type FunctionTool, type RunContext, type Tool } from '@openai/agents-core'; +import { Agent, tool, type FunctionTool, type RunContext, type Tool } from '@openai/agents-core'; import { Capability, + Environment, Manifest, SandboxAgent, isEnvValueReference, type SandboxSessionLike, } from '@openai/agents-core/sandbox'; import { TemporalOpenAIRunner, workerEnvValue, temporalSandboxClient } from '../../workflow'; +import { PrivateEnvValue } from '../stubs/private-env-value'; class TestSandboxCapability extends Capability { readonly type = 'test_sandbox'; @@ -120,6 +122,56 @@ export async function sandboxApprovalResumeWorkflow(): Promise { return `${resumed.finalOutput}`; } +/** + * A root agent pauses for approval before it hands off to a `SandboxAgent`. Then the target's manifest + * environment entry, an environment reference that keeps its value private, or its `runAs` user changes in + * place, and the resume must create the sandbox with the new value. + */ +export async function sandboxHandoffConfigResumeWorkflow( + change: 'environment' | 'reference' | 'runAs' +): Promise { + const target = new SandboxAgent({ + name: 'sandbox_target', + model: 'gpt-4o-mini', + capabilities: [new TestSandboxCapability()], + defaultManifest: new Manifest({ + entries: { 'data.bin': { type: 'file', content: new Uint8Array([0, 1, 2]) } }, + environment: { VERSION: change === 'reference' ? new PrivateEnvValue('before') : 'before' }, + }), + runAs: { name: 'before' }, + }); + const root = new Agent({ + name: 'root', + model: 'gpt-4o-mini', + tools: [ + tool({ + name: 'confirm', + description: 'confirm', + parameters: { type: 'object', properties: {}, required: [], additionalProperties: false }, + needsApproval: true, + execute: async () => 'confirmed', + }), + ], + handoffs: [target], + }); + const runner = new TemporalOpenAIRunner(); + const runConfig = { sandbox: { client: temporalSandboxClient('fake') } }; + + const result = await runner.run(root, 'go', { runConfig }); + if (result.interruptions.length === 0) return 'no-interruption'; + if (change === 'environment') { + target.defaultManifest!.environment.VERSION = new Environment('after'); + } else if (change === 'reference') { + target.defaultManifest!.environment.VERSION = new PrivateEnvValue('after'); + } else { + (target.runAs as { name: string }).name = 'after'; + } + for (const interruption of result.interruptions) result.state.approve(interruption); + + const resumed = await runner.run(root, result.state, { runConfig }); + return `${resumed.finalOutput}`; +} + export async function sandboxManifestResumeWorkflow(): Promise { const client = temporalSandboxClient('fake'); const session = await client.create( diff --git a/typescript/openai-agents/src/workflow/convert-agent.ts b/typescript/openai-agents/src/workflow/convert-agent.ts index c699db37..42aa4181 100644 --- a/typescript/openai-agents/src/workflow/convert-agent.ts +++ b/typescript/openai-agents/src/workflow/convert-agent.ts @@ -1,4 +1,5 @@ -import { type Agent, Handoff } from '@openai/agents-core'; +import { Agent, Handoff } from '@openai/agents-core'; +import { SandboxAgent } from '@openai/agents-core/sandbox'; import { ApplicationFailure } from '@temporalio/common'; import type { ModelActivityOptions } from '../common/model-activity-options'; import { ActivityBackedModel } from './activity-backed-model'; @@ -7,15 +8,56 @@ import { ActivityBackedModel } from './activity-backed-model'; * Recursively converts an agent graph: validate tools, replace each agent's * model with an ActivityBackedModel, and clone handoffs. Uses a seen-map to * handle circular handoff references. + * + * When `identities` maps a source agent to an agent an earlier conversion returned, that object is + * returned again, with the fields of this conversion's clone. Each agent then has the same fields as + * after a fresh conversion, but keeps its object identity, which approvals on a `RunContext` are keyed by. + * An object is only reused if its handoffs lead to the same objects as before, so a state saved on the + * earlier graph still reaches every agent it reached, and serializes the decisions those agents own. + * Other agents, and agents whose class overrides `clone`, get their fresh clone. Such a clone can hold + * state, such as private fields or entries keyed by the clone, that can't be moved onto another object. + * The objects only change once the whole graph has converted, so a conversion that throws leaves them + * as they were. */ export function convertAgent( agent: Agent, modelParams: ModelActivityOptions, seen?: Map, Agent>, - modelNameOverride?: string + modelNameOverride?: string, + identities?: ReadonlyMap, Agent> ): Agent { seen = seen ?? new Map(); if (seen.has(agent)) return seen.get(agent)!; + const clones = new Map, Agent>(); + cloneAgentGraph(agent, modelParams, seen, clones, modelNameOverride); + const reused = reusableIdentities(clones, identities); + for (const [source, clone] of clones) { + seen.set(source, reused.get(source) ?? clone); + } + for (const [source, clone] of clones) { + clone.handoffs = (source.handoffs ?? []).map((h) => { + const handoffAgent = h instanceof Handoff ? h.agent : h; + const converted = seen.get(handoffAgent)!; + if (!(h instanceof Handoff)) return converted; + // A reused target still has its earlier fields here, so take the name from its new clone. + return h.clone({ agent: converted, agentName: (clones.get(handoffAgent) ?? converted).name }); + }); + } + for (const [source, target] of reused) { + replaceFields(target, clones.get(source)!); + } + return seen.get(agent)!; +} + +/** Validates and clones each agent in `agent`'s graph that isn't in `seen`, into `clones`. */ +function cloneAgentGraph( + agent: Agent, + modelParams: ModelActivityOptions, + seen: ReadonlyMap, Agent>, + clones: Map, Agent>, + modelNameOverride: string | undefined +): void { + if (seen.has(agent) || clones.has(agent)) return; // Reject raw functions: authoring mistake — use tool() or activityAsTool(). const tools = agent.tools ?? []; @@ -57,17 +99,69 @@ export function convertAgent( // user-declared name and instructions. activityBackedModel.setAgent(agent); - const converted = agent.clone({ model: activityBackedModel }); - seen.set(agent, converted); + clones.set(agent, agent.clone({ model: activityBackedModel })); + for (const h of agent.handoffs ?? []) { + cloneAgentGraph(h instanceof Handoff ? h.agent : h, modelParams, seen, clones, modelNameOverride); + } +} - const convertedHandoffs = (agent.handoffs ?? []).map((h) => { - if (h instanceof Handoff) { - const convertedHandoffAgent = convertAgent(h.agent, modelParams, seen, modelNameOverride); - return h.clone({ agent: convertedHandoffAgent }); +/** + * The objects in `identities` this conversion reuses, keyed by source agent: those whose fields + * `canReplaceFields` allows replacing, and whose handoffs, once converted, lead to the same objects as + * before. Rewiring a reused object to a new one would hide the agent it led to from states saved on the + * earlier graph, which then lose that agent's approvals and rejections when serialized. + */ +function reusableIdentities( + clones: ReadonlyMap, Agent>, + identities: ReadonlyMap, Agent> | undefined +): Map, Agent> { + const reused = new Map, Agent>(); + for (const [source, clone] of clones) { + const identity = identities?.get(source); + if (identity !== undefined && canReplaceFields(source, identity, clone)) reused.set(source, identity); + } + // Dropping an agent can change the targets of agents that hand off to it, so repeat until none changes. + let dropped = true; + while (dropped) { + dropped = false; + for (const [source, identity] of reused) { + const before = new Set(handoffAgents(identity)); + const after = new Set(handoffAgents(source).map((target) => reused.get(target))); + if (after.size !== before.size || [...after].some((target) => target === undefined || !before.has(target))) { + reused.delete(source); + dropped = true; + } } - return convertAgent(h, modelParams, seen, modelNameOverride); - }); + } + return reused; +} - converted.handoffs = convertedHandoffs; - return converted; +function handoffAgents(agent: Agent): Agent[] { + return (agent.handoffs ?? []).map((h) => (h instanceof Handoff ? h.agent : h)); +} + +/** + * Whether `target` can take the fields of `clone`, which `agent.clone` returned. `Agent.clone` and + * `SandboxAgent.clone` return instances of exactly those classes, which keep all their state in own + * properties. Any other `clone`, even one that returns such an instance, can keep state elsewhere. + */ +function canReplaceFields(agent: Agent, target: Agent, clone: Agent): boolean { + const plainPrototypes: readonly object[] = [Agent.prototype, SandboxAgent.prototype]; + return ( + (agent.clone === Agent.prototype.clone || agent.clone === SandboxAgent.prototype.clone) && + plainPrototypes.includes(Object.getPrototypeOf(target)) && + plainPrototypes.includes(Object.getPrototypeOf(clone)) + ); +} + +/** Gives `target` exactly the own properties and prototype of `source`, and returns `target`. */ +function replaceFields(target: Agent, source: Agent): Agent { + for (const key of Reflect.ownKeys(target)) { + delete (target as any)[key]; + } + Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); + if (Object.getPrototypeOf(target) !== Object.getPrototypeOf(source)) { + Object.setPrototypeOf(target, Object.getPrototypeOf(source)); + } + return target; } diff --git a/typescript/openai-agents/src/workflow/runner.ts b/typescript/openai-agents/src/workflow/runner.ts index 732cfedc..006da968 100644 --- a/typescript/openai-agents/src/workflow/runner.ts +++ b/typescript/openai-agents/src/workflow/runner.ts @@ -3,6 +3,7 @@ import { type AgentInputItem, Handoff, MemorySession, + RunContext, Runner, RunState, type AgentOutputType, @@ -35,6 +36,196 @@ import { getCurrentPluginConfig } from './plugin-config-store'; const agentAsToolSandboxConfig = Symbol.for('@temporalio/openai-agents/agentAsToolSandboxConfig'); +/** A converted agent graph. */ +interface Conversion { + source: Agent; + /** Each source agent the graph has converted, mapped to its converted agent. */ + agents: Map, Agent>; + /** Runs that are using the graph. A resume converts into the graph only while no run uses it. */ + activeRuns: number; +} + +/** Each converted agent, mapped to the graph it belongs to. */ +const conversions = new WeakMap, Conversion>(); + +/** + * For each `RunContext`, the graph its latest resume used for each source agent. A state rehydrated onto + * the source agent doesn't point at a converted agent, so this is how its resume finds the graph that + * already owns the context's approvals. It's keyed by context so Workflows that share module-level agents + * under `reuseV8Context` never share a graph. + */ +const resumeConversions = new WeakMap, Conversion>>(); + +/** + * Converts `agent`'s graph. With `into`, the conversion reuses those of `into`'s agent objects that + * `convertAgent` can: each one gets the fields a fresh conversion would give it, and keeps its identity. + * `into` then maps each source agent to its latest converted agent. It keeps them even if the resume + * fails, since the restore may already have merged the state's approvals under them. + */ +function convertAgentGraph( + agent: Agent, + modelParams: ModelActivityOptions, + modelOverride: string | undefined, + sandbox: SandboxRunConfig | undefined, + into?: Conversion +): Conversion { + const convertedBySource = new Map, Agent>(); + const converted = convertAgent(agent, modelParams, convertedBySource, modelOverride, into?.agents); + if (sandbox !== undefined) { + propagateAgentToolSandboxConfig(converted, sandbox); + } + const conversion = into ?? { source: agent, agents: new Map(), activeRuns: 0 }; + for (const [source, convertedAgent] of convertedBySource) { + conversion.agents.set(source, convertedAgent); + conversions.set(convertedAgent, conversion); + } + return conversion; +} + +/** + * Returns the converted graph that `state` runs on, or else the one the latest resume on its context used, + * if it was converted from `agent`, no run is using it, and restoring the state into it keeps the state's + * decisions and rejection messages. + * + * Function-tool approvals are keyed by agent object on the state's `RunContext`, which the resume keeps. + * Rebuilding the state over new agent objects would add a copy of its approvals under every resume's + * graph, and the earlier copies can't be dropped, because saved states that still point at those agents + * read them. + */ +function reusableConversion( + state: RunState, + agent: Agent, + serializedState: string +): Conversion | undefined { + const candidates = [ + conversions.get(state._currentAgent as Agent), + resumeConversions.get(state._context)?.get(agent), + ]; + return candidates.find( + (conversion) => + conversion?.source === agent && + conversion.activeRuns === 0 && + !losesStateDecisions(conversion.agents.get(agent)!, serializedState, state._context) + ); +} + +/** A function-tool approval record, as a serialized `RunState` and `RunContext` store it. */ +interface ApprovalRecord { + approved: boolean | string[]; + rejected: boolean | string[]; + messages?: Record; + stickyRejectMessage?: string; +} + +interface FunctionApprovals { + agentIdentity: string; + approvals: Record; +} + +/** The `RunContext` internals of @openai/agents-core that read and merge per-agent approvals. */ +interface ApprovalContext { + _toJSONForRunState(keys: ReadonlyMap): { functionApprovals?: FunctionApprovals[] }; + _mergeFunctionApprovals(approvals: FunctionApprovals[], agentsByIdentity: ReadonlyMap): void; +} + +/** + * Whether the restore would lose one of the serialized state's decisions or rejection messages if it merged + * them into the decisions the agents of the graph rooted at `root` already own. The merge keeps an approval + * for good over a rejection, and an existing rejection message over the state's. So a state that always + * approved a tool and then always rejected it would run it again, and a state whose rejection message + * changed after a failed resume would send the model the earlier message. A fresh graph owns only the + * state's decisions. + */ +function losesStateDecisions(root: Agent, serializedState: string, context: RunContext): boolean { + const stateApprovals: FunctionApprovals[] = JSON.parse(serializedState).context.functionApprovals ?? []; + if (stateApprovals.length === 0) return false; + // Serializing a state that starts at `root` keys the decisions the graph's agents own by the same agent + // identities (`name`, or `name#n` for agents that share a name) the restore resolves the state's against. + const owned: FunctionApprovals[] = new RunState(context, '', root, null).toJSON().context.functionApprovals ?? []; + const ownedByIdentity = new Map(owned.map(({ agentIdentity, approvals }) => [agentIdentity, approvals])); + return stateApprovals.some(({ agentIdentity, approvals }) => { + const ownedApprovals = ownedByIdentity.get(agentIdentity); + return ( + ownedApprovals !== undefined && + Object.entries(approvals).some( + ([toolName, record]) => + ownedApprovals[toolName] !== undefined && + !keepsDecisions(record, mergeApprovalRecords(ownedApprovals[toolName], record, toolName)) + ) + ); + }); +} + +/** Merges `incoming` into `current` as the restore does, in a scratch `RunContext`. */ +function mergeApprovalRecords(current: ApprovalRecord, incoming: ApprovalRecord, toolName: string): ApprovalRecord { + const owner = {}; + const scratch = new RunContext() as unknown as ApprovalContext; + for (const record of [current, incoming]) { + scratch._mergeFunctionApprovals( + [{ agentIdentity: 'owner', approvals: { [toolName]: record } }], + new Map([['owner', owner]]) + ); + } + return scratch._toJSONForRunState(new Map([[owner, 'owner']])).functionApprovals![0]!.approvals[toolName]!; +} + +/** + * Whether `merged` resolves every call that `record` decides, and its rejection messages, as `record` does. + * That includes calls only `merged` names, which a decision for good in `record` also covers. + */ +function keepsDecisions(record: ApprovalRecord, merged: ApprovalRecord): boolean { + // Index each record's call IDs once, so long approval histories compare in linear time. + const recordDecision = approvalDecisions(record); + const mergedDecision = approvalDecisions(merged); + const callIds = new Set([undefined, ...namedCallIds(record), ...namedCallIds(merged)]); + for (const callId of callIds) { + const decision = recordDecision(callId); + if (decision === undefined) continue; + if (mergedDecision(callId) !== decision) return false; + if (!decision && rejectionMessage(merged, callId) !== rejectionMessage(record, callId)) return false; + } + return true; +} + +function namedCallIds(record: ApprovalRecord): string[] { + return [ + ...(Array.isArray(record.approved) ? record.approved : []), + ...(Array.isArray(record.rejected) ? record.rejected : []), + ...Object.keys(record.messages ?? {}), + ]; +} + +/** + * Resolves the decision a record gives a call, or a call it doesn't name if `callId` is undefined, as + * agents-core resolves it. + */ +function approvalDecisions(record: ApprovalRecord): (callId: string | undefined) => boolean | undefined { + const approved = new Set(Array.isArray(record.approved) ? record.approved : []); + const rejected = new Set(Array.isArray(record.rejected) ? record.rejected : []); + return (callId) => { + if (record.approved === true) return true; + if (record.rejected === true) return false; + if (callId !== undefined && approved.has(callId)) return true; + if (callId !== undefined && rejected.has(callId)) return false; + return undefined; + }; +} + +/** The rejection message a record gives a call, as agents-core resolves it. */ +function rejectionMessage(record: ApprovalRecord, callId: string | undefined): string | undefined { + if (callId !== undefined && typeof record.messages?.[callId] === 'string') return record.messages[callId]; + return record.rejected === true ? record.stickyRejectMessage : undefined; +} + +function recordResumeConversion(state: RunState, agent: Agent, conversion: Conversion): void { + let byAgent = resumeConversions.get(state._context); + if (byAgent === undefined) { + byAgent = new Map(); + resumeConversions.set(state._context, byAgent); + } + byAgent.set(agent, conversion); +} + export interface TemporalRunOptions { /** Run context passed to agents and tools */ context?: TContext; @@ -271,50 +462,82 @@ export class TemporalOpenAIRunner { const { model: modelOverride, ...runnerConfigOverrides } = options?.runConfig ?? {}; - const converted = convertAgent(agent, this.modelParams, undefined, modelOverride); - if (options?.runConfig?.sandbox !== undefined) { - propagateAgentToolSandboxConfig(converted, options.runConfig.sandbox); + const sandbox = options?.runConfig?.sandbox; + // Serialize the state before converting, from the graph it was saved on, which the conversion changes. + const serializedState = input instanceof RunState ? input.toString() : undefined; + const reusable = input instanceof RunState ? reusableConversion(input, agent, serializedState!) : undefined; + let conversion = convertAgentGraph(agent, this.modelParams, modelOverride, sandbox, reusable); + // The conversion can change which agents of the graph share a name identity, so check the graph it + // gave again. + if ( + reusable !== undefined && + losesStateDecisions(conversion.agents.get(agent)!, serializedState!, (input as RunState)._context) + ) { + conversion = convertAgentGraph(agent, this.modelParams, modelOverride, sandbox); } + const converted = conversion.agents.get(agent)!; - let preparedInput: string | AgentInputItem[] | RunState; - if (input instanceof RunState) { - // Round-trip through fromString so the rehydrated state's agent graph carries our converted - // agent (with ActivityBackedModel); setCurrentAgent only swaps the top-level ref, not nested ones. - // It also drops upstream's live-session identity, which would otherwise resolve manifest - // environment references on the Workflow thread rather than in the resume Activity. - const restored = (await RunState.fromString(converted, input.toString())) as RunState; - // Suppress upstream Runner.run's withTrace(state._trace, ...) branch; we want the current Workflow's trace. - restored.clearTrace(); - preparedInput = restored; - } else { - preparedInput = input; - } + conversion.activeRuns++; + // A streamed run keeps using the graph after `run` returns, until its result completes. + let releasedOnCompletion = false; + try { + let preparedInput: string | AgentInputItem[] | RunState; + if (input instanceof RunState) { + // Record before restoring, since the restore merges the state's approvals under this graph even if + // the resume then fails. + recordResumeConversion(input, agent, conversion); + // Round-trip through fromString so the rehydrated state's agent graph carries our converted + // agent (with ActivityBackedModel); setCurrentAgent only swaps the top-level ref, not nested ones. + // It also drops upstream's live-session identity, which would otherwise resolve manifest + // environment references on the Workflow thread rather than in the resume Activity. + // Pass the state's own context, as the upstream Runner uses it, so resumed tools write the + // caller's object rather than a JSON copy. + const restored = (await RunState.fromStringWithContext( + converted, + serializedState!, + input._context + )) as RunState; + // Suppress upstream Runner.run's withTrace(state._trace, ...) branch; we want the current Workflow's trace. + restored.clearTrace(); + preparedInput = restored; + } else { + preparedInput = input; + } - const innerRunner = new Runner({ ...runnerConfigOverrides }); + const innerRunner = new Runner({ ...runnerConfigOverrides }); - const sharedRunOptions = { - maxTurns: options?.maxTurns, - context: options?.context, - previousResponseId: options?.previousResponseId, - conversationId: options?.conversationId, - session: options?.session, - sessionInputCallback: options?.sessionInputCallback, - callModelInputFilter: options?.callModelInputFilter, - tracing: options?.tracing, - }; + const sharedRunOptions = { + maxTurns: options?.maxTurns, + context: options?.context, + previousResponseId: options?.previousResponseId, + conversationId: options?.conversationId, + session: options?.session, + sessionInputCallback: options?.sessionInputCallback, + callModelInputFilter: options?.callModelInputFilter, + tracing: options?.tracing, + }; - try { - if (options?.stream === true) { - return (await innerRunner.run(converted, preparedInput as any, { - ...sharedRunOptions, - stream: true, - })) as StreamedRunResult; + try { + if (options?.stream === true) { + const result = (await innerRunner.run(converted, preparedInput as any, { + ...sharedRunOptions, + stream: true, + })) as StreamedRunResult; + const release = () => { + conversion.activeRuns--; + }; + result.completed.then(release, release); + releasedOnCompletion = true; + return result; + } + return (await innerRunner.run(converted, preparedInput as any, sharedRunOptions)) as RunResult; + } catch (error) { + throw normalizeAgentsRunError(error); + } finally { + await flushOpenSpans(); } - return (await innerRunner.run(converted, preparedInput as any, sharedRunOptions)) as RunResult; - } catch (error) { - throw normalizeAgentsRunError(error); } finally { - await flushOpenSpans(); + if (!releasedOnCompletion) conversion.activeRuns--; } } }