diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 54b25d1248..a491a2a26e 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -1,8 +1,8 @@ schema_version: 1 -generation_id: 87f4db43-c675-4ba0-94cf-92419ca7e15f -openapi_spec_hash: 979c95586978a86f71fa536e4ea9c317 -openapi_transformed_spec_hash: 74543bae3cae0325d1a083a32a69fc8d -config_hash: d34bec56226dccb45e6006948ad0b8f3 -codegen_sha: 43ab162e33cae774aec636fa18809821c237db0d -codegen_hash: 25dcfd9472f37100aaac9abd7966ad96fbecaf5505f621cac1be0e1d31de2df0 -public_codegen_sha: 32b000d5c2c171055bb7b054e552581cbd60214d +generation_id: a24ee0c3-cc89-4f1d-94df-089f58b9c02c +openapi_spec_hash: a42624d4a38501daeaa1b9938b52a05c +openapi_transformed_spec_hash: 2a603926af5632209d15f252ab32aab2 +config_hash: 89a5b6f0f1764e7bea04cd6556cc44b0 +codegen_sha: 94e40827db42cbd4dc5f23f7d1dabc157017c2da +codegen_hash: 28a8d01092bd54e6f6340312270c083c4d247fcf5c94076bb22aa4a285364d0d +public_codegen_sha: ca9be3066fafeaf011fa09c2257edb7592f681d6 diff --git a/api_reference/openapi.transformed.yml b/api_reference/openapi.transformed.yml index edb426a338..4e8e7173ee 100644 --- a/api_reference/openapi.transformed.yml +++ b/api_reference/openapi.transformed.yml @@ -50637,6 +50637,7 @@ components: - $ref: '#/components/schemas/LiveInputAudioAppendEvent' - $ref: '#/components/schemas/LiveInputAudioMuteParam' - $ref: '#/components/schemas/LiveInputAudioUnmuteParam' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlayParam' - $ref: '#/components/schemas/LiveInstructionsAppendParam' - $ref: '#/components/schemas/LiveThinkingAppendParam' - $ref: '#/components/schemas/LiveCommentaryAppendParam' @@ -50694,6 +50695,7 @@ components: - $ref: '#/components/schemas/LiveInputAudioAppendEvent' - $ref: '#/components/schemas/LiveInputAudioMuteParam' - $ref: '#/components/schemas/LiveInputAudioUnmuteParam' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlayParam' - $ref: '#/components/schemas/LiveInstructionsAppendParam' - $ref: '#/components/schemas/LiveThinkingAppendParam' - $ref: '#/components/schemas/LiveCommentaryAppendParam' @@ -50881,13 +50883,14 @@ components: - session - transport LiveSidebandClientEvent: - description: Client events accepted by an attached Live sideband WebSocket. The session is already started; send audio over the primary connection. + description: Client events accepted by an attached Live sideband WebSocket. The session has already started. Send input audio over the primary connection. discriminator: propertyName: type anyOf: - $ref: '#/components/schemas/LiveSessionUpdateParam' - $ref: '#/components/schemas/LiveInputAudioMuteParam' - $ref: '#/components/schemas/LiveInputAudioUnmuteParam' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlayParam' - $ref: '#/components/schemas/LiveInstructionsAppendParam' - $ref: '#/components/schemas/LiveThinkingAppendParam' - $ref: '#/components/schemas/LiveCommentaryAppendParam' @@ -50920,6 +50923,8 @@ components: - $ref: '#/components/schemas/LiveSessionUpdated' - $ref: '#/components/schemas/LiveInputAudioMuted' - $ref: '#/components/schemas/LiveInputAudioUnmuted' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlaying' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlayed' - $ref: '#/components/schemas/LiveInstructionsAppended' - $ref: '#/components/schemas/LiveThinkingAppended' - $ref: '#/components/schemas/LiveCommentaryAppended' @@ -92107,6 +92112,39 @@ components: example: type: session.input_audio.unmute event_id: evt_unmute_001 + LiveOutputAudioFixturePlayParam: + description: Play one complete audio fixture through the session output. Send this event over the primary WebSocket or a server sideband. Audio must be raw mono PCM16 little-endian at 24 kHz, regardless of the session audio format, and at most 30 seconds. The entire payload is validated before model speech is interrupted and input is muted. Only one fixture can be active. fixture.playing acknowledges acceptance and fixture.played reports server mixer drain. These notifications are not exact transport-media boundaries. Input arriving during playback is dropped and not replayed. + example: + type: session.output_audio.fixture.play + event_id: evt_play_001 + audio: AAAAAA== + type: object + properties: + event_id: + anyOf: + - description: Optional client identifier for correlating this command with a server event's client_event_id or error.client_event_id. + type: string + maxLength: 512 + - type: 'null' + type: + default: session.output_audio.fixture.play + x-stainless-const: true + type: string + enum: + - session.output_audio.fixture.play + audio: + description: Base64-encoded raw mono PCM16 audio, little-endian, at 24 kHz. Do not include a WAV or other container header. + type: string + minLength: 1 + maxLength: 1920000 + required: + - type + - audio + x-oaiMeta: + example: + type: session.output_audio.fixture.play + event_id: evt_play_001 + audio: AAAAAA== LiveInstructionsAppendParam: description: Append instructions to the Live conversation while it is running, optionally associating them with an existing client delegation. example: @@ -92624,6 +92662,62 @@ components: type: session.input_audio.unmuted event_id: evt_unmuted_001 client_event_id: evt_unmute_001 + LiveOutputAudioFixturePlaying: + description: The existing playback path has accepted the validated audio fixture. This acknowledgment does not identify when the listener receives the first sample. The optional client_event_id echoes the play command. + example: + type: session.output_audio.fixture.playing + event_id: evt_playing_001 + client_event_id: evt_play_001 + type: object + properties: + event_id: + description: The unique ID of the Live server event. + type: string + client_event_id: + description: The event_id of the client command associated with this server event, when supplied. + type: string + type: + default: session.output_audio.fixture.playing + x-stainless-const: true + type: string + enum: + - session.output_audio.fixture.playing + required: + - event_id + - type + x-oaiMeta: + example: + type: session.output_audio.fixture.playing + event_id: evt_playing_001 + client_event_id: evt_play_001 + LiveOutputAudioFixturePlayed: + description: The complete audio fixture has drained from the server mixer and normal inference has resumed. Transport buffers may still contain audio; this event does not prove listener-side completion. The optional client_event_id echoes the play command. This event is not emitted for canceled or failed playback. + example: + type: session.output_audio.fixture.played + event_id: evt_played_001 + client_event_id: evt_play_001 + type: object + properties: + event_id: + description: The unique ID of the Live server event. + type: string + client_event_id: + description: The event_id of the client command associated with this server event, when supplied. + type: string + type: + default: session.output_audio.fixture.played + x-stainless-const: true + type: string + enum: + - session.output_audio.fixture.played + required: + - event_id + - type + x-oaiMeta: + example: + type: session.output_audio.fixture.played + event_id: evt_played_001 + client_event_id: evt_play_001 LiveInstructionsAppended: description: Returned when a session.instructions.append command is accepted into the Live session timeline. Acknowledges the appended instructions without guaranteeing that the model has acted on them. example: @@ -93487,6 +93581,8 @@ components: - $ref: '#/components/schemas/LiveSessionUpdated' - $ref: '#/components/schemas/LiveInputAudioMuted' - $ref: '#/components/schemas/LiveInputAudioUnmuted' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlaying' + - $ref: '#/components/schemas/LiveOutputAudioFixturePlayed' - $ref: '#/components/schemas/LiveInstructionsAppended' - $ref: '#/components/schemas/LiveThinkingAppended' - $ref: '#/components/schemas/LiveCommentaryAppended' @@ -107960,6 +108056,9 @@ x-oaiMeta: - type: object key: LiveInputAudioUnmuteParam path: + - type: object + key: LiveOutputAudioFixturePlayParam + path: - type: object key: LiveInstructionsAppendParam path: @@ -108004,6 +108103,12 @@ x-oaiMeta: - type: object key: LiveInputAudioUnmuted path: + - type: object + key: LiveOutputAudioFixturePlaying + path: + - type: object + key: LiveOutputAudioFixturePlayed + path: - type: object key: LiveInstructionsAppended path: diff --git a/scripts/castiron/generated-files.json b/scripts/castiron/generated-files.json index 6462cfec13..115dc2d082 100644 --- a/scripts/castiron/generated-files.json +++ b/scripts/castiron/generated-files.json @@ -15,7 +15,7 @@ }, "scripts/castiron/CUSTOM_CODE.md": { "mode": "100644", - "sha256": "ed9268b75d1714ff4627e7bd98ce1ab997199276b22fd26f2ba166af417902bf" + "sha256": "9c602fb42e00a40e21f3106c357fdaadfc1b9865de4105c71364354b451211d6" }, "scripts/castiron/README.md": { "mode": "100644", @@ -23,7 +23,7 @@ }, "scripts/castiron/custom_code_budget.py": { "mode": "100644", - "sha256": "4bffc531f6807ab95da5ef52e0f9deb69d81a42b467551e088df99516acff941" + "sha256": "e504abba2701f4733eb4b8ce26856cca596e13feab1e7e323adc8e7d2ccb64bc" }, "scripts/castiron/custom_code_report.py": { "mode": "100644", @@ -43,7 +43,7 @@ }, "scripts/castiron/test_custom_code_budget.py": { "mode": "100644", - "sha256": "e89192ddeb77ca8c3fb06ab2ae524c30fa3d83292438d7a7fabc4a0144e4f106" + "sha256": "f07fcaf6d10890a47403b784898e3f40a443fa551d66701a717de8382ebf0b9c" }, "scripts/castiron/test_custom_code_github.py": { "mode": "100644", @@ -743,7 +743,7 @@ }, "src/resources/live/api.md": { "mode": "100644", - "sha256": "26be4e640ca9734e59796027a764324f60a08c05aaa568c302bc75d5ea944060" + "sha256": "edabb404d86a7d4449fcc597c67b042539e0aabd546011eb84a661dcae872dcd" }, "src/resources/live/forks.ts": { "mode": "100644", @@ -751,7 +751,7 @@ }, "src/resources/live/forks/forks.ts": { "mode": "100644", - "sha256": "a3249f2d4873f58c883bdd5764de6aa3534e7e42cef59284dea90e7066693d5e" + "sha256": "624d42a82f3505ec0622c65526c20ed7cdb314a537754d99483391d1b1fed2cb" }, "src/resources/live/forks/index.ts": { "mode": "100644", @@ -779,7 +779,7 @@ }, "src/resources/live/live.ts": { "mode": "100644", - "sha256": "9e9af2c8cdfb58e568f0d5c4f0a9f56774802a026486b1a656e6e11d90252d25" + "sha256": "bec794f0d3266425425756048461d611641b77b77f21144f13f9356a1d8a1dfa" }, "src/resources/live/sessions.ts": { "mode": "100644", @@ -799,7 +799,7 @@ }, "src/resources/live/sideband/sideband.ts": { "mode": "100644", - "sha256": "9fd74ae12a5f4a9688e34cfdf4e9c96c1f96527f4be17ec1cd66b42643742e14" + "sha256": "2d16a82eaa3d6b71359bcf05f642e2d8a319b4b13d5673c1eef315c02e8f96ed" }, "src/resources/live/sideband/ws-base.ts": { "mode": "100644", diff --git a/src/lib/live/webrtc.ts b/src/lib/live/webrtc.ts index 89187876d1..b3b5ee418e 100644 --- a/src/lib/live/webrtc.ts +++ b/src/lib/live/webrtc.ts @@ -1,6 +1,6 @@ /* oxlint-disable max-classes-per-file -- Both thin facades belong to the same public Live WebRTC entrypoint. */ import type { - ConnectClientEvent as ClientEvent, + ConnectClientEvent, ConnectServerEvent as ServerEvent, } from '../../resources/live/sideband/sideband'; import { WebRTCConnection } from '../webrtc/connection'; @@ -16,6 +16,9 @@ export type { WebRTCState, } from '../webrtc/types'; +/** Fixture playback requires a primary WebSocket or server sideband. */ +type ClientEvent = Exclude; + /** Browser Live connection with application-owned SDP exchange and typed server-event subscriptions. */ export class OpenAILiveWebRTC extends WebRTCConnection {} diff --git a/src/resources/live/api.md b/src/resources/live/api.md index f912100ccb..917cf87b15 100644 --- a/src/resources/live/api.md +++ b/src/resources/live/api.md @@ -30,6 +30,9 @@ Types: - MediaSessionConfig - MediaSessionForkConfig - OutputAudioDeltaEvent +- OutputAudioFixturePlayEvent +- OutputAudioFixturePlayedEvent +- OutputAudioFixturePlayingEvent - OutputTranscriptDeltaEvent - ResponseCreateEvent - ResponseEvent diff --git a/src/resources/live/forks/forks.ts b/src/resources/live/forks/forks.ts index f2dc3903d5..1fa283c5de 100644 --- a/src/resources/live/forks/forks.ts +++ b/src/resources/live/forks/forks.ts @@ -17,6 +17,7 @@ export type ForkClientEvent = | LiveAPI.InputAudioAppendEvent | LiveAPI.InputAudioMuteEvent | LiveAPI.InputAudioUnmuteEvent + | LiveAPI.OutputAudioFixturePlayEvent | LiveAPI.InstructionsAppendEvent | LiveAPI.ThinkingAppendEvent | LiveAPI.CommentaryAppendEvent @@ -36,6 +37,8 @@ export type ForkServerEvent = | LiveAPI.SessionUpdatedEvent | LiveAPI.InputAudioMutedEvent | LiveAPI.InputAudioUnmutedEvent + | LiveAPI.OutputAudioFixturePlayingEvent + | LiveAPI.OutputAudioFixturePlayedEvent | LiveAPI.InstructionsAppendedEvent | LiveAPI.ThinkingAppendedEvent | LiveAPI.CommentaryAppendedEvent diff --git a/src/resources/live/live.ts b/src/resources/live/live.ts index 4a970ff03d..eed2c9a609 100644 --- a/src/resources/live/live.ts +++ b/src/resources/live/live.ts @@ -171,6 +171,7 @@ export type ClientEvent = | InputAudioAppendEvent | InputAudioMuteEvent | InputAudioUnmuteEvent + | OutputAudioFixturePlayEvent | InstructionsAppendEvent | ThinkingAppendEvent | CommentaryAppendEvent @@ -1110,6 +1111,73 @@ export interface OutputAudioDeltaEvent { start_ms?: number; } +/** + * Play one complete audio fixture through the session output. Send this event over + * the primary WebSocket or a server sideband. Audio must be raw mono PCM16 + * little-endian at 24 kHz, regardless of the session audio format, and at most 30 + * seconds. The entire payload is validated before model speech is interrupted and + * input is muted. Only one fixture can be active. fixture.playing acknowledges + * acceptance and fixture.played reports server mixer drain. These notifications + * are not exact transport-media boundaries. Input arriving during playback is + * dropped and not replayed. + */ +export interface OutputAudioFixturePlayEvent { + /** + * Base64-encoded raw mono PCM16 audio, little-endian, at 24 kHz. Do not include a + * WAV or other container header. + */ + audio: string; + + type: 'session.output_audio.fixture.play'; + + /** + * Optional client identifier for correlating this command with a server event's + * client_event_id or error.client_event_id. + */ + event_id?: string | null; +} + +/** + * The complete audio fixture has drained from the server mixer and normal + * inference has resumed. Transport buffers may still contain audio; this event + * does not prove listener-side completion. The optional client_event_id echoes the + * play command. This event is not emitted for canceled or failed playback. + */ +export interface OutputAudioFixturePlayedEvent { + /** + * The unique ID of the Live server event. + */ + event_id: string; + + type: 'session.output_audio.fixture.played'; + + /** + * The event_id of the client command associated with this server event, when + * supplied. + */ + client_event_id?: string; +} + +/** + * The existing playback path has accepted the validated audio fixture. This + * acknowledgment does not identify when the listener receives the first sample. + * The optional client_event_id echoes the play command. + */ +export interface OutputAudioFixturePlayingEvent { + /** + * The unique ID of the Live server event. + */ + event_id: string; + + type: 'session.output_audio.fixture.playing'; + + /** + * The event_id of the client command associated with this server event, when + * supplied. + */ + client_event_id?: string; +} + /** * A transcript fragment for assistant output audio in the Live session. Accumulate * fragments in delivery order; these events do not define complete turns or @@ -1470,6 +1538,8 @@ export type ServerEvent = | SessionUpdatedEvent | InputAudioMutedEvent | InputAudioUnmutedEvent + | OutputAudioFixturePlayingEvent + | OutputAudioFixturePlayedEvent | InstructionsAppendedEvent | ThinkingAppendedEvent | CommentaryAppendedEvent @@ -2271,6 +2341,9 @@ export declare namespace Live { type MediaSessionConfig as MediaSessionConfig, type MediaSessionForkConfig as MediaSessionForkConfig, type OutputAudioDeltaEvent as OutputAudioDeltaEvent, + type OutputAudioFixturePlayEvent as OutputAudioFixturePlayEvent, + type OutputAudioFixturePlayedEvent as OutputAudioFixturePlayedEvent, + type OutputAudioFixturePlayingEvent as OutputAudioFixturePlayingEvent, type OutputTranscriptDeltaEvent as OutputTranscriptDeltaEvent, type ResponseCreateEvent as ResponseCreateEvent, type ResponseEvent as ResponseEvent, diff --git a/src/resources/live/sideband/sideband.ts b/src/resources/live/sideband/sideband.ts index 40f8b468d7..50687d7f78 100644 --- a/src/resources/live/sideband/sideband.ts +++ b/src/resources/live/sideband/sideband.ts @@ -6,13 +6,14 @@ import * as LiveAPI from '../live'; export class Sideband extends APIResource {} /** - * Client events accepted by an attached Live sideband WebSocket. The session is - * already started; send audio over the primary connection. + * Client events accepted by an attached Live sideband WebSocket. The session has + * already started. Send input audio over the primary connection. */ export type ConnectClientEvent = | LiveAPI.SessionUpdateEvent | LiveAPI.InputAudioMuteEvent | LiveAPI.InputAudioUnmuteEvent + | LiveAPI.OutputAudioFixturePlayEvent | LiveAPI.InstructionsAppendEvent | LiveAPI.ThinkingAppendEvent | LiveAPI.CommentaryAppendEvent @@ -29,6 +30,8 @@ export type ConnectServerEvent = | LiveAPI.SessionUpdatedEvent | LiveAPI.InputAudioMutedEvent | LiveAPI.InputAudioUnmutedEvent + | LiveAPI.OutputAudioFixturePlayingEvent + | LiveAPI.OutputAudioFixturePlayedEvent | LiveAPI.InstructionsAppendedEvent | LiveAPI.ThinkingAppendedEvent | LiveAPI.CommentaryAppendedEvent diff --git a/tests/lib/webrtc.test.ts b/tests/lib/webrtc.test.ts index 12fe09ef78..0db3cc72f6 100644 --- a/tests/lib/webrtc.test.ts +++ b/tests/lib/webrtc.test.ts @@ -493,6 +493,15 @@ it('sends typed client messages and reports local failures synchronously, withou live.dispose(); }); +it('excludes WebSocket-only fixture playback from Live WebRTC client events', () => { + expectTypeOf<{ type: 'session.output_audio.fixture.play'; audio: string }>().not.toMatchTypeOf< + Parameters[0] + >(); + expectTypeOf[0]>().toEqualTypeOf< + Parameters[0] + >(); +}); + it('infers named payloads from each public helper’s generated server-event union', () => { const live = new OpenAILiveWebRTC(); const realtime = new OpenAIRealtimeWebRTC();