From c49e52d5e2084f91571451791a1c5c728af4e20b Mon Sep 17 00:00:00 2001 From: sinonchum Date: Sat, 16 May 2026 15:22:23 +0200 Subject: [PATCH 1/8] feat: integrate PatentFlow voice session --- .env.example | 6 +- .gitignore | 3 + docker-compose.yml | 8 +- frontend/.dockerignore | 13 + frontend/Dockerfile | 4 + .../voice-runtime/audio-output-worklet.js | 465 ++++++++++ .../public/voice-runtime/audio-processor.js | 420 +++++++++ .../public/voice-runtime/decoderWorker.min.js | 847 ++++++++++++++++++ .../voice-runtime/decoderWorker.min.wasm | Bin 0 -> 149534 bytes .../public/voice-runtime/encoderWorker.min.js | 1 + frontend/public/voice-runtime/opus-encoder.js | 154 ++++ .../voice-runtime/synced-audio-player.js | 399 +++++++++ frontend/src/app/page.tsx | 315 ++++++- src/api.py | 15 +- voice_pipeline/__init__.py | 4 + voice_pipeline/requirements.txt | 7 + voice_pipeline/server.py | 344 +++++++ voice_pipeline/static/index.html | 22 + 18 files changed, 3012 insertions(+), 15 deletions(-) create mode 100644 frontend/.dockerignore create mode 100644 frontend/public/voice-runtime/audio-output-worklet.js create mode 100644 frontend/public/voice-runtime/audio-processor.js create mode 100644 frontend/public/voice-runtime/decoderWorker.min.js create mode 100644 frontend/public/voice-runtime/decoderWorker.min.wasm create mode 100644 frontend/public/voice-runtime/encoderWorker.min.js create mode 100644 frontend/public/voice-runtime/opus-encoder.js create mode 100644 frontend/public/voice-runtime/synced-audio-player.js create mode 100644 voice_pipeline/__init__.py create mode 100644 voice_pipeline/requirements.txt create mode 100644 voice_pipeline/server.py create mode 100644 voice_pipeline/static/index.html diff --git a/.env.example b/.env.example index 7a6d735..4afbd14 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,21 @@ REDIS_URL=redis://redis:6379/0 CELERY_CONCURRENCY=1 NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 +NEXT_PUBLIC_VOICE_SERVER_URL=http://localhost:7860 +NEXT_PUBLIC_PATENTFLOW_API_KEY= ALLOWED_ORIGINS=http://localhost:3000,http://localhost:3001 STATUS_RESPONSE_MAX_BYTES=262144 RESULT_BACKEND_MAX_BYTES=262144 +PATENTFLOW_API_KEY= EPO_ENABLED=false EPO_CONSUMER_KEY= EPO_CONSUMER_SECRET= LLM_PROVIDER=local -LLM_BASE_URL=http://127.0.0.1:11434 +LLM_BASE_URL=http://host.docker.internal:11434/v1 LLM_MODEL= +LLM_API_KEY= RAG_EMBEDDING_BASE_URL= RAG_EMBEDDING_MODEL= diff --git a/.gitignore b/.gitignore index fd034f8..333c2c1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ dist/ *.egg-info/ .DS_Store +Icon? +**/Icon? +.voice-venv/ .vscode/ .idea/ diff --git a/docker-compose.yml b/docker-compose.yml index 5e761b5..ebaa4c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,8 @@ services: build: context: . dockerfile: Dockerfile.api + env_file: + - .env environment: - REDIS_URL=redis://redis:6379/0 networks: @@ -47,6 +49,8 @@ services: build: context: . dockerfile: Dockerfile.worker + env_file: + - .env environment: - REDIS_URL=redis://redis:6379/0 - CELERY_CONCURRENCY=1 @@ -63,7 +67,9 @@ services: context: ./frontend dockerfile: Dockerfile args: - NEXT_PUBLIC_API_BASE_URL: http://localhost:8000 + NEXT_PUBLIC_API_BASE_URL: ${NEXT_PUBLIC_API_BASE_URL:-http://localhost:8000} + NEXT_PUBLIC_PATENTFLOW_API_KEY: ${NEXT_PUBLIC_PATENTFLOW_API_KEY:-} + NEXT_PUBLIC_VOICE_SERVER_URL: ${NEXT_PUBLIC_VOICE_SERVER_URL:-http://localhost:7860} networks: - patentflow-net ports: diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 0000000..a9d4be7 --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,13 @@ +node_modules/ +.next/ +out/ +.git/ +.gitignore +.DS_Store +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +.env +.env.local +.env.*.local diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2fb6783..61e81f9 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -12,6 +12,10 @@ COPY . /app # NEXT_PUBLIC_* vars are baked at build time ARG NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 ENV NEXT_PUBLIC_API_BASE_URL=${NEXT_PUBLIC_API_BASE_URL} +ARG NEXT_PUBLIC_PATENTFLOW_API_KEY= +ENV NEXT_PUBLIC_PATENTFLOW_API_KEY=${NEXT_PUBLIC_PATENTFLOW_API_KEY} +ARG NEXT_PUBLIC_VOICE_SERVER_URL=http://localhost:7860 +ENV NEXT_PUBLIC_VOICE_SERVER_URL=${NEXT_PUBLIC_VOICE_SERVER_URL} RUN npm run build diff --git a/frontend/public/voice-runtime/audio-output-worklet.js b/frontend/public/voice-runtime/audio-output-worklet.js new file mode 100644 index 0000000..cabf5ca --- /dev/null +++ b/frontend/public/voice-runtime/audio-output-worklet.js @@ -0,0 +1,465 @@ +/** + * AudioWorklet processor for jitter-buffered audio playback. + * + * Handles: + * - Buffering incoming decoded audio frames + * - Adaptive buffer sizing (increases on underruns) + * - Packet dropping when buffer exceeds max + * - Exponential fade-in/out to avoid clicks (including slow fade on interruption) + * - Delay tracking for metrics + * + * Usage: + * await audioContext.audioWorklet.addModule('/path/to/audio-output-worklet.js'); + * const worklet = new AudioWorkletNode(audioContext, 'audio-output-processor'); + * worklet.connect(audioContext.destination); + * + * // Send decoded audio frames: + * worklet.port.postMessage({ type: 'audio', frame: float32Array }); + * + * // Reset state: + * worklet.port.postMessage({ type: 'reset' }); + */ + +function asMs(samples) { + return ((samples * 1000) / sampleRate).toFixed(1); +} + +function asSamples(ms) { + return Math.round((ms * sampleRate) / 1000); +} + +// Maximum buffer before we start dropping packets (30 seconds) +const DEFAULT_MAX_BUFFER_MS = 30 * 1000; + +// Fade durations in ms +const FADE_IN_MS = 15; +const FADE_OUT_MS = 10; +const FADE_OUT_INTERRUPTED_MS = 200; + +// Exponential fade threshold — fade is "complete" when it reaches this level (~-60dB) +const FADE_THRESHOLD = 0.001; + +// Compute per-sample exponential decay factor for a given duration in ms. +// After N samples, the fade reaches FADE_THRESHOLD (effectively silent). +function fadeDecay(ms) { + const N = asSamples(ms); + return Math.pow(FADE_THRESHOLD, 1.0 / Math.max(1, N)); +} + +// Set to true to enable debug logging +const DEBUG = false; +const debug = (...args) => { + if (DEBUG) console.debug('[AudioOutputWorklet]', ...args); +}; + +class AudioOutputProcessor extends AudioWorkletProcessor { + constructor() { + super(); + debug('AudioOutputProcessor created', currentFrame, sampleRate); + + // Buffer timing configuration (in samples) + const frameSize = asSamples(80); + + // Wait for at least this many samples before starting playback + this.initialBufferSamples = 1 * frameSize; + + // Additional wait after reaching initialBufferSamples (allows non-frame-aligned buffers) + this.partialBufferSamples = asSamples(10); + + // Drop oldest packets if buffer exceeds this + this.maxBufferSamples = asSamples(DEFAULT_MAX_BUFFER_MS); + + // How much to increase buffers on underrun/overflow + this.partialBufferIncrement = asSamples(5); + this.maxPartialWithIncrements = asSamples(80); + this.maxBufferSamplesIncrement = asSamples(5); + this.maxMaxBufferWithIncrements = asSamples(80); + + this.initState(); + + this.port.onmessage = (event) => { + if (event.data.type === 'reset') { + debug('Reset audio processor state'); + this.initState(); + return; + } + + if (event.data.type === 'audio') { + const frame = event.data.frame; + const stopS = event.data.stopS; + const turnIdx = event.data.turnIdx; + + console.log( + '[AudioWorklet] Audio received:', + frame.length + ' samples (' + asMs(frame.length) + 'ms),', + 'turn=' + turnIdx, + 'stopS=' + stopS + ); + + // Route audio to correct buffer based on turn + let buffered = false; + if (turnIdx === this.currentPlayingTurnIdx || this.currentPlayingTurnIdx === null) { + // Current turn - add to main buffer + this.pushFrame(this.frames, frame, turnIdx); + if (this.currentPlayingTurnIdx === null) { + this.currentPlayingTurnIdx = turnIdx; + } + buffered = true; + debug('Audio to MAIN buffer - turn', turnIdx, 'frames now:', this.frames.length); + } else if (turnIdx > this.currentPlayingTurnIdx) { + // Future turn - check if we can play immediately (buffer is empty) + if (this.frames.length === 0) { + // Buffer is empty, play new turn immediately + this.currentPlayingTurnIdx = turnIdx; + this.pushFrame(this.frames, frame, turnIdx); + buffered = true; + debug('Buffer empty - starting new turn immediately', turnIdx); + console.log('[AudioWorklet] Starting turn', turnIdx, 'immediately (buffer was empty)'); + } else { + // Buffer not empty - add to pending buffer + // If we get a newer turn than what's pending, replace pending (skip old turn) + if (this.pendingTurnIdx === null || turnIdx > this.pendingTurnIdx) { + if (this.pendingTurnIdx !== null) { + console.log('[AudioWorklet] Skipping turn', this.pendingTurnIdx, 'for newer turn', turnIdx); + } + debug('New pending turn', this.pendingTurnIdx, '->', turnIdx, '- clearing old pending'); + this.pendingFrames = []; + this.pendingTurnIdx = turnIdx; + } + // Add to pending buffer (either new or existing pending) + if (turnIdx === this.pendingTurnIdx) { + this.pushFrame(this.pendingFrames, frame, turnIdx); + buffered = true; + debug('Audio to PENDING buffer - turn', turnIdx, 'pending frames now:', this.pendingFrames.length); + } + // If turnIdx < pendingTurnIdx, it's stale - drop it + } + } else { + // Stale turn - drop it + debug('DROPPING audio - stale turn', turnIdx, '< current', this.currentPlayingTurnIdx); + console.log('[AudioWorklet] DROPPING audio for stale turn', turnIdx, '(current:', this.currentPlayingTurnIdx + ')'); + } + + // Only update tracking/state for audio that was actually buffered + if (buffered) { + if (stopS !== undefined) { + this.lastStopS = stopS; + } + + // Handle interrupted flag — switch to slow fade-out + // Only for current-turn audio (not pending), since it affects the active playback fade + if (event.data.interrupted && turnIdx === this.currentPlayingTurnIdx) { + const currentBuffered = this.currentSamples(); + const desiredFade = asSamples(FADE_OUT_INTERRUPTED_MS); + const effectiveFade = Math.min(desiredFade, currentBuffered); + const effectiveFadeMs = (effectiveFade * 1000) / sampleRate; + this.fadeOutDecay = fadeDecay(effectiveFadeMs); + console.log( + '[AudioWorklet] Interrupted fade-out:', + 'buffered=' + asMs(currentBuffered) + 'ms,', + 'fade=' + asMs(effectiveFade) + 'ms,', + 'decay=' + this.fadeOutDecay.toFixed(6) + ); + } + } + + // Start playback once we have enough buffered + if (this.currentSamples() >= this.initialBufferSamples && !this.started) { + this.start(); + } + + if (this.pidx < 20) { + debug( + this.timestamp(), + 'Got packet', + this.pidx++, + 'turn=' + turnIdx, + 'current=' + this.currentPlayingTurnIdx, + 'pending=' + this.pendingTurnIdx, + asMs(this.currentSamples()) + 'ms buffered', + asMs(frame.length) + 'ms frame' + ); + } + + // Drop packets if buffer is too full + if (this.currentSamples() >= this.totalMaxBufferSamples()) { + this.dropExcessPackets(); + } + + // Send metrics back to main thread (bufferMs and lastStopS are atomic) + this.port.postMessage({ + type: 'metrics', + totalAudioPlayed: this.totalAudioPlayed, + actualAudioPlayed: this.actualAudioPlayed, + bufferMs: (this.currentSamples() / sampleRate) * 1000, + lastStopS: this.lastStopS, + minDelay: this.minDelay, + maxDelay: this.maxDelay, + }); + } + }; + } + + initState() { + this.frames = []; + this.offsetInFirstBuffer = 0; + this.remainingPartialBufferSamples = 0; + this.fadePos = 0; // 0.0..1.0, current fade multiplier + this.fadeInDecay = fadeDecay(FADE_IN_MS); // per-sample decay for fade-in + this.fadeOutDecay = fadeDecay(FADE_OUT_MS); // per-sample decay for fade-out + this.timeInStream = 0; + this.resetStart(); + + // Metrics + this.totalAudioPlayed = 0; + this.actualAudioPlayed = 0; + this.maxDelay = 0; + this.minDelay = 2000; + this.lastStopS = 0; // stop_s of the last audio frame added to buffer + + // Turn tracking + this.currentPlayingTurnIdx = null; + + // Buffer empty detection + this.hadFrames = false; + + // Pending buffer for next turn (prevents mixing audio between turns) + this.pendingFrames = []; + this.pendingTurnIdx = null; + + // Debug counter + this.pidx = 0; + + // Reset buffer params + this.partialBufferSamples = asSamples(10); + this.maxBufferSamples = asSamples(DEFAULT_MAX_BUFFER_MS); + } + + totalMaxBufferSamples() { + return this.maxBufferSamples + this.partialBufferSamples + this.initialBufferSamples; + } + + timestamp() { + return Date.now() % 1000; + } + + currentSamples() { + let samples = 0; + for (let k = 0; k < this.frames.length; k++) { + samples += this.frames[k].samples.length; + } + samples -= this.offsetInFirstBuffer; + return samples; + } + + resetStart() { + this.started = false; + } + + start() { + this.started = true; + this.remainingPartialBufferSamples = this.partialBufferSamples; + } + + canPlay() { + return this.started && this.frames.length > 0 && this.remainingPartialBufferSamples <= 0; + } + + pushFrame(buffer, frame, turnIdx) { + buffer.push({ samples: frame, turnIdx: turnIdx }); + } + + dropExcessPackets() { + debug( + this.timestamp(), + 'Dropping packets', + asMs(this.currentSamples()) + 'ms buffered', + asMs(this.totalMaxBufferSamples()) + 'ms max' + ); + + const target = this.initialBufferSamples + this.partialBufferSamples; + while (this.currentSamples() > target) { + const first = this.frames[0]; + let toRemove = this.currentSamples() - target; + toRemove = Math.min(first.samples.length - this.offsetInFirstBuffer, toRemove); + this.offsetInFirstBuffer += toRemove; + this.timeInStream += toRemove / sampleRate; + + if (this.offsetInFirstBuffer === first.samples.length) { + this.frames.shift(); + this.offsetInFirstBuffer = 0; + } + } + + debug(this.timestamp(), 'After drop:', asMs(this.currentSamples()) + 'ms'); + + // Increase max buffer to reduce future drops + this.maxBufferSamples += this.maxBufferSamplesIncrement; + this.maxBufferSamples = Math.min(this.maxMaxBufferWithIncrements, this.maxBufferSamples); + debug('Increased maxBuffer to', asMs(this.maxBufferSamples) + 'ms'); + } + + process(inputs, outputs) { + const delay = this.currentSamples() / sampleRate; + if (this.canPlay()) { + this.maxDelay = Math.max(this.maxDelay, delay); + this.minDelay = Math.min(this.minDelay, delay); + } + + const output = outputs[0][0]; + + if (!this.canPlay()) { + if (this.actualAudioPlayed > 0) { + this.totalAudioPlayed += output.length / sampleRate; + } + this.fadePos = 0; + this.remainingPartialBufferSamples -= output.length; + // Output near-silent signal to prevent Chrome from suspending the audio output + // path (which causes an audible "switch" sound when audio resumes). + for (let i = 0; i < output.length; i++) { + output[i] = 1e-7; + } + return true; + } + + let outIdx = 0; + + // Copy frames to output buffer + // Note: All frames in this.frames now have the same turnIdx due to pending buffer system + while (outIdx < output.length && this.frames.length) { + const first = this.frames[0]; + + const toCopy = Math.min( + first.samples.length - this.offsetInFirstBuffer, + output.length - outIdx + ); + const subArray = first.samples.subarray( + this.offsetInFirstBuffer, + this.offsetInFirstBuffer + toCopy + ); + output.set(subArray, outIdx); + + this.offsetInFirstBuffer += toCopy; + outIdx += toCopy; + + if (this.offsetInFirstBuffer === first.samples.length) { + this.offsetInFirstBuffer = 0; + this.frames.shift(); + } + } + + // Apply per-sample exponential fade based on buffer fullness. + // fadePos (0.0..1.0) ramps up/down exponentially. + // Fade-out starts when remaining samples < samples needed to reach FADE_THRESHOLD. + const remaining = this.currentSamples(); + // How many samples needed for fadePos to decay to FADE_THRESHOLD: + // fadePos * fadeOutDecay^N = FADE_THRESHOLD => N = log(FADE_THRESHOLD/fadePos) / log(fadeOutDecay) + const fadeOutSamplesNeeded = this.fadePos > FADE_THRESHOLD + ? Math.log(FADE_THRESHOLD / this.fadePos) / Math.log(this.fadeOutDecay) + : 0; + const willFadeOut = remaining + outIdx < fadeOutSamplesNeeded; + const alreadyFull = this.fadePos >= 1 - FADE_THRESHOLD; + + if (!alreadyFull || willFadeOut) { + const fadePosStart = this.fadePos; + for (let i = 0; i < outIdx; i++) { + const samplesLeft = remaining + (outIdx - i); + // Re-check fade-out condition per sample (fadePos changes each iteration) + const neededNow = this.fadePos > FADE_THRESHOLD + ? Math.log(FADE_THRESHOLD / this.fadePos) / Math.log(this.fadeOutDecay) + : 0; + if (samplesLeft < neededNow) { + // Fade out: exponential decay toward 0 + this.fadePos *= this.fadeOutDecay; + if (this.fadePos < FADE_THRESHOLD) this.fadePos = 0; + } else { + // Fade in: exponential approach toward 1 + // fadePos = 1 - (1 - fadePos) * decay + this.fadePos = 1 - (1 - this.fadePos) * this.fadeInDecay; + if (this.fadePos > 1 - FADE_THRESHOLD) this.fadePos = 1; + } + output[i] *= this.fadePos; + } + console.log( + '[AudioWorklet] Fade active:', + fadePosStart.toFixed(3), '->', this.fadePos.toFixed(3), + willFadeOut ? 'fade-out' : 'fade-in', + 'remaining=' + asMs(remaining) + 'ms', + 'outIdx=' + outIdx, + 'fadeOutDecay=' + this.fadeOutDecay.toFixed(6) + ); + } + + // Handle buffer underrun + if (outIdx < output.length) { + debug(this.timestamp(), 'Buffer underrun:', output.length - outIdx, 'samples missed'); + + // Increase buffer to reduce future underruns + this.partialBufferSamples += this.partialBufferIncrement; + this.partialBufferSamples = Math.min(this.partialBufferSamples, this.maxPartialWithIncrements); + debug('Increased partial buffer to', asMs(this.partialBufferSamples) + 'ms'); + + // Reset to refill buffer + this.resetStart(); + } + + this.totalAudioPlayed += output.length / sampleRate; + this.actualAudioPlayed += outIdx / sampleRate; + this.timeInStream += outIdx / sampleRate; + + // Track buffer state and check for pending buffer swap + if (this.frames.length > 0) { + this.hadFrames = true; + } else { + // Main buffer is empty - check if we have pending audio to swap in + if (this.pendingFrames.length > 0) { + const oldTurnIdx = this.currentPlayingTurnIdx; + const newTurnIdx = this.pendingTurnIdx; + debug('Buffer empty - swapping pending turn', oldTurnIdx, '->', newTurnIdx, + 'with', this.pendingFrames.length, 'frames'); + + // Notify main thread of turn change + if (oldTurnIdx !== null) { + this.port.postMessage({ + type: 'turn_change', + oldTurnIdx: oldTurnIdx, + newTurnIdx: newTurnIdx, + }); + } + + // Swap pending to main + this.frames = this.pendingFrames; + this.pendingFrames = []; + this.currentPlayingTurnIdx = newTurnIdx; + this.pendingTurnIdx = null; + this.hadFrames = true; + + // Reset fade for new turn — start from silence + this.fadePos = 0; + this.fadeOutDecay = fadeDecay(FADE_OUT_MS); + + // Restart playback for the new turn + this.started = true; + this.remainingPartialBufferSamples = 0; + console.log('[AudioWorklet] Swapped to turn', newTurnIdx, 'and restarted playback'); + } else if (this.hadFrames) { + // Buffer just emptied with no pending audio — reset fade for next turn + this.hadFrames = false; + this.fadeOutDecay = fadeDecay(FADE_OUT_MS); + this.port.postMessage({ + type: 'metrics', + totalAudioPlayed: this.totalAudioPlayed, + actualAudioPlayed: this.actualAudioPlayed, + bufferMs: 0, + lastStopS: this.lastStopS, + minDelay: this.minDelay, + maxDelay: this.maxDelay, + }); + } + } + + return true; + } +} + +registerProcessor('audio-output-processor', AudioOutputProcessor); diff --git a/frontend/public/voice-runtime/audio-processor.js b/frontend/public/voice-runtime/audio-processor.js new file mode 100644 index 0000000..1ff8002 --- /dev/null +++ b/frontend/public/voice-runtime/audio-processor.js @@ -0,0 +1,420 @@ +/** + * Browser audio processor for real-time voice communication. + * + * Handles: + * - Microphone capture with echo cancellation + * - Opus encoding of microphone input + * - Opus decoding of incoming audio + * - Jitter-buffered playback via AudioWorklet + * - Audio visualization (input/output analyzers) + * + * Dependencies: + * - audio-output-worklet.js (AudioWorklet for playback) + * - Opus encoder/decoder workers (encoderWorker.min.js, decoderWorker.min.js) + * + * Usage: + * const processor = new AudioProcessor({ + * onEncodedAudio: (opusData) => websocket.send(opusData), + * onMetrics: (metrics) => console.log('Buffer:', metrics.bufferMs), + * basePath: '/static/js' // Path to worker files + * }); + * + * await processor.start(); + * processor.playOpusData(incomingOpusData); + * processor.stop(); + */ + +class AudioProcessor { + /** + * @param {Object} options + * @param {Function} options.onEncodedAudio - Callback when Opus data is available: (Uint8Array) => void + * @param {Function} [options.onMetrics] - Callback for playback metrics: (metrics) => void + * @param {Function} [options.onTurnChange] - Callback when playhead crosses turn boundary: ({oldTurnIdx, newTurnIdx}) => void + * @param {string} [options.basePath='/js'] - Path prefix for worker files + * @param {number} [options.sampleRate=24000] - Target sample rate for encoding + * @param {boolean} [options.echoCancellation=true] - Enable browser echo cancellation + */ + constructor(options) { + this.onEncodedAudio = options.onEncodedAudio; + this.onMetrics = options.onMetrics || (() => {}); + this.onTurnChange = options.onTurnChange || (() => {}); + this.basePath = options.basePath || '/js'; + this.sampleRate = options.sampleRate || 24000; + this.echoCancellation = options.echoCancellation !== false; // default true + + this.audioContext = null; + this.mediaStream = null; + this.encoder = null; + this.decoder = null; + this.outputWorklet = null; + this.inputAnalyser = null; + this.outputAnalyser = null; + + this._started = false; + this._decoderReady = false; + this._decoderQueue = []; + this._oggHeaderPages = []; // Cached Ogg header pages for decoder recreation + this._oggHeadersCaptured = false; + } + + /** + * Request microphone access and start audio processing. + * @returns {Promise} + */ + async start() { + if (this._started) return; + + // Request microphone access + this.mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: this.echoCancellation, + autoGainControl: true, + noiseSuppression: true, + }, + }); + + // Create audio context + this.audioContext = new AudioContext({ sampleRate: 48000 }); + + // Resume context before addModule — Safari hangs on addModule if context is suspended + await this.audioContext.resume(); + + // Set up output worklet for playback + await this.audioContext.audioWorklet.addModule(`${this.basePath}/audio-output-worklet.js`); + this.outputWorklet = new AudioWorkletNode(this.audioContext, 'audio-output-processor'); + this.outputWorklet.connect(this.audioContext.destination); + + // Handle messages from worklet + this.outputWorklet.port.onmessage = (event) => { + if (event.data.type === 'metrics') { + this.onMetrics(event.data); + } else if (event.data.type === 'turn_change') { + this.onTurnChange({ + oldTurnIdx: event.data.oldTurnIdx, + newTurnIdx: event.data.newTurnIdx, + }); + } + }; + + // Set up input chain + const source = this.audioContext.createMediaStreamSource(this.mediaStream); + + // Input analyzer for visualization + this.inputAnalyser = this.audioContext.createAnalyser(); + this.inputAnalyser.fftSize = 2048; + source.connect(this.inputAnalyser); + + // Output analyzer for visualization + this.outputAnalyser = this.audioContext.createAnalyser(); + this.outputAnalyser.fftSize = 2048; + this.outputWorklet.connect(this.outputAnalyser); + + // Use preloaded decoder if available, otherwise create new one + if (AudioProcessor._preloadWorker) { + console.debug('AudioProcessor: using preloaded decoder'); + this.decoder = AudioProcessor._preloadWorker; + AudioProcessor._preloadWorker = null; // Take ownership + this._decoderReady = true; // Preloaded = already initialized + this._setupDecoderHandlers(); + } else { + this._createDecoder(); + } + + // Set up Opus encoder using our standalone OpusEncoder + if (typeof OpusEncoder === 'undefined') { + throw new Error('OpusEncoder not loaded. Include opus-encoder.js via