diff --git a/.env.example b/.env.example index 7a6d735..a9d461c 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,30 @@ REDIS_URL=redis://redis:6379/0 CELERY_CONCURRENCY=1 NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 +NEXT_PUBLIC_VOICE_SERVER_URL=http://localhost:7860 +NEXT_PUBLIC_PATENTFLOW_API_KEY= ALLOWED_ORIGINS=http://localhost:3000,http://localhost:3001 STATUS_RESPONSE_MAX_BYTES=262144 RESULT_BACKEND_MAX_BYTES=262144 +PATENTFLOW_API_KEY= EPO_ENABLED=false EPO_CONSUMER_KEY= EPO_CONSUMER_SECRET= LLM_PROVIDER=local -LLM_BASE_URL=http://127.0.0.1:11434 +LLM_BASE_URL=http://host.docker.internal:11434/v1 LLM_MODEL= +LLM_API_KEY= + +# Fastino Pioneer fine-tuned PatentFlow attorney model +ENABLE_LOCAL_PRIVACY_MODE=false +FASTINO_API_KEY= +FASTINO_BASE_URL=https://api.pioneer.ai/v1 +FASTINO_INFERENCE_ENDPOINT=https://api.pioneer.ai/v1/inference +FASTINO_MODEL_NAME=PatentFlow-epo-patent-attorney-llama-3-1-8b +FASTINO_MODEL_ID=59d36fbf-6e40-4e07-96d5-617d321842e8 + RAG_EMBEDDING_BASE_URL= RAG_EMBEDDING_MODEL= diff --git a/.gitignore b/.gitignore index fd034f8..333c2c1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ dist/ *.egg-info/ .DS_Store +Icon? +**/Icon? +.voice-venv/ .vscode/ .idea/ diff --git a/README.md b/README.md index 52fc9ee..f9e3cde 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ Patent prosecution is not “generic writing.” It is **risk management**: - Inventive-step reasoning requires structured, repeatable mapping - Quality and traceability matter more than “chatty” UX -### 2) 100% offline operation for client confidentiality -PatentFlow is designed to run fully locally: -- Local LLM execution (air-gapped capable) -- No external SaaS dependencies required for core workflows +### 2) Privacy-aware hybrid operation for client confidentiality +PatentFlow supports both local and online inference paths: +- Sensitive files and privacy-mode requests can be routed to Local LLM or a controlled Pioneer by Fastino endpoint +- Non-sensitive tasks can use configured online OpenAI-compatible APIs such as OpenAI, OpenRouter, r9s.ai, or vLLM-compatible services - Local persistence for attorney-specific preferences ### 3) Enterprise UX: minimal, information-dense, institutional @@ -55,6 +55,12 @@ graph TD UI -->|POST /api/generate| API[FastAPI Gateway :8000] UI -->|GET /api/status/:id| API UI -->|GET/POST /api/memory/*| API + UI -->|Voice Controls| VoiceS[Voice Server :7860] + end + + subgraph Voice [Gradbot Voice Pipeline - Port 7860] + VoiceS -->|STT → LLM → TTS| GradbotRT[Gradbot Runtime] + GradbotRT -->|LLM Calls| API end subgraph Backend [FastAPI + Celery Workers] @@ -70,8 +76,11 @@ graph TD EPO[EPO API] -->|Prior Art Retrieval| API end - subgraph AI [Local AI Runtime] - Skills --> LLM[(Local LLM)] + subgraph AI [AI Runtime & Model Routing] + Skills --> Router[Privacy / Sensitivity Router] + Router -->|Sensitive files or Privacy Mode| LocalLLM[(Local LLM)] + Router -->|Non-sensitive tasks| OnlineAPI[(Online LLM API
OpenAI-compatible)] + Router -->|Optional trained route| Fastino[(Pioneer by Fastino
PatentFlow Attorney Model)] end ``` @@ -116,6 +125,45 @@ PatentFlow integrates EPO retrieval to support: - Increases completeness and consistency of cited-document context - Improves auditability of the evidence basis used in analysis +### 5) Real-Time Voice Interaction (Gradbot) +PatentFlow includes a real-time voice session for attorney-in-the-loop workflows: +- **Start the session** opens a browser-based voice channel to the PatentFlow engine +- Discuss claim charts, examiner objections, and draft strategy conversationally +- Voice pipeline uses Gradbot for streaming STT → LLM → TTS with sub-second latency +- Session state is ephemeral; no audio stored or logged + +**Business value** +- Reduces typing friction during document review +- Enables rapid "talk through" of objection strategy before committing to text +- Maintains client confidentiality: voice processing runs locally, no cloud transcription + +### 6) Privacy Mode (Pioneer by Fastino) +PatentFlow includes an opt-in inference path powered by our own Pioneer by Fastino trained model for EPO prosecution work. + +**Trained model** +- Platform: Pioneer by Fastino +- Model Name: `PatentFlow-epo-patent-attorney-llama-3-1-8b` +- Job ID / Model ID: `59d36fbf-6e40-4e07-96d5-617d321842e8` +- Base URL: `https://api.pioneer.ai/v1` +- Inference Endpoint: `https://api.pioneer.ai/v1/inference` +- Training dataset: the curated fine-tuning dataset is included in the repository root as `dataset for training model for PatentFlow Attorney.jsonl` + +**Why this model** +- EPO-specific: trained for European patent attorney workflows, including Art. 56 inventive-step reasoning and Art. 123(2) added-matter risk review +- Structured outputs: optimized for claim charts, objection extraction, severity labels, examiner reasoning, and recommended attorney actions +- Domain tone: produces prosecution-oriented analysis instead of generic legal summaries +- Privacy-aware routing: lets PatentFlow keep the default workflow unchanged while offering a dedicated controlled endpoint for sensitive patent analysis + +**How it is used** +- Toggle globally with `ENABLE_LOCAL_PRIVACY_MODE=true` in `.env` +- Or toggle per request from the frontend with `Privacy Mode (Local LLM)` +- When disabled, the existing LLM workflow operates unchanged +- When enabled, Pioneer by Fastino is tried first; timeout, invalid JSON, or API failure falls back gracefully + +**Business value** +- Adds patent-prosecution-specific model behavior without forking the pipeline +- Preserves the existing public/online workflow by default + --- ## Quick Start @@ -124,6 +172,7 @@ PatentFlow integrates EPO retrieval to support: 1) Configure environment: - Copy `.env.example` → `.env` - Set `NEXT_PUBLIC_API_BASE_URL`, `REDIS_URL`, and LLM configuration as needed +- For Fastino privacy mode, set `FASTINO_API_KEY`, `FASTINO_BASE_URL`, and `FASTINO_MODEL_ID`; leave `ENABLE_LOCAL_PRIVACY_MODE=false` unless you want Fastino as the default route 2) Start services: ```bash @@ -140,8 +189,8 @@ Typical services: #### 1) Backend (FastAPI) ```bash -python3 -m venv venv -source venv/bin/activate +python3 -m venv .venv +source .venv/bin/activate pip install -r requirements.txt export REDIS_URL=redis://localhost:6379/0 @@ -166,6 +215,14 @@ npm install npm run dev ``` +#### 5) Voice Server (Gradbot) +```bash +python3 -m venv .voice-venv +source .voice-venv/bin/activate +pip install -r voice_pipeline/requirements.txt +python -m voice_pipeline.server +``` + Open: - UI: `http://localhost:3000` - API: `http://localhost:8000/health` @@ -175,6 +232,7 @@ Open: ## API Overview (Selected) - `POST /api/generate` - Runs the async pipeline (Celery) for claim chart + verification + draft outputs + - Accepts optional `use_privacy_mode: true` to try Fastino Pioneer for this request - `GET /api/status/{task_id}` - Poll for progress and results - `GET /api/memory/{attorney_id}` @@ -184,17 +242,53 @@ Open: - `POST /api/generate-chart` - Deterministic + LLM-assisted chart generation with optional `attorney_id` +### Voice Session Endpoints +- `POST /start_bot` + - Initialize a voice session with attorney context prompt; returns `session_id` + WebSocket endpoint +- `POST /end_session/{session_id}` + - Terminate voice session and release server-side resources +- `WS /ws/chat` + - Bidirectional streaming channel for voice interaction + +--- + +## Pioneer by Fastino Configuration +Add these values to `.env` when using Privacy Mode: + +```env +ENABLE_LOCAL_PRIVACY_MODE=false +FASTINO_API_KEY= +FASTINO_BASE_URL=https://api.pioneer.ai/v1 +FASTINO_INFERENCE_ENDPOINT=https://api.pioneer.ai/v1/inference +FASTINO_MODEL_NAME=PatentFlow-epo-patent-attorney-llama-3-1-8b +FASTINO_MODEL_ID=59d36fbf-6e40-4e07-96d5-617d321842e8 +``` + +Notes: +- `ENABLE_LOCAL_PRIVACY_MODE=false` keeps the existing route as default. +- The frontend toggle sends `use_privacy_mode: true` for a single request. +- Do not commit `FASTINO_API_KEY` or local `.env`. + --- ## Security & Privacy Posture -- Designed for offline and air-gapped operation +- Privacy-aware routing: sensitive files and privacy-mode requests can be routed away from general online APIs - Local persistence only (SQLite) -- No dependency on third-party analytics, telemetry, or cloud inference for core workflows +- No dependency on third-party analytics or telemetry +- Online inference is explicit and configurable for non-sensitive tasks through OpenAI-compatible APIs +- Voice sessions are ephemeral: no audio retention, transcription, or logging +- Pioneer by Fastino privacy mode routes supported analysis through the configured Pioneer endpoint with fallback to the existing route --- ## Roadmap (Prioritized for Firm Integration) +- [ ] **Two-Tier AI Architecture (Performance & UX)** + Implement a dual-layer local inference pipeline using the Fastino platform to optimize both speed and compute efficiency for privacy-mode: + - **Layer 1 (Zero-Shot Triage):** Utilize GLiNER2 for ultra-fast, zero-shot entity extraction. Instantly highlight critical legal entities (e.g., Prior Art documents like D1/D2, disputed technical features, and statutory articles) directly in the UI milliseconds after document upload. + - **Layer 2 (Deep Legal Reasoning):** While the attorney engages with the instantly highlighted document, the fine-tuned local Llama-3.1-8B model asynchronously processes the complex Art. 56 / Art. 123(2) logic in the background via Celery. + This architecture provides immediate visual feedback to the attorney without blocking the UI or overloading the heavy LLM. + - [ ] **RAG with ChromaDB (Depth)** Local retrieval over firm-approved corpora (e.g., standards, prior OA templates) to improve long-document reasoning while controlling hallucination risk. diff --git a/dataset for training model for PatentFlow Attorney.jsonl b/dataset for training model for PatentFlow Attorney.jsonl new file mode 100644 index 0000000..41983a2 --- /dev/null +++ b/dataset for training model for PatentFlow Attorney.jsonl @@ -0,0 +1,4 @@ +{"messages":[{"role":"system","content":"You are a European Patent Attorney assistant. Extract EPC objections from the Office Action text and output ONLY valid JSON matching the specified schema."},{"role":"user","content":"Extract EPC objections: 'Regarding Claim 1, the term \"high temperature\" is vague and leaves the reader in doubt as to the meaning of the technical features to which it refers, thereby rendering the definition of the subject-matter of said claim unclear (Article 84 EPC). The applicant should specify the exact temperature range in degrees Celsius.'"},{"role":"assistant","content":"{\n \"article\": \"84\",\n \"affected_claims\": [1],\n \"severity\": \"high\",\n \"examiner_reasoning\": \"The term 'high temperature' is vague and does not define a clear boundary for the scope of protection.\",\n \"recommended_action\": \"Amend claim 1 to replace 'high temperature' with a specific temperature range in degrees Celsius, ensuring support in the original description.\"\n}"}]} +{"messages":[{"role":"system","content":"You are a European Patent Attorney assistant. Extract EPC objections from the Office Action text and output ONLY valid JSON matching the specified schema."},{"role":"user","content":"Extract EPC objections: 'The amendment to Claim 3 introduces the feature \"a casing made of titanium alloy\". However, the original description on page 5, line 12 only generically discloses a \"metal casing\". There is no basis in the application as filed for selecting titanium alloy. This constitutes added subject-matter extending beyond the content of the application as filed, contrary to Article 123(2) EPC.'"},{"role":"assistant","content":"{\n \"article\": \"123(2)\",\n \"affected_claims\": [3],\n \"severity\": \"high\",\n \"examiner_reasoning\": \"The specific material 'titanium alloy' was not disclosed in the original application, which only mentioned 'metal casing'.\",\n \"recommended_action\": \"Delete the feature 'made of titanium alloy' from Claim 3, or revert to the original broader term 'metal casing', unless a valid basis can be found elsewhere in the originally filed drawings or description.\"\n}"}]} +{"messages":[{"role":"system","content":"You are a European Patent Attorney assistant. Extract EPC objections from the Office Action text and output ONLY valid JSON matching the specified schema."},{"role":"user","content":"Extract EPC objections: 'The present application does not meet the requirements of Article 52(1) in conjunction with Article 56 EPC because the subject-matter of claim 1 does not involve an inventive step. Document D1 (US 2018/01234) is regarded as being the closest prior art and discloses a wireless charging pad. The subject-matter of claim 1 differs from D1 in that it includes an active cooling fan. The objective technical problem to be solved is how to prevent overheating. However, D2 (EP 312456) already teaches using a cooling fan in inductive chargers. It would be obvious to the skilled person to apply the teaching of D2 to D1.'"},{"role":"assistant","content":"{\n \"article\": \"56\",\n \"affected_claims\": [1],\n \"severity\": \"high\",\n \"examiner_reasoning\": \"D1 discloses the closest prior art. The distinguishing feature is an active cooling fan solving the problem of overheating. D2 renders this combination obvious to a person skilled in the art.\",\n \"recommended_action\": \"Argue why the combination of D1 and D2 would not be obvious (e.g., teaching away, incompatible structures), or amend claim 1 to include a further distinguishing feature not found in D1 or D2.\"\n}"}]} +{"messages":[{"role":"system","content":"You are a European Patent Attorney assistant. Extract EPC objections from the Office Action text and output ONLY valid JSON matching the specified schema."},{"role":"user","content":"Extract EPC objections: 'We are pleased to inform you that the renewal fee for the 4th year has been successfully processed and the receipt is attached herewith.'"},{"role":"assistant","content":"{\n \"article\": \"None\",\n \"affected_claims\": [],\n \"severity\": \"low\",\n \"examiner_reasoning\": \"The provided text is an administrative notification regarding fee payment, not a substantive examination report.\",\n \"recommended_action\": \"No substantive response required. Keep the receipt for records.\"\n}"}]} diff --git a/docker-compose.yml b/docker-compose.yml index 5e761b5..ebaa4c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,8 @@ services: build: context: . dockerfile: Dockerfile.api + env_file: + - .env environment: - REDIS_URL=redis://redis:6379/0 networks: @@ -47,6 +49,8 @@ services: build: context: . dockerfile: Dockerfile.worker + env_file: + - .env environment: - REDIS_URL=redis://redis:6379/0 - CELERY_CONCURRENCY=1 @@ -63,7 +67,9 @@ services: context: ./frontend dockerfile: Dockerfile args: - NEXT_PUBLIC_API_BASE_URL: http://localhost:8000 + NEXT_PUBLIC_API_BASE_URL: ${NEXT_PUBLIC_API_BASE_URL:-http://localhost:8000} + NEXT_PUBLIC_PATENTFLOW_API_KEY: ${NEXT_PUBLIC_PATENTFLOW_API_KEY:-} + NEXT_PUBLIC_VOICE_SERVER_URL: ${NEXT_PUBLIC_VOICE_SERVER_URL:-http://localhost:7860} networks: - patentflow-net ports: diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 0000000..a9d4be7 --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,13 @@ +node_modules/ +.next/ +out/ +.git/ +.gitignore +.DS_Store +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +.env +.env.local +.env.*.local diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2fb6783..61e81f9 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -12,6 +12,10 @@ COPY . /app # NEXT_PUBLIC_* vars are baked at build time ARG NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 ENV NEXT_PUBLIC_API_BASE_URL=${NEXT_PUBLIC_API_BASE_URL} +ARG NEXT_PUBLIC_PATENTFLOW_API_KEY= +ENV NEXT_PUBLIC_PATENTFLOW_API_KEY=${NEXT_PUBLIC_PATENTFLOW_API_KEY} +ARG NEXT_PUBLIC_VOICE_SERVER_URL=http://localhost:7860 +ENV NEXT_PUBLIC_VOICE_SERVER_URL=${NEXT_PUBLIC_VOICE_SERVER_URL} RUN npm run build diff --git a/frontend/public/voice-runtime/audio-output-worklet.js b/frontend/public/voice-runtime/audio-output-worklet.js new file mode 100644 index 0000000..cabf5ca --- /dev/null +++ b/frontend/public/voice-runtime/audio-output-worklet.js @@ -0,0 +1,465 @@ +/** + * AudioWorklet processor for jitter-buffered audio playback. + * + * Handles: + * - Buffering incoming decoded audio frames + * - Adaptive buffer sizing (increases on underruns) + * - Packet dropping when buffer exceeds max + * - Exponential fade-in/out to avoid clicks (including slow fade on interruption) + * - Delay tracking for metrics + * + * Usage: + * await audioContext.audioWorklet.addModule('/path/to/audio-output-worklet.js'); + * const worklet = new AudioWorkletNode(audioContext, 'audio-output-processor'); + * worklet.connect(audioContext.destination); + * + * // Send decoded audio frames: + * worklet.port.postMessage({ type: 'audio', frame: float32Array }); + * + * // Reset state: + * worklet.port.postMessage({ type: 'reset' }); + */ + +function asMs(samples) { + return ((samples * 1000) / sampleRate).toFixed(1); +} + +function asSamples(ms) { + return Math.round((ms * sampleRate) / 1000); +} + +// Maximum buffer before we start dropping packets (30 seconds) +const DEFAULT_MAX_BUFFER_MS = 30 * 1000; + +// Fade durations in ms +const FADE_IN_MS = 15; +const FADE_OUT_MS = 10; +const FADE_OUT_INTERRUPTED_MS = 200; + +// Exponential fade threshold — fade is "complete" when it reaches this level (~-60dB) +const FADE_THRESHOLD = 0.001; + +// Compute per-sample exponential decay factor for a given duration in ms. +// After N samples, the fade reaches FADE_THRESHOLD (effectively silent). +function fadeDecay(ms) { + const N = asSamples(ms); + return Math.pow(FADE_THRESHOLD, 1.0 / Math.max(1, N)); +} + +// Set to true to enable debug logging +const DEBUG = false; +const debug = (...args) => { + if (DEBUG) console.debug('[AudioOutputWorklet]', ...args); +}; + +class AudioOutputProcessor extends AudioWorkletProcessor { + constructor() { + super(); + debug('AudioOutputProcessor created', currentFrame, sampleRate); + + // Buffer timing configuration (in samples) + const frameSize = asSamples(80); + + // Wait for at least this many samples before starting playback + this.initialBufferSamples = 1 * frameSize; + + // Additional wait after reaching initialBufferSamples (allows non-frame-aligned buffers) + this.partialBufferSamples = asSamples(10); + + // Drop oldest packets if buffer exceeds this + this.maxBufferSamples = asSamples(DEFAULT_MAX_BUFFER_MS); + + // How much to increase buffers on underrun/overflow + this.partialBufferIncrement = asSamples(5); + this.maxPartialWithIncrements = asSamples(80); + this.maxBufferSamplesIncrement = asSamples(5); + this.maxMaxBufferWithIncrements = asSamples(80); + + this.initState(); + + this.port.onmessage = (event) => { + if (event.data.type === 'reset') { + debug('Reset audio processor state'); + this.initState(); + return; + } + + if (event.data.type === 'audio') { + const frame = event.data.frame; + const stopS = event.data.stopS; + const turnIdx = event.data.turnIdx; + + console.log( + '[AudioWorklet] Audio received:', + frame.length + ' samples (' + asMs(frame.length) + 'ms),', + 'turn=' + turnIdx, + 'stopS=' + stopS + ); + + // Route audio to correct buffer based on turn + let buffered = false; + if (turnIdx === this.currentPlayingTurnIdx || this.currentPlayingTurnIdx === null) { + // Current turn - add to main buffer + this.pushFrame(this.frames, frame, turnIdx); + if (this.currentPlayingTurnIdx === null) { + this.currentPlayingTurnIdx = turnIdx; + } + buffered = true; + debug('Audio to MAIN buffer - turn', turnIdx, 'frames now:', this.frames.length); + } else if (turnIdx > this.currentPlayingTurnIdx) { + // Future turn - check if we can play immediately (buffer is empty) + if (this.frames.length === 0) { + // Buffer is empty, play new turn immediately + this.currentPlayingTurnIdx = turnIdx; + this.pushFrame(this.frames, frame, turnIdx); + buffered = true; + debug('Buffer empty - starting new turn immediately', turnIdx); + console.log('[AudioWorklet] Starting turn', turnIdx, 'immediately (buffer was empty)'); + } else { + // Buffer not empty - add to pending buffer + // If we get a newer turn than what's pending, replace pending (skip old turn) + if (this.pendingTurnIdx === null || turnIdx > this.pendingTurnIdx) { + if (this.pendingTurnIdx !== null) { + console.log('[AudioWorklet] Skipping turn', this.pendingTurnIdx, 'for newer turn', turnIdx); + } + debug('New pending turn', this.pendingTurnIdx, '->', turnIdx, '- clearing old pending'); + this.pendingFrames = []; + this.pendingTurnIdx = turnIdx; + } + // Add to pending buffer (either new or existing pending) + if (turnIdx === this.pendingTurnIdx) { + this.pushFrame(this.pendingFrames, frame, turnIdx); + buffered = true; + debug('Audio to PENDING buffer - turn', turnIdx, 'pending frames now:', this.pendingFrames.length); + } + // If turnIdx < pendingTurnIdx, it's stale - drop it + } + } else { + // Stale turn - drop it + debug('DROPPING audio - stale turn', turnIdx, '< current', this.currentPlayingTurnIdx); + console.log('[AudioWorklet] DROPPING audio for stale turn', turnIdx, '(current:', this.currentPlayingTurnIdx + ')'); + } + + // Only update tracking/state for audio that was actually buffered + if (buffered) { + if (stopS !== undefined) { + this.lastStopS = stopS; + } + + // Handle interrupted flag — switch to slow fade-out + // Only for current-turn audio (not pending), since it affects the active playback fade + if (event.data.interrupted && turnIdx === this.currentPlayingTurnIdx) { + const currentBuffered = this.currentSamples(); + const desiredFade = asSamples(FADE_OUT_INTERRUPTED_MS); + const effectiveFade = Math.min(desiredFade, currentBuffered); + const effectiveFadeMs = (effectiveFade * 1000) / sampleRate; + this.fadeOutDecay = fadeDecay(effectiveFadeMs); + console.log( + '[AudioWorklet] Interrupted fade-out:', + 'buffered=' + asMs(currentBuffered) + 'ms,', + 'fade=' + asMs(effectiveFade) + 'ms,', + 'decay=' + this.fadeOutDecay.toFixed(6) + ); + } + } + + // Start playback once we have enough buffered + if (this.currentSamples() >= this.initialBufferSamples && !this.started) { + this.start(); + } + + if (this.pidx < 20) { + debug( + this.timestamp(), + 'Got packet', + this.pidx++, + 'turn=' + turnIdx, + 'current=' + this.currentPlayingTurnIdx, + 'pending=' + this.pendingTurnIdx, + asMs(this.currentSamples()) + 'ms buffered', + asMs(frame.length) + 'ms frame' + ); + } + + // Drop packets if buffer is too full + if (this.currentSamples() >= this.totalMaxBufferSamples()) { + this.dropExcessPackets(); + } + + // Send metrics back to main thread (bufferMs and lastStopS are atomic) + this.port.postMessage({ + type: 'metrics', + totalAudioPlayed: this.totalAudioPlayed, + actualAudioPlayed: this.actualAudioPlayed, + bufferMs: (this.currentSamples() / sampleRate) * 1000, + lastStopS: this.lastStopS, + minDelay: this.minDelay, + maxDelay: this.maxDelay, + }); + } + }; + } + + initState() { + this.frames = []; + this.offsetInFirstBuffer = 0; + this.remainingPartialBufferSamples = 0; + this.fadePos = 0; // 0.0..1.0, current fade multiplier + this.fadeInDecay = fadeDecay(FADE_IN_MS); // per-sample decay for fade-in + this.fadeOutDecay = fadeDecay(FADE_OUT_MS); // per-sample decay for fade-out + this.timeInStream = 0; + this.resetStart(); + + // Metrics + this.totalAudioPlayed = 0; + this.actualAudioPlayed = 0; + this.maxDelay = 0; + this.minDelay = 2000; + this.lastStopS = 0; // stop_s of the last audio frame added to buffer + + // Turn tracking + this.currentPlayingTurnIdx = null; + + // Buffer empty detection + this.hadFrames = false; + + // Pending buffer for next turn (prevents mixing audio between turns) + this.pendingFrames = []; + this.pendingTurnIdx = null; + + // Debug counter + this.pidx = 0; + + // Reset buffer params + this.partialBufferSamples = asSamples(10); + this.maxBufferSamples = asSamples(DEFAULT_MAX_BUFFER_MS); + } + + totalMaxBufferSamples() { + return this.maxBufferSamples + this.partialBufferSamples + this.initialBufferSamples; + } + + timestamp() { + return Date.now() % 1000; + } + + currentSamples() { + let samples = 0; + for (let k = 0; k < this.frames.length; k++) { + samples += this.frames[k].samples.length; + } + samples -= this.offsetInFirstBuffer; + return samples; + } + + resetStart() { + this.started = false; + } + + start() { + this.started = true; + this.remainingPartialBufferSamples = this.partialBufferSamples; + } + + canPlay() { + return this.started && this.frames.length > 0 && this.remainingPartialBufferSamples <= 0; + } + + pushFrame(buffer, frame, turnIdx) { + buffer.push({ samples: frame, turnIdx: turnIdx }); + } + + dropExcessPackets() { + debug( + this.timestamp(), + 'Dropping packets', + asMs(this.currentSamples()) + 'ms buffered', + asMs(this.totalMaxBufferSamples()) + 'ms max' + ); + + const target = this.initialBufferSamples + this.partialBufferSamples; + while (this.currentSamples() > target) { + const first = this.frames[0]; + let toRemove = this.currentSamples() - target; + toRemove = Math.min(first.samples.length - this.offsetInFirstBuffer, toRemove); + this.offsetInFirstBuffer += toRemove; + this.timeInStream += toRemove / sampleRate; + + if (this.offsetInFirstBuffer === first.samples.length) { + this.frames.shift(); + this.offsetInFirstBuffer = 0; + } + } + + debug(this.timestamp(), 'After drop:', asMs(this.currentSamples()) + 'ms'); + + // Increase max buffer to reduce future drops + this.maxBufferSamples += this.maxBufferSamplesIncrement; + this.maxBufferSamples = Math.min(this.maxMaxBufferWithIncrements, this.maxBufferSamples); + debug('Increased maxBuffer to', asMs(this.maxBufferSamples) + 'ms'); + } + + process(inputs, outputs) { + const delay = this.currentSamples() / sampleRate; + if (this.canPlay()) { + this.maxDelay = Math.max(this.maxDelay, delay); + this.minDelay = Math.min(this.minDelay, delay); + } + + const output = outputs[0][0]; + + if (!this.canPlay()) { + if (this.actualAudioPlayed > 0) { + this.totalAudioPlayed += output.length / sampleRate; + } + this.fadePos = 0; + this.remainingPartialBufferSamples -= output.length; + // Output near-silent signal to prevent Chrome from suspending the audio output + // path (which causes an audible "switch" sound when audio resumes). + for (let i = 0; i < output.length; i++) { + output[i] = 1e-7; + } + return true; + } + + let outIdx = 0; + + // Copy frames to output buffer + // Note: All frames in this.frames now have the same turnIdx due to pending buffer system + while (outIdx < output.length && this.frames.length) { + const first = this.frames[0]; + + const toCopy = Math.min( + first.samples.length - this.offsetInFirstBuffer, + output.length - outIdx + ); + const subArray = first.samples.subarray( + this.offsetInFirstBuffer, + this.offsetInFirstBuffer + toCopy + ); + output.set(subArray, outIdx); + + this.offsetInFirstBuffer += toCopy; + outIdx += toCopy; + + if (this.offsetInFirstBuffer === first.samples.length) { + this.offsetInFirstBuffer = 0; + this.frames.shift(); + } + } + + // Apply per-sample exponential fade based on buffer fullness. + // fadePos (0.0..1.0) ramps up/down exponentially. + // Fade-out starts when remaining samples < samples needed to reach FADE_THRESHOLD. + const remaining = this.currentSamples(); + // How many samples needed for fadePos to decay to FADE_THRESHOLD: + // fadePos * fadeOutDecay^N = FADE_THRESHOLD => N = log(FADE_THRESHOLD/fadePos) / log(fadeOutDecay) + const fadeOutSamplesNeeded = this.fadePos > FADE_THRESHOLD + ? Math.log(FADE_THRESHOLD / this.fadePos) / Math.log(this.fadeOutDecay) + : 0; + const willFadeOut = remaining + outIdx < fadeOutSamplesNeeded; + const alreadyFull = this.fadePos >= 1 - FADE_THRESHOLD; + + if (!alreadyFull || willFadeOut) { + const fadePosStart = this.fadePos; + for (let i = 0; i < outIdx; i++) { + const samplesLeft = remaining + (outIdx - i); + // Re-check fade-out condition per sample (fadePos changes each iteration) + const neededNow = this.fadePos > FADE_THRESHOLD + ? Math.log(FADE_THRESHOLD / this.fadePos) / Math.log(this.fadeOutDecay) + : 0; + if (samplesLeft < neededNow) { + // Fade out: exponential decay toward 0 + this.fadePos *= this.fadeOutDecay; + if (this.fadePos < FADE_THRESHOLD) this.fadePos = 0; + } else { + // Fade in: exponential approach toward 1 + // fadePos = 1 - (1 - fadePos) * decay + this.fadePos = 1 - (1 - this.fadePos) * this.fadeInDecay; + if (this.fadePos > 1 - FADE_THRESHOLD) this.fadePos = 1; + } + output[i] *= this.fadePos; + } + console.log( + '[AudioWorklet] Fade active:', + fadePosStart.toFixed(3), '->', this.fadePos.toFixed(3), + willFadeOut ? 'fade-out' : 'fade-in', + 'remaining=' + asMs(remaining) + 'ms', + 'outIdx=' + outIdx, + 'fadeOutDecay=' + this.fadeOutDecay.toFixed(6) + ); + } + + // Handle buffer underrun + if (outIdx < output.length) { + debug(this.timestamp(), 'Buffer underrun:', output.length - outIdx, 'samples missed'); + + // Increase buffer to reduce future underruns + this.partialBufferSamples += this.partialBufferIncrement; + this.partialBufferSamples = Math.min(this.partialBufferSamples, this.maxPartialWithIncrements); + debug('Increased partial buffer to', asMs(this.partialBufferSamples) + 'ms'); + + // Reset to refill buffer + this.resetStart(); + } + + this.totalAudioPlayed += output.length / sampleRate; + this.actualAudioPlayed += outIdx / sampleRate; + this.timeInStream += outIdx / sampleRate; + + // Track buffer state and check for pending buffer swap + if (this.frames.length > 0) { + this.hadFrames = true; + } else { + // Main buffer is empty - check if we have pending audio to swap in + if (this.pendingFrames.length > 0) { + const oldTurnIdx = this.currentPlayingTurnIdx; + const newTurnIdx = this.pendingTurnIdx; + debug('Buffer empty - swapping pending turn', oldTurnIdx, '->', newTurnIdx, + 'with', this.pendingFrames.length, 'frames'); + + // Notify main thread of turn change + if (oldTurnIdx !== null) { + this.port.postMessage({ + type: 'turn_change', + oldTurnIdx: oldTurnIdx, + newTurnIdx: newTurnIdx, + }); + } + + // Swap pending to main + this.frames = this.pendingFrames; + this.pendingFrames = []; + this.currentPlayingTurnIdx = newTurnIdx; + this.pendingTurnIdx = null; + this.hadFrames = true; + + // Reset fade for new turn — start from silence + this.fadePos = 0; + this.fadeOutDecay = fadeDecay(FADE_OUT_MS); + + // Restart playback for the new turn + this.started = true; + this.remainingPartialBufferSamples = 0; + console.log('[AudioWorklet] Swapped to turn', newTurnIdx, 'and restarted playback'); + } else if (this.hadFrames) { + // Buffer just emptied with no pending audio — reset fade for next turn + this.hadFrames = false; + this.fadeOutDecay = fadeDecay(FADE_OUT_MS); + this.port.postMessage({ + type: 'metrics', + totalAudioPlayed: this.totalAudioPlayed, + actualAudioPlayed: this.actualAudioPlayed, + bufferMs: 0, + lastStopS: this.lastStopS, + minDelay: this.minDelay, + maxDelay: this.maxDelay, + }); + } + } + + return true; + } +} + +registerProcessor('audio-output-processor', AudioOutputProcessor); diff --git a/frontend/public/voice-runtime/audio-processor.js b/frontend/public/voice-runtime/audio-processor.js new file mode 100644 index 0000000..1ff8002 --- /dev/null +++ b/frontend/public/voice-runtime/audio-processor.js @@ -0,0 +1,420 @@ +/** + * Browser audio processor for real-time voice communication. + * + * Handles: + * - Microphone capture with echo cancellation + * - Opus encoding of microphone input + * - Opus decoding of incoming audio + * - Jitter-buffered playback via AudioWorklet + * - Audio visualization (input/output analyzers) + * + * Dependencies: + * - audio-output-worklet.js (AudioWorklet for playback) + * - Opus encoder/decoder workers (encoderWorker.min.js, decoderWorker.min.js) + * + * Usage: + * const processor = new AudioProcessor({ + * onEncodedAudio: (opusData) => websocket.send(opusData), + * onMetrics: (metrics) => console.log('Buffer:', metrics.bufferMs), + * basePath: '/static/js' // Path to worker files + * }); + * + * await processor.start(); + * processor.playOpusData(incomingOpusData); + * processor.stop(); + */ + +class AudioProcessor { + /** + * @param {Object} options + * @param {Function} options.onEncodedAudio - Callback when Opus data is available: (Uint8Array) => void + * @param {Function} [options.onMetrics] - Callback for playback metrics: (metrics) => void + * @param {Function} [options.onTurnChange] - Callback when playhead crosses turn boundary: ({oldTurnIdx, newTurnIdx}) => void + * @param {string} [options.basePath='/js'] - Path prefix for worker files + * @param {number} [options.sampleRate=24000] - Target sample rate for encoding + * @param {boolean} [options.echoCancellation=true] - Enable browser echo cancellation + */ + constructor(options) { + this.onEncodedAudio = options.onEncodedAudio; + this.onMetrics = options.onMetrics || (() => {}); + this.onTurnChange = options.onTurnChange || (() => {}); + this.basePath = options.basePath || '/js'; + this.sampleRate = options.sampleRate || 24000; + this.echoCancellation = options.echoCancellation !== false; // default true + + this.audioContext = null; + this.mediaStream = null; + this.encoder = null; + this.decoder = null; + this.outputWorklet = null; + this.inputAnalyser = null; + this.outputAnalyser = null; + + this._started = false; + this._decoderReady = false; + this._decoderQueue = []; + this._oggHeaderPages = []; // Cached Ogg header pages for decoder recreation + this._oggHeadersCaptured = false; + } + + /** + * Request microphone access and start audio processing. + * @returns {Promise} + */ + async start() { + if (this._started) return; + + // Request microphone access + this.mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: this.echoCancellation, + autoGainControl: true, + noiseSuppression: true, + }, + }); + + // Create audio context + this.audioContext = new AudioContext({ sampleRate: 48000 }); + + // Resume context before addModule — Safari hangs on addModule if context is suspended + await this.audioContext.resume(); + + // Set up output worklet for playback + await this.audioContext.audioWorklet.addModule(`${this.basePath}/audio-output-worklet.js`); + this.outputWorklet = new AudioWorkletNode(this.audioContext, 'audio-output-processor'); + this.outputWorklet.connect(this.audioContext.destination); + + // Handle messages from worklet + this.outputWorklet.port.onmessage = (event) => { + if (event.data.type === 'metrics') { + this.onMetrics(event.data); + } else if (event.data.type === 'turn_change') { + this.onTurnChange({ + oldTurnIdx: event.data.oldTurnIdx, + newTurnIdx: event.data.newTurnIdx, + }); + } + }; + + // Set up input chain + const source = this.audioContext.createMediaStreamSource(this.mediaStream); + + // Input analyzer for visualization + this.inputAnalyser = this.audioContext.createAnalyser(); + this.inputAnalyser.fftSize = 2048; + source.connect(this.inputAnalyser); + + // Output analyzer for visualization + this.outputAnalyser = this.audioContext.createAnalyser(); + this.outputAnalyser.fftSize = 2048; + this.outputWorklet.connect(this.outputAnalyser); + + // Use preloaded decoder if available, otherwise create new one + if (AudioProcessor._preloadWorker) { + console.debug('AudioProcessor: using preloaded decoder'); + this.decoder = AudioProcessor._preloadWorker; + AudioProcessor._preloadWorker = null; // Take ownership + this._decoderReady = true; // Preloaded = already initialized + this._setupDecoderHandlers(); + } else { + this._createDecoder(); + } + + // Set up Opus encoder using our standalone OpusEncoder + if (typeof OpusEncoder === 'undefined') { + throw new Error('OpusEncoder not loaded. Include opus-encoder.js via