From 2db59d583ad10956d163ca3e9ade8f59d3a1dd54 Mon Sep 17 00:00:00 2001 From: 2witstudios <2witstudios@gmail.com> Date: Mon, 10 Aug 2026 23:16:10 -0500 Subject: [PATCH] feat(voice): the talk button, and voice as a mode on the chat surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first user-visible piece of the audio-native path. Everything beneath it was merged and untested by a human; this is the part a person actually touches. THE SHAPE, as settled: - ONE trigger, in the nav bar, on every route. Nothing connects until it is pressed. It doubles as the live indicator and as the way back to a minimized call — and it NEVER hangs up, because a toggle there means pressing the live indicator hides the call you pressed it to see. Ending a call is the End button on the call. - It binds to whatever assistant is in view: the sidebar's selected agent on a page route, the GlobalAssistantView conversation on the dashboard (where the sidebar has no chat tab at all). There is no second target picker anywhere. - Voice is a MODE on the chat surface, not a fourth tab and not an overlay: the same message list, with a live call header above it. Spoken turns arrive in that list as ordinary messages — the realtime server writes them through messageRepository, so they ride the `conversation:*` events every surface already subscribes to. No new client wiring, and no second history. - Closing the sidebar minimizes; navigating updates locationContext and does NOT rebind; the sidebar's agent switcher DOES rebind. Every decision is a pure module beside the wiring — `resolveVoiceBinding`, `rebindAction`, `decideReveal`, `describeCall`, `toVoiceLocationContext` — so the rules are testable without a browser, a peer connection or a microphone. `rebindAction` is the load-bearing one. Navigating and switching agent arrive as the SAME change to the derived target, so a "watch the target and call start" effect cannot tell them apart and would hang up the user's call every time they opened a page. A rebind is therefore driven by the switch EVENT: the switcher records an intent, and the intent is applied once the newly chosen agent's conversation has resolved. Navigation records nothing. TWO THINGS THE MERGED SEAM WAS MISSING, added at the provider rather than worked around in the UI: - `failure` — the classified reason beside `error`. A sentence cannot be branched on, and a denied prompt and an absent capture device need opposite affordances: one offers Try again, the other must not, because a retry that cannot conjure hardware teaches the user that voice is broken. - `muted`/`setMuted` — mute has to live with the microphone. Held by the chrome it would return un-muted every time the sidebar was collapsed. It also fixes a latent bug: the swap hardcoded `setMicrophoneEnabled(true)`, so a chain at the duration ceiling un-muted a deliberately muted call. `messages.source` now reaches the UI (conversion AND broadcast, so the glyph does not appear only after a refresh) and both renderers mark a spoken turn. The browser reads its own copy of the value with a drift guard, since client components here cannot import @pagespace/db/schema. Also: the right sidebar's page-context tab moved from panel-local state into the layout store, because a button in the header cannot reach a `useState` inside a panel that is unmounted while collapsed. Unrelated to this chunk, but the branch's knip gate was already red for it: two per-member request types in voice-bridge-contract.ts (from #2395) were exported and never imported. Removed — consumers narrow on the union's `kind`. Gate: `bun run typecheck` 17/17, `bun run lint` 15/15, `bun run knip:check` within baseline, web tests 17085 passed with only the 17 known Postgres-requiring integration files failing (no test DB in this worktree). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PyxC1FeYap8bU98cLzcXhH --- CHANGELOG.md | 24 ++ .../ai/shared/chat/CompactMessageRenderer.tsx | 10 +- .../ai/shared/chat/MessageRenderer.tsx | 10 +- .../ai/shared/chat/SpokenTurnGlyph.tsx | 37 +++ .../chat/__tests__/SpokenTurnGlyph.test.tsx | 56 ++++ .../ai/shared/chat/message-types.ts | 13 + .../ai/voice/realtime/VoiceCallBar.tsx | 170 ++++++++++ .../realtime/VoiceCallBarForConversation.tsx | 37 +++ .../ai/voice/realtime/VoiceNavTrigger.tsx | 196 ++++++++++++ .../ai/voice/realtime/VoiceSessionBridge.tsx | 95 ++++++ .../realtime/__tests__/VoiceCallBar.test.tsx | 182 +++++++++++ .../VoiceCallBarForConversation.test.tsx | 116 +++++++ .../__tests__/VoiceNavTrigger.test.tsx | 292 ++++++++++++++++++ .../__tests__/VoiceSessionBridge.test.tsx | 260 ++++++++++++++++ .../src/components/ai/voice/realtime/index.ts | 16 + apps/web/src/components/layout/Layout.tsx | 61 ++++ .../__tests__/Layout.voice-reveal.test.tsx | 184 +++++++++++ .../__tests__/Layout.voice-session.test.tsx | 16 + .../__tests__/TopBar.voice-trigger.test.tsx | 73 +++++ .../components/layout/main-header/index.tsx | 18 +- .../dashboard/GlobalAssistantView.tsx | 26 +- .../__tests__/RightPanel.page-tab.test.tsx | 90 ++++++ .../ai-assistant/SidebarChatTab.tsx | 29 +- .../components/layout/right-sidebar/index.tsx | 12 +- apps/web/src/contexts/VoiceSessionContext.tsx | 68 +++- .../__tests__/VoiceSessionContext.test.tsx | 84 +++++ apps/web/src/hooks/voice/useAudioLevel.ts | 114 +++++++ apps/web/src/hooks/voice/useVoiceBinding.ts | 54 ++++ .../ai/core/__tests__/message-utils.test.ts | 44 +++ apps/web/src/lib/ai/core/message-utils.ts | 13 +- .../ai/realtime/__tests__/call-chrome.test.ts | 103 ++++++ .../realtime/__tests__/voice-binding.test.ts | 154 +++++++++ .../realtime/__tests__/voice-location.test.ts | 83 +++++ .../realtime/__tests__/voice-rebind.test.ts | 95 ++++++ .../realtime/__tests__/voice-reveal.test.ts | 96 ++++++ apps/web/src/lib/ai/realtime/call-chrome.ts | 123 ++++++++ .../web/src/lib/ai/realtime/message-source.ts | 16 + apps/web/src/lib/ai/realtime/session-state.ts | 31 +- apps/web/src/lib/ai/realtime/voice-binding.ts | 133 ++++++++ .../web/src/lib/ai/realtime/voice-location.ts | 66 ++++ apps/web/src/lib/ai/realtime/voice-rebind.ts | 80 +++++ apps/web/src/lib/ai/realtime/voice-reveal.ts | 91 ++++++ .../lib/repositories/message-repository.ts | 21 +- apps/web/src/stores/useLayoutStore.ts | 22 ++ apps/web/src/stores/useVoiceRebindStore.ts | 41 +++ .../lib/src/realtime/voice-bridge-contract.ts | 8 +- 46 files changed, 3534 insertions(+), 29 deletions(-) create mode 100644 apps/web/src/components/ai/shared/chat/SpokenTurnGlyph.tsx create mode 100644 apps/web/src/components/ai/shared/chat/__tests__/SpokenTurnGlyph.test.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/VoiceCallBar.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/VoiceCallBarForConversation.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/VoiceNavTrigger.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/VoiceSessionBridge.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/__tests__/VoiceCallBar.test.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/__tests__/VoiceCallBarForConversation.test.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/__tests__/VoiceNavTrigger.test.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/__tests__/VoiceSessionBridge.test.tsx create mode 100644 apps/web/src/components/ai/voice/realtime/index.ts create mode 100644 apps/web/src/components/layout/__tests__/Layout.voice-reveal.test.tsx create mode 100644 apps/web/src/components/layout/main-header/__tests__/TopBar.voice-trigger.test.tsx create mode 100644 apps/web/src/components/layout/right-sidebar/__tests__/RightPanel.page-tab.test.tsx create mode 100644 apps/web/src/hooks/voice/useAudioLevel.ts create mode 100644 apps/web/src/hooks/voice/useVoiceBinding.ts create mode 100644 apps/web/src/lib/ai/realtime/__tests__/call-chrome.test.ts create mode 100644 apps/web/src/lib/ai/realtime/__tests__/voice-binding.test.ts create mode 100644 apps/web/src/lib/ai/realtime/__tests__/voice-location.test.ts create mode 100644 apps/web/src/lib/ai/realtime/__tests__/voice-rebind.test.ts create mode 100644 apps/web/src/lib/ai/realtime/__tests__/voice-reveal.test.ts create mode 100644 apps/web/src/lib/ai/realtime/call-chrome.ts create mode 100644 apps/web/src/lib/ai/realtime/message-source.ts create mode 100644 apps/web/src/lib/ai/realtime/voice-binding.ts create mode 100644 apps/web/src/lib/ai/realtime/voice-location.ts create mode 100644 apps/web/src/lib/ai/realtime/voice-rebind.ts create mode 100644 apps/web/src/lib/ai/realtime/voice-reveal.ts create mode 100644 apps/web/src/stores/useVoiceRebindStore.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a55d781fed..ad8f13cd54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable user-facing changes to PageSpace are documented here. Format follows ## [Unreleased] +### Added + +- **A microphone in the top bar, on every page, that talks to whichever assistant you are already + looking at** — there is no separate voice screen and nothing to set up first. Press it on a page + and the assistant sidebar opens in voice mode, talking to the agent you had selected there; press + it on the dashboard and it talks to the assistant already in the middle of your screen. Nothing + connects until you press it. +- **A spoken conversation is the same conversation you can read and type in** — voice is a way into + a thread that already existed and still exists after you hang up. What is said appears in that + thread as ordinary messages while the call is running, marked with a small microphone so you can + tell later what was spoken and what was typed. There is no separate voice history to go looking + for, and nothing to replay. +- **The call survives you walking around the app** — moving between pages does not end it or move it + to a different assistant; it just tells the assistant where you now are. Closing the sidebar + minimizes the call rather than hanging up, and the top-bar microphone stays lit so you can get + back to it. Deliberately choosing a different agent in the sidebar's switcher does move the call, + because that is a different conversation. Ending a call is the End button on the call itself, and + refreshing the page ends it too. +- **When voice cannot start, it says which problem you have** — a microphone you declined is + different from a microphone you do not have, and the two now get different advice and only the + fixable one offers to try again. If the call connects but the transcript service does not, the + call says so rather than letting you talk for ten minutes into something that was never going to + be saved. + ### Fixed - **A second agent in a session stays put instead of flashing up and vanishing** — opening a chat diff --git a/apps/web/src/components/ai/shared/chat/CompactMessageRenderer.tsx b/apps/web/src/components/ai/shared/chat/CompactMessageRenderer.tsx index db85918081..c991c53c07 100644 --- a/apps/web/src/components/ai/shared/chat/CompactMessageRenderer.tsx +++ b/apps/web/src/components/ai/shared/chat/CompactMessageRenderer.tsx @@ -13,6 +13,7 @@ import type { ConversationMessage, TextPart } from './message-types'; import { isTextGroupPart, isProcessedToolPart, isFileGroupPart, isCommandExecutionPart, isToolRunGroupPart } from './message-types'; import { CommandExecutionIndicator } from '@/components/messages/CommandExecutionIndicator'; import { ImageMessageContent } from './ImageMessageContent'; +import { SpokenTurnGlyph, isSpokenTurn } from './SpokenTurnGlyph'; import styles from './CompactMessageRenderer.module.css'; interface CompactTextBlockProps { @@ -30,6 +31,8 @@ interface CompactTextBlockProps { onCancelEdit?: () => void; /** Whether this message is currently being streamed (for progressive markdown rendering) */ isStreaming?: boolean; + /** Turn was spoken into a live voice call, not typed. */ + spoken?: boolean; } /** @@ -49,7 +52,8 @@ const CompactTextBlock: React.FC = React.memo(({ isEditing, onSaveEdit, onCancelEdit, - isStreaming = false + isStreaming = false, + spoken = false }) => { const content = parts.map(part => part.text).join(''); @@ -90,13 +94,14 @@ const CompactTextBlock: React.FC = React.memo(({ {/* Always show footer with buttons; timestamp only when createdAt exists */}
-
+
{createdAt && ( <> {new Date(createdAt).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} {editedAt && (edited)} )} + {spoken && }
{onEdit && onDelete && !isEditing && ( = Rea onSaveEdit={handleSaveEdit} onCancelEdit={() => setIsEditing(false)} isStreaming={isStreaming} + spoken={isSpokenTurn(message)} /> ); } else if (isFileGroupPart(group)) { diff --git a/apps/web/src/components/ai/shared/chat/MessageRenderer.tsx b/apps/web/src/components/ai/shared/chat/MessageRenderer.tsx index b8f553641e..3a2cad64c2 100644 --- a/apps/web/src/components/ai/shared/chat/MessageRenderer.tsx +++ b/apps/web/src/components/ai/shared/chat/MessageRenderer.tsx @@ -14,6 +14,7 @@ import { useMessageRendererState } from './useMessageRendererState'; import type { ConversationMessage, TextPart } from './message-types'; import { isTextGroupPart, isProcessedToolPart, isFileGroupPart, isCommandExecutionPart, isToolRunGroupPart } from './message-types'; import { ImageMessageContent } from './ImageMessageContent'; +import { SpokenTurnGlyph, isSpokenTurn } from './SpokenTurnGlyph'; import { CommandExecutionIndicator } from '@/components/messages/CommandExecutionIndicator'; interface TextBlockProps { @@ -31,6 +32,8 @@ interface TextBlockProps { onCancelEdit?: () => void; /** Whether this message is currently being streamed (for progressive markdown rendering) */ isStreaming?: boolean; + /** Turn was spoken into a live voice call, not typed. */ + spoken?: boolean; } /** @@ -49,7 +52,8 @@ const TextBlock: React.FC = React.memo(({ isEditing, onSaveEdit, onCancelEdit, - isStreaming = false + isStreaming = false, + spoken = false }) => { const content = parts.map(part => part.text).join(''); @@ -89,13 +93,14 @@ const TextBlock: React.FC = React.memo(({
{/* Always show footer with buttons; timestamp only when createdAt exists */}
-
+
{createdAt && ( <> {new Date(createdAt).toLocaleTimeString()} {editedAt && (edited)} )} + {spoken && }
{onEdit && onDelete && !isEditing && ( = React.memo(({ onSaveEdit={handleSaveEdit} onCancelEdit={() => setIsEditing(false)} isStreaming={isStreaming} + spoken={isSpokenTurn(message)} /> ); } else if (isFileGroupPart(group)) { diff --git a/apps/web/src/components/ai/shared/chat/SpokenTurnGlyph.tsx b/apps/web/src/components/ai/shared/chat/SpokenTurnGlyph.tsx new file mode 100644 index 0000000000..38edc41b34 --- /dev/null +++ b/apps/web/src/components/ai/shared/chat/SpokenTurnGlyph.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { Mic } from 'lucide-react'; + +import { cn } from '@/lib/utils'; +import { VOICE_MESSAGE_SOURCE } from '@/lib/ai/realtime/message-source'; +import type { ConversationMessage } from './message-types'; + +/** + * The mark on a turn that was SPOKEN rather than typed. + * + * WHY THE THREAD HAS TO SAY. Audio is ephemeral and the transcript is the + * artifact of record — so what lands in the conversation is all that survives a + * call, and it lands next to typed messages that look exactly like it. A + * dictated sentence reads differently: it runs long, it repeats itself, it + * carries whatever the transcriber heard. A reader who does not know it was + * spoken reads it as someone typing carelessly, and an AGENT re-reading the + * thread has the same problem. One glyph is the whole fix. + * + * Compared through the named constant rather than a `'voice'` literal: it + * exists precisely so the UI's check, the realtime writer and the seed builder + * cannot end up on three spellings of one string (see `message-source.ts` for + * why the browser reads its own copy, and where the drift guard lives). + */ +export const isSpokenTurn = (message: Pick): boolean => + message.source === VOICE_MESSAGE_SOURCE; + +export const SpokenTurnGlyph: React.FC<{ className?: string }> = ({ className }) => ( + + +); diff --git a/apps/web/src/components/ai/shared/chat/__tests__/SpokenTurnGlyph.test.tsx b/apps/web/src/components/ai/shared/chat/__tests__/SpokenTurnGlyph.test.tsx new file mode 100644 index 0000000000..d7411541e2 --- /dev/null +++ b/apps/web/src/components/ai/shared/chat/__tests__/SpokenTurnGlyph.test.tsx @@ -0,0 +1,56 @@ +/** + * A spoken turn is marked in the thread, in BOTH renderers. + * + * The transcript is the artifact of record — it is all that survives a call — + * and it lands beside typed messages that look exactly like it. A dictated + * sentence runs long, repeats itself, and carries whatever the transcriber + * heard; read as typing it looks careless. The glyph is the whole fix, and it + * has to be in both renderers because a conversation opened in the sidebar and + * the same conversation opened in the main view are the same conversation. + */ + +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; + +import { MESSAGE_SOURCE_VOICE } from '@pagespace/db/schema/conversations'; +import type { ConversationMessage } from '../message-types'; + +vi.mock('@/hooks/useAuth', () => ({ useAuth: () => ({ user: { id: 'u1', name: 'Me' } }) })); +vi.mock('@/hooks/useTasks', () => ({ useTasks: () => ({ tasks: [], isLoading: false }) })); + +import { CompactMessageRenderer } from '../CompactMessageRenderer'; +import { MessageRenderer } from '../MessageRenderer'; + +const message = (source: string | null): ConversationMessage => + ({ + id: 'm1', + role: 'user', + parts: [{ type: 'text', text: 'what is on this page' }], + createdAt: new Date('2026-01-01T10:00:00Z'), + messageType: 'standard', + source, + }) as ConversationMessage; + +describe.each([ + ['CompactMessageRenderer', CompactMessageRenderer], + ['MessageRenderer', MessageRenderer], +])('%s — marking a spoken turn', (_name, Renderer) => { + it('should mark a voice-authored row', () => { + render(); + expect(screen.getByTestId('spoken-turn-glyph')).toBeInTheDocument(); + // Named for screen readers, which get nothing from an icon. + expect(screen.getByText('Spoken in a voice call')).toBeInTheDocument(); + }); + + it('should leave a typed row unmarked', () => { + render(); + expect(screen.queryByTestId('spoken-turn-glyph')).not.toBeInTheDocument(); + }); + + it('should not mark a row that merely mentions voice', () => { + // The check is on the transport column, not on anything in the content. + render(); + expect(screen.queryByTestId('spoken-turn-glyph')).not.toBeInTheDocument(); + }); +}); diff --git a/apps/web/src/components/ai/shared/chat/message-types.ts b/apps/web/src/components/ai/shared/chat/message-types.ts index 73d312e2cf..de6bcbbdd0 100644 --- a/apps/web/src/components/ai/shared/chat/message-types.ts +++ b/apps/web/src/components/ai/shared/chat/message-types.ts @@ -23,6 +23,19 @@ export interface ConversationMessage extends UIMessage { * read as `'complete'` server-side by default. */ status?: 'streaming' | 'complete' | 'interrupted'; + /** + * The TRANSPORT this row was authored over (see `messages.source`) — + * `'voice'` for a turn spoken into a live realtime call, null/absent for a + * typed one. + * + * Rendered as a mic glyph rather than left invisible because a thread that + * mixes spoken and typed turns without saying which is which is a thread that + * misrepresents itself: the phrasing, the length and the errors of a spoken + * sentence are all different, and a reader who does not know it was dictated + * reads it as sloppy typing. Compared through {@link MESSAGE_SOURCE_VOICE}, + * never a bare string literal. + */ + source?: string | null; } /** diff --git a/apps/web/src/components/ai/voice/realtime/VoiceCallBar.tsx b/apps/web/src/components/ai/voice/realtime/VoiceCallBar.tsx new file mode 100644 index 0000000000..b4dec51cf0 --- /dev/null +++ b/apps/web/src/components/ai/voice/realtime/VoiceCallBar.tsx @@ -0,0 +1,170 @@ +'use client'; + +import { Loader2, Mic, MicOff, PhoneOff } from 'lucide-react'; + +import { Button } from '@/components/ui/button'; +import { cn } from '@/lib/utils'; +import { useAudioLevel } from '@/hooks/voice/useAudioLevel'; +import { useVoiceSession, useVoiceSessionControls } from '@/contexts/VoiceSessionContext'; +import { describeCall, isCallLive } from '@/lib/ai/realtime/call-chrome'; + +export interface VoiceCallBarProps { + /** The assistant this call is bound to, named the way the surface names it. */ + assistantName: string; + className?: string; +} + +const METER_BARS = 5; + +/** + * The live call, as a header on the chat surface it belongs to. + * + * VOICE IS A MODE, NOT A PANEL. This bar sits above the SAME message list, in + * the SAME conversation, on the surface that was already showing that + * conversation. It is not a fourth sidebar tab and not an overlay, because + * either of those would make the spoken turns a separate place with a separate + * history — and the whole claim of audio-native voice here is that there is one + * substrate and speaking is just another way into it. + * + * WHY IT SHOWS NO ERROR STATE. A failed connect clears the binding in the + * provider (`setTarget(null)`), so there is no longer a conversation for this + * bar to belong to — and a surface that reported an error for a call it can no + * longer identify would report OTHER surfaces' failures too. Failures are the + * nav trigger's job, as a toast, which reaches the user on every route + * including the ones where this panel is collapsed and this component does not + * exist. One reporter, and the one that is always mounted. + * + * WHAT IT DELIBERATELY DOES NOT RENDER: the transcript. Spoken turns are + * written into the conversation by the realtime server and arrive on the + * ordinary `conversation:*` socket events, so they appear in the message list + * below as ordinary messages, marked with a mic glyph. Rendering them here as + * well would be a second copy of the record, drifting from the first, and would + * be exactly the "second history" the design forbids. The one thing shown is + * the CURRENT utterance — which has not been persisted yet and therefore exists + * nowhere else. + */ +export function VoiceCallBar({ assistantName, className }: VoiceCallBarProps) { + const { status, error, failure, attached, target, userSpeaking, transcript, tools, localStream, muted } = + useVoiceSession(); + const { stop, setMuted } = useVoiceSessionControls(); + + const chrome = describeCall({ status, error, failure, attached, bound: target !== null }); + const live = isCallLive(chrome.state); + + // Metered from the user's own microphone, and only while unmuted: a meter + // that keeps moving after mute is a UI telling the user they are still being + // heard when the entire point of the button they just pressed is that they + // are not. + const level = useAudioLevel(localStream, live && !muted); + + const latest = transcript.length > 0 ? transcript[transcript.length - 1] : undefined; + const latestTool = tools.length > 0 ? tools[tools.length - 1] : undefined; + + const statusLine = (() => { + if (chrome.state === 'connecting') return 'Connecting…'; + if (muted) return 'Muted'; + if (latestTool) return latestTool.speech; + if (userSpeaking) return 'Listening…'; + return `Talking to ${assistantName}`; + })(); + + return ( +
+
+ {/* Level meter — the "is it hearing me" signal, before any words exist. */} + + +
+ {latest ? ( +

+ “{latest.text}” +

+ ) : ( +

{statusLine}

+ )} +
+ +
+ {chrome.state === 'connecting' ? ( + + ) : ( + + )} + + +
+
+ + {/* + The degradation notice. Told, never inferred: a call whose transcript is + not being written looks exactly like one whose transcript is, right up + until the user hangs up and finds nothing in the thread. It stays + visible for the whole call rather than fading, because the fact it + states is true for the whole call. + */} + {chrome.state === 'degraded' && chrome.detail && ( +

+ {chrome.detail} +

+ )} +
+ ); +} diff --git a/apps/web/src/components/ai/voice/realtime/VoiceCallBarForConversation.tsx b/apps/web/src/components/ai/voice/realtime/VoiceCallBarForConversation.tsx new file mode 100644 index 0000000000..f9c7316bfc --- /dev/null +++ b/apps/web/src/components/ai/voice/realtime/VoiceCallBarForConversation.tsx @@ -0,0 +1,37 @@ +'use client'; + +import { useVoiceSession } from '@/contexts/VoiceSessionContext'; +import { isCallOnConversation } from '@/lib/ai/realtime/voice-binding'; +import { VoiceCallBar } from './VoiceCallBar'; + +export interface VoiceCallBarForConversationProps { + /** The conversation this surface is showing. `null` while it is resolving. */ + conversationId: string | null; + /** How this surface names the assistant on screen. */ + assistantName: string; +} + +/** + * The call header, IF the live call is on this surface's conversation. + * + * WHY THIS EXISTS AS A COMPONENT rather than as an `&&` inside each chat + * surface. It is mounted by `SidebarChatTab` and `GlobalAssistantView`, both of + * which are ~1000-line files with enormous hook graphs that the repo does not + * render in tests. A gate written inline in each would be two copies of one + * decision, in the two files least likely to be covered — and the decision has + * a specific wrong answer that looks right: keying on WHICH SURFACE STARTED THE + * CALL instead of on the conversation. That version works until the user walks + * from the sidebar to the dashboard mid-call, at which point the call is live + * with no chrome anywhere and no way to end it but the nav trigger. + * + * Here, the decision is one line, in one place, and exercisable against a real + * session. + */ +export function VoiceCallBarForConversation({ + conversationId, + assistantName, +}: VoiceCallBarForConversationProps) { + const { target } = useVoiceSession(); + if (!isCallOnConversation(target, conversationId)) return null; + return ; +} diff --git a/apps/web/src/components/ai/voice/realtime/VoiceNavTrigger.tsx b/apps/web/src/components/ai/voice/realtime/VoiceNavTrigger.tsx new file mode 100644 index 0000000000..6d2e71b7a2 --- /dev/null +++ b/apps/web/src/components/ai/voice/realtime/VoiceNavTrigger.tsx @@ -0,0 +1,196 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { usePathname } from 'next/navigation'; +import { Loader2, Mic, MicOff } from 'lucide-react'; +import { toast } from 'sonner'; + +import { Button } from '@/components/ui/button'; +import { cn } from '@/lib/utils'; +import { useDriveStore } from '@/hooks/useDrive'; +import { useVoiceBinding } from '@/hooks/voice/useVoiceBinding'; +import { useVoiceSession, useVoiceSessionControls } from '@/contexts/VoiceSessionContext'; +import { resolveLocationContext } from '@/lib/ai/shared/resolveLocationContext'; +import { toVoiceLocationContext } from '@/lib/ai/realtime/voice-location'; +import { describeCall, isCallLive } from '@/lib/ai/realtime/call-chrome'; +import type { VoiceSurface } from '@/lib/ai/realtime/voice-binding'; + +export interface VoiceNavTriggerProps { + /** + * Bring the conversation the call is on into view. OPEN-ONLY — see + * `decideReveal`. Given the surface because the dashboard needs no panel + * opened at all: the conversation is already in the centre of the screen. + */ + onReveal: (surface: VoiceSurface) => void; + className?: string; +} + +/** + * THE one voice control, in the one piece of chrome that is on every route. + * + * It is a single button with two jobs and, deliberately, no third: + * - nothing running: start a call on whatever assistant is in view; + * - call running: it IS the live indicator, and pressing it brings that call + * back into view. + * + * IT NEVER HANGS UP. Ending a call is the call header's End button and only + * that. A trigger that toggled would mean the live indicator is also the + * destroy button — one control with two meanings, where the destructive one + * fires on exactly the press a user makes when they want to see the call again. + * + * NOTHING CONNECTS UNTIL IT IS PRESSED. No microphone, no negotiation, no + * permission prompt on page load — the whole voice stack is inert until this + * button is used, which is why it can be mounted app-wide without cost. + * + * WHY IT RESOLVES THE LOCATION BEFORE STARTING. The tools' sense of place + * reaches the model only when a call STARTS or CHAINS (see the KNOWN GAP note + * in `VoiceSessionContext`), so a call opened before the location is known + * answers "what's on this page?" about nowhere for its entire life. One resolve + * ahead of a WebRTC handshake and a permission prompt is a latency cost that + * buys the first spoken question actually working. + */ +export function VoiceNavTrigger({ onReveal, className }: VoiceNavTriggerProps) { + const pathname = usePathname(); + // No `drives` SUBSCRIPTION: the list is read fresh from the store at press + // time, below. Subscribing would re-render this button on every drive-store + // write for a value it only needs once, at the moment it is used. + const fetchDrives = useDriveStore((state) => state.fetchDrives); + + const { status, error, failure, attached, target, userSpeaking } = useVoiceSession(); + const { start, setLocationContext } = useVoiceSessionControls(); + const binding = useVoiceBinding(); + + const [isStarting, setIsStarting] = useState(false); + + const chrome = describeCall({ status, error, failure, attached, bound: target !== null }); + const live = isCallLive(chrome.state); + + /** + * The sidebar can be closed when a call fails, and the call header is the + * only other thing that would report it — so a failure with the panel shut + * would be a button that does nothing, forever, with no explanation. The + * toast is what makes the microphone honest on every route. + * + * Keyed on the SENTENCE, not on `status`: two consecutive attempts that fail + * the same way should not stack two identical toasts, and two that fail + * differently must both be told. + */ + const reportedRef = useRef(null); + /** + * `handlePress` is declared below and the toast needs to call it, so the + * toast's action reaches it through a ref rather than by hoisting the + * callback above the effect that would then have to depend on it. + */ + const pressRef = useRef<() => void>(() => {}); + useEffect(() => { + if (chrome.state !== 'error') { + reportedRef.current = null; + return; + } + const sentence = chrome.detail; + if (sentence === null || reportedRef.current === sentence) return; + reportedRef.current = sentence; + toast.error(sentence, { + // A Try again offered for a MISSING microphone is a button guaranteed to + // fail — it teaches the user that voice is broken when the truth is that + // they have no capture device. `canRetry` is that distinction, carried + // all the way from `classifyMicFailure`. + ...(chrome.canRetry + ? { action: { label: 'Try again', onClick: () => pressRef.current() } } + : {}), + }); + }, [chrome.state, chrome.detail, chrome.canRetry]); + + const handlePress = useCallback(async () => { + // Already up: this press is "show me the call", never "end it". + if (live) { + onReveal(binding.surface); + return; + } + + // The conversation for the assistant in view has not resolved yet. The + // button is disabled in this state; this is the belt to that suspenders. + if (binding.kind !== 'ready' || isStarting) return; + + onReveal(binding.surface); + setIsStarting(true); + try { + // Drives are cached for five minutes, so this is usually free — but the + // drive is half of what a location IS, and a call started before the + // store populated would be permanently placeless. + await fetchDrives(); + const { locationContext } = await resolveLocationContext( + pathname, + useDriveStore.getState().drives, + ); + setLocationContext(toVoiceLocationContext(locationContext)); + await start(binding.target); + } finally { + setIsStarting(false); + } + }, [ + live, + binding, + isStarting, + onReveal, + fetchDrives, + pathname, + setLocationContext, + start, + ]); + pressRef.current = () => void handlePress(); + + const busy = chrome.state === 'connecting' || isStarting; + // Disabled ONLY while the target is still resolving with nothing running. + // Never disabled during a live call: that press is the way back to it. + const disabled = !live && (binding.kind !== 'ready' || isStarting); + + return ( + + ); +} diff --git a/apps/web/src/components/ai/voice/realtime/VoiceSessionBridge.tsx b/apps/web/src/components/ai/voice/realtime/VoiceSessionBridge.tsx new file mode 100644 index 0000000000..4b60063fad --- /dev/null +++ b/apps/web/src/components/ai/voice/realtime/VoiceSessionBridge.tsx @@ -0,0 +1,95 @@ +'use client'; + +import { useEffect } from 'react'; +import { usePathname } from 'next/navigation'; + +import { useDriveStore } from '@/hooks/useDrive'; +import { useVoiceBinding } from '@/hooks/voice/useVoiceBinding'; +import { useVoiceRebindStore } from '@/stores/useVoiceRebindStore'; +import { useVoiceSession, useVoiceSessionControls } from '@/contexts/VoiceSessionContext'; +import { resolveLocationContext } from '@/lib/ai/shared/resolveLocationContext'; +import { toVoiceLocationContext } from '@/lib/ai/realtime/voice-location'; +import { rebindAction } from '@/lib/ai/realtime/voice-rebind'; +import { describeCall, isCallLive } from '@/lib/ai/realtime/call-chrome'; + +/** + * The two things that must happen to a live call while nobody is looking at it. + * + * Renders NOTHING, and that is the point. Both jobs below outlive every panel + * that could show a call — the sidebar is unmounted outright when collapsed — + * so neither can live in the chat surfaces. Mounted once, in `Layout`, beside + * the trigger that is likewise on every route. + * + * ── 1. WHERE THE USER IS STANDING ──────────────────────────────────────────── + * Navigating updates `locationContext` and MUST NOT touch the session. This is + * the effect that makes that true, and it is deliberately the ONLY thing in the + * app that reacts to a route change with a live call running. It calls + * `setLocationContext` and never `start` — a `start` here would be the settled + * rule inverted, and the symptom would be a call that hangs up every time the + * user opens a page. + * + * It runs only while a call is live. When nothing is running there is no + * session to inform, and resolving a location on every navigation for every + * user — two API calls a route — to serve a feature nobody has started would be + * a cost paid by people who never press the microphone. The FIRST call's + * location is resolved by the trigger, at press time, for the same reason. + * + * ── 2. FOLLOWING THE AGENT SWITCHER ────────────────────────────────────────── + * The switcher records an intent (`useVoiceRebindStore`); this applies it once + * the newly chosen agent's conversation has resolved. The decision itself is + * `rebindAction`, which is pure and tested — including the case this whole + * arrangement exists to make impossible, where a route change and a deliberate + * switch look identical in the derived target. + */ +export function VoiceSessionBridge() { + const pathname = usePathname(); + const drives = useDriveStore((state) => state.drives); + + const { status, error, failure, attached, target } = useVoiceSession(); + const { start, setLocationContext } = useVoiceSessionControls(); + + const binding = useVoiceBinding(); + const intent = useVoiceRebindStore((state) => state.intent); + const clearRebind = useVoiceRebindStore((state) => state.clearRebind); + + const chrome = describeCall({ + status, + error, + failure, + attached, + bound: target !== null, + }); + const callIsLive = isCallLive(chrome.state); + + // 1. Location, while a call is live. No `start`, ever. + useEffect(() => { + if (!callIsLive) return; + let ignore = false; + + void resolveLocationContext(pathname, drives).then(({ locationContext }) => { + if (ignore) return; + setLocationContext(toVoiceLocationContext(locationContext)); + }); + + return () => { + ignore = true; + }; + }, [callIsLive, pathname, drives, setLocationContext]); + + // 2. The switcher's intent, applied when the app has caught up. + useEffect(() => { + const action = rebindAction(intent, binding, callIsLive); + if (action.kind === 'wait') return; + if (action.kind === 'clear') { + clearRebind(); + return; + } + // Clear FIRST: `start` is async, and an intent still standing when this + // effect re-runs on the resulting state change would ask for the same + // rebind a second time. + clearRebind(); + void start(action.target); + }, [intent, binding, callIsLive, clearRebind, start]); + + return null; +} diff --git a/apps/web/src/components/ai/voice/realtime/__tests__/VoiceCallBar.test.tsx b/apps/web/src/components/ai/voice/realtime/__tests__/VoiceCallBar.test.tsx new file mode 100644 index 0000000000..4577ec0221 --- /dev/null +++ b/apps/web/src/components/ai/voice/realtime/__tests__/VoiceCallBar.test.tsx @@ -0,0 +1,182 @@ +/** + * The call header, driven through the REAL provider so that pressing Mute + * actually reaches a microphone track and pressing End actually closes a peer + * connection. A version of this file that mocked the session would pass with + * buttons wired to nothing. + */ + +import React from 'react'; +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + VoiceSessionProvider, + useVoiceSessionControls, + type VoiceSessionDeps, +} from '@/contexts/VoiceSessionContext'; +import { + FakePeerConnection, + asPeer, + asStream, + fakeCreateMediaStream, + fakeStream, + respondWith, +} from '@/lib/ai/realtime/__tests__/webrtc-fakes'; +import { VoiceCallBar } from '../VoiceCallBar'; + +const TARGET = { conversationId: 'conv-a', type: 'global' as const }; + +const harness = (attached = true) => { + const peers: FakePeerConnection[] = []; + const microphone = fakeStream('mic-device'); + const deps: VoiceSessionDeps = { + fetchImpl: async () => + respondWith({ + body: { callId: 'rtc_1', answerSdp: 'v=0 answer', attached }, + }), + createPeerConnection: () => { + const peer = new FakePeerConnection(); + peers.push(peer); + return asPeer(peer); + }, + getUserMedia: async () => asStream(microphone), + createMediaStream: fakeCreateMediaStream, + }; + return { peers, deps }; +}; + +function StartButton() { + const { start } = useVoiceSessionControls(); + return