Skip to content

Add assistant listen controls to Voice & Audio #800

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shanselman merged 3 commits into from
Jun 26, 2026
Merged

Add assistant listen controls to Voice & Audio #800

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5de300c
Add assistant controls to voice settings
RipauvGohil Jun 21, 2026
45e028c
fix(tray): keep assistant listen local by default
shanselman Jun 22, 2026
caf90c2
fix(tray): harden assistant bridge controls
shanselman Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions src/OpenClaw.Tray.WinUI/Pages/VoiceSettingsPage.xaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,76 @@
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
TextWrapping="Wrap" Margin="0,0,0,8"/>

<!-- Assistant loop -->
<Border Background="{ThemeResource CardBackgroundFillColorDefaultBrush}"
BorderBrush="{ThemeResource CardStrokeColorDefaultBrush}"
BorderThickness="1" CornerRadius="8" Padding="16">
<StackPanel Spacing="12">
<Grid ColumnSpacing="12">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*"/>
<ColumnDefinition Width="Auto"/>
</Grid.ColumnDefinitions>
<StackPanel Spacing="4">
<TextBlock x:Uid="VoiceSettingsPage_AssistantHeader" Text="Assistant" Style="{StaticResource BodyStrongTextBlockStyle}"/>
<TextBlock x:Uid="VoiceSettingsPage_AssistantDescription"
Text="Always-listening assistant service and latest conversation turns."
Style="{StaticResource CaptionTextBlockStyle}"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
TextWrapping="Wrap"/>
</StackPanel>
<StackPanel Grid.Column="1" Orientation="Horizontal" Spacing="8" VerticalAlignment="Top">
<Button x:Uid="VoiceSettingsPage_AssistantRefreshButton"
x:Name="AssistantRefreshButton" Click="OnAssistantRefreshClick" Padding="10,8"
ToolTipService.ToolTip="Refresh assistant status">
<FontIcon Glyph="&#xE72C;" FontSize="14"/>
</Button>
<Button x:Name="AssistantStartButton" Click="OnAssistantStartClick"
Style="{StaticResource AccentButtonStyle}">
<StackPanel Orientation="Horizontal" Spacing="6">
<FontIcon Glyph="&#xE768;" FontSize="14"/>
<TextBlock x:Uid="VoiceSettingsPage_AssistantStartButtonText" Text="Start"/>
</StackPanel>
</Button>
<Button x:Name="AssistantStopButton" Click="OnAssistantStopClick">
<StackPanel Orientation="Horizontal" Spacing="6">
<FontIcon Glyph="&#xE71A;" FontSize="14"/>
<TextBlock x:Uid="VoiceSettingsPage_AssistantStopButtonText" Text="Stop"/>
</StackPanel>
</Button>
</StackPanel>
</Grid>

<Border Background="{ThemeResource CardBackgroundFillColorSecondaryBrush}"
CornerRadius="8" Padding="12">
<Grid RowSpacing="4">
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
</Grid.RowDefinitions>
<TextBlock x:Uid="VoiceSettingsPage_AssistantStatusChecking"
x:Name="AssistantStatusText"
Text="Checking assistant status..."
Style="{StaticResource BodyStrongTextBlockStyle}"/>
<TextBlock x:Name="AssistantDetailText" Grid.Row="1"
Style="{StaticResource CaptionTextBlockStyle}"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
TextWrapping="Wrap"/>
<TextBlock x:Name="AssistantLastRefreshText" Grid.Row="2"
Style="{StaticResource CaptionTextBlockStyle}"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
TextWrapping="Wrap"/>
</Grid>
</Border>

<StackPanel Spacing="8">
<TextBlock x:Uid="VoiceSettingsPage_AssistantRecentTurnsHeader" Text="Recent turns" Style="{StaticResource BodyStrongTextBlockStyle}"/>
<StackPanel x:Name="AssistantTurnsPanel" Spacing="8"/>
</StackPanel>
</StackPanel>
</Border>

<!-- Enable STT -->
<Border Background="{ThemeResource CardBackgroundFillColorDefaultBrush}"
BorderBrush="{ThemeResource CardStrokeColorDefaultBrush}"
Expand Down
249 changes: 249 additions & 0 deletions src/OpenClaw.Tray.WinUI/Pages/VoiceSettingsPage.xaml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ public sealed partial class VoiceSettingsPage : Page
{
private static App CurrentApp => (App)Microsoft.UI.Xaml.Application.Current!;
private VoiceService? _voiceService;
private AssistantBridgeService? _assistantBridgeService;
private CancellationTokenSource? _assistantRequestCts;
private int _assistantOperationVersion;
private bool _suppressEvents = true; // suppress until Initialize/LoadSettings runs
// Per-asset CTS so a Piper download doesn't cancel an in-flight Whisper
// download (and vice versa). Each download type owns its own token.
Expand All @@ -34,9 +37,12 @@ public VoiceSettingsPage()
{
UpdateModelStatus();
UpdatePiperVoiceState();
_ = RefreshAssistantAsync();
};
Unloaded += async (_, _) =>
{
CancelAssistantRequest();

if (App.Current is App app)
app.SpeakerMuteChanged -= OnAppSpeakerMuteChanged;

Expand All @@ -52,6 +58,7 @@ public VoiceSettingsPage()
public void Initialize(VoiceService? voiceService)
{
_voiceService = voiceService;
_assistantBridgeService ??= new AssistantBridgeService(new AppLogger());
if (App.Current is App app)
{
app.SpeakerMuteChanged -= OnAppSpeakerMuteChanged;
Expand All @@ -64,6 +71,248 @@ public void Initialize(VoiceService? voiceService)
PiperPreviewLabel.Text = L("VoiceSettingsPage_PiperPreviewButtonContent");
PreviewVoiceLabel.Text = L("VoiceSettingsPage_PreviewVoiceButtonContent");
LoadSettings();
_ = RefreshAssistantAsync();
}

private void OnAssistantRefreshClick(object sender, RoutedEventArgs e) =>
AsyncEventHandlerGuard.Run(
RefreshAssistantAsync,
new OpenClawTray.AppLogger(),
nameof(OnAssistantRefreshClick));

private void OnAssistantStartClick(object sender, RoutedEventArgs e) =>
AsyncEventHandlerGuard.Run(
OnAssistantStartClickAsync,
new OpenClawTray.AppLogger(),
nameof(OnAssistantStartClick));

private async Task OnAssistantStartClickAsync()
{
var bridge = EnsureAssistantBridge();
var busyVersion = BeginAssistantOperation();
AssistantStatusText.Text = L("VoiceSettingsPage_AssistantStarting");
AssistantDetailText.Text = L("VoiceSettingsPage_AssistantStartingDetail");
try
{
var result = await bridge.StartListenServiceAsync(NewAssistantRequestToken());
if (!result.Success)
{
AssistantStatusText.Text = L("VoiceSettingsPage_AssistantStartFailed");
AssistantDetailText.Text = result.ErrorMessage;
return;
}

await RefreshAssistantAsync();
}
catch (OperationCanceledException)
{
}
finally
{
EndAssistantOperation(busyVersion);
}
}

private void OnAssistantStopClick(object sender, RoutedEventArgs e) =>
AsyncEventHandlerGuard.Run(
OnAssistantStopClickAsync,
new OpenClawTray.AppLogger(),
nameof(OnAssistantStopClick));

private async Task OnAssistantStopClickAsync()
{
var bridge = EnsureAssistantBridge();
var busyVersion = BeginAssistantOperation();
AssistantStatusText.Text = L("VoiceSettingsPage_AssistantStopping");
AssistantDetailText.Text = "";
try
{
var result = await bridge.StopListenServiceAsync(NewAssistantRequestToken());
if (!result.Success)
{
AssistantStatusText.Text = L("VoiceSettingsPage_AssistantStopFailed");
AssistantDetailText.Text = result.ErrorMessage;
return;
}

await RefreshAssistantAsync();
}
catch (OperationCanceledException)
{
}
finally
{
EndAssistantOperation(busyVersion);
}
}

private async Task RefreshAssistantAsync()
{
var bridge = EnsureAssistantBridge();
var busyVersion = BeginAssistantOperation();
try
{
var snapshot = await bridge.GetStatusAsync(NewAssistantRequestToken());
RenderAssistantSnapshot(snapshot);
}
catch (OperationCanceledException)
{
}
finally
{
EndAssistantOperation(busyVersion);
}
}

private AssistantBridgeService EnsureAssistantBridge() =>
_assistantBridgeService ??= new AssistantBridgeService(new AppLogger());

private CancellationToken NewAssistantRequestToken()
{
CancelAssistantRequest();
_assistantRequestCts = new CancellationTokenSource();
return _assistantRequestCts.Token;
}

private void CancelAssistantRequest()
{
try { _assistantRequestCts?.Cancel(); }
catch (Exception ex) { Logger.Debug($"VoiceSettingsPage: assistant request cancel failed: {ex.Message}"); }
_assistantRequestCts?.Dispose();
_assistantRequestCts = null;
}

private void SetAssistantBusy(bool busy)
{
AssistantRefreshButton.IsEnabled = !busy;
AssistantStartButton.IsEnabled = !busy;
AssistantStopButton.IsEnabled = !busy;
}

private int BeginAssistantOperation()
{
var version = Interlocked.Increment(ref _assistantOperationVersion);
SetAssistantBusy(true);
return version;
}

private void EndAssistantOperation(int version)
{
if (Volatile.Read(ref _assistantOperationVersion) == version)
SetAssistantBusy(false);
}

private void RenderAssistantSnapshot(AssistantBridgeSnapshot snapshot)
{
if (!snapshot.IsAvailable)
{
AssistantStatusText.Text = L("VoiceSettingsPage_AssistantBridgeUnavailable");
AssistantDetailText.Text = snapshot.ErrorMessage;
AssistantLastRefreshText.Text = "";
AssistantTurnsPanel.Children.Clear();
AssistantTurnsPanel.Children.Add(new TextBlock
{
Text = L("VoiceSettingsPage_AssistantNoTurnsLoaded"),
Style = (Style)Application.Current.Resources["CaptionTextBlockStyle"],
Foreground = (Brush)Application.Current.Resources["TextFillColorSecondaryBrush"],
TextWrapping = TextWrapping.Wrap
});
return;
}

var listen = snapshot.ListenService;
var unknown = L("VoiceSettingsPage_AssistantUnknown");
var status = string.IsNullOrWhiteSpace(listen.Status) ? unknown : listen.Status;
AssistantStatusText.Text = listen.IsRunning
? Lf("VoiceSettingsPage_AssistantListeningFormat", listen.Pid?.ToString(CultureInfo.CurrentCulture) ?? unknown)
: listen.IsStopped
? L("VoiceSettingsPage_AssistantStopped")
: Lf("VoiceSettingsPage_AssistantStatusFormat", status);

var details = new[]
{
string.IsNullOrWhiteSpace(listen.Transcriber) ? "" : Lf("VoiceSettingsPage_AssistantTranscriberFormat", listen.Transcriber),
string.IsNullOrWhiteSpace(snapshot.PreferredInputDevice) ? "" : Lf("VoiceSettingsPage_AssistantMicFormat", snapshot.PreferredInputDevice),
string.IsNullOrWhiteSpace(snapshot.PreferredOutputDevice) ? "" : Lf("VoiceSettingsPage_AssistantSpeakerFormat", snapshot.PreferredOutputDevice),
Lf("VoiceSettingsPage_AssistantCloudRoutingFormat", listen.AllowCloud ? L("VoiceSettingsPage_AssistantOn") : L("VoiceSettingsPage_AssistantOff")),
Lf("VoiceSettingsPage_AssistantSpeechOutputFormat", listen.SpeakAloud ? L("VoiceSettingsPage_AssistantOn") : L("VoiceSettingsPage_AssistantOff"))
}.Where(s => !string.IsNullOrWhiteSpace(s));
AssistantDetailText.Text = string.Join(" | ", details);
AssistantLastRefreshText.Text = string.IsNullOrWhiteSpace(snapshot.GeneratedAt)
? ""
: Lf("VoiceSettingsPage_AssistantLastBridgeUpdateFormat", snapshot.GeneratedAt);

AssistantTurnsPanel.Children.Clear();
if (snapshot.RecentTurns.Count == 0)
{
AssistantTurnsPanel.Children.Add(new TextBlock
{
Text = L("VoiceSettingsPage_AssistantNoConversationTurns"),
Style = (Style)Application.Current.Resources["CaptionTextBlockStyle"],
Foreground = (Brush)Application.Current.Resources["TextFillColorSecondaryBrush"],
TextWrapping = TextWrapping.Wrap
});
return;
}

foreach (var turn in snapshot.RecentTurns.Take(5))
AssistantTurnsPanel.Children.Add(CreateAssistantTurnView(turn));
}

private static Border CreateAssistantTurnView(AssistantTurnSnapshot turn)
{
var panel = new StackPanel { Spacing = 6 };
var source = string.IsNullOrWhiteSpace(turn.Source) ? L("VoiceSettingsPage_AssistantSourceDefault") : turn.Source;
var stage = string.IsNullOrWhiteSpace(turn.Stage) ? L("VoiceSettingsPage_AssistantUnknown") : turn.Stage;
var model = string.IsNullOrWhiteSpace(turn.Provider)
? turn.ModelProfile
: $"{turn.Provider} {turn.ModelProfile}".Trim();
var metadata = turn.TotalMs is int ms
? Lf("VoiceSettingsPage_AssistantTurnMetadataWithLatencyFormat", source, stage, ms.ToString(CultureInfo.CurrentCulture))
: Lf("VoiceSettingsPage_AssistantTurnMetadataFormat", source, stage);
panel.Children.Add(new TextBlock
{
Text = metadata,
Style = (Style)Application.Current.Resources["CaptionTextBlockStyle"],
Foreground = (Brush)Application.Current.Resources["TextFillColorSecondaryBrush"],
TextWrapping = TextWrapping.Wrap
});
panel.Children.Add(new TextBlock
{
Text = Lf("VoiceSettingsPage_AssistantUserTurnFormat", TrimForDisplay(turn.InputText)),
TextWrapping = TextWrapping.Wrap
});
panel.Children.Add(new TextBlock
{
Text = Lf("VoiceSettingsPage_AssistantResponseTurnFormat", TrimForDisplay(turn.ResponseText)),
TextWrapping = TextWrapping.Wrap
});
if (!string.IsNullOrWhiteSpace(model))
{
panel.Children.Add(new TextBlock
{
Text = model,
Style = (Style)Application.Current.Resources["CaptionTextBlockStyle"],
Foreground = (Brush)Application.Current.Resources["TextFillColorSecondaryBrush"],
TextWrapping = TextWrapping.Wrap
});
}

return new Border
{
Child = panel,
Background = (Brush)Application.Current.Resources["CardBackgroundFillColorSecondaryBrush"],
CornerRadius = new CornerRadius(8),
Padding = new Thickness(12)
};
}

private static string TrimForDisplay(string value)
{
if (string.IsNullOrWhiteSpace(value))
return L("VoiceSettingsPage_AssistantEmptyValue");
value = value.Trim();
return value.Length <= 220 ? value : value[..217] + "...";
}

private void OnAppSpeakerMuteChanged(bool muted)
Expand Down
Loading
Loading