From 71e59ceba1c418946fd4c1a8e84fee297533b22e Mon Sep 17 00:00:00 2001 From: notegen <525229509@qq.com> Date: Thu, 4 Jun 2026 16:01:33 +0800 Subject: [PATCH 1/3] docs: update offline captions readme --- .gitignore | 1 + README.md | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 82fc468b7c..294483d41e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ dist-ssr /electron/native/screencapturekit/.build/ /electron/native/screencapturekit/.swiftpm/ /electron/native/bin/ +/electron/native/captions/*/ # Native macOS generated files DerivedData/ diff --git a/README.md b/README.md index 7009a22098..e5d8c6786f 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Screen Studio is an awesome product and this is definitely not a 1:1 clone. Open - Blur effects to hide sensitive parts of the screen. - Cursor and click highlighting. - Text, arrow, and image annotations. +- Offline auto-captions on macOS release builds, generated locally with bundled whisper.cpp helpers and the default `ggml-small` model. - Save and reopen projects without re-recording. - Export to MP4 or GIF in multiple aspect ratios and resolutions. - Translated into Arabic, English, Spanish, French, Japanese, Korean, Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. @@ -154,6 +155,8 @@ System audio capture relies on Electron's [desktopCapturer](https://www.electron - **Windows**: Works out of the box. - **Linux**: Needs PipeWire (default on Ubuntu 22.04+, Fedora 34+). Older PulseAudio-only setups may not support system audio (mic should still work). +Offline auto-captions currently depend on packaged caption runtime assets. macOS release builds include the local whisper.cpp CLI, ffmpeg, and `ggml-small` model; other platforms report captions as unavailable until their helper assets are packaged. + ## Built with - Electron - React From 60801c1bb8c4ca58713143a58fe553e7d45db13c Mon Sep 17 00:00:00 2001 From: notegen <525229509@qq.com> Date: Fri, 5 Jun 2026 23:31:28 +0800 Subject: [PATCH 2/3] Add offline captions and recording fixes --- .gitignore | 1 + electron-builder.json5 | 35 +- electron/captions/jobs.test.ts | 55 +++ electron/captions/jobs.ts | 53 +++ electron/captions/whisper.test.ts | 223 ++++++++++ electron/captions/whisper.ts | 283 +++++++++++++ electron/electron-env.d.ts | 6 + electron/ipc/handlers.ts | 274 +++++++++++- .../macNativeCursorRecordingSession.test.ts | 53 +++ .../macNativeCursorRecordingSession.ts | 4 +- electron/preload.ts | 14 + .../AnnotationSettingsPanel.test.tsx | 59 +++ .../video-editor/AnnotationSettingsPanel.tsx | 35 ++ src/components/video-editor/VideoEditor.tsx | 64 +++ .../video-editor/annotationStickers.ts | 130 ++++++ .../video-editor/captionAnnotations.test.ts | 43 ++ .../video-editor/captionAnnotations.ts | 56 +++ .../useAutoCaptionGeneration.test.ts | 236 +++++++++++ .../video-editor/useAutoCaptionGeneration.ts | 92 +++++ src/hooks/useScreenRecorder.test.tsx | 390 ++++++++++++++++++ src/hooks/useScreenRecorder.ts | 55 ++- src/i18n/locales/ar/editor.json | 7 + src/i18n/locales/ar/settings.json | 2 + src/i18n/locales/en/editor.json | 7 + src/i18n/locales/en/settings.json | 2 + src/i18n/locales/es/editor.json | 7 + src/i18n/locales/es/settings.json | 2 + src/i18n/locales/fr/editor.json | 9 +- src/i18n/locales/fr/settings.json | 2 + src/i18n/locales/it/editor.json | 7 + src/i18n/locales/it/settings.json | 3 + src/i18n/locales/ja-JP/editor.json | 7 + src/i18n/locales/ja-JP/settings.json | 2 + src/i18n/locales/ko-KR/editor.json | 7 + src/i18n/locales/ko-KR/settings.json | 2 + src/i18n/locales/ru/editor.json | 7 + src/i18n/locales/ru/settings.json | 2 + src/i18n/locales/tr/editor.json | 7 + src/i18n/locales/tr/settings.json | 2 + src/i18n/locales/vi/editor.json | 7 + src/i18n/locales/vi/settings.json | 2 + src/i18n/locales/zh-CN/editor.json | 7 + src/i18n/locales/zh-CN/settings.json | 2 + src/i18n/locales/zh-TW/editor.json | 7 + src/i18n/locales/zh-TW/settings.json | 2 + src/lib/captions.test.ts | 103 +++++ src/lib/captions.ts | 167 ++++++++ 47 files changed, 2508 insertions(+), 34 deletions(-) create mode 100644 electron/captions/jobs.test.ts create mode 100644 electron/captions/jobs.ts create mode 100644 electron/captions/whisper.test.ts create mode 100644 electron/captions/whisper.ts create mode 100644 electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts create mode 100644 src/components/video-editor/AnnotationSettingsPanel.test.tsx create mode 100644 src/components/video-editor/annotationStickers.ts create mode 100644 src/components/video-editor/captionAnnotations.test.ts create mode 100644 src/components/video-editor/captionAnnotations.ts create mode 100644 src/components/video-editor/useAutoCaptionGeneration.test.ts create mode 100644 src/components/video-editor/useAutoCaptionGeneration.ts create mode 100644 src/hooks/useScreenRecorder.test.tsx create mode 100644 src/lib/captions.test.ts create mode 100644 src/lib/captions.ts diff --git a/.gitignore b/.gitignore index 294483d41e..e214654330 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ pnpm-debug.log* lerna-debug.log* node_modules +.worktrees/ dist dist-electron dist-ssr diff --git a/electron-builder.json5 b/electron-builder.json5 index 8ad4a80eb9..d53ab09736 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -38,21 +38,26 @@ "hardenedRuntime": true, "entitlements": "macos.entitlements", "entitlementsInherit": "macos.entitlements", - "target": [ - { - "target": "dmg", - "arch": ["x64", "arm64"] - } - ], - "icon": "icons/icons/mac/icon.icns", - "artifactName": "${productName}-Mac-${arch}-${version}-Installer.${ext}", - "extraResources": [ - { - "from": "electron/native/bin", - "to": "electron/native/bin", - "filter": ["darwin-*/*"] - } - ], + "target": [ + { + "target": "dmg", + "arch": ["arm64"] + } + ], + "icon": "icons/icons/mac/icon.icns", + "artifactName": "${productName}-Mac-${arch}-${version}-Installer.${ext}", + "extraResources": [ + { + "from": "electron/native/bin", + "to": "electron/native/bin", + "filter": ["darwin-arm64/**/*"] + }, + { + "from": "electron/native/captions", + "to": "electron/native/captions", + "filter": ["darwin-arm64/**/*"] + } + ], "extendInfo": { "NSAudioCaptureUsageDescription": "OpenScreen needs audio capture permission to record system audio.", "NSMicrophoneUsageDescription": "OpenScreen needs microphone access to record voice audio.", diff --git a/electron/captions/jobs.test.ts b/electron/captions/jobs.test.ts new file mode 100644 index 0000000000..540a74d0cf --- /dev/null +++ b/electron/captions/jobs.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, vi } from "vitest"; +import { CaptionJobRegistry } from "./jobs"; + +describe("CaptionJobRegistry", () => { + it("cleans up jobs after completion", async () => { + const registry = new CaptionJobRegistry({ + generate: vi.fn(async (request) => ({ + jobId: request.jobId, + status: "success", + segments: [], + })), + }); + + await expect(registry.start({ jobId: "job-1", videoPath: "/tmp/video.webm" })).resolves.toEqual( + { + jobId: "job-1", + status: "success", + segments: [], + }, + ); + expect(registry.has("job-1")).toBe(false); + }); + + it("aborts an active job", async () => { + let signal: AbortSignal | undefined; + const registry = new CaptionJobRegistry({ + generate: vi.fn(async (request) => { + signal = request.signal; + return { jobId: request.jobId, status: "cancelled", segments: [] }; + }), + }); + + const promise = registry.start({ + jobId: "job-2", + videoPath: "/tmp/video.webm", + }); + const cancelResult = registry.cancel("job-2"); + const result = await promise; + + expect(cancelResult).toEqual({ success: true, cancelled: true }); + expect(signal?.aborted).toBe(true); + expect(result.status).toBe("cancelled"); + }); + + it("reports missing jobs as not cancelled", () => { + const registry = new CaptionJobRegistry({ + generate: vi.fn(), + }); + + expect(registry.cancel("missing")).toEqual({ + success: true, + cancelled: false, + }); + }); +}); diff --git a/electron/captions/jobs.ts b/electron/captions/jobs.ts new file mode 100644 index 0000000000..c7adbb036d --- /dev/null +++ b/electron/captions/jobs.ts @@ -0,0 +1,53 @@ +import type { CaptionGenerationResult } from "../../src/lib/captions"; + +type CaptionJobStartRequest = { + jobId: string; + videoPath: string; + language?: string; +}; + +type CaptionJobGenerateRequest = CaptionJobStartRequest & { + signal: AbortSignal; +}; + +type CaptionJobRegistryDeps = { + generate: (request: CaptionJobGenerateRequest) => Promise; +}; + +export class CaptionJobRegistry { + private jobs = new Map(); + + constructor(private deps: CaptionJobRegistryDeps) {} + + has(jobId: string) { + return this.jobs.has(jobId); + } + + async start(request: CaptionJobStartRequest): Promise { + this.cancel(request.jobId); + const controller = new AbortController(); + this.jobs.set(request.jobId, controller); + + try { + return await this.deps.generate({ + ...request, + signal: controller.signal, + }); + } finally { + if (this.jobs.get(request.jobId) === controller) { + this.jobs.delete(request.jobId); + } + } + } + + cancel(jobId: string) { + const controller = this.jobs.get(jobId); + if (!controller) { + return { success: true, cancelled: false }; + } + + controller.abort(); + this.jobs.delete(jobId); + return { success: true, cancelled: true }; + } +} diff --git a/electron/captions/whisper.test.ts b/electron/captions/whisper.test.ts new file mode 100644 index 0000000000..27d4b4c5c9 --- /dev/null +++ b/electron/captions/whisper.test.ts @@ -0,0 +1,223 @@ +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { generateCaptionsWithWhisper } from "./whisper"; + +const baseRequest = { + jobId: "job-1", + videoPath: path.join("/recordings", "recording.webm"), + platform: "darwin", + arch: "arm64", + resourcesPath: "/resources", + tempDir: "/tmp", + env: {}, +}; + +describe("generateCaptionsWithWhisper", () => { + it("reports unavailable when helper assets are missing", async () => { + const result = await generateCaptionsWithWhisper(baseRequest, { + exists: vi.fn(async () => false), + readFile: vi.fn(), + rm: vi.fn(), + mkdir: vi.fn(), + runProcess: vi.fn(), + }); + + expect(result).toEqual({ + jobId: "job-1", + status: "unavailable", + segments: [], + message: "Local caption tools are not available.", + }); + }); + + it("uses process env helper overrides when request env is omitted", async () => { + const previousWhisperBin = process.env.OPENSCREEN_WHISPER_CPP_BIN; + const previousModelPath = process.env.OPENSCREEN_WHISPER_MODEL_PATH; + const previousFfmpegPath = process.env.OPENSCREEN_FFMPEG_PATH; + process.env.OPENSCREEN_WHISPER_CPP_BIN = "/env/whisper-cli"; + process.env.OPENSCREEN_WHISPER_MODEL_PATH = "/env/ggml-base.bin"; + process.env.OPENSCREEN_FFMPEG_PATH = "/env/ffmpeg"; + + const request = { + jobId: "job-1", + videoPath: path.join("/recordings", "recording.webm"), + platform: "darwin", + arch: "arm64", + resourcesPath: "/resources", + tempDir: "/tmp", + }; + const runProcess = vi + .fn() + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }) + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }); + + try { + const result = await generateCaptionsWithWhisper(request, { + exists: vi.fn(async (filePath) => filePath.startsWith("/env/")), + readFile: vi.fn(async () => + JSON.stringify({ + transcription: [{ offsets: { from: 0, to: 1000 }, text: "hello" }], + }), + ), + rm: vi.fn(), + mkdir: vi.fn(), + runProcess, + }); + + expect(result.status).toBe("success"); + expect(runProcess.mock.calls[0][0]).toBe("/env/ffmpeg"); + expect(runProcess.mock.calls[1][0]).toBe("/env/whisper-cli"); + expect(runProcess.mock.calls[1][1]).toContain("/env/ggml-base.bin"); + } finally { + if (previousWhisperBin === undefined) { + delete process.env.OPENSCREEN_WHISPER_CPP_BIN; + } else { + process.env.OPENSCREEN_WHISPER_CPP_BIN = previousWhisperBin; + } + if (previousModelPath === undefined) { + delete process.env.OPENSCREEN_WHISPER_MODEL_PATH; + } else { + process.env.OPENSCREEN_WHISPER_MODEL_PATH = previousModelPath; + } + if (previousFfmpegPath === undefined) { + delete process.env.OPENSCREEN_FFMPEG_PATH; + } else { + process.env.OPENSCREEN_FFMPEG_PATH = previousFfmpegPath; + } + } + }); + + it("prefers the bundled small model over the base fallback", async () => { + const runProcess = vi + .fn() + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }) + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }); + const readFile = vi.fn(async () => + JSON.stringify({ + transcription: [{ offsets: { from: 0, to: 1000 }, text: "hello" }], + }), + ); + + await generateCaptionsWithWhisper(baseRequest, { + exists: vi.fn(async () => true), + readFile, + rm: vi.fn(), + mkdir: vi.fn(), + runProcess, + }); + + expect(runProcess.mock.calls[1][1]).toContain( + path.join("/resources", "electron", "native", "captions", "darwin-arm64", "ggml-small.bin"), + ); + }); + + it("skips videos without an audio stream", async () => { + const runProcess = vi.fn().mockResolvedValueOnce({ + code: 1, + stdout: "", + stderr: "Stream map '0:a:0' matches no streams.", + }); + + const result = await generateCaptionsWithWhisper(baseRequest, { + exists: vi.fn(async () => true), + readFile: vi.fn(), + rm: vi.fn(), + mkdir: vi.fn(), + runProcess, + }); + + expect(result.status).toBe("skipped"); + expect(result.message).toBe("No audio track found; captions skipped."); + }); + + it("sanitizes job ids before building cleanup paths", async () => { + const runProcess = vi + .fn() + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }) + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }); + const rm = vi.fn(); + const request = { + ...baseRequest, + jobId: "foo/../../outside", + }; + + const result = await generateCaptionsWithWhisper(request, { + exists: vi.fn(async () => true), + readFile: vi.fn(async () => + JSON.stringify({ + transcription: [{ offsets: { from: 0, to: 1000 }, text: "hello" }], + }), + ), + rm, + mkdir: vi.fn(), + runProcess, + }); + + const cleanupPath = rm.mock.calls[0][0]; + const relativeCleanupPath = path.relative("/tmp", cleanupPath); + expect(result.jobId).toBe("foo/../../outside"); + expect(cleanupPath).toBe(path.join("/tmp", "openscreen-captions-foo-------outside")); + expect(relativeCleanupPath.startsWith("..")).toBe(false); + expect(relativeCleanupPath.split(path.sep)).not.toContain(".."); + }); + + it("extracts audio, runs whisper, and parses generated JSON", async () => { + const runProcess = vi + .fn() + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }) + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }); + const readFile = vi.fn(async () => + JSON.stringify({ + transcription: [{ offsets: { from: 0, to: 1000 }, text: "hello" }], + }), + ); + + const result = await generateCaptionsWithWhisper(baseRequest, { + exists: vi.fn(async () => true), + readFile, + rm: vi.fn(), + mkdir: vi.fn(), + runProcess, + }); + + expect(result).toEqual({ + jobId: "job-1", + status: "success", + segments: [{ id: "caption-1", startMs: 0, endMs: 1000, text: "hello" }], + message: "Generated 1 captions.", + }); + expect(runProcess).toHaveBeenCalledTimes(2); + expect(runProcess.mock.calls[0][1]).toContain("-map"); + expect(runProcess.mock.calls[1][1]).toContain("-oj"); + }); + + it("retries whisper on CPU when macOS GPU initialization fails", async () => { + const runProcess = vi + .fn() + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }) + .mockResolvedValueOnce({ + code: -1, + stdout: "", + stderr: "ggml_metal_buffer_init: error: failed to allocate buffer", + }) + .mockResolvedValueOnce({ code: 0, stdout: "", stderr: "" }); + const readFile = vi.fn(async () => + JSON.stringify({ + transcription: [{ offsets: { from: 0, to: 1000 }, text: "hello" }], + }), + ); + + const result = await generateCaptionsWithWhisper(baseRequest, { + exists: vi.fn(async () => true), + readFile, + rm: vi.fn(), + mkdir: vi.fn(), + runProcess, + }); + + expect(result.status).toBe("success"); + expect(runProcess).toHaveBeenCalledTimes(3); + expect(runProcess.mock.calls[1][1]).not.toContain("--no-gpu"); + expect(runProcess.mock.calls[2][1]).toContain("--no-gpu"); + }); +}); diff --git a/electron/captions/whisper.ts b/electron/captions/whisper.ts new file mode 100644 index 0000000000..b22d39144a --- /dev/null +++ b/electron/captions/whisper.ts @@ -0,0 +1,283 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { CaptionGenerationResult } from "../../src/lib/captions"; +import { parseWhisperJsonOutput } from "../../src/lib/captions"; + +type RunProcessResult = { + code: number | null; + stdout: string; + stderr: string; +}; + +type RunProcess = ( + command: string, + args: string[], + options?: { signal?: AbortSignal }, +) => Promise; + +export type GenerateCaptionsWithWhisperRequest = { + jobId: string; + videoPath: string; + platform: NodeJS.Platform | string; + arch: string; + resourcesPath: string; + tempDir?: string; + env?: NodeJS.ProcessEnv | Record; + language?: string; + signal?: AbortSignal; +}; + +export type WhisperCaptionDeps = { + exists?: (filePath: string) => Promise; + mkdir?: typeof fs.mkdir; + readFile?: typeof fs.readFile; + rm?: typeof fs.rm; + runProcess?: RunProcess; +}; + +async function exists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +function runProcess(command: string, args: string[], options: { signal?: AbortSignal } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + stdio: ["ignore", "pipe", "pipe"], + signal: options.signal, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => { + if (options.signal?.aborted) { + resolve({ code: null, stdout, stderr: String(error) }); + return; + } + reject(error); + }); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +function candidateResourcePaths( + request: GenerateCaptionsWithWhisperRequest, + fileName: string | string[], + envName: string, +) { + const envPath = (request.env ?? process.env)[envName]?.trim(); + const platformArch = `${request.platform}-${request.arch}`; + const fileNames = Array.isArray(fileName) ? fileName : [fileName]; + return [ + envPath || null, + ...fileNames.flatMap((name) => [ + path.join(request.resourcesPath, "electron", "native", "captions", platformArch, name), + path.join(request.resourcesPath, "captions", platformArch, name), + ]), + ].filter((value): value is string => Boolean(value)); +} + +async function firstExistingPath( + paths: string[], + deps: Required>, +) { + for (const candidate of paths) { + if (await deps.exists(candidate)) { + return candidate; + } + } + return null; +} + +async function resolveCaptionTools( + request: GenerateCaptionsWithWhisperRequest, + deps: Required>, +) { + const whisperBinaryName = request.platform === "win32" ? "whisper-cli.exe" : "whisper-cli"; + const ffmpegBinaryName = request.platform === "win32" ? "ffmpeg.exe" : "ffmpeg"; + const whisperPath = await firstExistingPath( + candidateResourcePaths(request, whisperBinaryName, "OPENSCREEN_WHISPER_CPP_BIN"), + deps, + ); + const modelPath = await firstExistingPath( + candidateResourcePaths( + request, + ["ggml-small.bin", "ggml-base.bin"], + "OPENSCREEN_WHISPER_MODEL_PATH", + ), + deps, + ); + const ffmpegPath = await firstExistingPath( + candidateResourcePaths(request, ffmpegBinaryName, "OPENSCREEN_FFMPEG_PATH"), + deps, + ); + + return { whisperPath, modelPath, ffmpegPath }; +} + +function isNoAudioError(stderr: string) { + return ( + stderr.includes("matches no streams") || + stderr.includes("Stream map") || + stderr.toLowerCase().includes("audio:0") + ); +} + +function isMacGpuInitializationError(request: GenerateCaptionsWithWhisperRequest, output: string) { + if (request.platform !== "darwin") { + return false; + } + + const normalizedOutput = output.toLowerCase(); + return ( + normalizedOutput.includes("ggml_metal") || + normalizedOutput.includes("metal") || + normalizedOutput.includes("no gpu found") + ); +} + +export async function generateCaptionsWithWhisper( + request: GenerateCaptionsWithWhisperRequest, + deps: WhisperCaptionDeps = {}, +): Promise { + const resolvedDeps = { + exists: deps.exists ?? exists, + mkdir: deps.mkdir ?? fs.mkdir, + readFile: deps.readFile ?? fs.readFile, + rm: deps.rm ?? fs.rm, + runProcess: deps.runProcess ?? runProcess, + }; + + const tools = await resolveCaptionTools(request, resolvedDeps); + if (!tools.whisperPath || !tools.modelPath || !tools.ffmpegPath) { + return { + jobId: request.jobId, + status: "unavailable", + segments: [], + message: "Local caption tools are not available.", + }; + } + + const tempRoot = request.tempDir ?? os.tmpdir(); + const safeJobId = request.jobId.replace(/[^a-zA-Z0-9_-]/g, "-") || "job"; + const jobDir = path.join(tempRoot, `openscreen-captions-${safeJobId}`); + const wavPath = path.join(jobDir, "audio.wav"); + const outputBase = path.join(jobDir, "transcript"); + const outputJsonPath = `${outputBase}.json`; + + try { + await resolvedDeps.mkdir(jobDir, { recursive: true }); + const extract = await resolvedDeps.runProcess( + tools.ffmpegPath, + [ + "-y", + "-i", + request.videoPath, + "-map", + "0:a:0", + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-f", + "wav", + wavPath, + ], + { signal: request.signal }, + ); + + if (request.signal?.aborted) { + return { jobId: request.jobId, status: "cancelled", segments: [] }; + } + + if (extract.code !== 0) { + if (isNoAudioError(extract.stderr)) { + return { + jobId: request.jobId, + status: "skipped", + segments: [], + message: "No audio track found; captions skipped.", + }; + } + return { + jobId: request.jobId, + status: "error", + segments: [], + message: "Failed to extract audio for captions.", + error: extract.stderr || extract.stdout, + }; + } + + const whisperArgs = [ + "-m", + tools.modelPath, + "-f", + wavPath, + "-oj", + "-of", + outputBase, + "--no-prints", + ]; + if (request.language) { + whisperArgs.push("-l", request.language); + } + + let whisper = await resolvedDeps.runProcess(tools.whisperPath, whisperArgs, { + signal: request.signal, + }); + + if (request.signal?.aborted) { + return { jobId: request.jobId, status: "cancelled", segments: [] }; + } + + if ( + whisper.code !== 0 && + isMacGpuInitializationError(request, `${whisper.stderr}\n${whisper.stdout}`) + ) { + whisper = await resolvedDeps.runProcess(tools.whisperPath, [...whisperArgs, "--no-gpu"], { + signal: request.signal, + }); + + if (request.signal?.aborted) { + return { jobId: request.jobId, status: "cancelled", segments: [] }; + } + } + + if (whisper.code !== 0) { + return { + jobId: request.jobId, + status: "error", + segments: [], + message: "Local caption transcription failed.", + error: whisper.stderr || whisper.stdout, + }; + } + + const jsonOutput = await resolvedDeps.readFile(outputJsonPath, "utf-8"); + const segments = parseWhisperJsonOutput(String(jsonOutput)); + return { + jobId: request.jobId, + status: "success", + segments, + message: `Generated ${segments.length} captions.`, + }; + } finally { + try { + await resolvedDeps.rm(jobDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup should not mask the caption result. + } + } +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index b86c4ff29a..ce24b5f637 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -171,6 +171,7 @@ interface Window { error?: string; }>; onStopRecordingFromTray: (callback: () => void) => () => void; + onNativeRecordingStopped: (callback: () => void) => () => void; openExternalUrl: (url: string) => Promise<{ success: boolean; error?: string }>; pickExportSavePath: ( fileName: string, @@ -217,6 +218,11 @@ interface Window { message?: string; error?: string; }>; + startCaptionGeneration: ( + videoPath: string, + options?: import("../src/lib/captions").CaptionGenerationOptions, + ) => Promise; + cancelCaptionGeneration: (jobId: string) => Promise<{ success: boolean; cancelled: boolean }>; clearCurrentVideoPath: () => Promise<{ success: boolean }>; saveProjectFile: ( projectData: unknown, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index d367728333..4673d4a9f7 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -35,6 +35,8 @@ import type { ProjectFileResult, ProjectPathResult, } from "../../src/native/contracts"; +import { CaptionJobRegistry } from "../captions/jobs"; +import { generateCaptionsWithWhisper } from "../captions/whisper"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; @@ -355,6 +357,20 @@ let selectedDesktopSource: DesktopCapturerSource | null = null; let lastEnumeratedSources = new Map(); let currentProjectPath: string | null = null; let currentRecordingSession: RecordingSession | null = null; +const captionJobRegistry = new CaptionJobRegistry({ + generate: (request) => + generateCaptionsWithWhisper({ + jobId: request.jobId, + videoPath: request.videoPath, + language: request.language, + signal: request.signal, + platform: process.platform, + arch: process.arch, + resourcesPath: process.resourcesPath, + tempDir: app.getPath("temp"), + env: process.env, + }), +}); /** * Returns the cached DesktopCapturerSource set when the user picked a source. @@ -408,6 +424,10 @@ let nativeWindowsCaptureOutput = ""; let nativeWindowsCaptureTargetPath: string | null = null; let nativeWindowsCaptureWebcamTargetPath: string | null = null; let nativeWindowsCaptureRecordingId: number | null = null; +let nativeWindowsCaptureStarted = false; +let nativeWindowsCaptureClosed = false; +let nativeWindowsCaptureCloseCode: number | null = null; +let nativeWindowsCaptureStopRequested = false; let nativeWindowsCursorOffsetMs = 0; let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; let nativeWindowsCursorRecordingStartMs = 0; @@ -419,6 +439,10 @@ let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null; let nativeMacCaptureOutput = ""; let nativeMacCaptureTargetPath: string | null = null; let nativeMacCaptureRecordingId: number | null = null; +let nativeMacCaptureStarted = false; +let nativeMacCaptureClosed = false; +let nativeMacCaptureCloseCode: number | null = null; +let nativeMacCaptureStopRequested = false; let nativeMacCursorOffsetMs = 0; let nativeMacCursorCaptureMode: CursorCaptureMode = "editable-overlay"; let nativeMacCursorRecordingStartMs = 0; @@ -922,8 +946,7 @@ function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) reject(new Error("Timed out waiting for native Windows capture to start")); }, 12000); - const onOutput = (chunk: Buffer) => { - nativeWindowsCaptureOutput += chunk.toString(); + const onOutput = () => { if (nativeWindowsCaptureOutput.includes("Recording started")) { cleanup(); resolve(); @@ -954,6 +977,10 @@ function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) proc.stderr.on("data", onOutput); proc.once("error", onError); proc.once("exit", onExit); + if (nativeWindowsCaptureOutput.includes("Recording started")) { + cleanup(); + resolve(); + } }); } @@ -972,14 +999,18 @@ function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) { ), ); }, NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS); - const onOutput = (chunk: Buffer) => { - nativeWindowsCaptureOutput += chunk.toString(); + const onOutput = () => { + const stoppedPath = getNativeWindowsStoppedPathFromOutput(); + if (stoppedPath) { + cleanup(); + resolve(stoppedPath); + } }; const onClose = (code: number | null) => { cleanup(); - const match = nativeWindowsCaptureOutput.match(/Recording stopped\. Output path: (.+)/); - if (match?.[1]) { - resolve(match[1].trim()); + const stoppedPath = getNativeWindowsStoppedPathFromOutput(); + if (stoppedPath) { + resolve(stoppedPath); return; } if (code === 0 && nativeWindowsCaptureTargetPath) { @@ -1009,9 +1040,60 @@ function waitForNativeWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) { proc.stderr.on("data", onOutput); proc.once("close", onClose); proc.once("error", onError); + + const stoppedPath = getNativeWindowsStoppedPathFromOutput(); + if (stoppedPath) { + cleanup(); + resolve(stoppedPath); + return; + } + if (nativeWindowsCaptureClosed) { + cleanup(); + if (nativeWindowsCaptureCloseCode === 0 && nativeWindowsCaptureTargetPath) { + resolve(nativeWindowsCaptureTargetPath); + return; + } + reject( + new Error( + nativeWindowsCaptureOutput.trim() || + `Native Windows capture exited with code=${nativeWindowsCaptureCloseCode ?? "unknown"}`, + ), + ); + } }); } +function attachNativeWindowsCaptureOutputDrain(proc: ChildProcessWithoutNullStreams) { + const drain = (chunk: Buffer) => { + nativeWindowsCaptureOutput += chunk.toString(); + }; + const cleanup = () => { + proc.stdout.off("data", drain); + proc.stderr.off("data", drain); + proc.off("close", cleanup); + proc.off("error", cleanup); + }; + + proc.stdout.on("data", drain); + proc.stderr.on("data", drain); + proc.once("close", cleanup); + proc.once("error", cleanup); +} + +function getNativeWindowsStoppedPathFromOutput() { + for (const line of nativeWindowsCaptureOutput.split(/\r?\n/).reverse()) { + const event = tryParseNativeHelperEvent(line.trim()); + if (event?.event === "recording-stopped" && typeof event.screenPath === "string") { + return event.screenPath; + } + } + + const matches = [ + ...nativeWindowsCaptureOutput.matchAll(/Recording stopped\. Output path: ([^\r\n]+)/g), + ]; + return matches.at(-1)?.[1]?.trim() ?? null; +} + function readNativeWindowsWebcamFormat(output: string) { const lines = output.split(/\r?\n/).filter((line) => line.includes('"event":"webcam-format"')); const lastLine = lines.at(-1); @@ -1179,9 +1261,40 @@ function waitForNativeMacCaptureStop(proc: ChildProcessWithoutNullStreams) { proc.once("close", onClose); proc.once("error", onError); inspectNativeMacCaptureOutput(); + + const stoppedPath = getNativeMacStoppedPathFromOutput(); + if (stoppedPath) { + cleanup(); + resolve(stoppedPath); + return; + } + if (nativeMacCaptureClosed) { + cleanup(); + if (nativeMacCaptureCloseCode === 0 && nativeMacCaptureTargetPath) { + resolve(nativeMacCaptureTargetPath); + return; + } + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited with code=${nativeMacCaptureCloseCode ?? "unknown"}`, + ), + ); + } }); } +function getNativeMacStoppedPathFromOutput() { + for (const line of nativeMacCaptureOutput.split(/\r?\n/).reverse()) { + const event = tryParseNativeHelperEvent(line.trim()); + if (event?.event === "recording-stopped" && typeof event.screenPath === "string") { + return event.screenPath; + } + } + + return null; +} + function setCurrentRecordingSessionState(session: RecordingSession | null) { currentRecordingSession = session; currentVideoPath = session?.screenVideoPath ?? null; @@ -1306,6 +1419,15 @@ export function registerIpcHandlers( } } + function notifyNativeRecordingStopped() { + const mainWindow = getMainWindow(); + if (!mainWindow || mainWindow.isDestroyed()) { + return; + } + + mainWindow.webContents.send("native-recording-stopped"); + } + ipcMain.handle("get-sources", async (_, opts) => { const sources = await desktopCapturer.getSources(opts); lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); @@ -1623,6 +1745,10 @@ export function registerIpcHandlers( nativeWindowsCaptureTargetPath = outputPath; nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamOutputPath : null; nativeWindowsCaptureRecordingId = recordingId; + nativeWindowsCaptureStarted = false; + nativeWindowsCaptureClosed = false; + nativeWindowsCaptureCloseCode = null; + nativeWindowsCaptureStopRequested = false; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = cursorCaptureMode; nativeWindowsCursorRecordingStartMs = 0; @@ -1648,8 +1774,30 @@ export function registerIpcHandlers( windowsHide: true, }); nativeWindowsCaptureProcess = proc; + attachNativeWindowsCaptureOutputDrain(proc); + proc.once("close", (code) => { + nativeWindowsCaptureClosed = true; + nativeWindowsCaptureCloseCode = code; + if ( + nativeWindowsCaptureStarted && + !nativeWindowsCaptureStopRequested && + (getNativeWindowsStoppedPathFromOutput() || + (code === 0 && nativeWindowsCaptureTargetPath)) + ) { + notifyNativeRecordingStopped(); + } + }); await waitForNativeWindowsCaptureStart(proc); + nativeWindowsCaptureStarted = true; + if ( + nativeWindowsCaptureClosed && + !nativeWindowsCaptureStopRequested && + (getNativeWindowsStoppedPathFromOutput() || + (nativeWindowsCaptureCloseCode === 0 && nativeWindowsCaptureTargetPath)) + ) { + notifyNativeRecordingStopped(); + } const captureStartedAtMs = Date.now(); nativeWindowsCursorOffsetMs = cursorCaptureMode === "editable-overlay" @@ -1680,6 +1828,10 @@ export function registerIpcHandlers( nativeWindowsCaptureTargetPath = null; nativeWindowsCaptureWebcamTargetPath = null; nativeWindowsCaptureRecordingId = null; + nativeWindowsCaptureStarted = false; + nativeWindowsCaptureClosed = false; + nativeWindowsCaptureCloseCode = null; + nativeWindowsCaptureStopRequested = false; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = "editable-overlay"; nativeWindowsCursorRecordingStartMs = 0; @@ -1778,6 +1930,10 @@ export function registerIpcHandlers( nativeMacCaptureOutput = ""; nativeMacCaptureTargetPath = outputPath; nativeMacCaptureRecordingId = recordingId; + nativeMacCaptureStarted = false; + nativeMacCaptureClosed = false; + nativeMacCaptureCloseCode = null; + nativeMacCaptureStopRequested = false; nativeMacCursorOffsetMs = 0; nativeMacCursorCaptureMode = cursorCaptureMode; nativeMacCursorRecordingStartMs = 0; @@ -1799,8 +1955,28 @@ export function registerIpcHandlers( }); nativeMacCaptureProcess = proc; attachNativeMacCaptureOutputDrain(proc); + proc.once("close", (code) => { + nativeMacCaptureClosed = true; + nativeMacCaptureCloseCode = code; + if ( + nativeMacCaptureStarted && + !nativeMacCaptureStopRequested && + (getNativeMacStoppedPathFromOutput() || (code === 0 && nativeMacCaptureTargetPath)) + ) { + notifyNativeRecordingStopped(); + } + }); await waitForNativeMacCaptureStart(proc); + nativeMacCaptureStarted = true; + if ( + nativeMacCaptureClosed && + !nativeMacCaptureStopRequested && + (getNativeMacStoppedPathFromOutput() || + (nativeMacCaptureCloseCode === 0 && nativeMacCaptureTargetPath)) + ) { + notifyNativeRecordingStopped(); + } const captureStartedAtMs = Date.now(); nativeMacCursorOffsetMs = cursorCaptureMode === "editable-overlay" @@ -1824,6 +2000,10 @@ export function registerIpcHandlers( nativeMacCaptureProcess = null; nativeMacCaptureTargetPath = null; nativeMacCaptureRecordingId = null; + nativeMacCaptureStarted = false; + nativeMacCaptureClosed = false; + nativeMacCaptureCloseCode = null; + nativeMacCaptureStopRequested = false; nativeMacCursorOffsetMs = 0; nativeMacCursorCaptureMode = "editable-overlay"; nativeMacCursorRecordingStartMs = 0; @@ -1944,9 +2124,19 @@ export function registerIpcHandlers( try { completeNativeWindowsCursorPauseRange(); - const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc); - proc.stdin.write("stop\n"); - const stoppedPath = await stoppedPathPromise; + let stoppedPath: string | null; + if (nativeWindowsCaptureClosed) { + stoppedPath = + getNativeWindowsStoppedPathFromOutput() ?? + (nativeWindowsCaptureCloseCode === 0 ? preferredPath : null); + } else { + nativeWindowsCaptureStopRequested = true; + const stoppedPathPromise = waitForNativeWindowsCaptureStop(proc); + if (proc.stdin.writable) { + proc.stdin.write("stop\n"); + } + stoppedPath = await stoppedPathPromise; + } const screenVideoPath = stoppedPath || preferredPath; if (!screenVideoPath) { throw new Error("Native Windows capture did not return an output path."); @@ -2008,6 +2198,10 @@ export function registerIpcHandlers( nativeWindowsCaptureTargetPath = null; nativeWindowsCaptureWebcamTargetPath = null; nativeWindowsCaptureRecordingId = null; + nativeWindowsCaptureStarted = false; + nativeWindowsCaptureClosed = false; + nativeWindowsCaptureCloseCode = null; + nativeWindowsCaptureStopRequested = false; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = "editable-overlay"; nativeWindowsCursorRecordingStartMs = 0; @@ -2037,9 +2231,19 @@ export function registerIpcHandlers( try { completeNativeMacCursorPauseRange(); - const stoppedPathPromise = waitForNativeMacCaptureStop(proc); - proc.stdin.write("stop\n"); - const stoppedPath = await stoppedPathPromise; + let stoppedPath: string | null; + if (nativeMacCaptureClosed) { + stoppedPath = + getNativeMacStoppedPathFromOutput() ?? + (nativeMacCaptureCloseCode === 0 ? preferredPath : null); + } else { + nativeMacCaptureStopRequested = true; + const stoppedPathPromise = waitForNativeMacCaptureStop(proc); + if (proc.stdin.writable) { + proc.stdin.write("stop\n"); + } + stoppedPath = await stoppedPathPromise; + } const screenVideoPath = stoppedPath || preferredPath; if (!screenVideoPath) { throw new Error("Native macOS capture did not return an output path."); @@ -2093,6 +2297,10 @@ export function registerIpcHandlers( nativeMacCaptureProcess = null; nativeMacCaptureTargetPath = null; nativeMacCaptureRecordingId = null; + nativeMacCaptureStarted = false; + nativeMacCaptureClosed = false; + nativeMacCaptureCloseCode = null; + nativeMacCaptureStopRequested = false; nativeMacCursorOffsetMs = 0; nativeMacCursorCaptureMode = "editable-overlay"; nativeMacCursorRecordingStartMs = 0; @@ -2530,6 +2738,46 @@ export function registerIpcHandlers( } }); + ipcMain.handle( + "start-caption-generation", + async ( + _, + videoPath: string, + options?: import("../../src/lib/captions").CaptionGenerationOptions, + ) => { + try { + const jobId = options?.jobId ?? `caption-${Date.now()}`; + const normalizedPath = resolveApprovedVideoPath(videoPath); + if (!normalizedPath) { + return { + jobId, + status: "error", + segments: [], + message: "File path is not approved or is not a supported video file", + }; + } + + return await captionJobRegistry.start({ + jobId, + videoPath: normalizedPath, + language: options?.language, + }); + } catch (error) { + return { + jobId: options?.jobId ?? `caption-${Date.now()}`, + status: "error", + segments: [], + message: "Failed to generate captions", + error: String(error), + }; + } + }, + ); + + ipcMain.handle("cancel-caption-generation", (_, jobId: string) => { + return captionJobRegistry.cancel(jobId); + }); + ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts new file mode 100644 index 0000000000..666033313b --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { requestMacCursorAccessibilityAccess } from "./macNativeCursorRecordingSession"; + +const mocks = vi.hoisted(() => ({ + accessSync: vi.fn(), + isTrustedAccessibilityClient: vi.fn(), + spawn: vi.fn(), +})); + +vi.mock("electron", () => ({ + screen: { + getCursorScreenPoint: vi.fn(() => ({ x: 0, y: 0 })), + getDisplayNearestPoint: vi.fn(() => ({ bounds: { x: 0, y: 0, width: 1, height: 1 } })), + }, + systemPreferences: { + isTrustedAccessibilityClient: mocks.isTrustedAccessibilityClient, + }, +})); + +vi.mock("node:child_process", () => ({ + default: { spawn: mocks.spawn }, + spawn: mocks.spawn, +})); + +vi.mock("node:fs", () => ({ + default: { + accessSync: mocks.accessSync, + constants: { X_OK: 1 }, + }, + accessSync: mocks.accessSync, + constants: { X_OK: 1 }, +})); + +describe("requestMacCursorAccessibilityAccess", () => { + beforeEach(() => { + vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + mocks.accessSync.mockImplementation(() => { + throw new Error("missing helper"); + }); + mocks.isTrustedAccessibilityClient.mockReset(); + mocks.spawn.mockReset(); + }); + + it("trusts the OpenScreen app accessibility grant without requiring the helper grant", async () => { + mocks.isTrustedAccessibilityClient.mockReturnValue(true); + + const result = await requestMacCursorAccessibilityAccess(); + + expect(result).toEqual({ success: true, granted: true, status: "granted" }); + expect(mocks.isTrustedAccessibilityClient).toHaveBeenCalledWith(true); + expect(mocks.spawn).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts index 95ed10cedb..bc1545ec1e 100644 --- a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -72,7 +72,9 @@ export async function requestMacCursorAccessibilityAccess() { } try { - systemPreferences.isTrustedAccessibilityClient(true); + if (systemPreferences.isTrustedAccessibilityClient(true)) { + return { success: true, granted: true, status: "granted" }; + } } catch { // Continue with helper probing; it can trigger the same macOS prompt. } diff --git a/electron/preload.ts b/electron/preload.ts index 4b69740940..73edfd9e21 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -133,6 +133,11 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("stop-recording-from-tray", listener); return () => ipcRenderer.removeListener("stop-recording-from-tray", listener); }, + onNativeRecordingStopped: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("native-recording-stopped", listener); + return () => ipcRenderer.removeListener("native-recording-stopped", listener); + }, openExternalUrl: (url: string) => { return ipcRenderer.invoke("open-external-url", url); }, @@ -163,6 +168,15 @@ contextBridge.exposeInMainWorld("electronAPI", { preparePreviewAudioTrack: (filePath: string) => { return ipcRenderer.invoke("prepare-preview-audio-track", filePath); }, + startCaptionGeneration: ( + videoPath: string, + options?: import("../src/lib/captions").CaptionGenerationOptions, + ) => { + return ipcRenderer.invoke("start-caption-generation", videoPath, options); + }, + cancelCaptionGeneration: (jobId: string) => { + return ipcRenderer.invoke("cancel-caption-generation", jobId); + }, clearCurrentVideoPath: () => { return ipcRenderer.invoke("clear-current-video-path"); }, diff --git a/src/components/video-editor/AnnotationSettingsPanel.test.tsx b/src/components/video-editor/AnnotationSettingsPanel.test.tsx new file mode 100644 index 0000000000..bb7d26e45b --- /dev/null +++ b/src/components/video-editor/AnnotationSettingsPanel.test.tsx @@ -0,0 +1,59 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; +import type { AnnotationRegion } from "./types"; + +function wrapper({ children }: { children: ReactNode }) { + return {children}; +} + +function createImageAnnotation(): AnnotationRegion { + return { + id: "annotation-1", + startMs: 0, + endMs: 1000, + type: "image", + content: "", + position: { x: 50, y: 50 }, + size: { width: 30, height: 20 }, + style: { + color: "#ffffff", + backgroundColor: "transparent", + fontSize: 32, + fontFamily: "Inter", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + }, + zIndex: 1, + }; +} + +describe("AnnotationSettingsPanel sticker presets", () => { + it("writes an SVG data URL when a built-in sticker is selected", async () => { + const user = userEvent.setup(); + const onContentChange = vi.fn(); + + render( + , + { wrapper }, + ); + + await user.click(screen.getByRole("button", { name: /check sticker/i })); + + expect(onContentChange).toHaveBeenCalledTimes(1); + const stickerDataUrl = onContentChange.mock.calls[0]?.[0] as string; + expect(stickerDataUrl).toMatch(/^data:image\/svg\+xml;charset=utf-8,/); + expect(decodeURIComponent(stickerDataUrl)).toContain('data-sticker-id="check"'); + }); +}); diff --git a/src/components/video-editor/AnnotationSettingsPanel.tsx b/src/components/video-editor/AnnotationSettingsPanel.tsx index 72e25a8b00..037020d059 100644 --- a/src/components/video-editor/AnnotationSettingsPanel.tsx +++ b/src/components/video-editor/AnnotationSettingsPanel.tsx @@ -33,6 +33,7 @@ import { cn } from "@/lib/utils"; import ColorPicker from "../ui/color-picker"; import { AddCustomFontDialog } from "./AddCustomFontDialog"; import { getArrowComponent } from "./ArrowSvgs"; +import { ANNOTATION_STICKER_PRESETS } from "./annotationStickers"; import { type AnnotationRegion, type AnnotationType, @@ -472,6 +473,40 @@ export function AnnotationSettingsPanel({ {t("annotation.uploadImage")} +
+
+ +

+ {t("annotation.stickerPresetsDescription")} +

+
+
+ {ANNOTATION_STICKER_PRESETS.map((sticker) => ( + + ))} +
+
+ {annotation.content && annotation.content.startsWith("data:image") && (
(null); const [videoSourcePath, setVideoSourcePath] = useState(null); + const [autoCaptionSourcePath, setAutoCaptionSourcePath] = useState(null); const [webcamVideoPath, setWebcamVideoPath] = useState(null); const [webcamVideoSourcePath, setWebcamVideoSourcePath] = useState(null); const [currentProjectPath, setCurrentProjectPath] = useState(null); @@ -276,6 +284,7 @@ export default function VideoEditor() { const t = useScopedT("editor"); const ts = useScopedT("settings"); const availableLocales = getAvailableLocales(); + const captionLanguage = useMemo(() => getWhisperLanguageForLocale(locale), [locale]); const nextAnnotationIdRef = useRef(1); const nextAnnotationZIndexRef = useRef(1); @@ -346,6 +355,7 @@ export default function VideoEditor() { setError(null); setVideoSourcePath(sourcePath); setVideoPath(toFileUrl(sourcePath)); + setAutoCaptionSourcePath(null); setWebcamVideoSourcePath(webcamSourcePath); setWebcamVideoPath(webcamSourcePath ? toFileUrl(webcamSourcePath) : null); setRecordingCursorCaptureMode(projectCursorCaptureMode); @@ -509,6 +519,7 @@ export default function VideoEditor() { INITIAL_EDITOR_STATE, ), ); + setAutoCaptionSourcePath(sourcePath); return; } @@ -521,6 +532,7 @@ export default function VideoEditor() { setLastSavedSnapshot( createProjectSnapshot({ screenVideoPath: result.path }, INITIAL_EDITOR_STATE), ); + setAutoCaptionSourcePath(null); } else { setError("No video to load. Please record or select a video."); } @@ -1938,6 +1950,53 @@ export default function VideoEditor() { } }, [exportError, editorState]); + const handleGeneratedCaptions = useCallback( + (segments: CaptionSegment[]) => { + if (segments.length === 0) return; + + pushState((prev) => { + const annotations = createCaptionAnnotations(segments, { + existingIds: prev.annotationRegions.map((region) => region.id), + startZIndex: nextAnnotationZIndexRef.current, + }); + nextAnnotationZIndexRef.current += annotations.length; + return { + annotationRegions: [...prev.annotationRegions, ...annotations], + }; + }); + }, + [pushState], + ); + + const handleCaptionStatusMessage = useCallback( + (result: CaptionGenerationResult) => { + if (result.status === "success" && result.segments.length > 0) { + toast.success(t("captions.generated", { count: result.segments.length })); + return; + } + if (result.status === "skipped") { + toast.info(t("captions.skippedNoAudio")); + return; + } + if (result.status === "unavailable") { + toast.warning(t("captions.unavailable")); + return; + } + if (result.status === "error") { + toast.error(result.message || t("captions.failed")); + } + }, + [t], + ); + + const captionStatus = useAutoCaptionGeneration({ + sourcePath: autoCaptionSourcePath, + enabled: Boolean(autoCaptionSourcePath), + language: captionLanguage, + onCaptions: handleGeneratedCaptions, + onStatusMessage: handleCaptionStatusMessage, + }); + if (loading) { return (
@@ -2041,6 +2100,11 @@ export default function VideoEditor() { {ts("project.save")} + {captionStatus === "running" && ( + + {t("captions.generating")} + + )}
diff --git a/src/components/video-editor/annotationStickers.ts b/src/components/video-editor/annotationStickers.ts new file mode 100644 index 0000000000..00bd8429fe --- /dev/null +++ b/src/components/video-editor/annotationStickers.ts @@ -0,0 +1,130 @@ +export interface AnnotationStickerPreset { + id: string; + label: string; + dataUrl: string; +} + +const SVG_SIZE = 128; + +function createStickerSvg(id: string, label: string, body: string): string { + return [ + ``, + body, + "", + ].join(""); +} + +function createStickerDataUrl(id: string, label: string, body: string): string { + return `data:image/svg+xml;charset=utf-8,${encodeURIComponent(createStickerSvg(id, label, body))}`; +} + +export const ANNOTATION_STICKER_PRESETS: AnnotationStickerPreset[] = [ + { + id: "check", + label: "Check", + dataUrl: createStickerDataUrl( + "check", + "Check", + '', + ), + }, + { + id: "cross", + label: "Cross", + dataUrl: createStickerDataUrl( + "cross", + "Cross", + '', + ), + }, + { + id: "warning", + label: "Warning", + dataUrl: createStickerDataUrl( + "warning", + "Warning", + '', + ), + }, + { + id: "question", + label: "Question", + dataUrl: createStickerDataUrl( + "question", + "Question", + '', + ), + }, + { + id: "number-1", + label: "Number 1", + dataUrl: createStickerDataUrl( + "number-1", + "Number 1", + '', + ), + }, + { + id: "number-2", + label: "Number 2", + dataUrl: createStickerDataUrl( + "number-2", + "Number 2", + '', + ), + }, + { + id: "number-3", + label: "Number 3", + dataUrl: createStickerDataUrl( + "number-3", + "Number 3", + '', + ), + }, + { + id: "click-target", + label: "Click target", + dataUrl: createStickerDataUrl( + "click-target", + "Click target", + '', + ), + }, + { + id: "arrow-up-right", + label: "Arrow up right", + dataUrl: createStickerDataUrl( + "arrow-up-right", + "Arrow up right", + '', + ), + }, + { + id: "command", + label: "Command", + dataUrl: createStickerDataUrl( + "command", + "Command", + '', + ), + }, + { + id: "spotlight", + label: "Spotlight", + dataUrl: createStickerDataUrl( + "spotlight", + "Spotlight", + '', + ), + }, + { + id: "play", + label: "Play", + dataUrl: createStickerDataUrl( + "play", + "Play", + '', + ), + }, +]; diff --git a/src/components/video-editor/captionAnnotations.test.ts b/src/components/video-editor/captionAnnotations.test.ts new file mode 100644 index 0000000000..3e36bdbbe9 --- /dev/null +++ b/src/components/video-editor/captionAnnotations.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { createCaptionAnnotations } from "./captionAnnotations"; + +describe("createCaptionAnnotations", () => { + it("maps caption segments to bottom-centered text annotations", () => { + const annotations = createCaptionAnnotations( + [ + { id: "caption-1", startMs: 100, endMs: 1200, text: "First line" }, + { id: "caption-2", startMs: 1300, endMs: 2600, text: "Second line" }, + ], + { existingIds: [], startZIndex: 7 }, + ); + + expect(annotations).toHaveLength(2); + expect(annotations[0]).toMatchObject({ + id: "caption-1", + type: "text", + content: "First line", + textContent: "First line", + startMs: 100, + endMs: 1200, + position: { x: 10, y: 78 }, + size: { width: 80, height: 14 }, + zIndex: 7, + }); + expect(annotations[0].style).toMatchObject({ + color: "#ffffff", + backgroundColor: "rgba(0, 0, 0, 0.58)", + fontWeight: "bold", + textAlign: "center", + }); + expect(annotations[1].zIndex).toBe(8); + }); + + it("keeps generated IDs unique when a project already contains caption IDs", () => { + const annotations = createCaptionAnnotations( + [{ id: "caption-1", startMs: 0, endMs: 1000, text: "Generated" }], + { existingIds: ["caption-1"], startZIndex: 1 }, + ); + + expect(annotations[0].id).toBe("caption-2"); + }); +}); diff --git a/src/components/video-editor/captionAnnotations.ts b/src/components/video-editor/captionAnnotations.ts new file mode 100644 index 0000000000..b857e0e4d5 --- /dev/null +++ b/src/components/video-editor/captionAnnotations.ts @@ -0,0 +1,56 @@ +import type { CaptionSegment } from "@/lib/captions"; +import { type AnnotationRegion, DEFAULT_ANNOTATION_STYLE } from "./types"; + +const CAPTION_POSITION = { x: 10, y: 78 }; +const CAPTION_SIZE = { width: 80, height: 14 }; +const CAPTION_BACKGROUND = "rgba(0, 0, 0, 0.58)"; + +type CreateCaptionAnnotationsOptions = { + existingIds: string[]; + startZIndex: number; +}; + +function nextCaptionId(existingIds: Set, preferredId: string): string { + if (!existingIds.has(preferredId)) { + existingIds.add(preferredId); + return preferredId; + } + + let index = 1; + while (existingIds.has(`caption-${index}`)) { + index += 1; + } + + const id = `caption-${index}`; + existingIds.add(id); + return id; +} + +export function createCaptionAnnotations( + segments: CaptionSegment[], + options: CreateCaptionAnnotationsOptions, +): AnnotationRegion[] { + const existingIds = new Set(options.existingIds); + + return segments.map((segment, index) => { + const id = nextCaptionId(existingIds, segment.id); + return { + id, + startMs: segment.startMs, + endMs: segment.endMs, + type: "text", + content: segment.text, + textContent: segment.text, + position: { ...CAPTION_POSITION }, + size: { ...CAPTION_SIZE }, + style: { + ...DEFAULT_ANNOTATION_STYLE, + color: "#ffffff", + backgroundColor: CAPTION_BACKGROUND, + fontWeight: "bold", + textAlign: "center", + }, + zIndex: options.startZIndex + index, + }; + }); +} diff --git a/src/components/video-editor/useAutoCaptionGeneration.test.ts b/src/components/video-editor/useAutoCaptionGeneration.test.ts new file mode 100644 index 0000000000..22075afe93 --- /dev/null +++ b/src/components/video-editor/useAutoCaptionGeneration.test.ts @@ -0,0 +1,236 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { createElement, type ReactNode, StrictMode } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { CaptionGenerationResult } from "@/lib/captions"; +import { useAutoCaptionGeneration } from "./useAutoCaptionGeneration"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function strictModeWrapper({ children }: { children: ReactNode }) { + return createElement(StrictMode, null, children); +} + +describe("useAutoCaptionGeneration", () => { + beforeEach(() => { + window.electronAPI = { + ...(window.electronAPI ?? {}), + startCaptionGeneration: vi.fn(), + cancelCaptionGeneration: vi.fn(async () => ({ success: true, cancelled: true })), + } as Window["electronAPI"]; + }); + + it("starts one caption job for a fresh recording source and forwards generated segments", async () => { + const result: CaptionGenerationResult = { + jobId: "caption-job", + status: "success", + segments: [{ id: "caption-1", startMs: 0, endMs: 1000, text: "Hello" }], + message: "Generated 1 captions.", + }; + vi.mocked(window.electronAPI.startCaptionGeneration).mockResolvedValue(result); + const onCaptions = vi.fn(); + const onStatusMessage = vi.fn(); + + const { result: hook } = renderHook(() => + useAutoCaptionGeneration({ + sourcePath: "/recordings/one.webm", + enabled: true, + onCaptions, + onStatusMessage, + }), + ); + + expect(hook.current).toBe("running"); + await waitFor(() => expect(onCaptions).toHaveBeenCalledWith(result.segments)); + expect(onStatusMessage).toHaveBeenCalledWith(result); + expect(hook.current).toBe("success"); + expect(window.electronAPI.startCaptionGeneration).toHaveBeenCalledTimes(1); + }); + + it("does not start again for the same source after rerender", async () => { + vi.mocked(window.electronAPI.startCaptionGeneration).mockResolvedValue({ + jobId: "caption-job", + status: "skipped", + segments: [], + message: "No audio track found; captions skipped.", + }); + const onCaptions = vi.fn(); + const onStatusMessage = vi.fn(); + + const { rerender } = renderHook(() => + useAutoCaptionGeneration({ + sourcePath: "/recordings/reused.webm", + enabled: true, + onCaptions, + onStatusMessage, + }), + ); + + rerender(); + await waitFor(() => expect(onStatusMessage).toHaveBeenCalledTimes(1)); + expect(window.electronAPI.startCaptionGeneration).toHaveBeenCalledTimes(1); + expect(onCaptions).not.toHaveBeenCalled(); + }); + + it("passes the requested transcription language to caption generation", async () => { + vi.mocked(window.electronAPI.startCaptionGeneration).mockResolvedValue({ + jobId: "caption-job", + status: "success", + segments: [{ id: "caption-1", startMs: 0, endMs: 1000, text: "你好" }], + }); + const onCaptions = vi.fn(); + const onStatusMessage = vi.fn(); + + renderHook(() => + useAutoCaptionGeneration({ + sourcePath: "/recordings/chinese.webm", + enabled: true, + language: "zh", + onCaptions, + onStatusMessage, + }), + ); + + await waitFor(() => expect(onStatusMessage).toHaveBeenCalledTimes(1)); + expect(window.electronAPI.startCaptionGeneration).toHaveBeenCalledWith( + "/recordings/chinese.webm", + expect.objectContaining({ language: "zh" }), + ); + }); + + it("cancels the active job and ignores stale results on unmount", async () => { + const pending = deferred(); + vi.mocked(window.electronAPI.startCaptionGeneration).mockReturnValue(pending.promise); + const onCaptions = vi.fn(); + const onStatusMessage = vi.fn(); + + const { unmount } = renderHook(() => + useAutoCaptionGeneration({ + sourcePath: "/recordings/stale.webm", + enabled: true, + onCaptions, + onStatusMessage, + }), + ); + + unmount(); + await act(async () => { + pending.resolve({ + jobId: "caption-job", + status: "success", + segments: [{ id: "caption-1", startMs: 0, endMs: 1000, text: "Late" }], + }); + await pending.promise; + }); + + expect(window.electronAPI.cancelCaptionGeneration).toHaveBeenCalledTimes(1); + expect(onCaptions).not.toHaveBeenCalled(); + expect(onStatusMessage).not.toHaveBeenCalled(); + }); + + it("keeps the active job running when callback references change", async () => { + const pending = deferred(); + vi.mocked(window.electronAPI.startCaptionGeneration).mockReturnValue(pending.promise); + const firstOnCaptions = vi.fn(); + const firstOnStatusMessage = vi.fn(); + const secondOnCaptions = vi.fn(); + const secondOnStatusMessage = vi.fn(); + + const { rerender } = renderHook( + ({ + onCaptions, + onStatusMessage, + }: { + onCaptions: Parameters[0]["onCaptions"]; + onStatusMessage: Parameters[0]["onStatusMessage"]; + }) => + useAutoCaptionGeneration({ + sourcePath: "/recordings/changing-callbacks.webm", + enabled: true, + onCaptions, + onStatusMessage, + }), + { + initialProps: { + onCaptions: firstOnCaptions, + onStatusMessage: firstOnStatusMessage, + }, + }, + ); + + rerender({ + onCaptions: secondOnCaptions, + onStatusMessage: secondOnStatusMessage, + }); + + const result: CaptionGenerationResult = { + jobId: "caption-job", + status: "success", + segments: [{ id: "caption-1", startMs: 0, endMs: 1000, text: "Latest" }], + }; + await act(async () => { + pending.resolve(result); + await pending.promise; + }); + + expect(window.electronAPI.startCaptionGeneration).toHaveBeenCalledTimes(1); + expect(window.electronAPI.cancelCaptionGeneration).not.toHaveBeenCalled(); + expect(firstOnCaptions).not.toHaveBeenCalled(); + expect(firstOnStatusMessage).not.toHaveBeenCalled(); + expect(secondOnCaptions).toHaveBeenCalledWith(result.segments); + expect(secondOnStatusMessage).toHaveBeenCalledWith(result); + }); + + it("restarts the job after React StrictMode replays the mount effect", async () => { + const firstPending = deferred(); + const secondPending = deferred(); + vi.mocked(window.electronAPI.startCaptionGeneration) + .mockReturnValueOnce(firstPending.promise) + .mockReturnValueOnce(secondPending.promise); + const onCaptions = vi.fn(); + const onStatusMessage = vi.fn(); + + renderHook( + () => + useAutoCaptionGeneration({ + sourcePath: "/recordings/strict-mode.webm", + enabled: true, + onCaptions, + onStatusMessage, + }), + { wrapper: strictModeWrapper }, + ); + + await waitFor(() => expect(window.electronAPI.startCaptionGeneration).toHaveBeenCalledTimes(2)); + + const staleResult: CaptionGenerationResult = { + jobId: "stale-caption-job", + status: "success", + segments: [{ id: "caption-1", startMs: 0, endMs: 1000, text: "Stale" }], + }; + const activeResult: CaptionGenerationResult = { + jobId: "active-caption-job", + status: "success", + segments: [{ id: "caption-2", startMs: 1000, endMs: 2000, text: "Active" }], + }; + await act(async () => { + firstPending.resolve(staleResult); + secondPending.resolve(activeResult); + await firstPending.promise; + await secondPending.promise; + }); + + expect(window.electronAPI.cancelCaptionGeneration).toHaveBeenCalledTimes(1); + expect(onCaptions).toHaveBeenCalledTimes(1); + expect(onCaptions).toHaveBeenCalledWith(activeResult.segments); + expect(onStatusMessage).toHaveBeenCalledTimes(1); + expect(onStatusMessage).toHaveBeenCalledWith(activeResult); + }); +}); diff --git a/src/components/video-editor/useAutoCaptionGeneration.ts b/src/components/video-editor/useAutoCaptionGeneration.ts new file mode 100644 index 0000000000..7ad28bfec5 --- /dev/null +++ b/src/components/video-editor/useAutoCaptionGeneration.ts @@ -0,0 +1,92 @@ +import { useEffect, useRef, useState } from "react"; +import type { CaptionGenerationResult, CaptionSegment } from "@/lib/captions"; + +export type AutoCaptionStatus = + | "idle" + | "running" + | "success" + | "skipped" + | "unavailable" + | "error"; + +type UseAutoCaptionGenerationOptions = { + sourcePath: string | null; + enabled: boolean; + language?: string; + onCaptions: (segments: CaptionSegment[]) => void; + onStatusMessage: (result: CaptionGenerationResult) => void; +}; + +export function useAutoCaptionGeneration({ + sourcePath, + enabled, + language, + onCaptions, + onStatusMessage, +}: UseAutoCaptionGenerationOptions) { + const startedSourcesRef = useRef(new Set()); + const onCaptionsRef = useRef(onCaptions); + const onStatusMessageRef = useRef(onStatusMessage); + const [status, setStatus] = useState("idle"); + + onCaptionsRef.current = onCaptions; + onStatusMessageRef.current = onStatusMessage; + + useEffect(() => { + const startKey = sourcePath ? `${sourcePath}\n${language ?? ""}` : null; + if (!enabled || !sourcePath || !startKey || startedSourcesRef.current.has(startKey)) { + return; + } + + startedSourcesRef.current.add(startKey); + const jobId = `caption-${Date.now()}`; + let cancelled = false; + let settled = false; + setStatus("running"); + + void window.electronAPI + .startCaptionGeneration(sourcePath, { jobId, ...(language ? { language } : {}) }) + .then((result) => { + if (cancelled) return; + settled = true; + + if (result.status === "success" && result.segments.length > 0) { + onCaptionsRef.current(result.segments); + setStatus("success"); + } else if (result.status === "skipped") { + setStatus("skipped"); + } else if (result.status === "unavailable") { + setStatus("unavailable"); + } else if (result.status === "cancelled") { + setStatus("idle"); + } else { + setStatus("error"); + } + + onStatusMessageRef.current(result); + }) + .catch((error) => { + if (cancelled) return; + settled = true; + + setStatus("error"); + onStatusMessageRef.current({ + jobId, + status: "error", + segments: [], + message: "Failed to generate captions", + error: String(error), + }); + }); + + return () => { + cancelled = true; + if (!settled) { + startedSourcesRef.current.delete(startKey); + } + void window.electronAPI.cancelCaptionGeneration(jobId); + }; + }, [enabled, sourcePath, language]); + + return status; +} diff --git a/src/hooks/useScreenRecorder.test.tsx b/src/hooks/useScreenRecorder.test.tsx new file mode 100644 index 0000000000..b0dba9fbc7 --- /dev/null +++ b/src/hooks/useScreenRecorder.test.tsx @@ -0,0 +1,390 @@ +import { act, cleanup, renderHook } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import { useScreenRecorder } from "./useScreenRecorder"; + +vi.mock("@fix-webm-duration/fix", () => ({ + fixWebmDuration: vi.fn(async (blob: Blob) => blob), +})); + +type ElectronAPI = Window["electronAPI"]; + +const countdownValuesMs = [1000, 1000, 1000]; +const flushPromises = () => Promise.resolve(); + +class FakeMediaStreamTrack extends EventTarget { + kind: string; + enabled = true; + readyState: "live" | "ended" = "live"; + onended: (() => void) | null = null; + + constructor(kind: string) { + super(); + this.kind = kind; + } + + stop(): void { + if (this.readyState === "ended") return; + this.readyState = "ended"; + } + + end(): void { + if (this.readyState === "ended") return; + this.readyState = "ended"; + this.onended?.(); + this.dispatchEvent(new Event("ended")); + } + + getSettings(): MediaTrackSettings { + return { width: 1920, height: 1080, frameRate: 60 }; + } + + applyConstraints(): Promise { + return Promise.resolve(); + } +} + +class FakeMediaStream { + private tracks: FakeMediaStreamTrack[]; + + constructor(tracks: FakeMediaStreamTrack[] = []) { + this.tracks = [...tracks]; + } + + addTrack(track: FakeMediaStreamTrack): void { + this.tracks.push(track); + } + + getTracks(): FakeMediaStreamTrack[] { + return [...this.tracks]; + } + + getVideoTracks(): FakeMediaStreamTrack[] { + return this.tracks.filter((track) => track.kind === "video"); + } + + getAudioTracks(): FakeMediaStreamTrack[] { + return this.tracks.filter((track) => track.kind === "audio"); + } +} + +class FakeMediaRecorder extends EventTarget { + static instances: FakeMediaRecorder[] = []; + static isTypeSupported = vi.fn(() => true); + + ondataavailable: ((event: BlobEvent) => void) | null = null; + onstop: (() => void) | null = null; + onerror: (() => void) | null = null; + state: "inactive" | "recording" | "paused" = "inactive"; + + constructor() { + super(); + FakeMediaRecorder.instances.push(this); + } + + start(): void { + this.state = "recording"; + } + + stop(): void { + if (this.state === "inactive") return; + this.ondataavailable?.({ + data: new Blob(["screen-data"], { type: "video/webm" }), + } as BlobEvent); + this.state = "inactive"; + this.onstop?.(); + this.dispatchEvent(new Event("stop")); + } + + pause(): void { + this.state = "paused"; + } + + resume(): void { + this.state = "recording"; + } +} + +function wrapper({ children }: { children: ReactNode }) { + return {children}; +} + +function stubElectronAPI(api: Partial): void { + window.electronAPI = api as unknown as ElectronAPI; +} + +describe("useScreenRecorder", () => { + let screenTrack: FakeMediaStreamTrack; + let screenStream: FakeMediaStream; + let storeRecordedSession: ReturnType; + let switchToEditor: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + FakeMediaRecorder.instances = []; + screenTrack = new FakeMediaStreamTrack("video"); + screenStream = new FakeMediaStream([screenTrack]); + storeRecordedSession = vi.fn(async () => ({ + success: true, + path: "/tmp/recording.webm", + session: { + screenVideoPath: "/tmp/recording.webm", + createdAt: 1, + cursorCaptureMode: "editable-overlay", + }, + })); + switchToEditor = vi.fn(async () => undefined); + + vi.stubGlobal("MediaRecorder", FakeMediaRecorder); + vi.stubGlobal("MediaStream", FakeMediaStream); + Object.defineProperty(navigator, "mediaDevices", { + value: { + getDisplayMedia: vi.fn(async () => screenStream), + getUserMedia: vi.fn(async () => screenStream), + }, + configurable: true, + }); + stubElectronAPI({ + getSelectedSource: vi.fn(async () => ({ + id: "screen:1:0", + name: "Entire Screen", + display_id: "1", + thumbnail: null, + appIcon: null, + })), + getPlatform: vi.fn(async () => "linux"), + showCountdownOverlay: vi.fn(async () => undefined), + setCountdownOverlayValue: vi.fn(async () => undefined), + hideCountdownOverlay: vi.fn(async () => undefined), + setRecordingState: vi.fn(async () => undefined), + openRecordingStream: vi.fn(async () => ({ success: false })), + appendRecordingChunk: vi.fn(async () => ({ success: true })), + storeRecordedSession, + setCurrentRecordingSession: vi.fn(async () => undefined), + setCurrentVideoPath: vi.fn(async () => undefined), + switchToEditor, + onStopRecordingFromTray: vi.fn(() => () => undefined), + setLocale: vi.fn(async () => undefined), + }); + }); + + afterEach(() => { + cleanup(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + window.electronAPI = undefined as unknown as ElectronAPI; + }); + + it("opens the editor when a full-screen display track ends", async () => { + const { result } = renderHook(() => useScreenRecorder(), { wrapper }); + + await act(async () => { + result.current.toggleRecording(); + for (const ms of countdownValuesMs) { + await vi.advanceTimersByTimeAsync(ms); + } + await flushPromises(); + }); + + expect(result.current.recording).toBe(true); + + await act(async () => { + screenTrack.end(); + await flushPromises(); + }); + + expect(storeRecordedSession).toHaveBeenCalled(); + expect(switchToEditor).toHaveBeenCalled(); + }); + + it("opens the editor when a native macOS full-screen recording stops externally", async () => { + let nativeStoppedHandler: (() => void) | undefined; + const stopNativeMacRecording = vi.fn(async () => ({ + success: true, + path: "/tmp/native-recording.mp4", + session: { + screenVideoPath: "/tmp/native-recording.mp4", + createdAt: 2, + cursorCaptureMode: "editable-overlay", + }, + })); + + stubElectronAPI({ + ...window.electronAPI, + getPlatform: vi.fn(async () => "darwin"), + requestNativeMacCursorAccess: vi.fn(async () => ({ + success: true, + granted: true, + status: "granted", + })), + isNativeMacCaptureAvailable: vi.fn(async () => ({ + success: true, + available: true, + helperPath: "/tmp/helper", + })), + startNativeMacRecording: vi.fn(async () => ({ + success: true, + recordingId: 2, + path: "/tmp/native-recording.mp4", + helperPath: "/tmp/helper", + })), + stopNativeMacRecording, + onNativeRecordingStopped: vi.fn((callback: () => void) => { + nativeStoppedHandler = callback; + return () => undefined; + }), + } as Partial & { + onNativeRecordingStopped: (callback: () => void) => () => void; + }); + + const { result } = renderHook(() => useScreenRecorder(), { wrapper }); + + await act(async () => { + result.current.toggleRecording(); + for (const ms of countdownValuesMs) { + await vi.advanceTimersByTimeAsync(ms); + } + await flushPromises(); + }); + + expect(result.current.recording).toBe(true); + expect(nativeStoppedHandler).toBeDefined(); + + await act(async () => { + nativeStoppedHandler?.(); + await flushPromises(); + }); + + expect(stopNativeMacRecording).toHaveBeenCalledWith(false); + expect(window.electronAPI.setCurrentRecordingSession).toHaveBeenCalledWith({ + screenVideoPath: "/tmp/native-recording.mp4", + createdAt: 2, + cursorCaptureMode: "editable-overlay", + }); + expect(switchToEditor).toHaveBeenCalled(); + }); + + it("falls back to browser recording when native macOS capture lacks screen permission", async () => { + const startNativeMacRecording = vi.fn(async () => ({ + success: false, + error: "Screen recording permission is required for ScreenCaptureKit capture.", + })); + + stubElectronAPI({ + ...window.electronAPI, + getPlatform: vi.fn(async () => "darwin"), + requestNativeMacCursorAccess: vi.fn(async () => ({ + success: true, + granted: true, + status: "granted", + })), + isNativeMacCaptureAvailable: vi.fn(async () => ({ + success: true, + available: true, + helperPath: "/tmp/helper", + })), + startNativeMacRecording, + stopNativeMacRecording: vi.fn(async () => ({ + success: true, + discarded: true, + })), + }); + + const { result } = renderHook(() => useScreenRecorder(), { wrapper }); + + await act(async () => { + result.current.toggleRecording(); + for (const ms of countdownValuesMs) { + await vi.advanceTimersByTimeAsync(ms); + } + await flushPromises(); + }); + + expect(startNativeMacRecording).toHaveBeenCalled(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalled(); + expect(result.current.recording).toBe(true); + + await act(async () => { + screenTrack.end(); + await flushPromises(); + }); + + expect(storeRecordedSession).toHaveBeenCalled(); + expect(switchToEditor).toHaveBeenCalled(); + }); + + it("uses browser recording on macOS when microphone recording is enabled", async () => { + const micStream = new FakeMediaStream([new FakeMediaStreamTrack("audio")]); + const getUserMedia = vi.fn(async (constraints: MediaStreamConstraints) => { + if (constraints.video === false) { + return micStream; + } + return screenStream; + }); + Object.defineProperty(navigator, "mediaDevices", { + value: { + getDisplayMedia: vi.fn(async () => screenStream), + getUserMedia, + }, + configurable: true, + }); + const startNativeMacRecording = vi.fn(async () => ({ + success: true, + recordingId: 3, + path: "/tmp/native-recording.mp4", + helperPath: "/tmp/helper", + })); + + stubElectronAPI({ + ...window.electronAPI, + getPlatform: vi.fn(async () => "darwin"), + requestNativeMacCursorAccess: vi.fn(async () => ({ + success: true, + granted: true, + status: "granted", + })), + isNativeMacCaptureAvailable: vi.fn(async () => ({ + success: true, + available: true, + helperPath: "/tmp/helper", + })), + startNativeMacRecording, + stopNativeMacRecording: vi.fn(async () => ({ + success: true, + discarded: true, + })), + }); + + const { result } = renderHook(() => useScreenRecorder(), { wrapper }); + + await act(async () => { + result.current.setMicrophoneEnabled(true); + }); + expect(result.current.microphoneEnabled).toBe(true); + + await act(async () => { + result.current.toggleRecording(); + for (const ms of countdownValuesMs) { + await vi.advanceTimersByTimeAsync(ms); + } + await flushPromises(); + }); + + expect(startNativeMacRecording).not.toHaveBeenCalled(); + expect(getUserMedia).toHaveBeenCalledWith( + expect.objectContaining({ + video: false, + }), + ); + expect(result.current.recording).toBe(true); + + await act(async () => { + screenTrack.end(); + await flushPromises(); + }); + + expect(storeRecordedSession).toHaveBeenCalled(); + expect(switchToEditor).toHaveBeenCalled(); + }); +}); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index f5fb920323..e348f5239b 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -46,6 +46,8 @@ const AUDIO_BITRATE_SYSTEM = 192_000; const MIC_GAIN_BOOST = 1.4; const WEBCAM_TARGET_FRAME_RATE = 30; +const SCREEN_CAPTUREKIT_PERMISSION_ERROR = + "Screen recording permission is required for ScreenCaptureKit capture"; type UseScreenRecorderReturn = { recording: boolean; @@ -87,6 +89,13 @@ type NativeMacRecordingHandle = { paused: boolean; }; +function isRecoverableNativeMacScreenPermissionError(message: string) { + return ( + message.includes(SCREEN_CAPTUREKIT_PERMISSION_ERROR) || + message.includes("screen-permission-denied") + ); +} + export function useScreenRecorder(): UseScreenRecorderReturn { const t = useScopedT("editor"); const [recording, setRecording] = useState(false); @@ -104,6 +113,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const webcamRecorder = useRef(null); const nativeWindowsRecording = useRef(null); const nativeMacRecording = useRef(null); + const cleanupScreenTrackEndedListener = useRef<(() => void) | null>(null); const stream = useRef(null); const screenStream = useRef(null); const microphoneStream = useRef(null); @@ -167,6 +177,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; const teardownMedia = useCallback(() => { + cleanupScreenTrackEndedListener.current?.(); + cleanupScreenTrackEndedListener.current = null; if (stream.current) { stream.current.getTracks().forEach((track) => track.stop()); stream.current = null; @@ -679,17 +691,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }, []); useEffect(() => { - let cleanup: (() => void) | undefined; + let cleanupStopFromTray: (() => void) | undefined; + let cleanupNativeRecordingStopped: (() => void) | undefined; if (window.electronAPI?.onStopRecordingFromTray) { - cleanup = window.electronAPI.onStopRecordingFromTray(() => { + cleanupStopFromTray = window.electronAPI.onStopRecordingFromTray(() => { + stopRecording.current(); + }); + } + + if (window.electronAPI?.onNativeRecordingStopped) { + cleanupNativeRecordingStopped = window.electronAPI.onNativeRecordingStopped(() => { stopRecording.current(); }); } return () => { const activeRunId = countdownRunId.current; - if (cleanup) cleanup(); + if (cleanupStopFromTray) cleanupStopFromTray(); + if (cleanupNativeRecordingStopped) cleanupNativeRecordingStopped(); countdownRunId.current += 1; void safeHideCountdownOverlay(activeRunId); allowAutoFinalize.current = false; @@ -905,6 +925,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return false; } + // The current ScreenCaptureKit helper can write microphone samples as a + // separate track, but it does not yet mix them into the primary AAC track. + // Use the browser path for microphone recordings so the mic is captured + // into the MediaRecorder audio track reliably. + if (microphoneEnabled) { + return false; + } + const availability = await window.electronAPI.isNativeMacCaptureAvailable(); if (!availability.success || !availability.available) { if (availability.reason === "unsupported-platform") { @@ -1009,7 +1037,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (nativeWebcamRecorder && nativeWebcamRecorder.recorder.state !== "inactive") { nativeWebcamRecorder.recorder.stop(); } - throw new Error(result.error ?? "Native macOS capture failed."); + const nativeError = result.error ?? "Native macOS capture failed."; + if (isRecoverableNativeMacScreenPermissionError(nativeError)) { + console.warn( + "Native macOS ScreenCaptureKit permission is unavailable; falling back to browser recording.", + nativeError, + ); + return false; + } + throw new Error(nativeError); } if (!isCountdownRunActive(countdownRunToken)) { if (nativeWebcamRecorder && nativeWebcamRecorder.recorder.state !== "inactive") { @@ -1272,6 +1308,17 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (!videoTrack) { throw new Error("Video track is not available."); } + const handleScreenTrackEnded = () => { + if (!allowAutoFinalize.current) { + return; + } + stopRecording.current(); + }; + cleanupScreenTrackEndedListener.current?.(); + videoTrack.addEventListener("ended", handleScreenTrackEnded, { once: true }); + cleanupScreenTrackEndedListener.current = () => { + videoTrack.removeEventListener("ended", handleScreenTrackEnded); + }; stream.current.addTrack(videoTrack); const systemAudioTrack = screenMediaStream.getAudioTracks()[0]; diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 1eec625b59..fc242bbdee 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -42,5 +42,12 @@ "cameraNotFound": "لم يتم العثور على كاميرا.", "permissionDenied": "تم رفض إذن التسجيل. يرجى السماح بتسجيل الشاشة.", "accessibilityAllowAndRetry": "اسمح بوصول تسهيلات الاستخدام لـ OpenScreen، ثم اضغط على التسجيل مرة أخرى لبدء العد التنازلي." + }, + "captions": { + "generating": "جارٍ إنشاء الترجمات...", + "generated": "تم إنشاء {{count}} ترجمة.", + "skippedNoAudio": "لم يتم العثور على مسار صوتي؛ تم تخطي الترجمات.", + "unavailable": "نموذج الترجمات المحلي غير متاح.", + "failed": "فشل إنشاء الترجمات." } } diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index 6cd90b6a29..55f573dc1a 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -139,6 +139,8 @@ "clearBackground": "مسح الخلفية", "uploadImage": "رفع صورة", "supportedFormats": "الصيغ المدعومة: JPG, PNG, GIF, WebP", + "stickerPresets": "ملصقات جاهزة", + "stickerPresetsDescription": "انقر على ملصق لاستخدامه كتعليق صورة.", "arrowDirection": "اتجاه السهم", "strokeWidth": "عرض الخط: {{width}}px", "arrowColor": "لون السهم", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index aad37003fd..811574b937 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -42,5 +42,12 @@ "cameraNotFound": "Camera not found.", "permissionDenied": "Recording permission denied. Please allow screen recording.", "accessibilityAllowAndRetry": "Allow Accessibility access for OpenScreen, then press record again to start the countdown." + }, + "captions": { + "generating": "Generating captions...", + "generated": "Generated {{count}} captions.", + "skippedNoAudio": "No audio track found; captions skipped.", + "unavailable": "Local caption model is not available.", + "failed": "Failed to generate captions." } } diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 3e2541aa15..5d6debe47a 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -139,6 +139,8 @@ "clearBackground": "Clear Background", "uploadImage": "Upload Image", "supportedFormats": "Supported formats: JPG, PNG, GIF, WebP", + "stickerPresets": "Sticker presets", + "stickerPresetsDescription": "Click a sticker to use it as this image annotation.", "arrowDirection": "Arrow Direction", "strokeWidth": "Stroke Width: {{width}}px", "arrowColor": "Arrow Color", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 27e0caeda2..a9255c5aa7 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -42,5 +42,12 @@ "description": "Tu sesión actual ha sido guardada.", "cancel": "Cancelar", "confirm": "Confirmar" + }, + "captions": { + "generating": "Generando subtítulos...", + "generated": "{{count}} subtítulos generados.", + "skippedNoAudio": "No se encontró pista de audio; se omitieron los subtítulos.", + "unavailable": "El modelo local de subtítulos no está disponible.", + "failed": "Error al generar subtítulos." } } diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 99cff77e52..a1577e9e05 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -139,6 +139,8 @@ "clearBackground": "Quitar fondo", "uploadImage": "Subir imagen", "supportedFormats": "Formatos compatibles: JPG, PNG, GIF, WebP", + "stickerPresets": "Stickers predefinidos", + "stickerPresetsDescription": "Haz clic en un sticker para usarlo como esta anotación de imagen.", "arrowDirection": "Dirección de la flecha", "strokeWidth": "Grosor del trazo: {{width}}px", "arrowColor": "Color de la flecha", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 7195aedffe..26fb4bf743 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -42,5 +42,12 @@ "permissionDenied": "Permission d'enregistrement refusée. Veuillez autoriser l'enregistrement d'écran.", "accessibilityAllowAndRetry": "Autorisez l'accès Accessibilité pour OpenScreen, puis appuyez de nouveau sur enregistrer pour lancer le compte à rebours." }, - "loadingVideo": "Chargement de la vidéo..." + "loadingVideo": "Chargement de la vidéo...", + "captions": { + "generating": "Génération des sous-titres...", + "generated": "{{count}} sous-titres générés.", + "skippedNoAudio": "Aucune piste audio trouvée ; sous-titres ignorés.", + "unavailable": "Le modèle de sous-titres local n'est pas disponible.", + "failed": "Échec de la génération des sous-titres." + } } diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index c968c68dbb..c9e1bc6691 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -140,6 +140,8 @@ "clearBackground": "Supprimer l'arrière-plan", "uploadImage": "Téléverser une image", "supportedFormats": "Formats supportés : JPG, PNG, GIF, WebP", + "stickerPresets": "Préréglages de stickers", + "stickerPresetsDescription": "Cliquez sur un sticker pour l'utiliser comme annotation d'image.", "arrowDirection": "Direction de la flèche", "strokeWidth": "Épaisseur du trait : {{width}}px", "arrowColor": "Couleur de la flèche", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 336d3e6ba8..882c4ecab9 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -42,5 +42,12 @@ "cameraNotFound": "Fotocamera non trovata.", "permissionDenied": "Autorizzazione di registrazione negata. Consenti la registrazione dello schermo.", "accessibilityAllowAndRetry": "Consenti l'accesso all'accessibilità per OpenScreen, poi premi di nuovo registra per avviare il conto alla rovescia." + }, + "captions": { + "generating": "Generazione sottotitoli...", + "generated": "{{count}} sottotitoli generati.", + "skippedNoAudio": "Nessuna traccia audio trovata; sottotitoli saltati.", + "unavailable": "Il modello locale per i sottotitoli non è disponibile.", + "failed": "Impossibile generare i sottotitoli." } } diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index 0515a7653a..17dd6663bd 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -1,5 +1,6 @@ { "zoom": { + "previewHold": "Tieni premuto per vedere l'anteprima dell'effetto zoom", "level": "Livello zoom", "customScale": "Zoom personalizzato", "selectRegion": "Seleziona una regione zoom da regolare", @@ -138,6 +139,8 @@ "clearBackground": "Rimuovi sfondo", "uploadImage": "Carica immagine", "supportedFormats": "Formati supportati: JPG, PNG, GIF, WebP", + "stickerPresets": "Sticker predefiniti", + "stickerPresetsDescription": "Fai clic su uno sticker per usarlo come annotazione immagine.", "arrowDirection": "Direzione freccia", "strokeWidth": "Larghezza tratto: {{width}}px", "arrowColor": "Colore freccia", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index d37e132c22..5beb531371 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -42,5 +42,12 @@ "cameraDisconnected": "ウェブカメラが切断されました。", "cameraNotFound": "カメラが見つかりません。", "accessibilityAllowAndRetry": "OpenScreenにアクセシビリティアクセスを許可してから、もう一度録画を押してカウントダウンを開始してください。" + }, + "captions": { + "generating": "字幕を生成しています...", + "generated": "{{count}} 件の字幕を生成しました。", + "skippedNoAudio": "音声トラックが見つからないため、字幕生成をスキップしました。", + "unavailable": "ローカル字幕モデルを利用できません。", + "failed": "字幕の生成に失敗しました。" } } diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index 697e1ac5e8..4b7fd264d2 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -139,6 +139,8 @@ "clearBackground": "背景をクリア", "uploadImage": "画像を読み込む", "supportedFormats": "サポートされている形式: JPG, PNG, GIF, WebP", + "stickerPresets": "ステッカープリセット", + "stickerPresetsDescription": "ステッカーをクリックすると、この画像注釈として使用します。", "arrowDirection": "矢印の方向", "strokeWidth": "線の太さ: {{width}}px", "arrowColor": "矢印の色", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 13c8bfd2d4..632b9f0bbf 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -42,5 +42,12 @@ "cameraDisconnected": "웹캠 연결이 끊어졌습니다.", "cameraNotFound": "카메라를 찾을 수 없습니다.", "accessibilityAllowAndRetry": "OpenScreen의 손쉬운 사용 접근을 허용한 다음, 카운트다운을 시작하려면 다시 녹화를 누르세요." + }, + "captions": { + "generating": "자막을 생성하는 중...", + "generated": "자막 {{count}}개를 생성했습니다.", + "skippedNoAudio": "오디오 트랙을 찾을 수 없어 자막 생성을 건너뛰었습니다.", + "unavailable": "로컬 자막 모델을 사용할 수 없습니다.", + "failed": "자막 생성에 실패했습니다." } } diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index df4c6a2733..5c57036293 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -138,6 +138,8 @@ "clearBackground": "배경 지우기", "uploadImage": "이미지 업로드", "supportedFormats": "지원 형식: JPG, PNG, GIF, WebP", + "stickerPresets": "스티커 프리셋", + "stickerPresetsDescription": "스티커를 클릭하면 이 이미지 주석으로 사용됩니다.", "arrowDirection": "화살표 방향", "strokeWidth": "선 두께: {{width}}px", "arrowColor": "화살표 색상", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 5452124f46..e72092068c 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -42,5 +42,12 @@ "cameraNotFound": "Камера не найдена.", "permissionDenied": "Разрешение на запись запрещено. Пожалуйста, разрешите запись экрана.", "accessibilityAllowAndRetry": "Разрешите OpenScreen доступ к Универсальному доступу, затем снова нажмите запись, чтобы начать обратный отсчет." + }, + "captions": { + "generating": "Создание субтитров...", + "generated": "Создано субтитров: {{count}}.", + "skippedNoAudio": "Аудиодорожка не найдена; субтитры пропущены.", + "unavailable": "Локальная модель субтитров недоступна.", + "failed": "Не удалось создать субтитры." } } diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index c0dbba8f09..349fe53a83 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -139,6 +139,8 @@ "clearBackground": "Очистить фон", "uploadImage": "Загрузить изображение", "supportedFormats": "Поддерживаемые форматы: JPG, PNG, GIF, WebP", + "stickerPresets": "Готовые стикеры", + "stickerPresetsDescription": "Нажмите стикер, чтобы использовать его как аннотацию-изображение.", "arrowDirection": "Направление стрелки", "strokeWidth": "Толщина линии: {{width}}px", "arrowColor": "Цвет стрелки", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index b50630a931..40767d06e5 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -42,5 +42,12 @@ "description": "Mevcut oturumunuz kaydedildi.", "cancel": "İptal", "confirm": "Onayla" + }, + "captions": { + "generating": "Altyazılar oluşturuluyor...", + "generated": "{{count}} altyazı oluşturuldu.", + "skippedNoAudio": "Ses parçası bulunamadı; altyazılar atlandı.", + "unavailable": "Yerel altyazı modeli kullanılamıyor.", + "failed": "Altyazılar oluşturulamadı." } } diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 587f266889..264d483fbf 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -139,6 +139,8 @@ "clearBackground": "Arka Planı Temizle", "uploadImage": "Görüntü Yükle", "supportedFormats": "Desteklenen biçimler: JPG, PNG, GIF, WebP", + "stickerPresets": "Hazır çıkartmalar", + "stickerPresetsDescription": "Bu görsel açıklama için kullanmak üzere bir çıkartmaya tıklayın.", "arrowDirection": "Ok Yönü", "strokeWidth": "Çizgi Kalınlığı: {{width}}px", "arrowColor": "Ok Rengi", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 03e909f414..d57ba90573 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -42,5 +42,12 @@ "cameraNotFound": "Không tìm thấy máy ảnh.", "permissionDenied": "Quyền ghi hình bị từ chối. Vui lòng cho phép ghi màn hình.", "accessibilityAllowAndRetry": "Cho phép OpenScreen truy cập Trợ năng, sau đó nhấn ghi lại để bắt đầu đếm ngược." + }, + "captions": { + "generating": "Đang tạo phụ đề...", + "generated": "Đã tạo {{count}} phụ đề.", + "skippedNoAudio": "Không tìm thấy bản âm thanh; đã bỏ qua phụ đề.", + "unavailable": "Mô hình phụ đề cục bộ không khả dụng.", + "failed": "Tạo phụ đề thất bại." } } diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index e83c799d82..7b473c696f 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -137,6 +137,8 @@ "clearBackground": "Xóa nền", "uploadImage": "Tải lên hình ảnh", "supportedFormats": "Định dạng hỗ trợ: JPG, PNG, GIF, WebP", + "stickerPresets": "Mẫu nhãn dán", + "stickerPresetsDescription": "Nhấp vào nhãn dán để dùng làm chú thích hình ảnh này.", "arrowDirection": "Hướng mũi tên", "strokeWidth": "Độ dày nét: {{width}}px", "arrowColor": "Màu mũi tên", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 56a36f8d76..96e6f5526a 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -42,5 +42,12 @@ "cameraNotFound": "未找到摄像头。", "permissionDenied": "录屏权限被拒绝。请允许屏幕录制。", "accessibilityAllowAndRetry": "允许 OpenScreen 使用辅助功能权限，然后再次按录制以开始倒计时。" + }, + "captions": { + "generating": "正在生成字幕...", + "generated": "已生成 {{count}} 条字幕。", + "skippedNoAudio": "未找到音轨，已跳过字幕生成。", + "unavailable": "本地字幕模型不可用。", + "failed": "字幕生成失败。" } } diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index b9b516a045..20b4948c99 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -139,6 +139,8 @@ "clearBackground": "清除背景", "uploadImage": "上传图片", "supportedFormats": "支持的格式：JPG、PNG、GIF、WebP", + "stickerPresets": "内置贴纸", + "stickerPresetsDescription": "点击贴纸即可用作当前图片标注。", "arrowDirection": "箭头方向", "strokeWidth": "描边宽度：{{width}}px", "arrowColor": "箭头颜色", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index d4ad23f15c..f64dc915a9 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -42,5 +42,12 @@ "cameraDisconnected": "網路攝影機已中斷連線。", "cameraNotFound": "找不到攝影機。", "accessibilityAllowAndRetry": "允許 OpenScreen 使用輔助使用權限，然後再次按下錄製以開始倒數。" + }, + "captions": { + "generating": "正在產生字幕...", + "generated": "已產生 {{count}} 條字幕。", + "skippedNoAudio": "找不到音軌，已略過字幕產生。", + "unavailable": "本機字幕模型不可用。", + "failed": "字幕產生失敗。" } } diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index ee56459e78..d229030d29 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -140,6 +140,8 @@ "clearBackground": "清除背景", "uploadImage": "上傳圖片", "supportedFormats": "支援的格式：JPG、PNG、GIF、WebP", + "stickerPresets": "內建貼紙", + "stickerPresetsDescription": "點擊貼紙即可用作目前圖片標註。", "arrowDirection": "箭頭方向", "strokeWidth": "描邊寬度：{{width}}px", "arrowColor": "箭頭顏色", diff --git a/src/lib/captions.test.ts b/src/lib/captions.test.ts new file mode 100644 index 0000000000..52cc91f1fe --- /dev/null +++ b/src/lib/captions.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import { + getWhisperLanguageForLocale, + MIN_CAPTION_DURATION_MS, + normalizeCaptionSegments, + parseWhisperJsonOutput, +} from "./captions"; + +describe("parseWhisperJsonOutput", () => { + it("parses whisper.cpp transcription entries that use offsets in milliseconds", () => { + const output = JSON.stringify({ + transcription: [ + { + offsets: { from: 1200, to: 3450 }, + text: " Hello world ", + }, + ], + }); + + expect(parseWhisperJsonOutput(output)).toEqual([ + { id: "caption-1", startMs: 1200, endMs: 3450, text: "Hello world" }, + ]); + }); + + it("parses whisper.cpp timestamp strings", () => { + const output = JSON.stringify({ + transcription: [ + { + timestamps: { from: "00:01:02.500", to: "00:01:04.000" }, + text: "timestamp text", + }, + ], + }); + + expect(parseWhisperJsonOutput(output)).toEqual([ + { id: "caption-1", startMs: 62500, endMs: 64000, text: "timestamp text" }, + ]); + }); + + it("parses common segment arrays that use seconds", () => { + const output = JSON.stringify({ + segments: [{ start: 1.25, end: 2.5, text: "segment text" }], + }); + + expect(parseWhisperJsonOutput(output)).toEqual([ + { id: "caption-1", startMs: 1250, endMs: 2500, text: "segment text" }, + ]); + }); + + it("returns an empty list for invalid JSON", () => { + expect(parseWhisperJsonOutput("{not-json")).toEqual([]); + }); +}); + +describe("normalizeCaptionSegments", () => { + it("drops empty text and invalid timing", () => { + expect( + normalizeCaptionSegments([ + { startMs: 0, endMs: 1000, text: " " }, + { startMs: 1000, endMs: 900, text: "reversed" }, + { startMs: Number.NaN, endMs: 2000, text: "nan" }, + { startMs: 2000, endMs: 3000, text: "valid" }, + ]), + ).toEqual([{ id: "caption-1", startMs: 2000, endMs: 3000, text: "valid" }]); + }); + + it("sorts by start time and clamps very short captions", () => { + const segments = normalizeCaptionSegments([ + { startMs: 2000, endMs: 2100, text: "second" }, + { startMs: 1000, endMs: 1200, text: "first" }, + ]); + + expect(segments).toEqual([ + { + id: "caption-1", + startMs: 1000, + endMs: 1000 + MIN_CAPTION_DURATION_MS, + text: "first", + }, + { + id: "caption-2", + startMs: 2000, + endMs: 2000 + MIN_CAPTION_DURATION_MS, + text: "second", + }, + ]); + }); +}); + +describe("getWhisperLanguageForLocale", () => { + it("maps supported app locales to whisper language codes", () => { + expect(getWhisperLanguageForLocale("zh-CN")).toBe("zh"); + expect(getWhisperLanguageForLocale("zh-TW")).toBe("zh"); + expect(getWhisperLanguageForLocale("ja-JP")).toBe("ja"); + expect(getWhisperLanguageForLocale("ko-KR")).toBe("ko"); + expect(getWhisperLanguageForLocale("fr")).toBe("fr"); + }); + + it("returns undefined for empty or invalid locale values", () => { + expect(getWhisperLanguageForLocale("")).toBeUndefined(); + expect(getWhisperLanguageForLocale(" ")).toBeUndefined(); + }); +}); diff --git a/src/lib/captions.ts b/src/lib/captions.ts new file mode 100644 index 0000000000..43bed0e4bf --- /dev/null +++ b/src/lib/captions.ts @@ -0,0 +1,167 @@ +export const MIN_CAPTION_DURATION_MS = 500; + +export type CaptionGenerationStatus = "success" | "skipped" | "unavailable" | "error" | "cancelled"; + +export interface CaptionSegment { + id: string; + startMs: number; + endMs: number; + text: string; +} + +export interface CaptionGenerationOptions { + jobId?: string; + language?: string; +} + +export interface CaptionGenerationResult { + jobId: string; + status: CaptionGenerationStatus; + segments: CaptionSegment[]; + message?: string; + error?: string; +} + +const WHISPER_LANGUAGE_BY_LOCALE: Record = { + ar: "ar", + en: "en", + es: "es", + fr: "fr", + it: "it", + "ja-jp": "ja", + "ko-kr": "ko", + ru: "ru", + tr: "tr", + vi: "vi", + "zh-cn": "zh", + "zh-tw": "zh", +}; + +interface RawCaptionSegment { + startMs: number; + endMs: number; + text: string; +} + +export function getWhisperLanguageForLocale(locale: string): string | undefined { + const normalizedLocale = locale.trim().toLowerCase(); + if (!normalizedLocale) { + return undefined; + } + + return WHISPER_LANGUAGE_BY_LOCALE[normalizedLocale]; +} + +export function normalizeCaptionSegments(segments: RawCaptionSegment[]): CaptionSegment[] { + return segments + .map((segment) => ({ + startMs: segment.startMs, + endMs: segment.endMs, + text: segment.text.trim(), + })) + .filter( + (segment) => + segment.text.length > 0 && + Number.isFinite(segment.startMs) && + Number.isFinite(segment.endMs) && + segment.endMs >= segment.startMs, + ) + .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs) + .map((segment, index) => ({ + id: `caption-${index + 1}`, + startMs: segment.startMs, + endMs: Math.max(segment.endMs, segment.startMs + MIN_CAPTION_DURATION_MS), + text: segment.text, + })); +} + +export function parseWhisperJsonOutput(output: string): CaptionSegment[] { + let parsed: unknown; + + try { + parsed = JSON.parse(output); + } catch { + return []; + } + + if (!isRecord(parsed)) { + return []; + } + + if (Array.isArray(parsed.transcription)) { + return normalizeCaptionSegments(parsed.transcription.map(parseTranscriptionEntry)); + } + + if (Array.isArray(parsed.segments)) { + return normalizeCaptionSegments(parsed.segments.map(parseSecondsSegment)); + } + + return []; +} + +function parseTranscriptionEntry(entry: unknown): RawCaptionSegment { + if (!isRecord(entry)) { + return invalidCaptionSegment(); + } + + const text = typeof entry.text === "string" ? entry.text : ""; + + if (isRecord(entry.offsets)) { + return { + startMs: numberOrNaN(entry.offsets.from), + endMs: numberOrNaN(entry.offsets.to), + text, + }; + } + + if (isRecord(entry.timestamps)) { + return { + startMs: parseTimestampMs(entry.timestamps.from), + endMs: parseTimestampMs(entry.timestamps.to), + text, + }; + } + + return invalidCaptionSegment(); +} + +function parseSecondsSegment(entry: unknown): RawCaptionSegment { + if (!isRecord(entry)) { + return invalidCaptionSegment(); + } + + return { + startMs: numberOrNaN(entry.start) * 1000, + endMs: numberOrNaN(entry.end) * 1000, + text: typeof entry.text === "string" ? entry.text : "", + }; +} + +function parseTimestampMs(value: unknown): number { + if (typeof value !== "string") { + return Number.NaN; + } + + const match = value.trim().match(/^(\d+):(\d{2}):(\d{2})(?:\.(\d{1,3}))?$/); + if (!match) { + return Number.NaN; + } + + const [, hours, minutes, seconds, milliseconds = "0"] = match; + return ( + (Number(hours) * 60 * 60 + Number(minutes) * 60 + Number(seconds)) * 1000 + + Number(milliseconds.padEnd(3, "0")) + ); +} + +function numberOrNaN(value: unknown): number { + return typeof value === "number" ? value : Number.NaN; +} + +function invalidCaptionSegment(): RawCaptionSegment { + return { startMs: Number.NaN, endMs: Number.NaN, text: "" }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} From d60e42f9ec36c53a530d46e8fb98dc34da3e4755 Mon Sep 17 00:00:00 2001 From: notegen <525229509@qq.com> Date: Sat, 6 Jun 2026 12:48:34 +0800 Subject: [PATCH 3/3] Fix live recording annotations --- electron/electron-env.d.ts | 20 + electron/ipc/handlers.ts | 108 +++++ electron/main.ts | 24 +- electron/preload.ts | 32 ++ electron/windows.ts | 50 ++- src/App.tsx | 10 +- src/components/launch/LaunchWindow.tsx | 115 +++++- .../RecordingAnnotationOverlay.test.tsx | 110 +++++ .../launch/RecordingAnnotationOverlay.tsx | 381 ++++++++++++++++++ .../launch/recordingAnnotations.test.ts | 96 +++++ src/components/launch/recordingAnnotations.ts | 234 +++++++++++ src/hooks/useScreenRecorder.test.tsx | 29 ++ src/hooks/useScreenRecorder.ts | 45 ++- src/main.tsx | 3 +- 14 files changed, 1247 insertions(+), 10 deletions(-) create mode 100644 src/components/launch/RecordingAnnotationOverlay.test.tsx create mode 100644 src/components/launch/RecordingAnnotationOverlay.tsx create mode 100644 src/components/launch/recordingAnnotations.test.ts create mode 100644 src/components/launch/recordingAnnotations.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index ce24b5f637..031062b2d4 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -267,6 +267,26 @@ interface Window { showCountdownOverlay: (value: number, runId: number) => Promise; setCountdownOverlayValue: (value: number, runId: number) => Promise; hideCountdownOverlay: (runId: number) => Promise; + showRecordingAnnotationOverlay: () => Promise<{ success: boolean; error?: string }>; + hideRecordingAnnotationOverlay: () => Promise<{ success: boolean; error?: string }>; + setRecordingAnnotationTool: ( + tool: import("../src/components/launch/recordingAnnotations").RecordingAnnotationTool | null, + ) => Promise<{ + success: boolean; + tool?: import("../src/components/launch/recordingAnnotations").RecordingAnnotationTool | null; + error?: string; + }>; + clearRecordingAnnotations: () => Promise<{ success: boolean; error?: string }>; + undoRecordingAnnotation: () => Promise<{ success: boolean; error?: string }>; + onRecordingAnnotationToolChange: ( + callback: ( + tool: + | import("../src/components/launch/recordingAnnotations").RecordingAnnotationTool + | null, + ) => void, + ) => () => void; + onRecordingAnnotationClear: (callback: () => void) => () => void; + onRecordingAnnotationUndo: (callback: () => void) => () => void; onCountdownOverlayValue: (callback: (value: number | null) => void) => () => void; setMicrophoneExpanded: (expanded: boolean) => void; setHasUnsavedChanges: (hasChanges: boolean) => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 4673d4a9f7..48be3ab64c 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1376,12 +1376,62 @@ export function registerIpcHandlers( createEditorWindow: () => void, createSourceSelectorWindow: () => BrowserWindow, createCountdownOverlayWindow: () => BrowserWindow, + createRecordingAnnotationOverlayWindow: (bounds?: Electron.Rectangle) => BrowserWindow, getMainWindow: () => BrowserWindow | null, getSourceSelectorWindow: () => BrowserWindow | null, getCountdownOverlayWindow?: () => BrowserWindow | null, + getRecordingAnnotationOverlayWindow?: () => BrowserWindow | null, onRecordingStateChange?: (recording: boolean, sourceName: string) => void, _switchToHud?: () => void, ) { + const recordingAnnotationTools = new Set([ + "pen", + "arrow", + "rectangle", + "ellipse", + "highlight", + "text", + ]); + + function notifyRecordingAnnotationTool(tool: string | null) { + const mainWindow = getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send("recording-annotation-tool", tool); + } + + const overlayWindow = getRecordingAnnotationOverlayWindow?.(); + if (overlayWindow && !overlayWindow.isDestroyed()) { + overlayWindow.webContents.send("recording-annotation-tool", tool); + } + } + + async function ensureRecordingAnnotationOverlayWindow() { + const bounds = getSelectedSourceBounds(); + const overlayWindow = + getRecordingAnnotationOverlayWindow?.() ?? createRecordingAnnotationOverlayWindow(bounds); + + if (overlayWindow.isDestroyed()) { + return null; + } + + overlayWindow.setBounds(bounds, false); + if (!overlayWindow.isVisible()) { + overlayWindow.showInactive(); + } + if (overlayWindow.webContents.isLoading()) { + await new Promise((resolve) => { + overlayWindow.webContents.once("did-finish-load", () => resolve()); + }); + } + + const mainWindow = getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed() && mainWindow.isVisible()) { + mainWindow.moveTop(); + } + + return overlayWindow; + } + async function requestScreenAccess() { if (process.platform !== "darwin") { return { success: true, granted: true, status: "granted" }; @@ -1607,6 +1657,64 @@ export function registerIpcHandlers( overlayWindow.hide(); }); + ipcMain.handle("recording-annotation-overlay-show", async () => { + const overlayWindow = await ensureRecordingAnnotationOverlayWindow(); + if (!overlayWindow) { + return { success: false, error: "Recording annotation overlay is unavailable." }; + } + + overlayWindow.setIgnoreMouseEvents(true, { forward: true }); + overlayWindow.webContents.send("recording-annotation-clear"); + notifyRecordingAnnotationTool(null); + return { success: true }; + }); + + ipcMain.handle("recording-annotation-overlay-hide", () => { + const overlayWindow = getRecordingAnnotationOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return { success: true }; + } + + notifyRecordingAnnotationTool(null); + overlayWindow.webContents.send("recording-annotation-clear"); + overlayWindow.setIgnoreMouseEvents(true, { forward: true }); + overlayWindow.hide(); + return { success: true }; + }); + + ipcMain.handle("recording-annotation-tool-set", async (_, tool: unknown) => { + const normalizedTool = + typeof tool === "string" && recordingAnnotationTools.has(tool) ? tool : null; + const overlayWindow = await ensureRecordingAnnotationOverlayWindow(); + if (!overlayWindow) { + return { success: false, error: "Recording annotation overlay is unavailable." }; + } + + overlayWindow.setIgnoreMouseEvents(!normalizedTool, { forward: true }); + notifyRecordingAnnotationTool(normalizedTool); + return { success: true, tool: normalizedTool }; + }); + + ipcMain.handle("recording-annotation-clear", () => { + const overlayWindow = getRecordingAnnotationOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return { success: true }; + } + + overlayWindow.webContents.send("recording-annotation-clear"); + return { success: true }; + }); + + ipcMain.handle("recording-annotation-undo", () => { + const overlayWindow = getRecordingAnnotationOverlayWindow?.(); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return { success: true }; + } + + overlayWindow.webContents.send("recording-annotation-undo"); + return { success: true }; + }); + ipcMain.handle("is-native-windows-capture-available", async () => { if (!isWindowsGraphicsCaptureOsSupported()) { return { success: true, available: false, reason: "unsupported-os" }; diff --git a/electron/main.ts b/electron/main.ts index 3e2258f8f4..09198c7a83 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -17,6 +17,7 @@ import { createCountdownOverlayWindow, createEditorWindow, createHudOverlayWindow, + createRecordingAnnotationOverlayWindow, createSourceSelectorWindow, } from "./windows"; @@ -77,6 +78,7 @@ process.env.VITE_PUBLIC = VITE_DEV_SERVER_URL let mainWindow: BrowserWindow | null = null; let sourceSelectorWindow: BrowserWindow | null = null; let countdownOverlayWindow: BrowserWindow | null = null; +let recordingAnnotationOverlayWindow: BrowserWindow | null = null; let tray: Tray | null = null; let selectedSourceName = ""; const isMac = process.platform === "darwin"; @@ -416,6 +418,21 @@ function createCountdownOverlayWindowWrapper() { return countdownOverlayWindow; } +function createRecordingAnnotationOverlayWindowWrapper(bounds?: Electron.Rectangle) { + if (recordingAnnotationOverlayWindow && !recordingAnnotationOverlayWindow.isDestroyed()) { + if (bounds) { + recordingAnnotationOverlayWindow.setBounds(bounds, false); + } + return recordingAnnotationOverlayWindow; + } + + recordingAnnotationOverlayWindow = createRecordingAnnotationOverlayWindow(bounds); + recordingAnnotationOverlayWindow.on("closed", () => { + recordingAnnotationOverlayWindow = null; + }); + return recordingAnnotationOverlayWindow; +} + // Closing every window quits the app entirely (tray icon goes too). // The in-app "Return to Recorder" button covers the editor → HUD round-trip, // so closing the last window is an explicit "I'm done" signal. @@ -433,7 +450,10 @@ app.on("activate", () => { const url = window.webContents.getURL(); const isCountdownOverlayWindow = url.includes("windowType=countdown-overlay"); - return !isCountdownOverlayWindow; + const isRecordingAnnotationOverlayWindow = url.includes( + "windowType=recording-annotation-overlay", + ); + return !isCountdownOverlayWindow && !isRecordingAnnotationOverlayWindow; }); if (!hasVisibleWindow) { showMainWindow(); @@ -532,9 +552,11 @@ app.whenReady().then(async () => { createEditorWindowWrapper, createSourceSelectorWindowWrapper, createCountdownOverlayWindowWrapper, + createRecordingAnnotationOverlayWindowWrapper, () => mainWindow, () => sourceSelectorWindow, () => countdownOverlayWindow, + () => recordingAnnotationOverlayWindow, (recording: boolean, sourceName: string) => { selectedSourceName = sourceName; if (!tray) createTray(); diff --git a/electron/preload.ts b/electron/preload.ts index 73edfd9e21..f13f78f0e0 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,4 +1,5 @@ import { contextBridge, ipcRenderer } from "electron"; +import type { RecordingAnnotationTool } from "../src/components/launch/recordingAnnotations"; import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; @@ -242,6 +243,37 @@ contextBridge.exposeInMainWorld("electronAPI", { hideCountdownOverlay: (runId: number) => { return ipcRenderer.invoke("countdown-overlay-hide", runId); }, + showRecordingAnnotationOverlay: () => { + return ipcRenderer.invoke("recording-annotation-overlay-show"); + }, + hideRecordingAnnotationOverlay: () => { + return ipcRenderer.invoke("recording-annotation-overlay-hide"); + }, + setRecordingAnnotationTool: (tool: RecordingAnnotationTool | null) => { + return ipcRenderer.invoke("recording-annotation-tool-set", tool); + }, + clearRecordingAnnotations: () => { + return ipcRenderer.invoke("recording-annotation-clear"); + }, + undoRecordingAnnotation: () => { + return ipcRenderer.invoke("recording-annotation-undo"); + }, + onRecordingAnnotationToolChange: (callback: (tool: RecordingAnnotationTool | null) => void) => { + const listener = (_event: unknown, tool: unknown) => + callback((typeof tool === "string" ? tool : null) as RecordingAnnotationTool | null); + ipcRenderer.on("recording-annotation-tool", listener); + return () => ipcRenderer.removeListener("recording-annotation-tool", listener); + }, + onRecordingAnnotationClear: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("recording-annotation-clear", listener); + return () => ipcRenderer.removeListener("recording-annotation-clear", listener); + }, + onRecordingAnnotationUndo: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("recording-annotation-undo", listener); + return () => ipcRenderer.removeListener("recording-annotation-undo", listener); + }, onCountdownOverlayValue: (callback: (value: number | null) => void) => { const listener = (_event: unknown, value: number | null) => callback(value); ipcRenderer.on("countdown-overlay-value", listener); diff --git a/electron/windows.ts b/electron/windows.ts index 3a7350edf0..58da0dd45c 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -53,7 +53,7 @@ export function createHudOverlayWindow(): BrowserWindow { const primaryDisplay = screen.getPrimaryDisplay(); const { workArea } = primaryDisplay; - const windowWidth = 600; + const windowWidth = Math.min(880, Math.max(600, workArea.width - 24)); const windowHeight = 160; const x = Math.floor(workArea.x + (workArea.width - windowWidth) / 2); @@ -62,8 +62,8 @@ export function createHudOverlayWindow(): BrowserWindow { const win = new BrowserWindow({ width: windowWidth, height: windowHeight, - minWidth: 600, - maxWidth: 600, + minWidth: windowWidth, + maxWidth: windowWidth, minHeight: 160, maxHeight: 160, x: x, @@ -261,3 +261,47 @@ export function createCountdownOverlayWindow(): BrowserWindow { return win; } + +export function createRecordingAnnotationOverlayWindow(bounds?: Electron.Rectangle): BrowserWindow { + const targetBounds = bounds ?? screen.getPrimaryDisplay().bounds; + + const win = new BrowserWindow({ + x: targetBounds.x, + y: targetBounds.y, + width: targetBounds.width, + height: targetBounds.height, + frame: false, + resizable: false, + alwaysOnTop: true, + skipTaskbar: true, + focusable: true, + transparent: true, + backgroundColor: "#00000000", + hasShadow: false, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + backgroundThrottling: false, + }, + }); + + win.setIgnoreMouseEvents(true, { forward: true }); + win.setAlwaysOnTop(true, "floating"); + + if (process.platform === "darwin") { + win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); + } + + if (VITE_DEV_SERVER_URL) { + win.loadURL(VITE_DEV_SERVER_URL + "?windowType=recording-annotation-overlay"); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { + query: { windowType: "recording-annotation-overlay" }, + }); + } + + return win; +} diff --git a/src/App.tsx b/src/App.tsx index 6f737b9b0a..79dc4756ef 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,6 +1,7 @@ import { lazy, Suspense, useEffect, useState } from "react"; import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx"; import { LaunchWindow } from "./components/launch/LaunchWindow"; +import { RecordingAnnotationOverlay } from "./components/launch/RecordingAnnotationOverlay"; import { SourceSelector } from "./components/launch/SourceSelector"; import { Toaster } from "./components/ui/sonner"; import { TooltipProvider } from "./components/ui/tooltip"; @@ -25,7 +26,12 @@ export default function App() { setWindowType(type); } - if (type === "hud-overlay" || type === "source-selector" || type === "countdown-overlay") { + if ( + type === "hud-overlay" || + type === "source-selector" || + type === "countdown-overlay" || + type === "recording-annotation-overlay" + ) { document.body.style.background = "transparent"; document.documentElement.style.background = "transparent"; document.getElementById("root")?.style.setProperty("background", "transparent"); @@ -61,6 +67,8 @@ export default function App() { return ; case "countdown-overlay": return ; + case "recording-annotation-overlay": + return ; case "editor": return ( diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 570ec2809f..6735d61976 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,4 +1,16 @@ -import { Check, ChevronDown, Languages } from "lucide-react"; +import { + ArrowUpRight, + Check, + ChevronDown, + Circle, + Eraser, + Highlighter, + Languages, + Pencil, + Square, + Type, + Undo2, +} from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs"; @@ -32,6 +44,7 @@ import { AudioLevelMeter } from "../ui/audio-level-meter"; import { Button } from "../ui/button"; import { Tooltip } from "../ui/tooltip"; import styles from "./LaunchWindow.module.css"; +import type { RecordingAnnotationTool } from "./recordingAnnotations"; const ICON_SIZE = 20; @@ -78,6 +91,19 @@ const windowBtnClasses = const hudSidebarClasses = "ml-0.5 pl-1.5 border-l border-white/10 flex items-center gap-0.5"; +const annotationToolConfig = [ + { tool: "pen", label: "Pen / brush", icon: Pencil }, + { tool: "arrow", label: "Arrow", icon: ArrowUpRight }, + { tool: "rectangle", label: "Rectangle", icon: Square }, + { tool: "ellipse", label: "Circle", icon: Circle }, + { tool: "highlight", label: "Highlight", icon: Highlighter }, + { tool: "text", label: "Text", icon: Type }, +] satisfies Array<{ + tool: RecordingAnnotationTool; + label: string; + icon: typeof Pencil; +}>; + export function LaunchWindow() { const t = useScopedT("launch"); const availableLocales = getAvailableLocales(); @@ -129,6 +155,9 @@ export function LaunchWindow() { const webcamExpanded = isWebcamHovered || isWebcamFocused; const [isLanguageMenuOpen, setIsLanguageMenuOpen] = useState(false); const [supportsCursorModeToggle, setSupportsCursorModeToggle] = useState(false); + const [activeAnnotationTool, setActiveAnnotationTool] = useState( + null, + ); const languageTriggerRef = useRef(null); const languageMenuPanelRef = useRef(null); const [languageMenuStyle, setLanguageMenuStyle] = useState<{ @@ -217,6 +246,20 @@ export function LaunchWindow() { }); }, []); + useEffect(() => { + if (!recording && activeAnnotationTool) { + setActiveAnnotationTool(null); + } + }, [activeAnnotationTool, recording]); + + useEffect(() => { + const cleanup = window.electronAPI?.onRecordingAnnotationToolChange?.((nextTool) => { + setActiveAnnotationTool(nextTool); + }); + + return () => cleanup?.(); + }, []); + useEffect(() => { if (!isLanguageMenuOpen) return; @@ -371,6 +414,30 @@ export function LaunchWindow() { setMicrophoneEnabled(!microphoneEnabled); } }; + + const selectAnnotationTool = useCallback( + (tool: RecordingAnnotationTool) => { + const nextTool = activeAnnotationTool === tool ? null : tool; + setActiveAnnotationTool(nextTool); + window.electronAPI?.setRecordingAnnotationTool(nextTool).catch((error) => { + console.warn("Failed to update recording annotation tool:", error); + setActiveAnnotationTool(activeAnnotationTool); + }); + }, + [activeAnnotationTool], + ); + + const undoRecordingAnnotation = useCallback(() => { + window.electronAPI?.undoRecordingAnnotation?.().catch((error) => { + console.warn("Failed to undo recording annotation:", error); + }); + }, []); + + const clearRecordingAnnotations = useCallback(() => { + window.electronAPI?.clearRecordingAnnotations?.().catch((error) => { + console.warn("Failed to clear recording annotations:", error); + }); + }, []); const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null); const handleHudDragPointerDown = (event: React.PointerEvent) => { event.preventDefault(); @@ -694,6 +761,52 @@ export function LaunchWindow() { )} + {recording && ( +
+ {annotationToolConfig.map(({ tool, label, icon: Icon }) => { + const isActive = activeAnnotationTool === tool; + return ( + + + + ); + })} +
+ + + + + + +
+ )} + {/* Record/Stop group */}