diff --git a/.env.example b/.env.example index 6b442a1..eb58a3a 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,5 @@ DEEPGRAM_API_KEY=your_deepgram_key_here ANTHROPIC_API_KEY=your_anthropic_key_here LLM_PROVIDER=deepseek DEEPSEEK_API_KEY= +OPENAI_API_KEY= +GEMINI_API_KEY= diff --git a/.gitignore b/.gitignore index b6c81a6..5a9af55 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ node_modules dist +bun.lock +src-tauri/gen/schemas/* +package-lock.json target src-tauri/target .env diff --git a/README.md b/README.md index af94aed..96dce43 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repository implements the desktop app foundation using **Tauri 2 + React + ## Key Features -- **Dynamic Multi-LLM Support**: Supports both **DeepSeek** (default) and **Claude** (anthropic) for viral moment detection and hooks analysis. +- **Dynamic Multi-LLM Support**: Supports **DeepSeek** (default), **Claude** (anthropic), **OpenAI**, and **Google Gemini** for cloud-based viral moment detection and hooks analysis, plus **Ollama** for fully-local inference. Pick your provider and model in API Settings. - **Automated Pipeline**: Imports media, extracts audio, transcribes using Deepgram, and automatically analyzes and ranks moments in a single automated chain. - **Local SQLite Storage**: Saves transcripts, candidates, custom names, and rendering data locally. - **Native Project Manager**: Create, open, rename, and delete projects from the dashboard. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 567d3d4..01375a8 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -51,9 +51,8 @@ async fn environment_status(state: tauri::State<'_, AppState>) -> Result { - let key = api_key - .or_else(|| std::env::var("GEMINI_API_KEY").ok()) - .ok_or_else(|| "Set GEMINI_API_KEY or supply Gemini API Key to generate candidates.".to_string())?; - llm::detect_candidates_with_gemini(&normalized, &key) - .await - .map_err(to_command_error)? - } "openai" => { let key = api_key .or_else(|| std::env::var("OPENAI_API_KEY").ok()) .ok_or_else(|| "Set OPENAI_API_KEY or supply OpenAI API Key to generate candidates.".to_string())?; - llm::detect_candidates_with_openai(&normalized, &key) + let model = model_name + .or_else(|| std::env::var("OPENAI_MODEL").ok()) + .unwrap_or_else(|| "gpt-4o-mini".to_string()); + llm::detect_candidates_with_openai(&normalized, &key, &model) .await .map_err(to_command_error)? } - "openrouter" => { + "gemini" => { let key = api_key - .or_else(|| std::env::var("OPENROUTER_API_KEY").ok()) - .ok_or_else(|| "Set OPENROUTER_API_KEY or supply OpenRouter API Key to generate candidates.".to_string())?; - llm::detect_candidates_with_openrouter(&normalized, &key) + .or_else(|| std::env::var("GEMINI_API_KEY").ok()) + .ok_or_else(|| "Set GEMINI_API_KEY or supply Gemini API Key to generate candidates.".to_string())?; + let model = model_name + .or_else(|| std::env::var("GEMINI_MODEL").ok()) + .unwrap_or_else(|| "gemini-2.5-flash".to_string()); + llm::detect_candidates_with_gemini(&normalized, &key, &model) .await .map_err(to_command_error)? } @@ -893,12 +890,7 @@ fn build_drawtext_filters( let mut font_option = String::new(); for path in &font_paths { if std::path::Path::new(path).exists() { - let escaped_path = path - .replace('\\', "\\\\") - .replace(':', "\\:") - .replace('\'', "\\'") - .replace(' ', "\\ "); - font_option = format!("fontfile={}:", escaped_path); + font_option = format!("fontfile='{}':", path.replace(':', r"\:")); break; } } diff --git a/src-tauri/src/llm.rs b/src-tauri/src/llm.rs index 0a2cd5f..9735ff1 100644 --- a/src-tauri/src/llm.rs +++ b/src-tauri/src/llm.rs @@ -86,106 +86,24 @@ Avoid rambling setup, context-dependent references, and pure filler. Return up t } #[derive(Debug, Deserialize)] -struct GeminiResponse { - candidates: Vec, -} - -#[derive(Debug, Deserialize)] -struct GeminiCandidate { - content: GeminiContent, -} - -#[derive(Debug, Deserialize)] -struct GeminiContent { - parts: Vec, -} - -#[derive(Debug, Deserialize)] -struct GeminiPart { - text: Option, -} - -pub async fn detect_candidates_with_gemini( - transcript: &NormalizedTranscript, - api_key: &str, -) -> Result> { - let segments = compact_segments(&transcript.segments); - let prompt = format!( - "You are identifying the most viral moments and strongest short-form clip candidates from a long-form transcript. \ -For each candidate, the clip must be self-contained, starting with an extremely engaging hook within the first 3 seconds (to capture immediate attention on social feeds), \ -30-90 seconds long, and cut at clean sentence/thought boundaries. Favor highly shareable content: concrete stories, \ -strong opinions, emotional turns, surprising or counter-intuitive claims, clear payoffs, and high-energy/dramatic peaks. \ -Avoid rambling setup, context-dependent references, and pure filler. Return up to 10 candidates as JSON matching this schema: \ -{{\"candidates\":[{{\"start\":0.0,\"end\":0.0,\"score\":0.0,\"hook\":\"...\",\"rationale\":\"...\"}}]}}\n\nTranscript:\n{segments}" - ); - - let model = std::env::var("GEMINI_MODEL").unwrap_or_else(|_| "gemini-2.5-flash".to_string()); - let url = format!( - "https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}", - model, api_key - ); - - let response = reqwest::Client::new() - .post(&url) - .json(&json!({ - "contents": [ - { - "parts": [ - { - "text": prompt - } - ] - } - ], - "generationConfig": { - "responseMimeType": "application/json", - "temperature": 0.2 - } - })) - .send() - .await - .context("calling Gemini")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow!("Gemini request failed ({status}): {body}")); - } - - let res_body: GeminiResponse = response.json().await.context("parsing Gemini response")?; - let text = res_body - .candidates - .first() - .and_then(|c| c.content.parts.first()) - .and_then(|p| p.text.clone()) - .ok_or_else(|| anyhow!("Gemini response did not include content text"))?; - - let min_duration = if transcript.duration < 60.0 { - (transcript.duration * 0.5).max(5.0) - } else { - 30.0 - }; - parse_candidate_json(&text, min_duration) -} - -#[derive(Debug, Deserialize)] -struct ChatCompletionResponse { - choices: Vec, +struct OpenAiMessage { + content: String, } #[derive(Debug, Deserialize)] -struct ChatCompletionChoice { - message: ChatCompletionMessage, +struct OpenAiChoice { + message: OpenAiMessage, } #[derive(Debug, Deserialize)] -struct ChatCompletionMessage { - content: String, +struct OpenAiResponse { + choices: Vec, } pub async fn detect_candidates_with_openai( transcript: &NormalizedTranscript, api_key: &str, + model: &str, ) -> Result> { let segments = compact_segments(&transcript.segments); let prompt = format!( @@ -197,8 +115,6 @@ Avoid rambling setup, context-dependent references, and pure filler. Return up t {{\"candidates\":[{{\"start\":0.0,\"end\":0.0,\"score\":0.0,\"hook\":\"...\",\"rationale\":\"...\"}}]}}\n\nTranscript:\n{segments}" ); - let model = std::env::var("OPENAI_MODEL").unwrap_or_else(|_| "gpt-4o-mini".to_string()); - let response = reqwest::Client::new() .post("https://api.openai.com/v1/chat/completions") .header("Authorization", format!("Bearer {api_key}")) @@ -225,8 +141,7 @@ Avoid rambling setup, context-dependent references, and pure filler. Return up t return Err(anyhow!("OpenAI request failed ({status}): {body}")); } - let res_body: ChatCompletionResponse = - response.json().await.context("parsing OpenAI response")?; + let res_body: OpenAiResponse = response.json().await.context("parsing OpenAI response")?; let text = res_body .choices .first() @@ -241,9 +156,30 @@ Avoid rambling setup, context-dependent references, and pure filler. Return up t parse_candidate_json(&text, min_duration) } -pub async fn detect_candidates_with_openrouter( +#[derive(Debug, Deserialize)] +struct GeminiPart { + text: Option, +} + +#[derive(Debug, Deserialize)] +struct GeminiContent { + parts: Vec, +} + +#[derive(Debug, Deserialize)] +struct GeminiCandidate { + content: GeminiContent, +} + +#[derive(Debug, Deserialize)] +struct GeminiResponse { + candidates: Vec, +} + +pub async fn detect_candidates_with_gemini( transcript: &NormalizedTranscript, api_key: &str, + model: &str, ) -> Result> { let segments = compact_segments(&transcript.segments); let prompt = format!( @@ -255,44 +191,46 @@ Avoid rambling setup, context-dependent references, and pure filler. Return up t {{\"candidates\":[{{\"start\":0.0,\"end\":0.0,\"score\":0.0,\"hook\":\"...\",\"rationale\":\"...\"}}]}}\n\nTranscript:\n{segments}" ); - let model = - std::env::var("OPENROUTER_MODEL").unwrap_or_else(|_| "google/gemini-2.5-flash".to_string()); + let url = format!("https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"); let response = reqwest::Client::new() - .post("https://openrouter.ai/api/v1/chat/completions") - .header("Authorization", format!("Bearer {api_key}")) + .post(&url) + .header("x-goog-api-key", api_key) .json(&json!({ - "model": model, - "messages": [ + "contents": [ { - "role": "user", - "content": prompt, + "parts": [ + { "text": prompt } + ] } ], - "temperature": 0.2, - "response_format": { - "type": "json_object" + "generationConfig": { + "temperature": 0.2, + "responseMimeType": "application/json" } })) .send() .await - .context("calling OpenRouter")?; + .context("calling Gemini")?; if !response.status().is_success() { let status = response.status(); let body = response.text().await.unwrap_or_default(); - return Err(anyhow!("OpenRouter request failed ({status}): {body}")); + return Err(anyhow!("Gemini request failed ({status}): {body}")); } - let res_body: ChatCompletionResponse = response - .json() - .await - .context("parsing OpenRouter response")?; + let res_body: GeminiResponse = response.json().await.context("parsing Gemini response")?; let text = res_body - .choices - .first() - .map(|c| c.message.content.clone()) - .ok_or_else(|| anyhow!("OpenRouter response did not include choices content"))?; + .candidates + .into_iter() + .find_map(|candidate| { + candidate + .content + .parts + .into_iter() + .find_map(|part| part.text) + }) + .ok_or_else(|| anyhow!("Gemini response did not include text content"))?; let min_duration = if transcript.duration < 60.0 { (transcript.duration * 0.5).max(5.0) diff --git a/src-tauri/src/models.rs b/src-tauri/src/models.rs index 87d8ddc..5353a7a 100644 --- a/src-tauri/src/models.rs +++ b/src-tauri/src/models.rs @@ -9,9 +9,8 @@ pub struct EnvironmentStatus { pub has_deepgram_key: bool, pub has_anthropic_key: bool, pub has_deepseek_key: bool, - pub has_gemini_key: bool, pub has_openai_key: bool, - pub has_openrouter_key: bool, + pub has_gemini_key: bool, pub llm_provider: String, pub has_local_whisper_model: bool, pub has_ollama: bool, diff --git a/src/main.tsx b/src/main.tsx index 13f561f..fdadf18 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -32,9 +32,8 @@ type EnvironmentStatus = { hasDeepgramKey: boolean; hasAnthropicKey: boolean; hasDeepseekKey: boolean; - hasGeminiKey: boolean; hasOpenaiKey: boolean; - hasOpenrouterKey: boolean; + hasGeminiKey: boolean; llmProvider: string; hasLocalWhisperModel: boolean; hasOllama: boolean; @@ -111,6 +110,14 @@ type BusyState = | "clipCount" | "cut"; +function cloudLlmLabel(engine: string): string { + if (engine === "claude") return "Claude"; + if (engine === "deepseek") return "DeepSeek"; + if (engine === "openai") return "OpenAI"; + if (engine === "gemini") return "Gemini"; + return engine; +} + function App() { const [environment, setEnvironment] = useState(null); const [projects, setProjects] = useState([]); @@ -128,8 +135,8 @@ function App() { const [transcriptionEngine, setTranscriptionEngine] = useState<"deepgram" | "local">(() => { return (localStorage.getItem("autoshorts_transcription_engine") as "deepgram" | "local") || "local"; }); - const [llmEngine, setLlmEngine] = useState<"claude" | "deepseek" | "local" | "gemini" | "openai" | "openrouter">(() => { - return (localStorage.getItem("autoshorts_llm_engine") as "claude" | "deepseek" | "local" | "gemini" | "openai" | "openrouter") || "local"; + const [llmEngine, setLlmEngine] = useState<"claude" | "deepseek" | "local" | "openai" | "gemini">(() => { + return (localStorage.getItem("autoshorts_llm_engine") as "claude" | "deepseek" | "local" | "openai" | "gemini") || "local"; }); const [localLlmModel, setLocalLlmModel] = useState(() => { return localStorage.getItem("autoshorts_local_llm_model") || "llama3.2"; @@ -143,14 +150,17 @@ function App() { const [deepseekKey, setDeepseekKey] = useState(() => { return localStorage.getItem("autoshorts_deepseek_key") || ""; }); + const [openaiKey, setOpenaiKey] = useState(() => { + return localStorage.getItem("autoshorts_openai_key") || ""; + }); const [geminiKey, setGeminiKey] = useState(() => { return localStorage.getItem("autoshorts_gemini_key") || ""; }); - const [openaiKey, setOpenaiKey] = useState(() => { - return localStorage.getItem("autoshorts_openai_key") || ""; + const [openaiModel, setOpenaiModel] = useState(() => { + return localStorage.getItem("autoshorts_openai_model") || "gpt-4o-mini"; }); - const [openrouterKey, setOpenrouterKey] = useState(() => { - return localStorage.getItem("autoshorts_openrouter_key") || ""; + const [geminiModel, setGeminiModel] = useState(() => { + return localStorage.getItem("autoshorts_gemini_model") || "gemini-2.5-flash"; }); const [downloadingModelName, setDownloadingModelName] = useState(null); @@ -182,9 +192,8 @@ function App() { const canUseCloudKey = environment?.hasDeepgramKey || deepgramKey.trim().length > 0; const canUseClaude = environment?.hasAnthropicKey || anthropicKey.trim().length > 0; const canUseDeepseek = environment?.hasDeepseekKey || deepseekKey.trim().length > 0; - const canUseGemini = environment?.hasGeminiKey || geminiKey.trim().length > 0; const canUseOpenai = environment?.hasOpenaiKey || openaiKey.trim().length > 0; - const canUseOpenrouter = environment?.hasOpenrouterKey || openrouterKey.trim().length > 0; + const canUseGemini = environment?.hasGeminiKey || geminiKey.trim().length > 0; const canTranscribe = transcriptionEngine === "local" ? Boolean(environment?.hasLocalWhisperModel) @@ -192,15 +201,10 @@ function App() { const canUseActiveLlm = llmEngine === "local" ? Boolean(environment?.hasOllama) - : llmEngine === "claude" - ? canUseClaude - : llmEngine === "deepseek" - ? canUseDeepseek - : llmEngine === "gemini" - ? canUseGemini - : llmEngine === "openai" - ? canUseOpenai - : canUseOpenrouter; + : llmEngine === "claude" ? canUseClaude + : llmEngine === "deepseek" ? canUseDeepseek + : llmEngine === "openai" ? canUseOpenai + : canUseGemini; useEffect(() => { void refresh(); @@ -236,17 +240,21 @@ function App() { localStorage.setItem("autoshorts_deepseek_key", deepseekKey); }, [deepseekKey]); + useEffect(() => { + localStorage.setItem("autoshorts_openai_key", openaiKey); + }, [openaiKey]); + useEffect(() => { localStorage.setItem("autoshorts_gemini_key", geminiKey); }, [geminiKey]); useEffect(() => { - localStorage.setItem("autoshorts_openai_key", openaiKey); - }, [openaiKey]); + localStorage.setItem("autoshorts_openai_model", openaiModel); + }, [openaiModel]); useEffect(() => { - localStorage.setItem("autoshorts_openrouter_key", openrouterKey); - }, [openrouterKey]); + localStorage.setItem("autoshorts_gemini_model", geminiModel); + }, [geminiModel]); const pullModelDirectly = async (modelName: string) => { setDownloadingModelName(modelName); @@ -366,26 +374,13 @@ function App() { return; } } else { - const activeKey = - llmEngine === "claude" ? anthropicKey : - llmEngine === "deepseek" ? deepseekKey : - llmEngine === "gemini" ? geminiKey : - llmEngine === "openai" ? openaiKey : - llmEngine === "openrouter" ? openrouterKey : ""; const hasActiveKey = - llmEngine === "claude" ? (env.hasAnthropicKey || activeKey.trim().length > 0) : - llmEngine === "deepseek" ? (env.hasDeepseekKey || activeKey.trim().length > 0) : - llmEngine === "gemini" ? (env.hasGeminiKey || activeKey.trim().length > 0) : - llmEngine === "openai" ? (env.hasOpenaiKey || activeKey.trim().length > 0) : - llmEngine === "openrouter" ? (env.hasOpenrouterKey || activeKey.trim().length > 0) : false; + llmEngine === "claude" ? (env.hasAnthropicKey || anthropicKey.trim().length > 0) + : llmEngine === "deepseek" ? (env.hasDeepseekKey || deepseekKey.trim().length > 0) + : llmEngine === "openai" ? (env.hasOpenaiKey || openaiKey.trim().length > 0) + : (env.hasGeminiKey || geminiKey.trim().length > 0); if (!hasActiveKey) { - const engineName = - llmEngine === "claude" ? "Claude" : - llmEngine === "deepseek" ? "DeepSeek" : - llmEngine === "gemini" ? "Gemini" : - llmEngine === "openai" ? "OpenAI" : - llmEngine === "openrouter" ? "OpenRouter" : "LLM"; - setError(`Transcription complete. ${engineName} API Key is missing. Please add it in settings to analyze viral moments.`); + setError(`Transcription complete. ${cloudLlmLabel(llmEngine)} API Key is missing. Please add it in settings to analyze viral moments.`); return; } } @@ -408,12 +403,12 @@ function App() { // 2. LLM Moments try { setBusy("moments"); - const activeKey = llmEngine === "claude" ? anthropicKey.trim() : (llmEngine === "deepseek" ? deepseekKey.trim() : ""); + const activeKey = llmEngine === "claude" ? anthropicKey.trim() : llmEngine === "deepseek" ? deepseekKey.trim() : llmEngine === "openai" ? openaiKey.trim() : llmEngine === "gemini" ? geminiKey.trim() : ""; await invoke("generate_candidates", { projectId, apiKey: activeKey || null, provider: llmEngine, - modelName: llmEngine === "local" ? localLlmModel.trim() : null, + modelName: llmEngine === "local" ? localLlmModel.trim() : llmEngine === "openai" ? (openaiModel.trim() || null) : llmEngine === "gemini" ? (geminiModel.trim() || null) : null, allowDemo: false, }); await refresh(projectId); @@ -489,13 +484,13 @@ function App() { async function moments(allowDemo: boolean) { if (!detail) return; await run("moments", async () => { - const activeKey = llmEngine === "claude" ? anthropicKey.trim() : (llmEngine === "deepseek" ? deepseekKey.trim() : ""); + const activeKey = llmEngine === "claude" ? anthropicKey.trim() : llmEngine === "deepseek" ? deepseekKey.trim() : llmEngine === "openai" ? openaiKey.trim() : llmEngine === "gemini" ? geminiKey.trim() : ""; try { await invoke("generate_candidates", { projectId: detail.project.id, apiKey: activeKey || null, provider: llmEngine, - modelName: llmEngine === "local" ? localLlmModel.trim() : null, + modelName: llmEngine === "local" ? localLlmModel.trim() : llmEngine === "openai" ? (openaiModel.trim() || null) : llmEngine === "gemini" ? (geminiModel.trim() || null) : null, allowDemo, }); await refresh(detail.project.id); @@ -580,9 +575,13 @@ function App() { setDeepgramKey={setDeepgramKey} setAnthropicKey={setAnthropicKey} setDeepseekKey={setDeepseekKey} + setOpenaiKey={setOpenaiKey} + setGeminiKey={setGeminiKey} deepgramKey={deepgramKey} anthropicKey={anthropicKey} deepseekKey={deepseekKey} + openaiKey={openaiKey} + geminiKey={geminiKey} refreshEnv={() => refresh()} /> ); @@ -674,14 +673,13 @@ function App() { LLM Engine {transcriptionEngine === "deepgram" && ( @@ -717,40 +715,50 @@ function App() { /> )} - {llmEngine === "gemini" && ( - - )} {llmEngine === "openai" && ( - + <> + + + )} - {llmEngine === "openrouter" && ( - + {llmEngine === "gemini" && ( + <> + + + )} - {llmEngine === "local" && (