Skip to content

feat: Add Gemini Integration in Frontend, implement media processing utilities and Deepgram transcription #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pranjalaman wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: Add Gemini Integration in Frontend, implement media processing utilities and Deepgram transcription #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@ dist
target
src-tauri/target
.env
.env.*
.vscode/
.DS_Store
*.log
2,358 changes: 2,358 additions & 0 deletions src-tauri/gen/schemas/windows-schema.json
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions src-tauri/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -894,8 +894,8 @@ fn build_drawtext_filters(
for path in &font_paths {
if std::path::Path::new(path).exists() {
let escaped_path = path
.replace('\\', "\\\\")
.replace(':', "\\:")
.replace('\\', "/")
.replace(':', "\\\\:")
.replace('\'', "\\'")
.replace(' ', "\\ ");
font_option = format!("fontfile={}:", escaped_path);
Expand Down
4 changes: 2 additions & 2 deletions src-tauri/src/media.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,10 @@ pub fn extract_audio(source_path: &str, project_dir: &Path) -> Result<PathBuf> {
}

std::fs::create_dir_all(project_dir)?;
let output_path = project_dir.join("transcription_audio.wav");
let output_path = project_dir.join("transcription_audio.mp3");

let output = Command::new("ffmpeg")
.args(["-y", "-i", source_path, "-vn", "-ac", "1", "-ar", "16000"])
.args(["-y", "-i", source_path, "-vn", "-c:a", "libmp3lame", "-b:a", "64k", "-ac", "1", "-ar", "16000"])
.arg(&output_path)
.output()
.context("running ffmpeg audio extraction")?;
Expand Down
2 changes: 1 addition & 1 deletion src-tauri/src/transcription.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub async fn transcribe_deepgram(audio_path: &str, api_key: &str) -> Result<Norm
let response = reqwest::Client::new()
.post("https://api.deepgram.com/v1/listen?model=nova-2&smart_format=true&diarize=true&punctuate=true&filler_words=true")
.header("Authorization", format!("Token {api_key}"))
.header("Content-Type", "audio/wav")
.header("Content-Type", "audio/mpeg")
.body(bytes)
.send()
.await
Expand Down
36 changes: 29 additions & 7 deletions src/main.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ function App() {
// 2. LLM Moments
try {
setBusy("moments");
const activeKey = llmEngine === "claude" ? anthropicKey.trim() : (llmEngine === "deepseek" ? deepseekKey.trim() : "");
const activeKey = llmEngine === "claude" ? anthropicKey.trim() : llmEngine === "deepseek" ? deepseekKey.trim() : llmEngine === "gemini" ? geminiKey.trim() : llmEngine === "openai" ? openaiKey.trim() : llmEngine === "openrouter" ? openrouterKey.trim() : "";
await invoke<Candidate[]>("generate_candidates", {
projectId,
apiKey: activeKey || null,
Expand Down Expand Up @@ -489,7 +489,7 @@ function App() {
async function moments(allowDemo: boolean) {
if (!detail) return;
await run("moments", async () => {
const activeKey = llmEngine === "claude" ? anthropicKey.trim() : (llmEngine === "deepseek" ? deepseekKey.trim() : "");
const activeKey = llmEngine === "claude" ? anthropicKey.trim() : llmEngine === "deepseek" ? deepseekKey.trim() : llmEngine === "gemini" ? geminiKey.trim() : llmEngine === "openai" ? openaiKey.trim() : llmEngine === "openrouter" ? openrouterKey.trim() : "";
try {
await invoke<Candidate[]>("generate_candidates", {
projectId: detail.project.id,
Expand Down Expand Up @@ -580,9 +580,11 @@ function App() {
setDeepgramKey={setDeepgramKey}
setAnthropicKey={setAnthropicKey}
setDeepseekKey={setDeepseekKey}
setGeminiKey={setGeminiKey}
deepgramKey={deepgramKey}
anthropicKey={anthropicKey}
deepseekKey={deepseekKey}
geminiKey={geminiKey}
refreshEnv={() => refresh()}
/>
);
Expand Down Expand Up @@ -1185,14 +1187,16 @@ interface OnboardingProps {
environment: EnvironmentStatus | null;
onComplete: () => void;
setTranscriptionEngine: (engine: "deepgram" | "local") => void;
setLlmEngine: (engine: "claude" | "deepseek" | "local") => void;
setLlmEngine: (engine: "claude" | "deepseek" | "local" | "gemini" | "openai" | "openrouter") => void;
setLocalLlmModel: (model: string) => void;
setDeepgramKey: (key: string) => void;
setAnthropicKey: (key: string) => void;
setDeepseekKey: (key: string) => void;
setGeminiKey: (key: string) => void;
deepgramKey: string;
anthropicKey: string;
deepseekKey: string;
geminiKey: string;
refreshEnv: () => Promise<void>;
}

Expand All @@ -1205,9 +1209,11 @@ function Onboarding({
setDeepgramKey,
setAnthropicKey,
setDeepseekKey,
setGeminiKey,
deepgramKey: initialDeepgramKey,
anthropicKey: initialAnthropicKey,
deepseekKey: initialDeepseekKey,
geminiKey: initialGeminiKey,
refreshEnv,
}: OnboardingProps) {
const [setupMode, setSetupMode] = useState<"choose" | "local" | "cloud" | "downloading">("choose");
Expand All @@ -1216,6 +1222,7 @@ function Onboarding({
const [dgKey, setDgKey] = useState(initialDeepgramKey);
const [antKey, setAntKey] = useState(initialAnthropicKey);
const [dsKey, setDsKey] = useState(initialDeepseekKey);
const [gmKey, setGmKey] = useState(initialGeminiKey);

const [downloadStatus, setDownloadStatus] = useState("Initializing download...");
const [downloadProgress, setDownloadProgress] = useState(0);
Expand All @@ -1235,8 +1242,8 @@ function Onboarding({
setError("Deepgram API Key is required for cloud mode.");
return;
}
if (!antKey.trim() && !dsKey.trim()) {
setError("Please provide at least one LLM Key (Claude or DeepSeek).");
if (!antKey.trim() && !dsKey.trim() && !gmKey.trim()) {
setError("Please provide at least one LLM Key (Claude, DeepSeek, or Gemini).");
return;
}

Expand All @@ -1245,7 +1252,12 @@ function Onboarding({
localStorage.setItem("autoshorts_deepgram_key", dgKey.trim());
localStorage.setItem("autoshorts_transcription_engine", "deepgram");

if (antKey.trim()) {
if (gmKey.trim()) {
setLlmEngine("gemini");
setGeminiKey(gmKey.trim());
localStorage.setItem("autoshorts_gemini_key", gmKey.trim());
localStorage.setItem("autoshorts_llm_engine", "gemini");
} else if (antKey.trim()) {
setLlmEngine("claude");
setAnthropicKey(antKey.trim());
localStorage.setItem("autoshorts_anthropic_key", antKey.trim());
Expand Down Expand Up @@ -1500,7 +1512,17 @@ function Onboarding({
placeholder="Insert your DeepSeek API Key (alternative moment detection)"
/>
</div>
<p className="form-help">* Deepgram Key + at least one LLM Key (Claude or DeepSeek) is required.</p>

<div className="input-group">
<label>Gemini API Key</label>
<input
type="password"
value={gmKey}
onChange={(e) => setGmKey(e.target.value)}
placeholder="Insert your Google Gemini API Key (moment detection)"
/>
</div>
<p className="form-help">* Deepgram Key + at least one LLM Key (Claude, DeepSeek, or Gemini) is required.</p>
</div>

<div className="onboarding-actions">
Expand Down