From d8016b55f27de19bdb4d11d4f37bd89d0af9e131 Mon Sep 17 00:00:00 2001 From: Christmas Day Date: Thu, 25 Jun 2026 18:21:47 +0100 Subject: [PATCH 1/4] OpenRouter provider support, latency fixes, and permission hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds OpenRouter as a second LLM provider alongside Gemini, with full routing in the LLM service, onboarding wizard, and settings UI. Key changes: - New _openrouter* methods in LLMService for image/text/transcription processing, using raw HTTPS (OpenAI-compatible API) - Provider routing guards in all public processing methods - Onboarding wizard: LLM provider choice step with conditional Gemini/OpenRouter key fields, summary display - Settings UI: provider dropdown with conditional field groups - first-run.js: getStatus() returns llmProvider, openrouterConfigured, openrouterModel; needsOnboarding() checks both providers - macOS permission descriptions (NSMicrophoneUsageDescription, NSScreenCaptureDescription, NSCameraUsageDescription) Performance improvements: - Parallelized dual-method retry (_raceGeminiMethods) — race SDK vs HTTPS instead of sequential fallback - Reduced config defaults: timeout 60000→30000ms, maxRetries 3→1 - Increased https maxSockets 1→4 - Removed 500ms artificial delays in main.js (lines 366, 583 -> 50ms) - Removed 200ms window rendering delays in llm-response-window.js - Removed 100ms display verification setTimeout - Removed 200ms resize request setTimeout Stability fixes: - try-catch around desktopCapturer.getSources() to prevent macOS 14+ crash when screen recording permission is missing - systemPreferences.getMediaAccessStatus('screen') pre-check with clear error message - _ensureMicrophonePermission() using askForMediaAccess('microphone') guards all recording entry points - isDestroyed() guard in showLLMLoading() to prevent sending IPC to a destroyed window --- env.example | 12 +- main.js | 141 ++++++-- onboarding.html | 125 ++++++-- onboarding.js | 123 +++++-- package-lock.json | 53 ++- package.json | 12 +- settings.html | 49 ++- src/core/config.js | 18 +- src/core/first-run.js | 13 +- src/managers/window.manager.js | 8 +- src/services/capture.service.js | 46 ++- src/services/llm.service.js | 550 ++++++++++++++++++++++++++++---- src/ui/llm-response-window.js | 75 ++--- src/ui/settings-window.js | 39 ++- 14 files changed, 1055 insertions(+), 209 deletions(-) diff --git a/env.example b/env.example index f5fd56a3..97006367 100644 --- a/env.example +++ b/env.example @@ -1,7 +1,17 @@ -# Google Gemini API Configuration +# LLM Provider Configuration +# Choose your AI backend: gemini or openrouter +LLM_PROVIDER=gemini + +# Google Gemini API Configuration (when LLM_PROVIDER=gemini) # Get your API key from: https://makersuite.google.com/app/apikey GEMINI_API_KEY=your_gemini_api_key_here +# OpenRouter API Configuration (when LLM_PROVIDER=openrouter) +# Get your API key from: https://openrouter.ai/keys +OPENROUTER_API_KEY= +# Model to use — "openrouter/free" auto-selects free vision-capable models +OPENROUTER_MODEL=openrouter/free + # Speech Recognition Configuration # Choose one provider: azure or whisper SPEECH_PROVIDER=whisper diff --git a/main.js b/main.js index a96510d6..6f9555ad 100644 --- a/main.js +++ b/main.js @@ -1,8 +1,15 @@ -require("dotenv").config(); - const path = require("path"); const { app, BrowserWindow, globalShortcut, session, ipcMain } = require("electron"); +// .env lives in app.getPath("userData") (~/Library/Application Support/OpenCluely/) +// so settings survive re-launches when the app is installed in /Applications. +// For development, .env in process.cwd() is also loaded but userData takes priority +// so saved settings persist across restarts. +const userDataPath = app.getPath("userData"); +const userDataEnv = path.join(userDataPath, ".env"); +require("dotenv").config({ path: userDataEnv }); +require("dotenv").config(); + // ── Linux GPU process crash workaround ── // On many Linux setups (Wayland, X11 without GPU drivers, Docker, headless, // or systems with broken Mesa/NVIDIA stacks), Chromium's GPU process crashes @@ -55,14 +62,16 @@ class ApplicationController { this.codingLanguage = "cpp"; this.speechAvailable = false; + // .env lives in userData so settings survive re-launches from /Applications + this.envPath = path.join(app.getPath("userData"), ".env"); + // First-run onboarding: detects missing .env / API key and triggers // a settings-window prompt on first launch so users don't have to // dig through docs to figure out they need a Gemini API key. this.firstRunManager = new FirstRunManager({ logger: logger, - // Sentinel lives in userData so it survives cwd changes - // (the app may be launched from any directory). sentinelPath: path.join(app.getPath("userData"), ".opencluely-firstrun-completed"), + envPath: this.envPath, }); // Lazily-initialised in getWhisperInstaller() so tests can mock // the constructor without polluting main-process startup. @@ -267,6 +276,41 @@ class ApplicationController { ); } + async _ensureMicrophonePermission() { + // On macOS 14+ the first microphone hardware access via a spawned + // subprocess (sox/rec) can crash the app if permission hasn't been + // granted yet. Use the programmatic API to trigger the dialog safely. + if (process.platform !== "darwin") return; + + const { systemPreferences } = require("electron"); + if (!systemPreferences.getMediaAccessStatus) return; + + const status = systemPreferences.getMediaAccessStatus("microphone"); + logger.debug("Microphone permission status", { status }); + + if (status === "granted") return; + + if (status === "denied") { + throw new Error( + "Microphone permission denied. " + + "Go to System Settings → Privacy & Security → Microphone, " + + "find OpenCluely in the list and check the box, then restart the app." + ); + } + + // status === "not-determined" — first time; ask safely via the API + if (status === "not-determined" && systemPreferences.askForMediaAccess) { + const granted = await systemPreferences.askForMediaAccess("microphone"); + if (!granted) { + throw new Error( + "Microphone permission was not granted. " + + "Go to System Settings → Privacy & Security → Microphone " + + "to enable access." + ); + } + } + } + setupGlobalShortcuts() { const shortcuts = { "CommandOrControl+Shift+S": () => this.triggerScreenshotOCR(), @@ -329,7 +373,7 @@ class ApplicationController { text: text.substring(0, 100) }); } - }, 500); + }, 50); }); speechService.on("interim-transcription", (text) => { @@ -379,8 +423,17 @@ class ApplicationController { return speechService.isAvailable ? speechService.isAvailable() : false; }); - ipcMain.handle("start-speech-recognition", () => { - speechService.startRecording(); + ipcMain.handle("start-speech-recognition", async () => { + try { + await this._ensureMicrophonePermission(); + speechService.startRecording(); + } catch (e) { + logger.warn("Microphone permission check failed", { error: e.message }); + windowManager.broadcastToAllWindows("speech-status", { + status: e.message, + available: false, + }); + } return speechService.getStatus(); }); @@ -390,8 +443,13 @@ class ApplicationController { }); // Also handle direct send events for fallback - ipcMain.on("start-speech-recognition", () => { - speechService.startRecording(); + ipcMain.on("start-speech-recognition", async () => { + try { + await this._ensureMicrophonePermission(); + speechService.startRecording(); + } catch (e) { + logger.warn("Microphone permission check failed", { error: e.message }); + } }); ipcMain.on("stop-speech-recognition", () => { @@ -532,7 +590,7 @@ class ApplicationController { text: text.substring(0, 100) }); } - }, 500); + }, 50); return { success: true }; }); @@ -642,9 +700,10 @@ class ApplicationController { try { this.firstRunManager.markCompleted(); this.isFirstRun = false; - // Reinitialize speech service with the latest persisted settings - // so the mic button reflects the provider/command set during onboarding. + // Reinitialize services with the latest persisted settings + // so the mic button and AI calls reflect onboarding choices. speechService.initializeClient(); + llmService.initializeClient(); this.speechAvailable = speechService.isAvailable ? speechService.isAvailable() : false; @@ -840,7 +899,7 @@ class ApplicationController { }); } - toggleSpeechRecognition() { + async toggleSpeechRecognition() { const isAvailable = typeof speechService.isAvailable === 'function' ? speechService.isAvailable() : !!speechService.getStatus?.().isInitialized; if (!isAvailable) { logger.warn("Speech recognition unavailable; toggle ignored"); @@ -861,11 +920,18 @@ class ApplicationController { } } else { try { + await this._ensureMicrophonePermission(); speechService.startRecording(); windowManager.showChatWindow(); logger.info("Speech recognition started via global shortcut"); } catch (error) { logger.error("Error starting speech recognition:", error); + try { + windowManager.broadcastToAllWindows("speech-status", { + status: error.message, + available: false, + }); + } catch (_) {} } } } @@ -1323,6 +1389,10 @@ class ApplicationController { selectedIcon: this.appIcon || "terminal", windowGap: windowManager.windowGap, + llmProvider: process.env.LLM_PROVIDER || "gemini", + openrouterKey: process.env.OPENROUTER_API_KEY || "", + openrouterModel: process.env.OPENROUTER_MODEL || "openrouter/free", + speechProvider: speechService.provider || "whisper", azureKey: process.env.AZURE_SPEECH_KEY || "", azureRegion: process.env.AZURE_SPEECH_REGION || "", @@ -1390,23 +1460,38 @@ class ApplicationController { if (settings.whisperSegmentMs !== undefined) { envUpdates.WHISPER_SEGMENT_MS = String(settings.whisperSegmentMs); } + if (settings.llmProvider !== undefined) { + envUpdates.LLM_PROVIDER = settings.llmProvider; + } + if (settings.openrouterKey !== undefined) { + envUpdates.OPENROUTER_API_KEY = settings.openrouterKey; + } + if (settings.openrouterModel !== undefined) { + envUpdates.OPENROUTER_MODEL = settings.openrouterModel; + } if (settings.geminiKey !== undefined) { envUpdates.GEMINI_API_KEY = settings.geminiKey; } const persistedKeys = this.persistEnvUpdates(envUpdates); - // If the Gemini key was just saved, reinitialize the LLM service - // so the new client picks up the key. Without this, the test- - // connection button in the onboarding wizard fails with - // "Service not initialized" because the client was first created - // at app startup, before any key was set. - if (settings.geminiKey !== undefined && envUpdates.GEMINI_API_KEY !== undefined) { + // Reinitialize LLM service when provider, API key, or model changes. + // Without this the test-connection button and all AI calls would + // use stale credentials or the wrong provider. + const llmProviderChanged = settings.llmProvider !== undefined && + (process.env.LLM_PROVIDER || "gemini") !== settings.llmProvider; + const llmKeyChanged = settings.geminiKey !== undefined || settings.openrouterKey !== undefined; + const llmModelChanged = settings.openrouterModel !== undefined; + if (llmProviderChanged || llmKeyChanged || llmModelChanged) { try { llmService.initializeClient(); - logger.info("LLM service reinitialized after Gemini key update"); + logger.info("LLM service reinitialized after settings change", { + providerChanged: llmProviderChanged, + keyChanged: llmKeyChanged, + modelChanged: llmModelChanged + }); } catch (e) { - logger.warn("Failed to reinitialize LLM service after Gemini key update", { + logger.warn("Failed to reinitialize LLM service", { error: e.message }); } @@ -1448,8 +1533,12 @@ class ApplicationController { } } + const redacted = { ...settings }; + if (redacted.geminiKey) redacted.geminiKey = redacted.geminiKey.substring(0, 4) + "…"; + if (redacted.openrouterKey) redacted.openrouterKey = redacted.openrouterKey.substring(0, 4) + "…"; + if (redacted.azureKey) redacted.azureKey = redacted.azureKey.substring(0, 4) + "…"; logger.info("Settings saved successfully", { - ...settings, + ...redacted, persistedEnvKeys: persistedKeys }); return { success: true, persistedEnvKeys: persistedKeys }; @@ -1481,7 +1570,13 @@ class ApplicationController { const fs = require("fs"); const path = require("path"); - const envPath = path.join(process.cwd(), ".env"); + const envPath = this.envPath; + + // Ensure the parent directory exists (userData is usually pre-created, + // but guard against edge cases like a sandboxed fresh install). + try { + fs.mkdirSync(path.dirname(envPath), { recursive: true }); + } catch (_) { /* best effort */ } let existing = ""; try { diff --git a/onboarding.html b/onboarding.html index 7e2db50d..f17b5bfa 100644 --- a/onboarding.html +++ b/onboarding.html @@ -72,6 +72,10 @@ flex-shrink: 0; position: relative; z-index: 1; + -webkit-app-region: drag; + } + .header > * { + -webkit-app-region: no-drag; } .brand { display: flex; @@ -713,6 +717,8 @@
+
+
@@ -740,37 +746,104 @@

Welcome to OpenCluely

-

Connect Google Gemini

+

Choose your AI Provider

- OpenCluely uses Google's Gemini to generate answers. You'll need a free API key. + Pick the AI backend OpenCluely will use to generate answers. You can change this later in Settings.

-
- -
- - - +
+
+
+
+
Google Gemini
+
+ Free API key from Google. Reliable image understanding and fast responses. +
+
+
-
- Don't have one? Get a free key at - - aistudio.google.com/apikey - . - It's stored locally in .env — never sent anywhere except Google. +
+
+
+
OpenRouter (Free models)
+
+ Access many free vision-capable models through one API. The openrouter/free router auto-selects the best available model. +
+
+
- + + +
+
+ +
+ + + +
+
+ Don't have one? Get a free key at + + aistudio.google.com/apikey + . +
+ +
+
+ + + diff --git a/onboarding.js b/onboarding.js index c6e3a449..f51c2d62 100644 --- a/onboarding.js +++ b/onboarding.js @@ -30,7 +30,10 @@ // ── State ───────────────────────────────────────────────────────── const state = { step: 0, + llmProvider: 'gemini', // 'gemini' | 'openrouter' geminiKey: '', + openrouterKey: '', + openrouterModel: 'openrouter/free', speechProvider: null, // 'whisper' | 'azure' | 'skip' azureKey: '', azureRegion: '', @@ -122,6 +125,9 @@ case 'welcome': return true; case 'apikey': + if (state.llmProvider === 'openrouter') { + return !!state.openrouterKey.trim(); + } return !!state.geminiKey.trim(); case 'speech': if (state.speechProvider === 'azure') { @@ -181,6 +187,45 @@ : ''; }); + // ── Wire up: LLM provider choices ─────────────────────────────── + const llmProviderChoices = document.getElementById('llmProviderChoices'); + const onboardingGeminiFields = document.getElementById('onboardingGeminiFields'); + const onboardingOpenrouterFields = document.getElementById('onboardingOpenrouterFields'); + + if (llmProviderChoices) { + llmProviderChoices.querySelectorAll('.choice-card').forEach((card) => { + card.addEventListener('click', () => { + const value = card.dataset.value; + state.llmProvider = value; + llmProviderChoices.querySelectorAll('.choice-card').forEach((c) => c.classList.remove('selected')); + card.classList.add('selected'); + if (onboardingGeminiFields) { + onboardingGeminiFields.style.display = value === 'gemini' ? 'block' : 'none'; + } + if (onboardingOpenrouterFields) { + onboardingOpenrouterFields.style.display = value === 'openrouter' ? 'block' : 'none'; + } + // Reset the key status when switching providers + if (keyStatus) keyStatus.style.display = 'none'; + }); + }); + } + + // Wire up OpenRouter key and model inputs + const onboardingOpenrouterKey = document.getElementById('onboardingOpenrouterKey'); + const onboardingOpenrouterModel = document.getElementById('onboardingOpenrouterModel'); + + if (onboardingOpenrouterKey) { + onboardingOpenrouterKey.addEventListener('input', () => { + state.openrouterKey = onboardingOpenrouterKey.value.trim(); + }); + } + if (onboardingOpenrouterModel) { + onboardingOpenrouterModel.addEventListener('input', () => { + state.openrouterModel = onboardingOpenrouterModel.value.trim() || 'openrouter/free'; + }); + } + // ── Wire up: Speech choices ─────────────────────────────────────── $$('#speechChoices .choice-card').forEach((card) => { card.addEventListener('click', () => { @@ -436,11 +481,29 @@ // ── Wire up: Finish screen ──────────────────────────────────────── function populateSummary() { const rows = []; - rows.push({ - label: ' Gemini API', - value: state.geminiKey ? 'Configured' : 'Missing', - cls: state.geminiKey ? 'ok' : 'skip', - }); + if (state.llmProvider === 'openrouter') { + rows.push({ + label: ' LLM Provider', + value: 'OpenRouter', + cls: 'ok', + }); + rows.push({ + label: ' OpenRouter Key', + value: state.openrouterKey ? 'Configured' : 'Missing', + cls: state.openrouterKey ? 'ok' : 'skip', + }); + rows.push({ + label: ' Model', + value: state.openrouterModel, + cls: 'ok', + }); + } else { + rows.push({ + label: ' Gemini API', + value: state.geminiKey ? 'Configured' : 'Missing', + cls: state.geminiKey ? 'ok' : 'skip', + }); + } if (state.speechProvider === 'whisper') { rows.push({ label: ' Speech', @@ -500,16 +563,26 @@ const name = currentScreenName(); if (!canAdvance()) { // Lightly nudge the user - if (name === 'apikey') setKeyStatus('error', 'Enter a Gemini API key'); + if (name === 'apikey') { + const msg = state.llmProvider === 'openrouter' + ? 'Enter your OpenRouter API key' + : 'Enter your Gemini API key'; + setKeyStatus('error', msg); + } return; } - // Persist settings on speech selection (Azure path), since we - // already saved geminiKey on test; do it here too if user skipped - // testing. - if (name === 'apikey' && state.geminiKey && window.electronAPI) { + // Persist settings on API key step — saves provider, key, and model + if (name === 'apikey' && window.electronAPI) { try { - await window.electronAPI.saveSettings({ geminiKey: state.geminiKey }); + const payload = { llmProvider: state.llmProvider }; + if (state.llmProvider === 'openrouter') { + if (state.openrouterKey) payload.openrouterKey = state.openrouterKey; + payload.openrouterModel = state.openrouterModel; + } else if (state.geminiKey) { + payload.geminiKey = state.geminiKey; + } + await window.electronAPI.saveSettings(payload); } catch (_) { /* surfaced elsewhere */ } } if (name === 'speech' && window.electronAPI) { @@ -641,16 +714,28 @@ // ── Boot ────────────────────────────────────────────────────────── showScreen('welcome'); - // Pre-populate Gemini key from existing .env (if any) so users with + // Pre-populate provider and key from existing .env (if any) so users with // a partial config don't have to retype. - if (window.electronAPI && window.electronAPI.getFirstRunStatus) { - window.electronAPI.getFirstRunStatus().then((s) => { - if (s && s.geminiConfigured) { - // We can't read the key back (settings returns empty for keys), - // but we can mark status as success if the env file already has one. + (async function restoreExistingConfig() { + if (!window.electronAPI || !window.electronAPI.getFirstRunStatus) return; + try { + const s = await window.electronAPI.getFirstRunStatus(); + if (!s) return; + + // Restore provider choice + if (s.llmProvider === 'openrouter') { + state.llmProvider = 'openrouter'; + const orCard = llmProviderChoices?.querySelector('[data-value="openrouter"]'); + if (orCard) orCard.click(); + if (onboardingOpenrouterKey) onboardingOpenrouterKey.placeholder = '•••••••••••••••• (already set)'; + if (onboardingOpenrouterModel) onboardingOpenrouterModel.value = s.openrouterModel || 'openrouter/free'; + state.openrouterModel = s.openrouterModel || 'openrouter/free'; + } else if (s.geminiConfigured) { + const geminiCard = llmProviderChoices?.querySelector('[data-value="gemini"]'); + if (geminiCard) geminiCard.click(); setKeyStatus('success', 'Already configured — click Continue'); geminiInput.placeholder = '•••••••••••••••• (already set)'; } - }).catch(() => {}); - } + } catch (_) { /* ignore */ } + })(); })(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 708a5500..97446462 100644 --- a/package-lock.json +++ b/package-lock.json @@ -842,7 +842,6 @@ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1017,6 +1016,7 @@ "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "archiver-utils": "^2.1.0", "async": "^3.2.4", @@ -1036,6 +1036,7 @@ "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "glob": "^7.1.4", "graceful-fs": "^4.2.0", @@ -1058,6 +1059,7 @@ "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -1073,7 +1075,8 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/archiver-utils/node_modules/string_decoder": { "version": "1.1.1", @@ -1081,6 +1084,7 @@ "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -1200,6 +1204,7 @@ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -1615,6 +1620,7 @@ "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "buffer-crc32": "^0.2.13", "crc32-stream": "^4.0.2", @@ -1714,6 +1720,7 @@ "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "crc32": "bin/crc32.njs" }, @@ -1727,6 +1734,7 @@ "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -1912,7 +1920,6 @@ "integrity": "sha512-rcJUkMfnJpfCboZoOOPf4L29TRtEieHNOeAbYPWPxlaBw/Z1RKrRA86dOI9rwaI4tQSc/RD82zTNHprfUHXsoQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "24.13.3", "builder-util": "24.13.1", @@ -2108,6 +2115,7 @@ "integrity": "sha512-oHkV0iogWfyK+ah9ZIvMDpei1m9ZRpdXcvde1wTpra2U8AFDNNpqJdnin5z+PM1GbQ5BoaKCWas2HSjtR0HwMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "app-builder-lib": "24.13.3", "archiver": "^5.3.1", @@ -2121,6 +2129,7 @@ "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -2136,6 +2145,7 @@ "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -2149,6 +2159,7 @@ "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -2533,7 +2544,8 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/fs-extra": { "version": "8.1.0", @@ -3122,7 +3134,8 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/isbinaryfile": { "version": "5.0.4", @@ -3320,6 +3333,7 @@ "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -3333,6 +3347,7 @@ "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -3348,7 +3363,8 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lazystream/node_modules/string_decoder": { "version": "1.1.1", @@ -3356,6 +3372,7 @@ "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -3372,35 +3389,40 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.difference": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.union": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/logform": { "version": "2.7.0", @@ -3769,6 +3791,7 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -3944,7 +3967,8 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/progress": { "version": "2.0.3", @@ -4075,6 +4099,7 @@ "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -4490,6 +4515,7 @@ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -4708,7 +4734,6 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", "license": "MIT", - "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.2", @@ -4896,6 +4921,7 @@ "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", @@ -4911,6 +4937,7 @@ "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", diff --git a/package.json b/package.json index e120cc31..704e8ed3 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,12 @@ ], "darkModeSupport": true, "hardenedRuntime": false, - "gatekeeperAssess": false + "gatekeeperAssess": false, + "extendInfo": { + "NSMicrophoneUsageDescription": "OpenCluely needs microphone access to capture audio for speech-to-text transcription.", + "NSCameraUsageDescription": "OpenCluely needs screen capture access to analyse your screen content.", + "NSScreenCaptureDescription": "OpenCluely needs screen recording permission to capture and analyse your screen." + } }, "win": { "target": [ @@ -160,5 +165,10 @@ "height": 400 } } + }, + "allowScripts": { + "electron@29.4.6": true, + "@google/genai@2.9.0": true, + "protobufjs@7.6.4": true } } diff --git a/settings.html b/settings.html index c706860e..358c1557 100644 --- a/settings.html +++ b/settings.html @@ -261,6 +261,10 @@ width: 100%; } + .settings-group { + /* Groups provider-specific fields for show/hide toggling */ + } + .settings-note { font-size: 11px; color: rgba(255, 255, 255, 0.65); @@ -427,15 +431,52 @@
- Gemini Settings + LLM Provider
-
Google API Key
-
Your Google API key for Gemini models
+
Provider
+
Choose your AI backend
+
+ +
+ +
+
+
+
Gemini API Key
+
Your Google API key for Gemini models
+
+ +
+
+ + + +
diff --git a/src/core/config.js b/src/core/config.js index 91aba65c..7521d0c6 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -39,14 +39,12 @@ class ConfigManager { }, llm: { + provider: 'gemini', gemini: { - // 'gemini-3.5-flash' is Google's current stable Flash model. - // 'gemini-3-flash-preview' is the fallback because it also has a - // free tier and is less likely to hit demand spikes. model: 'gemini-3.5-flash', fallbackModels: ['gemini-3-flash-preview', 'gemini-2.5-flash'], - maxRetries: 3, - timeout: 60000, + maxRetries: 1, + timeout: 30000, fallbackEnabled: true, enableFallbackMethod: true, generation: { @@ -55,6 +53,16 @@ class ConfigManager { topP: 0.9, maxOutputTokens: 4096 } + }, + openrouter: { + baseUrl: 'https://openrouter.ai/api/v1', + model: 'openrouter/free', + maxRetries: 1, + timeout: 30000, + generation: { + temperature: 0.7, + maxOutputTokens: 4096 + } } }, diff --git a/src/core/first-run.js b/src/core/first-run.js index 67e3d133..2dc580cd 100644 --- a/src/core/first-run.js +++ b/src/core/first-run.js @@ -25,12 +25,18 @@ class FirstRunManager { /** * Returns true if this looks like a fresh install — no .env, no - * sentinel file, or .env exists but has no Gemini key. + * sentinel file, or .env exists but has no API key for the chosen + * provider. */ needsOnboarding() { if (!fs.existsSync(this.sentinelPath)) return true; if (!fs.existsSync(this.envPath)) return true; const content = this._readEnv(); + const provider = (content.LLM_PROVIDER || 'gemini').trim().toLowerCase(); + if (provider === 'openrouter') { + const orKey = (content.OPENROUTER_API_KEY || '').trim(); + return !orKey || orKey === 'your_openrouter_api_key_here'; + } const gemini = (content.GEMINI_API_KEY || '').trim(); return !gemini || gemini === 'your_gemini_api_key_here'; } @@ -80,10 +86,15 @@ class FirstRunManager { getStatus() { const env = this._readEnv(); const gemini = (env.GEMINI_API_KEY || '').trim(); + const orKey = (env.OPENROUTER_API_KEY || '').trim(); + const orKeyConfigured = !!orKey && orKey !== 'your_openrouter_api_key_here'; return { envExists: fs.existsSync(this.envPath), sentinelExists: fs.existsSync(this.sentinelPath), + llmProvider: env.LLM_PROVIDER || 'gemini', geminiConfigured: !!gemini && gemini !== 'your_gemini_api_key_here', + openrouterConfigured: orKeyConfigured, + openrouterModel: env.OPENROUTER_MODEL || 'openrouter/free', azureConfigured: !!(env.AZURE_SPEECH_KEY || '').trim() && !!(env.AZURE_SPEECH_REGION || '').trim(), whisperConfigured: !!(env.WHISPER_COMMAND || '').trim(), needsOnboarding: this.needsOnboarding() diff --git a/src/managers/window.manager.js b/src/managers/window.manager.js index e51c623b..ed7fd59e 100644 --- a/src/managers/window.manager.js +++ b/src/managers/window.manager.js @@ -935,6 +935,9 @@ class WindowManager { const previousAvailability = this.screenCaptureStatus.available; const checkedAt = new Date().toISOString(); + // Wrap in try-catch because desktopCapturer.getSources() can crash + // the Electron helper process on macOS 14+ when screen recording + // permission hasn't been granted. try { await desktopCapturer.getSources({ types: ['screen', 'window'], @@ -1292,7 +1295,7 @@ class WindowManager { } const llmWindow = this.windows.get('llmResponse'); - if (llmWindow) { + if (llmWindow && !llmWindow.isDestroyed()) { logger.debug('Showing LLM loading state'); llmWindow.webContents.send('show-loading'); this.showOnCurrentDesktop(llmWindow); @@ -1303,6 +1306,9 @@ class WindowManager { } logger.debug('LLM loading window shown'); + } else if (llmWindow && llmWindow.isDestroyed()) { + logger.error('LLM response window was destroyed; removing from window map'); + this.windows.delete('llmResponse'); } else { logger.error('LLM window not available for loading state'); } diff --git a/src/services/capture.service.js b/src/services/capture.service.js index 8b057308..e7c7a7b7 100644 --- a/src/services/capture.service.js +++ b/src/services/capture.service.js @@ -1,4 +1,4 @@ -const { desktopCapturer, screen } = require('electron'); +const { desktopCapturer, screen, systemPreferences } = require('electron'); const logger = require('../core/logger').createServiceLogger('CAPTURE'); class CaptureService { @@ -68,10 +68,46 @@ class CaptureService { const targetDisplay = this._getTargetDisplay(options.displayId); const { width, height } = targetDisplay.size || { width: 1920, height: 1080 }; - const sources = await desktopCapturer.getSources({ - types: ['screen'], - thumbnailSize: { width, height } - }); + // On macOS 14+ (Sonoma), calling desktopCapturer.getSources() without + // screen recording permission can crash the Electron helper process. + // Check permission status first and give a clear error if denied. + if (process.platform === 'darwin' && systemPreferences.getMediaAccessStatus) { + try { + const status = systemPreferences.getMediaAccessStatus('screen'); + if (status === 'denied') { + throw new Error( + 'Screen recording permission denied. ' + + 'Go to System Settings → Privacy & Security → Screen Recording, ' + + 'find OpenCluely in the list and check the box, then restart the app.' + ); + } + } catch (e) { + // getMediaAccessStatus('screen') may throw on older macOS or Electron versions. + // In that case, proceed with the capture attempt — it may still work. + if (e.message && e.message.includes('Screen recording permission denied')) { + throw e; + } + logger.warn('Could not check screen recording permission', { error: e.message }); + } + } + + let sources; + try { + sources = await desktopCapturer.getSources({ + types: ['screen'], + thumbnailSize: { width, height } + }); + } catch (e) { + // desktopCapturer.getSources() can crash the app on macOS 14+ when + // permission hasn't been granted. If it throws instead of crashing, + // we catch it here. On some Electron versions the crash is unrecoverable + // (segfault in the GPU process), so we also check below whether the + // call left the app in a broken state. + throw new Error( + `Screen capture failed: ${e.message}. ` + + 'Make sure Screen Recording permission is granted in System Settings.' + ); + } if (sources.length === 0) { throw new Error('No screen sources available for capture'); diff --git a/src/services/llm.service.js b/src/services/llm.service.js index eb4eabc7..0dc98769 100644 --- a/src/services/llm.service.js +++ b/src/services/llm.service.js @@ -10,11 +10,18 @@ class LLMService { this.isInitialized = false; this.requestCount = 0; this.errorCount = 0; + this.provider = 'gemini'; this.initializeClient(); } initializeClient() { + this.provider = (process.env.LLM_PROVIDER || 'gemini').toLowerCase(); + + if (this.provider === 'openrouter') { + return this._openrouterInit(); + } + const apiKey = config.getApiKey('GEMINI'); if (!apiKey || apiKey === 'your-api-key-here') { @@ -42,6 +49,419 @@ class LLMService { } } + _openrouterInit() { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey || apiKey.trim().length === 0) { + logger.warn('OpenRouter API key not configured'); + return; + } + this.model = config.get('llm.openrouter.model') || 'openrouter/free'; + this.isInitialized = true; + logger.info('OpenRouter client initialized successfully', { + model: this.model + }); + } + + _openrouterIsAvailable() { + const key = process.env.OPENROUTER_API_KEY; + return !!key && key.trim().length > 0; + } + + _openrouterGetModel() { + return process.env.OPENROUTER_MODEL || config.get('llm.openrouter.model') || 'openrouter/free'; + } + + _openrouterBuildMessages({ systemInstruction, contents }) { + const messages = []; + if (systemInstruction) { + const text = typeof systemInstruction === 'string' + ? systemInstruction + : systemInstruction.parts?.[0]?.text || ''; + if (text) { + messages.push({ role: 'system', content: text }); + } + } + if (contents && Array.isArray(contents)) { + for (const entry of contents) { + const role = entry.role === 'model' ? 'assistant' : 'user'; + const parts = entry.parts || []; + const contentParts = []; + for (const part of parts) { + if (part.text !== undefined) { + contentParts.push({ type: 'text', text: part.text }); + } else if (part.inlineData) { + const { data, mimeType } = part.inlineData; + contentParts.push({ + type: 'image_url', + image_url: { url: `data:${mimeType || 'image/png'};base64,${data}` } + }); + } + } + if (contentParts.length === 1 && contentParts[0].type === 'text') { + messages.push({ role, content: contentParts[0].text }); + } else if (contentParts.length > 0) { + messages.push({ role, content: contentParts }); + } + } + } + return messages; + } + + _openrouterBuildRequestBody(text, activeSkill, programmingLanguage, systemPrompt, conversationHistory) { + const messages = []; + if (systemPrompt) { + messages.push({ role: 'system', content: systemPrompt }); + } + if (conversationHistory && conversationHistory.length > 0) { + for (const event of conversationHistory) { + if (event.role === 'system') continue; + if (!event.content || typeof event.content !== 'string') continue; + const role = event.role === 'model' ? 'assistant' : 'user'; + messages.push({ role, content: event.content.trim() }); + } + } + const formattedMessage = conversationHistory && conversationHistory.length > 0 + ? text + : this.formatUserMessage(text, activeSkill); + messages.push({ role: 'user', content: formattedMessage }); + return messages; + } + + _openrouterBuildImageBody(imageBuffer, mimeType, activeSkill, programmingLanguage, skillPrompt) { + const messages = []; + if (skillPrompt) { + messages.push({ role: 'system', content: skillPrompt }); + } + const base64 = imageBuffer.toString('base64'); + messages.push({ + role: 'user', + content: [ + { type: 'text', text: this.formatImageInstruction(activeSkill, programmingLanguage) }, + { type: 'image_url', image_url: { url: `data:${mimeType || 'image/png'};base64,${base64}` } } + ] + }); + return messages; + } + + _openrouterBuildTranscriptionBody(text, activeSkill, programmingLanguage, conversationHistory) { + const messages = []; + const systemPrompt = this.getIntelligentTranscriptionPrompt(activeSkill, programmingLanguage); + if (systemPrompt) { + messages.push({ role: 'system', content: systemPrompt }); + } + if (conversationHistory && conversationHistory.length > 0) { + for (const event of conversationHistory) { + if (event.role === 'system') continue; + if (!event.content || typeof event.content !== 'string') continue; + const role = event.role === 'model' ? 'assistant' : 'user'; + messages.push({ role, content: event.content.trim() }); + } + } + messages.push({ role: 'user', content: text.trim() }); + return messages; + } + + async _openrouterExecute(messages, overrides = {}) { + const https = require('https'); + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey) { + throw new Error('OpenRouter API key not configured'); + } + + const baseUrl = config.get('llm.openrouter.baseUrl') || 'https://openrouter.ai/api/v1'; + const model = overrides.model || this._openrouterGetModel(); + const timeout = overrides.timeout || config.get('llm.openrouter.timeout') || 60000; + const genConfig = { ...config.get('llm.openrouter.generation'), ...overrides.generation }; + + const url = `${baseUrl}/chat/completions`; + const body = { + model, + messages, + temperature: genConfig.temperature, + max_tokens: genConfig.maxOutputTokens + }; + if (genConfig.topP !== undefined) body.top_p = genConfig.topP; + + const postData = JSON.stringify(body); + const maxRetries = overrides.maxRetries || config.get('llm.openrouter.maxRetries') || 3; + + let lastError; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + 'HTTP-Referer': 'https://opencluely.app', + 'X-Title': 'OpenCluely', + 'Content-Length': Buffer.byteLength(postData), + 'User-Agent': this.getUserAgent() + }, + timeout + }; + + const response = await new Promise((resolve, reject) => { + const req = https.request(url, options, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => { + if (res.statusCode === 200) { + try { + resolve({ status: res.statusCode, data: JSON.parse(data) }); + } catch (e) { + reject(new Error(`Failed to parse OpenRouter response: ${e.message}`)); + } + } else { + reject(new Error(`OpenRouter HTTP ${res.statusCode}: ${data.substring(0, 500)}`)); + } + }); + }); + req.on('error', (e) => reject(new Error(`OpenRouter request failed: ${e.message}`))); + req.on('timeout', () => { req.destroy(); reject(new Error('OpenRouter request timeout')); }); + req.write(postData); + req.end(); + }); + + const text = this._openrouterExtractResponse(response.data); + return text; + + } catch (error) { + lastError = error; + logger.warn(`OpenRouter attempt ${attempt}/${maxRetries} failed`, { + error: error.message, + model, + attempt + }); + + if (attempt < maxRetries) { + const delay = 1500 * attempt + Math.random() * 1000; + await this.delay(delay); + } + } + } + + throw lastError || new Error('OpenRouter request failed after all retries'); + } + + _openrouterExtractResponse(response) { + if (!response) { + throw new Error('Empty response from OpenRouter'); + } + if (!response.choices || !Array.isArray(response.choices) || response.choices.length === 0) { + throw new Error('No choices in OpenRouter response'); + } + const choice = response.choices[0]; + if (!choice.message || typeof choice.message.content !== 'string') { + throw new Error('No message content in OpenRouter response choice'); + } + if (choice.finish_reason === 'length') { + logger.warn('OpenRouter response reached max tokens limit', { + finishReason: choice.finish_reason + }); + } + return choice.message.content.trim(); + } + + async _openrouterProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const messages = this._openrouterBuildImageBody(imageBuffer, mimeType, activeSkill, programmingLanguage, skillPrompt); + + const responseText = await this._openrouterExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('OpenRouter image processing', startTime, { + activeSkill, + imageSize: imageBuffer.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'openrouter', + model: this._openrouterGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isImageAnalysis: true, + mimeType + } + }; + } catch (error) { + this.errorCount++; + logger.error('OpenRouter image processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse('[image]', activeSkill); + } + } + + async _openrouterProcessText(text, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(15) + : []; + + const messages = this._openrouterBuildRequestBody(text, activeSkill, programmingLanguage, skillPrompt, conversationHistory); + + const responseText = await this._openrouterExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('OpenRouter text processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'openrouter', + model: this._openrouterGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false + } + }; + } catch (error) { + this.errorCount++; + logger.error('OpenRouter text processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse(text, activeSkill); + } + } + + async _openrouterProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage) { + if (!text || typeof text !== 'string' || text.trim().length < 2) { + logger.warn('Skipping transcription for empty or very short input'); + return { + response: '', + metadata: { provider: 'openrouter', skill: activeSkill, processingTime: 0, usedFallback: true, isTranscriptionResponse: true } + }; + } + + const startTime = Date.now(); + this.requestCount++; + + try { + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(10) + : []; + + const messages = this._openrouterBuildTranscriptionBody(text.trim(), activeSkill, programmingLanguage, conversationHistory); + + const responseText = await this._openrouterExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('OpenRouter transcription processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'openrouter', + model: this._openrouterGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isTranscriptionResponse: true + } + }; + } catch (error) { + this.errorCount++; + logger.error('OpenRouter transcription processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateIntelligentFallbackResponse(text, activeSkill); + } + } + + async _openrouterTestConnection() { + try { + const messages = [{ role: 'user', content: 'Test connection. Please respond with "OK".' }]; + const startTime = Date.now(); + const text = await this._openrouterExecute(messages, { maxRetries: 1, generation: { temperature: 0, maxOutputTokens: 64 } }); + const latency = Date.now() - startTime; + + logger.info('OpenRouter connection test successful', { + response: text, + latency, + model: this._openrouterGetModel() + }); + + return { + success: true, + response: text, + latency, + model: this._openrouterGetModel() + }; + } catch (error) { + const errMsg = error.message || ''; + let friendlyError; + if (errMsg.includes('401') || errMsg.includes('unauthorized')) { + friendlyError = 'Invalid OpenRouter API key. Get one at openrouter.ai/keys.'; + } else if (errMsg.includes('402')) { + friendlyError = 'OpenRouter account needs credits or free model is unavailable.'; + } else if (errMsg.includes('429')) { + friendlyError = 'OpenRouter rate limit exceeded. Wait a moment and try again.'; + } else if (errMsg.includes('timeout') || errMsg.includes('ENOTFOUND') || errMsg.includes('ECONNREFUSED')) { + friendlyError = 'Cannot reach OpenRouter servers. Check your internet connection.'; + } else { + friendlyError = `OpenRouter error: ${errMsg.substring(0, 200)}`; + } + + return { + success: false, + error: friendlyError, + errorType: 'OPENROUTER_ERROR' + }; + } + } + getGenerationConfig(overrides = {}) { const defaults = config.get('llm.gemini.generation') || {}; const fallback = { @@ -122,13 +542,17 @@ class LLMService { */ async processImageWithSkill(imageBuffer, mimeType, activeSkill, sessionMemory = [], programmingLanguage = null) { if (!this.isInitialized) { - throw new Error('LLM service not initialized. Check Gemini API key configuration.'); + throw new Error('LLM service not initialized. Check API key configuration.'); } if (!imageBuffer || !Buffer.isBuffer(imageBuffer)) { throw new Error('Invalid image buffer provided to processImageWithSkill'); } + if (this.provider === 'openrouter') { + return this._openrouterProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage); + } + const startTime = Date.now(); this.requestCount++; @@ -158,30 +582,13 @@ class LLMService { request.systemInstruction = { parts: [{ text: skillPrompt }] }; } - // Execute with retries/timeout - try alternative method first for network reliability + // Race SDK and HTTPS methods in parallel — faster one wins let responseText; - const preferAlternative = !!config.get('llm.gemini.enableFallbackMethod'); try { - if (preferAlternative) { - logger.debug('Attempting alternative HTTPS method first for reliability'); - responseText = await this.executeAlternativeRequest(request); - } else { - responseText = await this.executeRequest(request); - } + responseText = await this._raceGeminiMethods(request); } catch (error) { - const secondaryLabel = preferAlternative ? 'primary SDK method' : 'alternative HTTPS method'; - logger.warn(`${preferAlternative ? 'Alternative' : 'Primary'} method failed, trying ${secondaryLabel}`, { error: error.message }); - const secondaryFn = preferAlternative ? this.executeRequest.bind(this) : this.executeAlternativeRequest.bind(this); - - try { - responseText = await secondaryFn(request); - } catch (secondaryError) { - logger.error('Both Gemini request methods failed', { - firstError: error.message, - secondError: secondaryError.message - }); - throw secondaryError; - } + logger.error('Both Gemini request methods failed', { error: error.message }); + throw error; } // Enforce language in code fences if provided @@ -231,7 +638,11 @@ class LLMService { async processTextWithSkill(text, activeSkill, sessionMemory = [], programmingLanguage = null) { if (!this.isInitialized) { - throw new Error('LLM service not initialized. Check Gemini API key configuration.'); + throw new Error('LLM service not initialized. Check API key configuration.'); + } + + if (this.provider === 'openrouter') { + return this._openrouterProcessText(text, activeSkill, sessionMemory, programmingLanguage); } const startTime = Date.now(); @@ -248,32 +659,15 @@ class LLMService { const geminiRequest = this.buildGeminiRequest(text, activeSkill, sessionMemory, programmingLanguage); - const preferAlternative = !!config.get('llm.gemini.enableFallbackMethod'); let response; try { - if (preferAlternative) { - logger.debug('Attempting alternative HTTPS method first for text processing'); - response = await this.executeAlternativeRequest(geminiRequest); - } else { - response = await this.executeRequest(geminiRequest); - } + response = await this._raceGeminiMethods(geminiRequest); } catch (error) { - const secondaryLabel = preferAlternative ? 'primary SDK method' : 'alternative HTTPS method'; - logger.warn(`${preferAlternative ? 'Alternative' : 'Primary'} method failed, trying ${secondaryLabel}`, { + logger.error('Both Gemini request methods failed for text processing', { error: error.message, requestId: this.requestCount }); - const secondaryFn = preferAlternative ? this.executeRequest.bind(this) : this.executeAlternativeRequest.bind(this); - try { - response = await secondaryFn(geminiRequest); - } catch (secondaryError) { - logger.error('Both Gemini request methods failed for text processing', { - firstError: error.message, - secondError: secondaryError.message, - requestId: this.requestCount - }); - throw secondaryError; - } + throw error; } // Enforce language in code fences if programmingLanguage specified @@ -318,7 +712,11 @@ class LLMService { async processTranscriptionWithIntelligentResponse(text, activeSkill, sessionMemory = [], programmingLanguage = null) { if (!this.isInitialized) { - throw new Error('LLM service not initialized. Check Gemini API key configuration.'); + throw new Error('LLM service not initialized. Check API key configuration.'); + } + + if (this.provider === 'openrouter') { + return this._openrouterProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage); } const startTime = Date.now(); @@ -335,32 +733,15 @@ class LLMService { const geminiRequest = this.buildIntelligentTranscriptionRequest(text, activeSkill, sessionMemory, programmingLanguage); - const preferAlternative = !!config.get('llm.gemini.enableFallbackMethod'); let response; try { - if (preferAlternative) { - logger.debug('Attempting alternative HTTPS method first for transcription processing'); - response = await this.executeAlternativeRequest(geminiRequest); - } else { - response = await this.executeRequest(geminiRequest); - } + response = await this._raceGeminiMethods(geminiRequest); } catch (error) { - const secondaryLabel = preferAlternative ? 'primary SDK method' : 'alternative HTTPS method'; - logger.warn(`${preferAlternative ? 'Alternative' : 'Primary'} method failed, trying ${secondaryLabel}`, { + logger.error('Both Gemini request methods failed for transcription processing', { error: error.message, requestId: this.requestCount }); - const secondaryFn = preferAlternative ? this.executeRequest.bind(this) : this.executeAlternativeRequest.bind(this); - try { - response = await secondaryFn(geminiRequest); - } catch (secondaryError) { - logger.error('Both Gemini request methods failed for transcription processing', { - firstError: error.message, - secondError: secondaryError.message, - requestId: this.requestCount - }); - throw secondaryError; - } + throw error; } // Enforce language in code fences if programmingLanguage specified @@ -1059,6 +1440,10 @@ Remember: Be intelligent about filtering - only provide detailed responses when return { success: false, error: 'Service not initialized' }; } + if (this.provider === 'openrouter') { + return this._openrouterTestConnection(); + } + try { // First check network connectivity const networkCheck = await this.checkNetworkConnectivity(); @@ -1175,19 +1560,36 @@ Remember: Be intelligent about filtering - only provide detailed responses when } updateApiKey(newApiKey) { - process.env.GEMINI_API_KEY = newApiKey; + if (this.provider === 'openrouter') { + process.env.OPENROUTER_API_KEY = newApiKey; + } else { + process.env.GEMINI_API_KEY = newApiKey; + } this.isInitialized = false; this.initializeClient(); - logger.info('API key updated and client reinitialized'); + logger.info('API key updated and client reinitialized', { provider: this.provider }); } getStats() { + if (this.provider === 'openrouter') { + return { + provider: 'openrouter', + isInitialized: this.isInitialized, + requestCount: this.requestCount, + errorCount: this.errorCount, + successRate: this.requestCount > 0 ? ((this.requestCount - this.errorCount) / this.requestCount) * 100 : 0, + model: this._openrouterGetModel(), + config: config.get('llm.openrouter') + }; + } return { + provider: 'gemini', isInitialized: this.isInitialized, requestCount: this.requestCount, errorCount: this.errorCount, successRate: this.requestCount > 0 ? ((this.requestCount - this.errorCount) / this.requestCount) * 100 : 0, + model: this.model, config: config.get('llm.gemini') }; } @@ -1196,6 +1598,20 @@ Remember: Be intelligent about filtering - only provide detailed responses when return new Promise(resolve => setTimeout(resolve, ms)); } + /** + * Race the SDK and HTTPS methods in parallel. The first to succeed + * wins. If both fail, throw the error from the primary (SDK) method. + */ + async _raceGeminiMethods(request) { + const results = await Promise.allSettled([ + this.executeRequest(request), + this.executeAlternativeRequest(request) + ]); + const fulfilled = results.find(r => r.status === 'fulfilled'); + if (fulfilled) return fulfilled.value; + throw results[0].reason || new Error('Both Gemini request methods failed'); + } + async executeAlternativeRequest(geminiRequest) { const https = require('https'); const apiKey = config.getApiKey('GEMINI'); @@ -1238,7 +1654,7 @@ Remember: Be intelligent about filtering - only provide detailed responses when const postData = JSON.stringify(geminiRequest); - const agent = new https.Agent({ keepAlive: true, maxSockets: 1 }); + const agent = new https.Agent({ keepAlive: true, maxSockets: 4 }); const options = { method: 'POST', diff --git a/src/ui/llm-response-window.js b/src/ui/llm-response-window.js index cad0fd82..07e44b03 100644 --- a/src/ui/llm-response-window.js +++ b/src/ui/llm-response-window.js @@ -92,10 +92,7 @@ class LLMResponseWindowUI { timestamp: new Date().toISOString(), }); - // Add a small delay to ensure DOM is ready - setTimeout(() => { - this.handleDisplayResponse(data); - }, 50); + this.handleDisplayResponse(data); }); // Interaction state handlers @@ -247,8 +244,6 @@ class LLMResponseWindowUI { result, }); - // Use a more reliable delay mechanism - await new Promise((resolve) => setTimeout(resolve, 200)); this.displayResponseContent(data); } catch (error) { logger.error("Failed to expand window", { @@ -345,29 +340,27 @@ class LLMResponseWindowUI { this.requestWindowResize(contentMetrics); // Final verification - setTimeout(() => { - const isLoadingHidden = this.elements.loading - ? this.elements.loading.classList.contains("hidden") - : true; - const isContentVisible = this.elements.responseContent - ? !this.elements.responseContent.classList.contains("hidden") - : false; - - logger.info("Display state verification", { + const isLoadingHidden = this.elements.loading + ? this.elements.loading.classList.contains("hidden") + : true; + const isContentVisible = this.elements.responseContent + ? !this.elements.responseContent.classList.contains("hidden") + : false; + + logger.info("Display state verification", { + component: "LLMResponseWindowUI", + loadingHidden: isLoadingHidden, + contentVisible: isContentVisible, + windowVisible: !document.hidden, + }); + + if (!isLoadingHidden || !isContentVisible) { + logger.warn("Display state inconsistent - forcing correction", { component: "LLMResponseWindowUI", - loadingHidden: isLoadingHidden, - contentVisible: isContentVisible, - windowVisible: !document.hidden, }); - - if (!isLoadingHidden || !isContentVisible) { - logger.warn("Display state inconsistent - forcing correction", { - component: "LLMResponseWindowUI", - }); - this.hideLoadingState(); - this.showResponseContent(); - } - }, 100); + this.hideLoadingState(); + this.showResponseContent(); + } logger.debug("displayResponseContent completed - END"); } catch (error) { @@ -400,23 +393,21 @@ class LLMResponseWindowUI { async requestWindowResize(contentMetrics) { try { - setTimeout(async () => { - logger.debug("Requesting window resize based on content metrics", { - component: "LLMResponseWindowUI", - metrics: contentMetrics, - }); + logger.debug("Requesting window resize based on content metrics", { + component: "LLMResponseWindowUI", + metrics: contentMetrics, + }); - const { ipcRenderer } = require("electron"); - const result = await ipcRenderer.invoke( - "resize-llm-window-for-content", - contentMetrics - ); + const { ipcRenderer } = require("electron"); + const result = await ipcRenderer.invoke( + "resize-llm-window-for-content", + contentMetrics + ); - logger.debug("Window resize result", { - component: "LLMResponseWindowUI", - result, - }); - }, 200); + logger.debug("Window resize result", { + component: "LLMResponseWindowUI", + result, + }); } catch (error) { logger.error("Failed to resize window", { component: "LLMResponseWindowUI", diff --git a/src/ui/settings-window.js b/src/ui/settings-window.js index e0270258..a8b1852b 100644 --- a/src/ui/settings-window.js +++ b/src/ui/settings-window.js @@ -19,6 +19,11 @@ document.addEventListener('DOMContentLoaded', () => { const activeSkillSelect = document.getElementById('activeSkill'); const iconGrid = document.getElementById('iconGrid'); + // LLM provider fields + const llmProviderSelect = document.getElementById('llmProvider'); + const openrouterKeyInput = document.getElementById('openrouterKey'); + const openrouterModelInput = document.getElementById('openrouterModel'); + // Check if window.api exists if (!window.api) { console.error('window.api not available'); @@ -83,6 +88,10 @@ document.addEventListener('DOMContentLoaded', () => { if (whisperSegmentMsInput) whisperSegmentMsInput.value = settings.whisperSegmentMs || ''; if (geminiKeyInput) geminiKeyInput.value = settings.geminiKey || ''; if (windowGapInput) windowGapInput.value = settings.windowGap || ''; + if (llmProviderSelect) llmProviderSelect.value = settings.llmProvider || 'gemini'; + if (openrouterKeyInput) openrouterKeyInput.value = settings.openrouterKey || ''; + if (openrouterModelInput) openrouterModelInput.value = settings.openrouterModel || 'openrouter/free'; + updateLLMFieldStates(); // Set C++ as default if no coding language is specified if (codingLanguageSelect) { @@ -127,6 +136,21 @@ document.addEventListener('DOMContentLoaded', () => { }); } + const updateLLMFieldStates = () => { + const provider = llmProviderSelect ? llmProviderSelect.value : 'gemini'; + const geminiGroup = document.getElementById('geminiLlmFields'); + const openrouterGroup = document.getElementById('openrouterLlmFields'); + const openrouterNote = document.getElementById('openrouterNote'); + + if (geminiGroup) geminiGroup.style.display = provider === 'gemini' ? '' : 'none'; + if (openrouterGroup) openrouterGroup.style.display = provider === 'openrouter' ? '' : 'none'; + if (openrouterNote) openrouterNote.style.display = provider === 'openrouter' ? '' : 'none'; + + if (geminiKeyInput) geminiKeyInput.disabled = provider !== 'gemini'; + if (openrouterKeyInput) openrouterKeyInput.disabled = provider !== 'openrouter'; + if (openrouterModelInput) openrouterModelInput.disabled = provider !== 'openrouter'; + }; + // Save settings helper function const saveSettings = () => { const settings = {}; @@ -141,6 +165,9 @@ document.addEventListener('DOMContentLoaded', () => { if (windowGapInput) settings.windowGap = windowGapInput.value; if (codingLanguageSelect) settings.codingLanguage = codingLanguageSelect.value; if (activeSkillSelect) settings.activeSkill = activeSkillSelect.value; + if (llmProviderSelect) settings.llmProvider = llmProviderSelect.value; + if (openrouterKeyInput) settings.openrouterKey = openrouterKeyInput.value; + if (openrouterModelInput) settings.openrouterModel = openrouterModelInput.value; window.api.send('save-settings', settings); }; @@ -184,7 +211,10 @@ document.addEventListener('DOMContentLoaded', () => { whisperLanguageInput, whisperSegmentMsInput, geminiKeyInput, - windowGapInput + windowGapInput, + llmProviderSelect, + openrouterKeyInput, + openrouterModelInput ]; inputs.forEach(input => { @@ -201,6 +231,13 @@ document.addEventListener('DOMContentLoaded', () => { }); } + if (llmProviderSelect) { + llmProviderSelect.addEventListener('change', () => { + updateLLMFieldStates(); + saveSettings(); + }); + } + // Language selection handler if (codingLanguageSelect) { codingLanguageSelect.addEventListener('change', (e) => { From 30815c8177e7098af120b71161e82b73b4f2c3ba Mon Sep 17 00:00:00 2001 From: Christmas Day Date: Thu, 25 Jun 2026 18:50:50 +0100 Subject: [PATCH 2/4] Fix _raceGeminiMethods to use Promise.any instead of Promise.allSettled Promise.allSettled waited for ALL promises to settle before returning the fastest result. This caused the user to wait for the slowest method (e.g. 30s HTTPS timeout) even when the SDK method succeeded in 2s. Promise.any returns as soon as the FIRST method fulfills, so there is zero latency penalty from the slower method on the happy path. --- src/services/llm.service.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/services/llm.service.js b/src/services/llm.service.js index 0dc98769..36532e65 100644 --- a/src/services/llm.service.js +++ b/src/services/llm.service.js @@ -1603,13 +1603,14 @@ Remember: Be intelligent about filtering - only provide detailed responses when * wins. If both fail, throw the error from the primary (SDK) method. */ async _raceGeminiMethods(request) { - const results = await Promise.allSettled([ - this.executeRequest(request), - this.executeAlternativeRequest(request) - ]); - const fulfilled = results.find(r => r.status === 'fulfilled'); - if (fulfilled) return fulfilled.value; - throw results[0].reason || new Error('Both Gemini request methods failed'); + try { + return await Promise.any([ + this.executeRequest(request), + this.executeAlternativeRequest(request) + ]); + } catch (err) { + throw err.errors?.[0] || new Error('Both Gemini request methods failed'); + } } async executeAlternativeRequest(geminiRequest) { From 0a07fe3a58b40e307cade82c6cfefb2fab47fed4 Mon Sep 17 00:00:00 2001 From: Christmas Day Date: Thu, 25 Jun 2026 19:44:18 +0100 Subject: [PATCH 3/4] Replace DSA-only skill system with General, Coding, Meeting modes BREAKING: The previous single-skill (DSA) system is replaced by three distinct modes that users can navigate via keyboard shortcuts or settings. New skills: - General: versatile assistant for a wide variety of questions - Coding: programming-focused mode with language enforcement - Meeting: passive listener that detects questions during meetings Changes: - prompts/: replaced dsa.md + unused programming.md with general.md, coding.md, meeting.md - prompt-loader.js: - skillsRequiringProgrammingLanguage: dsa -> coding - Loads all three .md prompts (drops the dsa-only filter) - normalizeSkillName maps old names to new skills - getAvailableSkills returns [general, coding, meeting] - main.js: default skill 'general'; navigateSkill cycles 3 skills; programming language gates on 'coding' - session.manager.js: default skill 'general' - src/ui/main-window.js: skill names, icons, language selector visibility (shown only in Coding mode), click handler cycles forward - settings.html: dropdown options for three skills - src/ui/chat-window.js: updated skill icon map - src/services/llm.service.js: formatImageInstruction, formatUserMessage, getIntelligentTranscriptionPrompt updated per mode; fallback responses and skill keywords updated for new skills --- llm-response.html | 2 +- main.js | 14 +++--- prompt-loader.js | 56 +++++++++--------------- prompts/coding.md | 24 +++++++++++ prompts/dsa.md | 28 ------------ prompts/general.md | 19 +++++++++ prompts/meeting.md | 48 +++++++++++++++++++++ prompts/programming.md | 52 ---------------------- settings.html | 4 +- src/managers/session.manager.js | 2 +- src/services/llm.service.js | 38 ++++++++++------- src/ui/chat-window.js | 12 ++---- src/ui/llm-response-window.js | 2 +- src/ui/main-window.js | 76 ++++++++++++++++----------------- 14 files changed, 187 insertions(+), 190 deletions(-) create mode 100644 prompts/coding.md delete mode 100644 prompts/dsa.md create mode 100644 prompts/general.md create mode 100644 prompts/meeting.md delete mode 100644 prompts/programming.md diff --git a/llm-response.html b/llm-response.html index 40c2669c..b340c3ee 100644 --- a/llm-response.html +++ b/llm-response.html @@ -462,7 +462,7 @@ // Global variables let currentLayout = 'split'; let hasCode = false; - let currentSkill = 'dsa'; + let currentSkill = 'general'; let isInteractive = false; let scrollableElements = []; let initialized = false; diff --git a/main.js b/main.js index 6f9555ad..ae2ff468 100644 --- a/main.js +++ b/main.js @@ -57,7 +57,7 @@ class ApplicationController { constructor() { this.isReady = false; this.starting = false; - this.activeSkill = "dsa"; + this.activeSkill = "general"; // Default to C++ so language is enforced from first run this.codingLanguage = "cpp"; this.speechAvailable = false; @@ -992,7 +992,9 @@ class ApplicationController { navigateSkill(direction) { const availableSkills = [ - "dsa", + "general", + "coding", + "meeting", ]; const currentIndex = availableSkills.indexOf(this.activeSkill); @@ -1050,7 +1052,7 @@ class ApplicationController { // Use image directly with LLM and active skill; do not send chat messages here const sessionHistory = sessionManager.getOptimizedHistory(); - const skillsRequiringProgrammingLanguage = ['dsa']; + const skillsRequiringProgrammingLanguage = ['coding']; const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill); const llmResult = await llmService.processImageWithSkill( @@ -1103,7 +1105,7 @@ class ApplicationController { sessionManager.addUserInput(text, 'llm_input'); // Check if current skill needs programming language context - const skillsRequiringProgrammingLanguage = ['dsa']; + const skillsRequiringProgrammingLanguage = ['coding']; const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill); const llmResult = await llmService.processTextWithSkill( @@ -1182,7 +1184,7 @@ class ApplicationController { }); // Check if current skill needs programming language context - const skillsRequiringProgrammingLanguage = ['dsa']; + const skillsRequiringProgrammingLanguage = ['coding']; const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill); const llmResult = await llmService.processTranscriptionWithIntelligentResponse( @@ -1384,7 +1386,7 @@ class ApplicationController { // distinguish "unset" from "stale value from a previous load". return { codingLanguage: this.codingLanguage || "cpp", - activeSkill: this.activeSkill || "dsa", + activeSkill: this.activeSkill || "general", appIcon: this.appIcon || "terminal", selectedIcon: this.appIcon || "terminal", windowGap: windowManager.windowGap, diff --git a/prompt-loader.js b/prompt-loader.js index e57259a0..ad802410 100644 --- a/prompt-loader.js +++ b/prompt-loader.js @@ -7,7 +7,7 @@ class PromptLoader { this.promptsLoaded = false; this.skillPromptSent = new Set(); // Focus only on DSA - this.skillsRequiringProgrammingLanguage = ['dsa']; + this.skillsRequiringProgrammingLanguage = ['coding']; } /** @@ -28,7 +28,7 @@ class PromptLoader { for (const file of files) { if (file.endsWith('.md')) { const skillName = path.basename(file, '.md'); - if (skillName !== 'dsa') continue; // only keep DSA + if (!['general', 'coding', 'meeting'].includes(skillName)) continue; const filePath = path.join(promptsDir, file); const promptContent = fs.readFileSync(filePath, 'utf8'); @@ -88,7 +88,7 @@ class PromptLoader { let languageInjection = ''; switch (skillName) { - case 'dsa': + case 'coding': languageInjection = `\n\n## IMPLEMENTATION LANGUAGE: ${languageUpper} STRICT REQUIREMENTS: - Respond ONLY in ${languageTitle}. Do not include any snippets or alternatives in other languages. @@ -322,39 +322,23 @@ STRICT REQUIREMENTS: // Convert to lowercase and handle common variations const normalized = skillName.toLowerCase().trim(); - // Map common variations to standard names + // Map common variations to canonical skill names const skillMap = { - 'dsa': 'dsa', - 'data-structures': 'dsa', - 'algorithms': 'dsa', - 'data-structures-algorithms': 'dsa', - 'behavioral': 'behavioral', - 'behavioral-interview': 'behavioral', - 'behavior': 'behavioral', - 'sales': 'sales', - 'selling': 'sales', - 'business-development': 'sales', - 'presentation': 'presentation', - 'presentations': 'presentation', - 'public-speaking': 'presentation', - 'data-science': 'data-science', - 'datascience': 'data-science', - 'machine-learning': 'data-science', - 'ml': 'data-science', - 'programming': 'programming', - 'coding': 'programming', - 'software-development': 'programming', - 'development': 'programming', - 'devops': 'devops', - 'dev-ops': 'devops', - 'infrastructure': 'devops', - 'system-design': 'system-design', - 'systems-design': 'system-design', - 'architecture': 'system-design', - 'distributed-systems': 'system-design', - 'negotiation': 'negotiation', - 'negotiating': 'negotiation', - 'conflict-resolution': 'negotiation' + 'general': 'general', + 'gen': 'general', + 'default': 'general', + 'coding': 'coding', + 'code': 'coding', + 'programming': 'coding', + 'software': 'coding', + 'development': 'coding', + 'dsa': 'coding', + 'algorithms': 'coding', + 'data-structures': 'coding', + 'meeting': 'meeting', + 'meetings': 'meeting', + 'conference': 'meeting', + 'call': 'meeting' }; return skillMap[normalized] || normalized; @@ -368,7 +352,7 @@ STRICT REQUIREMENTS: if (!this.promptsLoaded) { this.loadPrompts(); } - return ['dsa']; + return ['general', 'coding', 'meeting']; } /** diff --git a/prompts/coding.md b/prompts/coding.md new file mode 100644 index 00000000..d3558cc1 --- /dev/null +++ b/prompts/coding.md @@ -0,0 +1,24 @@ +# Coding Assistant Agent + +You are a programming expert that helps users solve coding problems, debug issues, and understand code. + +## STRICT RULES + +- Output code ONLY in the user-selected language. No alternatives unless asked. +- Use triple backticks with the correct language tag. +- Ensure all code is syntactically correct and handles edge cases. +- If there's pre-code or a template in the question, strictly use that template to structure your answer. +- Avoid unnecessary commentary in code; keep code clean and readable. +- Your code should have minimal comments unless the logic is non-obvious. + +## Response Structure + +1. Understand the problem by restating it briefly +2. Explain your approach with the reasoning behind it +3. Provide the implementation in the selected language +4. Note time and space complexity where applicable +5. Mention edge cases or testing considerations + +## Programming Language + +When a programming language is specified, ALL code must be in that language. Use the appropriate language tag for code fences. Do not provide alternatives in other languages unless explicitly asked. diff --git a/prompts/dsa.md b/prompts/dsa.md deleted file mode 100644 index f2300fcb..00000000 --- a/prompts/dsa.md +++ /dev/null @@ -1,28 +0,0 @@ -# DSA Interview Helper Agent (Focused & Optimal) - -You are a competitive programming expert that outputs the most optimal solution with minimal time and space complexity. - -STRICT RULES -- Output code ONLY in the user-selected language. No alternatives unless asked. -- Use triple backticks with the correct language tag. -- Prefer O(n) or O(n log n) where feasible; call out if optimal lower bound is higher. -- if there's some pre-code or template in Question then strictly use that template to answer it. -- Avoid extra commentary; be concise and implementation-focused. -- Your code must not contain any comments. - -Workflow -1) Identify the problem pattern quickly (Array, Hashing, Two Pointers, Sliding Window, Binary Search, Stack/Queue, Linked List, Tree/Graph, Heap, Greedy, DP). -2) State naive idea in 1–2 lines with complexity. -3) Give optimal approach with 3–5 bullet steps. -4) Provide clean, production-ready, comment-free implementation in the selected language. -5) State time and space complexity precisely. -6) Optional: 1 short dry-run example if non-obvious. - -Implementation Template -```lang -``` - -Notes -- Prefer iterative over recursive when it reduces stack usage or improves clarity. -- Use built-in data structures and libraries idiomatically for the selected language. -- For DP, specify state, transition, and memory optimization opportunities. \ No newline at end of file diff --git a/prompts/general.md b/prompts/general.md new file mode 100644 index 00000000..470bc1e1 --- /dev/null +++ b/prompts/general.md @@ -0,0 +1,19 @@ +# General Assistant Agent + +You are a versatile AI assistant capable of answering a wide variety of questions across different domains. Adapt your response style to the question being asked. + +## Response Guidelines + +- Be concise and direct when the question is straightforward +- Provide detailed explanations when the topic requires depth +- Use examples to illustrate complex concepts +- If asked for code, provide clean, well-structured implementations +- If asked for analysis, break down the problem step by step +- Stay factual and acknowledge uncertainty when appropriate + +## Communication Style + +- Professional but approachable +- Tailor technical depth to the question's implied audience +- Prioritize clarity and correctness over brevity for complex topics +- Use bullet points, steps, or structured formats when they improve clarity diff --git a/prompts/meeting.md b/prompts/meeting.md new file mode 100644 index 00000000..d9afc64a --- /dev/null +++ b/prompts/meeting.md @@ -0,0 +1,48 @@ +# Meeting Assistant Agent + +You are a meeting assistant that listens to conversations and provides helpful information when needed. + +## Role + +- Listen to meeting conversations passively +- Detect when a question is asked that you can answer +- Provide concise, relevant answers only when helpful +- Do not interrupt or respond to casual conversation, chit-chat, or statements +- Be unobtrusive — answers should be informative but brief + +## Question Detection + +A question is being asked when the speaker: +- Uses question words (what, why, how, when, where, who, which) +- Asks for an explanation, definition, or example +- Requests a suggestion, recommendation, or opinion +- Asks for data, facts, or information +- Uses a rising intonation or phrasing that indicates uncertainty + +## Response Guidelines + +When a question is detected: +- Provide a direct, concise answer +- Include relevant facts, data, or explanations +- Use examples if they help clarify +- Avoid tangential information +- Keep responses under 3-4 sentences when possible + +When no question is detected: +- Do not respond +- Stay silent and continue listening +- Do not acknowledge statements or casual conversation + +## Examples of When to Respond + +- "What's the best way to handle this situation?" → Provide suggestions +- "Can someone explain how this works?" → Provide explanation +- "What are the trade-offs here?" → List pros and cons +- "How do you implement this?" → Provide implementation approach + +## Examples of When NOT to Respond + +- "Let's move to the next agenda item" → Silent +- "I think we should consider this approach" → Silent +- "Good morning everyone" → Silent +- "That's a good point" → Silent diff --git a/prompts/programming.md b/prompts/programming.md deleted file mode 100644 index 42efc53d..00000000 --- a/prompts/programming.md +++ /dev/null @@ -1,52 +0,0 @@ -# Programming Interview Helper Agent - -You are a concise programming interview assistant. Provide quick, actionable guidance without revealing you're an AI helper. - -## Response Structure - -### 1. Naive Approach (30 seconds) -- State the simplest solution first -- Mention time/space complexity -- One-line reasoning why it works - -### 2. Optimized Solution (2 minutes) -- Best approach with clear explanation -- Step-by-step algorithm breakdown -- Time/space complexity analysis - -### 3. Dry Run (1 minute) -- Walk through with a concrete example -- Show key variable states at each step -- Highlight the core insight - -### 4. Production Code -```language -// Clean, interview-ready implementation -// Include edge case handling -// Add meaningful comments -``` - -### 5. Quick Validation -- 2-3 test cases (edge cases included) -- Alternative approaches if time permits - -## Communication Style -- Start with "Let me think through this step by step" -- Use "First, the straightforward approach would be..." -- Transition with "But we can optimize this by..." -- Be conversational, not robotic -- Show your thought process naturally - -## Key Technologies to Reference -**Data Structures**: Arrays, HashMaps, Trees, Graphs, Heaps, Stacks, Queues -**Algorithms**: Two Pointers, Sliding Window, DFS/BFS, Dynamic Programming, Binary Search -**Patterns**: Divide & Conquer, Greedy, Backtracking, Memoization - -## Common Optimizations -- HashMap for O(1) lookups instead of nested loops -- Two pointers for array problems -- Binary search for sorted data -- DP for overlapping subproblems -- BFS/DFS for tree/graph traversal - -Give direct, implementable solutions with clear reasoning. Focus on demonstrating problem-solving skills naturally. \ No newline at end of file diff --git a/settings.html b/settings.html index 358c1557..1db9092b 100644 --- a/settings.html +++ b/settings.html @@ -321,7 +321,9 @@
Choose your current focus area
diff --git a/src/managers/session.manager.js b/src/managers/session.manager.js index 9c9d971e..8b122cd2 100644 --- a/src/managers/session.manager.js +++ b/src/managers/session.manager.js @@ -8,7 +8,7 @@ class SessionManager { this.compressionEnabled = true; this.maxSize = config.get('session.maxMemorySize'); this.compressionThreshold = config.get('session.compressionThreshold'); - this.currentSkill = 'dsa'; // Default skill is DSA + this.currentSkill = 'general'; this.isInitialized = false; this.initializeWithSkillPrompts(); diff --git a/src/services/llm.service.js b/src/services/llm.service.js index 36532e65..f27b4bd6 100644 --- a/src/services/llm.service.js +++ b/src/services/llm.service.js @@ -535,7 +535,7 @@ class LLMService { * For image-based queries, we include the skill prompt (e.g., DSA) as systemInstruction. * @param {Buffer} imageBuffer - PNG/JPEG image bytes * @param {string} mimeType - e.g., 'image/png' or 'image/jpeg' - * @param {string} activeSkill - current skill (e.g. 'dsa') + * @param {string} activeSkill - current skill (e.g. 'coding') * @param {Array} sessionMemory - optional (not required for image) * @param {string|null} programmingLanguage - optional language context for skills that need it * @returns {Promise<{response: string, metadata: object}>} @@ -633,7 +633,10 @@ class LLMService { formatImageInstruction(activeSkill, programmingLanguage) { const langNote = programmingLanguage ? ` Use only ${programmingLanguage.toUpperCase()} for any code.` : ''; - return `Analyze this image for a ${activeSkill.toUpperCase()} question. Extract the problem concisely and provide the best possible solution with explanation and final code.${langNote}`; + const modePrefix = activeSkill === 'meeting' + ? 'Analyze this image from a meeting or presentation.' + : `Analyze this image for a ${activeSkill.toUpperCase()} question.`; + return `${modePrefix} Extract the problem concisely and provide the best possible solution with explanation and final code.${langNote}`; } async processTextWithSkill(text, activeSkill, sessionMemory = [], programmingLanguage = null) { @@ -1059,6 +1062,12 @@ Always respond to the point, do not repeat the question or unnecessary informati ## Response Rules: +### If in MEETING mode: +- Detect if the speaker is asking a question +- If a question is detected, provide a concise, direct answer +- If no question is detected, respond with a brief acknowledgment or stay silent +- Do NOT respond to statements, chit-chat, or agenda items — only questions + ### If the transcription is casual conversation, greetings, or NOT related to ${activeSkill}: - Respond with: "Yeah, I'm listening. Ask your question relevant to ${activeSkill}." - Or similar brief acknowledgments like: "I'm here, what's your ${activeSkill} question?" @@ -1087,7 +1096,7 @@ Always respond to the point, do not repeat the question or unnecessary informati - Be encouraging and helpful - Stay focused on ${activeSkill} -If the user's input is a coding or DSA problem statement and contains no code, produce a complete, runnable solution in the selected programming language without asking for more details. Always include the final implementation in a properly tagged code block. +If the user's input is a coding problem statement and contains no code, produce a complete, runnable solution in the selected programming language without asking for more details. Always include the final implementation in a properly tagged code block. Remember: Be intelligent about filtering - only provide detailed responses when the user actually needs help with ${activeSkill}.`; @@ -1095,6 +1104,9 @@ Remember: Be intelligent about filtering - only provide detailed responses when } formatUserMessage(text, activeSkill) { + if (activeSkill === 'meeting') { + return `Context: Meeting conversation\n\nDetected speech:\n${text}`; + } return `Context: ${activeSkill.toUpperCase()} analysis request\n\nText to analyze:\n${text}`; } @@ -1373,10 +1385,10 @@ Remember: Be intelligent about filtering - only provide detailed responses when logger.info('Generating fallback response', { activeSkill }); const fallbackResponses = { - 'dsa': 'This appears to be a data structures and algorithms problem. Consider breaking it down into smaller components and identifying the appropriate algorithm or data structure to use.', - 'system-design': 'For this system design question, consider scalability, reliability, and the trade-offs between different architectural approaches.', - 'programming': 'This looks like a programming challenge. Focus on understanding the requirements, edge cases, and optimal time/space complexity.', - 'default': 'I can help analyze this content. Please ensure your Gemini API key is properly configured for detailed analysis.' + 'general': 'I can help analyze this content. Here are my thoughts on the matter.', + 'coding': 'This looks like a coding challenge. Focus on understanding the requirements, edge cases, and optimal time/space complexity.', + 'meeting': 'I heard a question in the meeting. Here is my response based on the context.', + 'default': 'I can help analyze this content. Please ensure your API key is properly configured for detailed analysis.' }; const response = fallbackResponses[activeSkill] || fallbackResponses.default; @@ -1397,15 +1409,9 @@ Remember: Be intelligent about filtering - only provide detailed responses when // Simple heuristic to determine if message seems skill-related const skillKeywords = { - 'dsa': ['algorithm', 'data structure', 'array', 'tree', 'graph', 'sort', 'search', 'complexity', 'big o'], - 'programming': ['code', 'function', 'variable', 'class', 'method', 'bug', 'debug', 'syntax'], - 'system-design': ['scalability', 'database', 'architecture', 'microservice', 'load balancer', 'cache'], - 'behavioral': ['interview', 'experience', 'situation', 'leadership', 'conflict', 'team'], - 'sales': ['customer', 'deal', 'negotiation', 'price', 'revenue', 'prospect'], - 'presentation': ['slide', 'audience', 'public speaking', 'presentation', 'nervous'], - 'data-science': ['data', 'model', 'machine learning', 'statistics', 'analytics', 'python', 'pandas'], - 'devops': ['deployment', 'ci/cd', 'docker', 'kubernetes', 'infrastructure', 'monitoring'], - 'negotiation': ['negotiate', 'compromise', 'agreement', 'terms', 'conflict resolution'] + 'general': ['help', 'question', 'explain', 'what', 'how', 'why', 'tell me', 'can you'], + 'coding': ['code', 'function', 'variable', 'class', 'method', 'bug', 'debug', 'syntax', 'algorithm', 'array', 'sort'], + 'meeting': ['question', 'how', 'what', 'why', 'could you', 'can you', 'suggestion', 'recommend'] }; const textLower = text.toLowerCase(); diff --git a/src/ui/chat-window.js b/src/ui/chat-window.js index 5836ce09..d61ccd0e 100644 --- a/src/ui/chat-window.js +++ b/src/ui/chat-window.js @@ -280,15 +280,9 @@ class ChatWindowUI { // Show a brief activation message with the skill title const icons = { - 'dsa': '🧠', - 'behavioral': '💼', - 'sales': '💰', - 'presentation': '🎤', - 'data-science': '📊', - 'programming': '💻', - 'devops': '🚀', - 'system-design': '🏗️', - 'negotiation': '🤝' + 'general': '🧠', + 'coding': '💻', + 'meeting': '📋' }; const icon = icons[skillName] || '🎯'; diff --git a/src/ui/llm-response-window.js b/src/ui/llm-response-window.js index 07e44b03..f0ac46dc 100644 --- a/src/ui/llm-response-window.js +++ b/src/ui/llm-response-window.js @@ -4,7 +4,7 @@ class LLMResponseWindowUI { constructor() { this.currentLayout = "split"; this.hasCode = false; - this.currentSkill = "dsa"; + this.currentSkill = "general"; this.isInteractive = false; this.scrollableElements = []; diff --git a/src/ui/main-window.js b/src/ui/main-window.js index e6ab7cc4..9bf63fc9 100644 --- a/src/ui/main-window.js +++ b/src/ui/main-window.js @@ -10,7 +10,7 @@ class MainWindowUI { constructor() { this.isInteractive = false; this.isHidden = false; - this.currentSkill = 'dsa'; // Default, will be updated from settings + this.currentSkill = 'general'; this.statusDot = null; this.skillIndicator = null; this.micButton = null; @@ -18,9 +18,10 @@ class MainWindowUI { this.speechAvailable = false; // track availability this._popoverHideTimeout = null; - // Define available skills for navigation this.availableSkills = [ - 'dsa' + 'general', + 'coding', + 'meeting' ]; this.init(); @@ -283,17 +284,10 @@ class MainWindowUI { } }); - // Skill indicator click handler toggles DSA skill + // Skill indicator click handler cycles to next skill this.skillIndicator.addEventListener('click', () => { if (!this.isInteractive) return; - const newSkill = 'dsa'; - if (window.electronAPI && window.electronAPI.updateActiveSkill) { - window.electronAPI.updateActiveSkill(newSkill).then(() => { - this.handleSkillActivated(newSkill); - }); - } else { - this.handleSkillActivated(newSkill); - } + this.navigateSkill(1); }); // Check for required elements (settingsIndicator is optional) @@ -505,15 +499,9 @@ class MainWindowUI { handleLLMResponse(data) { const skill = data.skill || data.metadata?.skill || 'General'; const skillNames = { - 'dsa': 'DSA', - 'behavioral': 'Behavioral', - 'sales': 'Sales', - 'presentation': 'Presentation', - 'data-science': 'Data Science', - 'programming': 'Programming', - 'devops': 'DevOps', - 'system-design': 'System Design', - 'negotiation': 'Negotiation' + 'general': 'General', + 'coding': 'Coding', + 'meeting': 'Meeting' }; const displaySkill = skillNames[skill] || skill.toUpperCase(); @@ -645,15 +633,9 @@ class MainWindowUI { updateSkillIndicator() { const skillNames = { - 'dsa': 'DSA', - 'behavioral': 'Behavioral', - 'sales': 'Sales', - 'presentation': 'Presentation', - 'data-science': 'Data Science', - 'programming': 'Programming', - 'devops': 'DevOps', - 'system-design': 'System Design', - 'negotiation': 'Negotiation' + 'general': 'General', + 'coding': 'Coding', + 'meeting': 'Meeting' }; logger.info('Updating skill indicator', { @@ -669,6 +651,7 @@ class MainWindowUI { const skillName = skillNames[this.currentSkill] || this.currentSkill.toUpperCase(); const skillSpan = this.skillIndicator.querySelector('span'); + const skillIcon = this.skillIndicator.querySelector('i'); logger.info('Looking for skill span element', { component: 'MainWindowUI', @@ -676,6 +659,27 @@ class MainWindowUI { skillName: skillName }); + // Update the icon based on skill + const skillIcons = { + 'general': 'fa-brain', + 'coding': 'fa-code', + 'meeting': 'fa-users' + }; + if (skillIcon) { + // Remove all existing fa-* classes except fa-solid + skillIcon.className = skillIcon.className + .split(' ') + .filter(c => c === 'fas' || c === 'fa-solid') + .join(' '); + skillIcon.classList.add(skillIcons[this.currentSkill] || 'fa-brain'); + } + + // Show/hide language selector based on skill + const languageSelector = document.getElementById('languageSelector'); + if (languageSelector) { + languageSelector.style.display = this.currentSkill === 'coding' ? '' : 'none'; + } + if (skillSpan) { const oldText = skillSpan.textContent; skillSpan.textContent = skillName; @@ -758,15 +762,9 @@ class MainWindowUI { showSkillChangeNotification(skill, direction) { const skillNames = { - 'dsa': 'DSA', - 'behavioral': 'Behavioral', - 'sales': 'Sales', - 'presentation': 'Presentation', - 'data-science': 'Data Science', - 'programming': 'Programming', - 'devops': 'DevOps', - 'system-design': 'System Design', - 'negotiation': 'Negotiation' + 'general': 'General', + 'coding': 'Coding', + 'meeting': 'Meeting' }; const displayName = skillNames[skill] || skill.toUpperCase(); From b4de7638fbc658305d371eaea24b372630073bd9 Mon Sep 17 00:00:00 2001 From: Christmas Day Date: Mon, 29 Jun 2026 20:43:14 +0100 Subject: [PATCH 4/4] Fix microphone and screen capture permissions and migrate to main-process audio capture --- chat.html | 118 ++++++++- env.example | 26 +- main.js | 150 ++++++++++- onboarding.html | 52 ++++ onboarding.js | 56 +++- package-lock.json | 127 +++++++++ package.json | 1 + preload.js | 27 +- settings.html | 83 ++++++ src/core/config.js | 16 ++ src/core/first-run.js | 8 + src/services/llm.service.js | 461 +++++++++++++++++++++++++++------ src/services/speech.service.js | 297 +++++++++++++++------ src/services/tts.service.js | 181 +++++++++++++ src/ui/audio-capture.html | 75 ++++++ src/ui/settings-window.js | 37 ++- 16 files changed, 1541 insertions(+), 174 deletions(-) create mode 100644 src/services/tts.service.js create mode 100644 src/ui/audio-capture.html diff --git a/chat.html b/chat.html index 47edcaff..6785873b 100644 --- a/chat.html +++ b/chat.html @@ -732,6 +732,11 @@ let listeningTimer = null; let speechAvailable = false; + // Renderer-based audio capture (getUserMedia in chat window) + let rendererAudioStream = null; + let rendererAudioContext = null; + let rendererAudioProcessor = null; + // Chat persistence and deduplication const CHAT_HISTORY_KEY = 'opencluely_chat_history_v1'; let chatHistory = []; @@ -798,12 +803,15 @@ } async function initSpeechAvailability() { + console.log('[Chat] initSpeechAvailability() called'); try { if (window.electronAPI && window.electronAPI.getSpeechAvailability) { speechAvailable = await window.electronAPI.getSpeechAvailability(); + console.log('[Chat] Speech availability:', speechAvailable); applyMicVisibility(); } } catch (e) { + console.error('[Chat] Failed to get speech availability:', e); speechAvailable = false; applyMicVisibility(); } @@ -817,12 +825,58 @@ }); } + // TTS: speak assistant responses aloud + let ttsEnabled = true; + + // Load TTS enabled state from settings + if (whysperAPI && typeof whysperAPI.getSettings === 'function') { + whysperAPI.getSettings().then(settings => { + if (settings && settings.ttsEnabled !== undefined) { + ttsEnabled = settings.ttsEnabled; + } + }).catch(() => {}); + } + + function maybeSpeakResponse(text) { + if (!ttsEnabled || !text || !whysperAPI || typeof whysperAPI.synthesizeSpeech !== 'function') return; + whysperAPI.synthesizeSpeech(text).catch(() => {}); + } + + // Listen for TTS events to update UI + if (whysperAPI && whysperAPI.onTtsStarted) { + whysperAPI.onTtsStarted(() => { + const ttsIndicator = document.getElementById('ttsIndicator'); + if (ttsIndicator) ttsIndicator.style.display = 'inline-block'; + }); + } + if (whysperAPI && whysperAPI.onTtsCompleted) { + whysperAPI.onTtsCompleted(() => { + const ttsIndicator = document.getElementById('ttsIndicator'); + if (ttsIndicator) ttsIndicator.style.display = 'none'; + }); + } + if (whysperAPI && whysperAPI.onTtsError) { + whysperAPI.onTtsError((event, data) => { + const ttsIndicator = document.getElementById('ttsIndicator'); + if (ttsIndicator) ttsIndicator.style.display = 'none'; + console.warn('TTS error:', data?.error); + }); + } + if (whysperAPI && whysperAPI.receive) { + whysperAPI.receive('tts-settings-changed', (data) => { + if (data && data.ttsEnabled !== undefined) { + ttsEnabled = data.ttsEnabled; + } + }); + } + // Also capture generic LLM responses (from OCR/screenshot flows) if (whysperAPI && whysperAPI.onLlmResponse) { whysperAPI.onLlmResponse((event, data) => { try { hideThinkingIndicator(); } catch (_) {} if (data && data.response) { renderAssistantResponse(data.response); + maybeSpeakResponse(data.response); } }); } @@ -1173,7 +1227,7 @@ // Listen for speech errors whysperAPI.onSpeechError((event, data) => { if (data && data.error) { - addMessage(`Error in recognizing speech`, 'error'); + addMessage(`Speech error: ${data.error}`, 'error'); handleRecordingStopped(); } }); @@ -1196,6 +1250,60 @@ if (data && data.response) { try { hideThinkingIndicator(); } catch (_) {} renderAssistantResponse(data.response); + maybeSpeakResponse(data.response); + } + }); + + // Renderer-based audio capture (getUserMedia in chat window) + whysperAPI.onStartRendererCapture(() => { + console.log('[Chat] Starting renderer audio capture'); + if (rendererAudioStream) { + console.log('[Chat] Renderer capture already active'); + return; + } + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + console.log('[Chat] getUserMedia succeeded'); + rendererAudioStream = stream; + + const audioCtx = new AudioContext(); + rendererAudioContext = audioCtx; + const source = audioCtx.createMediaStreamSource(stream); + const processor = audioCtx.createScriptProcessor(4096, 1, 1); + rendererAudioProcessor = processor; + + source.connect(processor); + processor.onaudioprocess = (e) => { + const input = e.inputBuffer.getChannelData(0); + const int16 = new Int16Array(input.length); + for (let i = 0; i < input.length; i++) { + const s = Math.max(-1, Math.min(1, input[i])); + int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + if (whysperAPI && whysperAPI.sendAudioChunk) { + whysperAPI.sendAudioChunk(int16.buffer); + } + }; + }).catch((err) => { + console.error('[Chat] getUserMedia failed:', err.message); + if (whysperAPI && whysperAPI.sendAudioCaptureError) { + whysperAPI.sendAudioCaptureError(err.message); + } + }); + }); + + whysperAPI.onStopRendererCapture(() => { + console.log('[Chat] Stopping renderer audio capture'); + if (rendererAudioProcessor) { + try { rendererAudioProcessor.disconnect(); } catch (_) {} + rendererAudioProcessor = null; + } + if (rendererAudioContext) { + try { rendererAudioContext.close(); } catch (_) {} + rendererAudioContext = null; + } + if (rendererAudioStream) { + rendererAudioStream.getTracks().forEach(t => t.stop()); + rendererAudioStream = null; } }); } @@ -1207,6 +1315,7 @@ if (micButton) { micButton.addEventListener('click', async () => { + console.log('[Chat] Mic button clicked, isInteractive:', isInteractive, 'speechAvailable:', speechAvailable, 'isRecording:', isRecording); if (!isInteractive) { addMessage('Window is in non-interactive mode. Press Alt+A to enable interaction.', 'error'); return; @@ -1218,16 +1327,21 @@ try { if (isRecording) { + console.log('[Chat] Stopping speech recognition'); if (whysperAPI) { const result = await whysperAPI.stopSpeechRecognition(); + console.log('[Chat] Stop result:', result); } } else { + console.log('[Chat] Starting speech recognition'); if (whysperAPI) { const result = await whysperAPI.startSpeechRecognition(); + console.log('[Chat] Start result:', result); } } } catch (error) { - addMessage("Error in recognizing speech", 'error'); + console.error('[Chat] Speech recognition error:', error); + addMessage(`Speech error: ${error.message || error}`, 'error'); } }); } diff --git a/env.example b/env.example index 97006367..75561255 100644 --- a/env.example +++ b/env.example @@ -1,5 +1,5 @@ # LLM Provider Configuration -# Choose your AI backend: gemini or openrouter +# Choose your AI backend: gemini, openrouter, or groq LLM_PROVIDER=gemini # Google Gemini API Configuration (when LLM_PROVIDER=gemini) @@ -12,15 +12,21 @@ OPENROUTER_API_KEY= # Model to use — "openrouter/free" auto-selects free vision-capable models OPENROUTER_MODEL=openrouter/free -# Speech Recognition Configuration -# Choose one provider: azure or whisper +# Groq API Configuration (when LLM_PROVIDER=groq or SPEECH_PROVIDER=groq) +# Get your API key from: https://console.groq.com/keys +GROQ_API_KEY= +# LLM model — "llama-3.3-70b-versatile" is fast and free +GROQ_MODEL=llama-3.3-70b-versatile + +# Speech / STT Configuration +# Choose one provider: whisper, azure, or groq SPEECH_PROVIDER=whisper -# Optional: Azure Speech Services Configuration +# Azure Speech Services Configuration (when SPEECH_PROVIDER=azure) AZURE_SPEECH_KEY=your_azure_speech_key_here AZURE_SPEECH_REGION=your_azure_region_here -# Optional: Local OpenAI Whisper Configuration +# Local OpenAI Whisper Configuration (when SPEECH_PROVIDER=whisper) # Requires a local Whisper CLI installation, for example: # pip install openai-whisper # brew install ffmpeg sox @@ -30,3 +36,13 @@ WHISPER_MODEL_DIR=.whisper-models WHISPER_MODEL=turbo WHISPER_LANGUAGE=en WHISPER_SEGMENT_MS=4000 + +# Groq STT Configuration (when SPEECH_PROVIDER=groq) +# Uses Whisper Large V3 via Groq's API — requires GROQ_API_KEY above +GROQ_STT_MODEL=whisper-large-v3-turbo + +# Groq TTS Configuration (Text-to-Speech via Orpheus) +# Uses Orpheus TTS — requires GROQ_API_KEY above +GROQ_TTS_MODEL=orpheus-tts-0.1-ayane +GROQ_TTS_VOICE=tara +GROQ_TTS_SPEED=1.0 diff --git a/main.js b/main.js index ae2ff468..c9cb3166 100644 --- a/main.js +++ b/main.js @@ -47,6 +47,7 @@ const FirstRunManager = require("./src/core/first-run"); // Screen capture (image-based) const captureService = require("./src/services/capture.service"); const speechService = require("./src/services/speech.service"); +const ttsService = require("./src/services/tts.service"); const llmService = require("./src/services/llm.service"); // Managers @@ -267,30 +268,45 @@ class ApplicationController { setupPermissions() { session.defaultSession.setPermissionRequestHandler( (webContents, permission, callback) => { - const allowedPermissions = ["microphone", "camera", "display-capture"]; + console.log('[Main] Session permission request:', permission); + const allowedPermissions = ["microphone", "camera", "display-capture", "media"]; const granted = allowedPermissions.includes(permission); logger.debug("Permission request", { permission, granted }); callback(granted); } ); + + session.defaultSession.setPermissionCheckHandler(() => true); } async _ensureMicrophonePermission() { + console.log('[Main] _ensureMicrophonePermission() called'); // On macOS 14+ the first microphone hardware access via a spawned // subprocess (sox/rec) can crash the app if permission hasn't been // granted yet. Use the programmatic API to trigger the dialog safely. - if (process.platform !== "darwin") return; + if (process.platform !== "darwin") { + console.log('[Main] Not macOS, skipping mic permission check'); + return; + } const { systemPreferences } = require("electron"); - if (!systemPreferences.getMediaAccessStatus) return; + if (!systemPreferences.getMediaAccessStatus) { + console.log('[Main] No getMediaAccessStatus API, skipping'); + return; + } const status = systemPreferences.getMediaAccessStatus("microphone"); logger.debug("Microphone permission status", { status }); + console.log('[Main] Microphone permission status:', status); - if (status === "granted") return; + if (status === "granted") { + console.log('[Main] Mic permission already granted'); + return; + } if (status === "denied") { + console.log('[Main] Mic permission denied'); throw new Error( "Microphone permission denied. " + "Go to System Settings → Privacy & Security → Microphone, " + @@ -300,7 +316,9 @@ class ApplicationController { // status === "not-determined" — first time; ask safely via the API if (status === "not-determined" && systemPreferences.askForMediaAccess) { + console.log('[Main] Asking for mic permission...'); const granted = await systemPreferences.askForMediaAccess("microphone"); + console.log('[Main] askForMediaAccess result:', granted); if (!granted) { throw new Error( "Microphone permission was not granted. " + @@ -424,15 +442,23 @@ class ApplicationController { }); ipcMain.handle("start-speech-recognition", async () => { + console.log('[Main] start-speech-recognition invoked'); try { await this._ensureMicrophonePermission(); + console.log('[Main] Mic permission OK, calling speechService.startRecording()'); speechService.startRecording(); + console.log('[Main] speechService.startRecording() returned'); } catch (e) { + console.error('[Main] start-speech-recognition failed:', e.message); logger.warn("Microphone permission check failed", { error: e.message }); windowManager.broadcastToAllWindows("speech-status", { status: e.message, available: false, }); + windowManager.broadcastToAllWindows("speech-error", { + error: e.message, + available: false, + }); } return speechService.getStatus(); }); @@ -456,6 +482,57 @@ class ApplicationController { speechService.stopRecording(); }); + // Audio capture IPC from renderer-based getUserMedia + let chunkCount = 0; + ipcMain.on("audio-chunk", (_event, buffer) => { + chunkCount++; + if (chunkCount % 50 === 1) { + console.log('[Main] Received audio-chunk #', chunkCount, 'size:', buffer?.length || 0); + } + speechService._handleRendererAudioChunk(buffer); + }); + + ipcMain.on("audio-capture-error", (_event, msg) => { + logger.error("Renderer audio capture error", { error: msg }); + speechService.emit("error", `Microphone error: ${msg}`); + speechService.stopRecording(); + }); + + ipcMain.on("audio-capture-ready", (event) => { + const win = event.sender.getOwnerBrowserWindow(); + if (win && !win.isDestroyed()) { + win.hide(); + } + }); + + // TTS (Text-to-Speech) IPC + ipcMain.handle("synthesize-speech", async (_event, text, options) => { + try { + await ttsService.synthesizeSpeech(text, options); + return { success: true }; + } catch (error) { + return { success: false, error: error.message }; + } + }); + + ipcMain.handle("stop-tts-playback", () => { + ttsService.stopPlayback(); + return { success: true }; + }); + + // Broadcast TTS events to all windows + ttsService.on("tts-started", () => { + windowManager.broadcastToAllWindows("tts-started"); + }); + + ttsService.on("tts-completed", () => { + windowManager.broadcastToAllWindows("tts-completed"); + }); + + ttsService.on("tts-error", (error) => { + windowManager.broadcastToAllWindows("tts-error", { error }); + }); + ipcMain.on("chat-window-ready", () => { // Send a test message to confirm communication setTimeout(() => { @@ -1394,6 +1471,8 @@ class ApplicationController { llmProvider: process.env.LLM_PROVIDER || "gemini", openrouterKey: process.env.OPENROUTER_API_KEY || "", openrouterModel: process.env.OPENROUTER_MODEL || "openrouter/free", + groqKey: process.env.GROQ_API_KEY || "", + groqModel: process.env.GROQ_MODEL || "llama-3.3-70b-versatile", speechProvider: speechService.provider || "whisper", azureKey: process.env.AZURE_SPEECH_KEY || "", @@ -1402,6 +1481,11 @@ class ApplicationController { whisperModel: process.env.WHISPER_MODEL || "turbo", whisperLanguage: process.env.WHISPER_LANGUAGE || "en", whisperSegmentMs: process.env.WHISPER_SEGMENT_MS || "4000", + groqSttModel: process.env.GROQ_STT_MODEL || "whisper-large-v3-turbo", + ttsEnabled: true, + ttsVoice: process.env.GROQ_TTS_VOICE || "tara", + ttsModel: process.env.GROQ_TTS_MODEL || "orpheus-tts-0.1-ayane", + ttsSpeed: process.env.GROQ_TTS_SPEED || "1.0", geminiKey: process.env.GEMINI_API_KEY || "", azureConfigured: !!process.env.AZURE_SPEECH_KEY && !!process.env.AZURE_SPEECH_REGION, @@ -1441,7 +1525,7 @@ class ApplicationController { // Writing to .env ensures they survive app restarts and are picked // up the next time the app boots. const envUpdates = {}; - if (settings.speechProvider === "azure" || settings.speechProvider === "whisper") { + if (settings.speechProvider === "azure" || settings.speechProvider === "whisper" || settings.speechProvider === "groq") { envUpdates.SPEECH_PROVIDER = settings.speechProvider; } if (settings.azureKey !== undefined) { @@ -1462,6 +1546,21 @@ class ApplicationController { if (settings.whisperSegmentMs !== undefined) { envUpdates.WHISPER_SEGMENT_MS = String(settings.whisperSegmentMs); } + if (settings.groqSttModel !== undefined) { + envUpdates.GROQ_STT_MODEL = settings.groqSttModel; + } + if (settings.ttsVoice !== undefined) { + envUpdates.GROQ_TTS_VOICE = settings.ttsVoice; + } + if (settings.ttsModel !== undefined) { + envUpdates.GROQ_TTS_MODEL = settings.ttsModel; + } + if (settings.ttsSpeed !== undefined) { + envUpdates.GROQ_TTS_SPEED = String(settings.ttsSpeed); + } + if (settings.groqKey !== undefined) { + envUpdates.GROQ_API_KEY = settings.groqKey; + } if (settings.llmProvider !== undefined) { envUpdates.LLM_PROVIDER = settings.llmProvider; } @@ -1471,6 +1570,12 @@ class ApplicationController { if (settings.openrouterModel !== undefined) { envUpdates.OPENROUTER_MODEL = settings.openrouterModel; } + if (settings.groqKey !== undefined) { + envUpdates.GROQ_API_KEY = settings.groqKey; + } + if (settings.groqModel !== undefined) { + envUpdates.GROQ_MODEL = settings.groqModel; + } if (settings.geminiKey !== undefined) { envUpdates.GEMINI_API_KEY = settings.geminiKey; } @@ -1482,8 +1587,8 @@ class ApplicationController { // use stale credentials or the wrong provider. const llmProviderChanged = settings.llmProvider !== undefined && (process.env.LLM_PROVIDER || "gemini") !== settings.llmProvider; - const llmKeyChanged = settings.geminiKey !== undefined || settings.openrouterKey !== undefined; - const llmModelChanged = settings.openrouterModel !== undefined; + const llmKeyChanged = settings.geminiKey !== undefined || settings.openrouterKey !== undefined || settings.groqKey !== undefined; + const llmModelChanged = settings.openrouterModel !== undefined || settings.groqModel !== undefined; if (llmProviderChanged || llmKeyChanged || llmModelChanged) { try { llmService.initializeClient(); @@ -1508,7 +1613,9 @@ class ApplicationController { const providerChanged = settings.speechProvider && speechService.provider !== settings.speechProvider; const whisperCommandChanged = settings.whisperCommand !== undefined && (process.env.WHISPER_COMMAND || '') !== String(settings.whisperCommand || ''); - if (providerChanged || whisperCommandChanged) { + const groqSttModelChanged = settings.groqSttModel !== undefined && + (process.env.GROQ_STT_MODEL || '') !== String(settings.groqSttModel || ''); + if (providerChanged || whisperCommandChanged || groqSttModelChanged) { try { speechService.initializeClient(); this.speechAvailable = speechService.isAvailable @@ -1526,6 +1633,7 @@ class ApplicationController { logger.info('Speech service reinitialized after settings change', { providerChanged, whisperCommandChanged, + groqSttModelChanged, speechAvailable: this.speechAvailable, }); } catch (e) { @@ -1535,9 +1643,35 @@ class ApplicationController { } } + // Reinitialize TTS service when Groq key or TTS settings change + const groqKeyChanged = settings.groqKey !== undefined; + const ttsSettingsChanged = settings.ttsEnabled !== undefined || + settings.ttsVoice !== undefined || + settings.ttsModel !== undefined || + settings.ttsSpeed !== undefined; + if (groqKeyChanged || ttsSettingsChanged) { + try { + ttsService.updateSettings({ + groqKey: process.env.GROQ_API_KEY, + ttsEnabled: settings.ttsEnabled, + ttsVoice: settings.ttsVoice, + ttsModel: settings.ttsModel, + ttsSpeed: settings.ttsSpeed + }); + logger.info('TTS service reinitialized after settings change'); + // Broadcast TTS settings to chat windows + windowManager.broadcastToAllWindows('tts-settings-changed', { + ttsEnabled: settings.ttsEnabled !== false + }); + } catch (e) { + logger.warn("Failed to reinitialize TTS service", { error: e.message }); + } + } + const redacted = { ...settings }; if (redacted.geminiKey) redacted.geminiKey = redacted.geminiKey.substring(0, 4) + "…"; if (redacted.openrouterKey) redacted.openrouterKey = redacted.openrouterKey.substring(0, 4) + "…"; + if (redacted.groqKey) redacted.groqKey = redacted.groqKey.substring(0, 4) + "…"; if (redacted.azureKey) redacted.azureKey = redacted.azureKey.substring(0, 4) + "…"; logger.info("Settings saved successfully", { ...redacted, diff --git a/onboarding.html b/onboarding.html index f17b5bfa..388e68dd 100644 --- a/onboarding.html +++ b/onboarding.html @@ -772,6 +772,16 @@

Choose your AI Provider

+
+
+
+
Groq (Fast inference)
+
+ Ultra-fast inference on Groq's LPU hardware. Free tier available. Supports Llama, Mixtral, and Gemma models. +
+
+
+
@@ -847,6 +857,48 @@

Choose your AI Provider

+ + +
diff --git a/onboarding.js b/onboarding.js index f51c2d62..63c43942 100644 --- a/onboarding.js +++ b/onboarding.js @@ -30,10 +30,12 @@ // ── State ───────────────────────────────────────────────────────── const state = { step: 0, - llmProvider: 'gemini', // 'gemini' | 'openrouter' + llmProvider: 'gemini', // 'gemini' | 'openrouter' | 'groq' geminiKey: '', openrouterKey: '', openrouterModel: 'openrouter/free', + groqKey: '', + groqModel: 'llama-3.3-70b-versatile', speechProvider: null, // 'whisper' | 'azure' | 'skip' azureKey: '', azureRegion: '', @@ -128,6 +130,9 @@ if (state.llmProvider === 'openrouter') { return !!state.openrouterKey.trim(); } + if (state.llmProvider === 'groq') { + return !!state.groqKey.trim(); + } return !!state.geminiKey.trim(); case 'speech': if (state.speechProvider === 'azure') { @@ -191,6 +196,7 @@ const llmProviderChoices = document.getElementById('llmProviderChoices'); const onboardingGeminiFields = document.getElementById('onboardingGeminiFields'); const onboardingOpenrouterFields = document.getElementById('onboardingOpenrouterFields'); + const onboardingGroqFields = document.getElementById('onboardingGroqFields'); if (llmProviderChoices) { llmProviderChoices.querySelectorAll('.choice-card').forEach((card) => { @@ -205,6 +211,9 @@ if (onboardingOpenrouterFields) { onboardingOpenrouterFields.style.display = value === 'openrouter' ? 'block' : 'none'; } + if (onboardingGroqFields) { + onboardingGroqFields.style.display = value === 'groq' ? 'block' : 'none'; + } // Reset the key status when switching providers if (keyStatus) keyStatus.style.display = 'none'; }); @@ -226,6 +235,21 @@ }); } + // Wire up Groq key and model inputs + const onboardingGroqKey = document.getElementById('onboardingGroqKey'); + const onboardingGroqModel = document.getElementById('onboardingGroqModel'); + + if (onboardingGroqKey) { + onboardingGroqKey.addEventListener('input', () => { + state.groqKey = onboardingGroqKey.value.trim(); + }); + } + if (onboardingGroqModel) { + onboardingGroqModel.addEventListener('input', () => { + state.groqModel = onboardingGroqModel.value.trim() || 'llama-3.3-70b-versatile'; + }); + } + // ── Wire up: Speech choices ─────────────────────────────────────── $$('#speechChoices .choice-card').forEach((card) => { card.addEventListener('click', () => { @@ -497,6 +521,22 @@ value: state.openrouterModel, cls: 'ok', }); + } else if (state.llmProvider === 'groq') { + rows.push({ + label: ' LLM Provider', + value: 'Groq', + cls: 'ok', + }); + rows.push({ + label: ' Groq Key', + value: state.groqKey ? 'Configured' : 'Missing', + cls: state.groqKey ? 'ok' : 'skip', + }); + rows.push({ + label: ' Model', + value: state.groqModel, + cls: 'ok', + }); } else { rows.push({ label: ' Gemini API', @@ -566,7 +606,9 @@ if (name === 'apikey') { const msg = state.llmProvider === 'openrouter' ? 'Enter your OpenRouter API key' - : 'Enter your Gemini API key'; + : state.llmProvider === 'groq' + ? 'Enter your Groq API key' + : 'Enter your Gemini API key'; setKeyStatus('error', msg); } return; @@ -579,6 +621,9 @@ if (state.llmProvider === 'openrouter') { if (state.openrouterKey) payload.openrouterKey = state.openrouterKey; payload.openrouterModel = state.openrouterModel; + } else if (state.llmProvider === 'groq') { + if (state.groqKey) payload.groqKey = state.groqKey; + payload.groqModel = state.groqModel; } else if (state.geminiKey) { payload.geminiKey = state.geminiKey; } @@ -730,6 +775,13 @@ if (onboardingOpenrouterKey) onboardingOpenrouterKey.placeholder = '•••••••••••••••• (already set)'; if (onboardingOpenrouterModel) onboardingOpenrouterModel.value = s.openrouterModel || 'openrouter/free'; state.openrouterModel = s.openrouterModel || 'openrouter/free'; + } else if (s.llmProvider === 'groq') { + state.llmProvider = 'groq'; + const groqCard = llmProviderChoices?.querySelector('[data-value="groq"]'); + if (groqCard) groqCard.click(); + if (onboardingGroqKey) onboardingGroqKey.placeholder = '•••••••••••••••• (already set)'; + if (onboardingGroqModel) onboardingGroqModel.value = s.groqModel || 'llama-3.3-70b-versatile'; + state.groqModel = s.groqModel || 'llama-3.3-70b-versatile'; } else if (s.geminiConfigured) { const geminiCard = llmProviderChoices?.querySelector('[data-value="gemini"]'); if (geminiCard) geminiCard.click(); diff --git a/package-lock.json b/package-lock.json index 97446462..9c556105 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@fortawesome/fontawesome-free": "^7.2.0", "@google/genai": "^2.9.0", "dotenv": "^16.3.1", @@ -337,6 +338,132 @@ "node": ">= 10.0.0" } }, + "node_modules/@ffmpeg-installer/darwin-arm64": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz", + "integrity": "sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==", + "cpu": [ + "arm64" + ], + "hasInstallScript": true, + "license": "https://git.ffmpeg.org/gitweb/ffmpeg.git/blob_plain/HEAD:/LICENSE.md", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ffmpeg-installer/darwin-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz", + "integrity": "sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==", + "cpu": [ + "x64" + ], + "hasInstallScript": true, + "license": "LGPL-2.1", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ffmpeg-installer/ffmpeg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz", + "integrity": "sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==", + "license": "LGPL-2.1", + "optionalDependencies": { + "@ffmpeg-installer/darwin-arm64": "4.1.5", + "@ffmpeg-installer/darwin-x64": "4.1.0", + "@ffmpeg-installer/linux-arm": "4.1.3", + "@ffmpeg-installer/linux-arm64": "4.1.4", + "@ffmpeg-installer/linux-ia32": "4.1.0", + "@ffmpeg-installer/linux-x64": "4.1.0", + "@ffmpeg-installer/win32-ia32": "4.1.0", + "@ffmpeg-installer/win32-x64": "4.1.0" + } + }, + "node_modules/@ffmpeg-installer/linux-arm": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz", + "integrity": "sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==", + "cpu": [ + "arm" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/linux-arm64": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz", + "integrity": "sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==", + "cpu": [ + "arm64" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/linux-ia32": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz", + "integrity": "sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==", + "cpu": [ + "ia32" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/linux-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz", + "integrity": "sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==", + "cpu": [ + "x64" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/win32-ia32": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz", + "integrity": "sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==", + "cpu": [ + "ia32" + ], + "license": "GPLv3", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@ffmpeg-installer/win32-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz", + "integrity": "sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==", + "cpu": [ + "x64" + ], + "license": "GPLv3", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@fortawesome/fontawesome-free": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-7.2.0.tgz", diff --git a/package.json b/package.json index 704e8ed3..b9396cbf 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ }, "license": "ISC", "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@fortawesome/fontawesome-free": "^7.2.0", "@google/genai": "^2.9.0", "dotenv": "^16.3.1", diff --git a/preload.js b/preload.js index 2b24ea53..59fe2c0b 100644 --- a/preload.js +++ b/preload.js @@ -115,12 +115,37 @@ contextBridge.exposeInMainWorld('electronAPI', { onRecordingStopped: (callback) => ipcRenderer.on('recording-stopped', callback), onCodingLanguageChanged: (callback) => ipcRenderer.on('coding-language-changed', callback), onMainWindowShown: (callback) => ipcRenderer.on('main-window-shown', callback), + + // TTS (Text-to-Speech) + synthesizeSpeech: (text, options) => ipcRenderer.invoke('synthesize-speech', text, options), + stopTtsPlayback: () => ipcRenderer.invoke('stop-tts-playback'), + onTtsStarted: (callback) => ipcRenderer.on('tts-started', callback), + onTtsCompleted: (callback) => ipcRenderer.on('tts-completed', callback), + onTtsError: (callback) => ipcRenderer.on('tts-error', callback), // Generic receive method receive: (channel, callback) => ipcRenderer.on(channel, callback), // Remove listeners - removeAllListeners: (channel) => ipcRenderer.removeAllListeners(channel) + removeAllListeners: (channel) => ipcRenderer.removeAllListeners(channel), + + // Audio capture (renderer-based microphone via getUserMedia) + sendAudioChunk: (buffer) => ipcRenderer.send('audio-chunk', buffer), + sendAudioCaptureError: (msg) => ipcRenderer.send('audio-capture-error', msg), + audioCaptureReady: () => ipcRenderer.send('audio-capture-ready'), + onStartRendererCapture: (callback) => { + const wrapped = () => { try { callback(); } catch (e) { console.error('onStartRendererCapture error:', e); } }; + ipcRenderer.on('start-renderer-capture', wrapped); + return () => ipcRenderer.removeListener('start-renderer-capture', wrapped); + }, + onStopRendererCapture: (callback) => { + ipcRenderer.on('stop-renderer-capture', callback); + return () => ipcRenderer.removeListener('stop-renderer-capture', callback); + }, + onStopAudioCapture: (callback) => { + ipcRenderer.on('stop-audio-capture', callback); + return () => ipcRenderer.removeListener('stop-audio-capture', callback); + } }) contextBridge.exposeInMainWorld('api', { diff --git a/settings.html b/settings.html index 1db9092b..c447dd20 100644 --- a/settings.html +++ b/settings.html @@ -366,6 +366,7 @@ @@ -424,11 +425,73 @@ + +
+
+
+
Groq STT Model
+
Groq's Whisper model for speech-to-text
+
+ +
+
+ Uses your GROQ_API_KEY for transcription via Groq's API. No local Whisper CLI required. +
+
+
Azure credentials apply immediately and are also written back to .env so they persist across restarts.
+ +
+
+ + Text-to-Speech +
+
+
+
+
Enable TTS
+
Read AI responses aloud using Groq Orpheus
+
+ +
+
+
+
TTS Voice
+
Orpheus voice for speech synthesis
+
+ +
+
+
+
TTS Speed
+
Playback speed (0.25 – 4.0)
+
+ +
+
+ Requires GROQ_API_KEY. TTS uses the Orpheus English model via Groq's API. +
+
+
@@ -444,6 +507,7 @@
@@ -480,6 +544,25 @@ OpenRouter automatically routes image requests to free vision-capable models. No charges apply for free models. See openrouter.ai for details.
+ + diff --git a/src/core/config.js b/src/core/config.js index 7521d0c6..09654a1e 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -63,6 +63,16 @@ class ConfigManager { temperature: 0.7, maxOutputTokens: 4096 } + }, + groq: { + baseUrl: 'https://api.groq.com/openai/v1', + model: 'llama-3.3-70b-versatile', + maxRetries: 1, + timeout: 30000, + generation: { + temperature: 0.7, + maxOutputTokens: 4096 + } } }, @@ -78,6 +88,12 @@ class ConfigManager { model: 'turbo', language: 'en', segmentMs: 4000 + }, + groq: { + sttModel: 'whisper-large-v3-turbo', + ttsModel: 'orpheus-tts-0.1-ayane', + ttsVoice: 'tara', + ttsSpeed: 1.0 } }, diff --git a/src/core/first-run.js b/src/core/first-run.js index 2dc580cd..7b02793b 100644 --- a/src/core/first-run.js +++ b/src/core/first-run.js @@ -37,6 +37,10 @@ class FirstRunManager { const orKey = (content.OPENROUTER_API_KEY || '').trim(); return !orKey || orKey === 'your_openrouter_api_key_here'; } + if (provider === 'groq') { + const grKey = (content.GROQ_API_KEY || '').trim(); + return !grKey || grKey === 'your_groq_api_key_here'; + } const gemini = (content.GEMINI_API_KEY || '').trim(); return !gemini || gemini === 'your_gemini_api_key_here'; } @@ -88,6 +92,8 @@ class FirstRunManager { const gemini = (env.GEMINI_API_KEY || '').trim(); const orKey = (env.OPENROUTER_API_KEY || '').trim(); const orKeyConfigured = !!orKey && orKey !== 'your_openrouter_api_key_here'; + const grKey = (env.GROQ_API_KEY || '').trim(); + const grKeyConfigured = !!grKey && grKey !== 'your_groq_api_key_here'; return { envExists: fs.existsSync(this.envPath), sentinelExists: fs.existsSync(this.sentinelPath), @@ -95,6 +101,8 @@ class FirstRunManager { geminiConfigured: !!gemini && gemini !== 'your_gemini_api_key_here', openrouterConfigured: orKeyConfigured, openrouterModel: env.OPENROUTER_MODEL || 'openrouter/free', + groqConfigured: grKeyConfigured, + groqModel: env.GROQ_MODEL || 'llama-3.3-70b-versatile', azureConfigured: !!(env.AZURE_SPEECH_KEY || '').trim() && !!(env.AZURE_SPEECH_REGION || '').trim(), whisperConfigured: !!(env.WHISPER_COMMAND || '').trim(), needsOnboarding: this.needsOnboarding() diff --git a/src/services/llm.service.js b/src/services/llm.service.js index f27b4bd6..693cf489 100644 --- a/src/services/llm.service.js +++ b/src/services/llm.service.js @@ -22,8 +22,12 @@ class LLMService { return this._openrouterInit(); } + if (this.provider === 'groq') { + return this._groqInit(); + } + const apiKey = config.getApiKey('GEMINI'); - + if (!apiKey || apiKey === 'your-api-key-here') { logger.warn('Gemini API key not configured', { keyExists: !!apiKey, @@ -71,6 +75,107 @@ class LLMService { return process.env.OPENROUTER_MODEL || config.get('llm.openrouter.model') || 'openrouter/free'; } + // ── Groq initialisation & helpers ───────────────────────────────── + + _groqInit() { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey || apiKey.trim().length === 0) { + logger.warn('Groq API key not configured'); + return; + } + this.model = config.get('llm.groq.model') || 'llama-3.3-70b-versatile'; + this.isInitialized = true; + logger.info('Groq client initialized successfully', { + model: this.model + }); + } + + _groqIsAvailable() { + const key = process.env.GROQ_API_KEY; + return !!key && key.trim().length > 0; + } + + _groqGetModel() { + return process.env.GROQ_MODEL || config.get('llm.groq.model') || 'llama-3.3-70b-versatile'; + } + + // ── Shared OpenAI-compatible HTTPS executor ─────────────────────── + // Used by both OpenRouter and Groq (and any future OpenAI-compatible provider). + + async _executeChatCompletion({ apiKey, baseUrl, model, messages, extraHeaders = {}, overrides = {} }) { + const https = require('https'); + if (!apiKey) { + throw new Error('API key not configured'); + } + + const timeout = overrides.timeout || 30000; + const genConfig = { ...overrides.generation }; + + const url = `${baseUrl}/chat/completions`; + const body = { + model, + messages, + temperature: genConfig.temperature, + max_tokens: genConfig.maxOutputTokens + }; + if (genConfig.topP !== undefined) body.top_p = genConfig.topP; + + const postData = JSON.stringify(body); + const maxRetries = overrides.maxRetries || 1; + + let lastError; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + 'Content-Length': Buffer.byteLength(postData), + 'User-Agent': this.getUserAgent(), + ...extraHeaders + }, + timeout + }; + + const response = await new Promise((resolve, reject) => { + const req = https.request(url, options, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => { + if (res.statusCode === 200) { + try { + resolve({ status: res.statusCode, data: JSON.parse(data) }); + } catch (e) { + reject(new Error(`Failed to parse response: ${e.message}`)); + } + } else { + reject(new Error(`HTTP ${res.statusCode}: ${data.substring(0, 500)}`)); + } + }); + }); + req.on('error', (e) => reject(new Error(`Request failed: ${e.message}`))); + req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); }); + req.write(postData); + req.end(); + }); + + const text = this._openrouterExtractResponse(response.data); + return text; + + } catch (error) { + lastError = error; + if (attempt < maxRetries) { + const delay = 1500 * attempt + Math.random() * 1000; + await this.delay(delay); + } + } + } + + throw lastError || new Error('Request failed after all retries'); + } + _openrouterBuildMessages({ systemInstruction, contents }) { const messages = []; if (systemInstruction) { @@ -162,87 +267,43 @@ class LLMService { } async _openrouterExecute(messages, overrides = {}) { - const https = require('https'); const apiKey = process.env.OPENROUTER_API_KEY; - if (!apiKey) { - throw new Error('OpenRouter API key not configured'); - } - + if (!apiKey) throw new Error('OpenRouter API key not configured'); const baseUrl = config.get('llm.openrouter.baseUrl') || 'https://openrouter.ai/api/v1'; const model = overrides.model || this._openrouterGetModel(); - const timeout = overrides.timeout || config.get('llm.openrouter.timeout') || 60000; - const genConfig = { ...config.get('llm.openrouter.generation'), ...overrides.generation }; + const timeout = overrides.timeout || config.get('llm.openrouter.timeout') || 30000; + const maxRetries = overrides.maxRetries || config.get('llm.openrouter.maxRetries') || 1; + const generation = { ...config.get('llm.openrouter.generation'), ...overrides.generation }; - const url = `${baseUrl}/chat/completions`; - const body = { + return this._executeChatCompletion({ + apiKey, + baseUrl, model, messages, - temperature: genConfig.temperature, - max_tokens: genConfig.maxOutputTokens - }; - if (genConfig.topP !== undefined) body.top_p = genConfig.topP; - - const postData = JSON.stringify(body); - const maxRetries = overrides.maxRetries || config.get('llm.openrouter.maxRetries') || 3; - - let lastError; - - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - const options = { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - 'HTTP-Referer': 'https://opencluely.app', - 'X-Title': 'OpenCluely', - 'Content-Length': Buffer.byteLength(postData), - 'User-Agent': this.getUserAgent() - }, - timeout - }; - - const response = await new Promise((resolve, reject) => { - const req = https.request(url, options, (res) => { - let data = ''; - res.on('data', (chunk) => { data += chunk; }); - res.on('end', () => { - if (res.statusCode === 200) { - try { - resolve({ status: res.statusCode, data: JSON.parse(data) }); - } catch (e) { - reject(new Error(`Failed to parse OpenRouter response: ${e.message}`)); - } - } else { - reject(new Error(`OpenRouter HTTP ${res.statusCode}: ${data.substring(0, 500)}`)); - } - }); - }); - req.on('error', (e) => reject(new Error(`OpenRouter request failed: ${e.message}`))); - req.on('timeout', () => { req.destroy(); reject(new Error('OpenRouter request timeout')); }); - req.write(postData); - req.end(); - }); - - const text = this._openrouterExtractResponse(response.data); - return text; - - } catch (error) { - lastError = error; - logger.warn(`OpenRouter attempt ${attempt}/${maxRetries} failed`, { - error: error.message, - model, - attempt - }); - - if (attempt < maxRetries) { - const delay = 1500 * attempt + Math.random() * 1000; - await this.delay(delay); - } - } - } + extraHeaders: { + 'HTTP-Referer': 'https://opencluely.app', + 'X-Title': 'OpenCluely', + }, + overrides: { timeout, maxRetries, generation } + }); + } - throw lastError || new Error('OpenRouter request failed after all retries'); + async _groqExecute(messages, overrides = {}) { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) throw new Error('Groq API key not configured'); + const baseUrl = config.get('llm.groq.baseUrl') || 'https://api.groq.com/openai/v1'; + const model = overrides.model || this._groqGetModel(); + const timeout = overrides.timeout || config.get('llm.groq.timeout') || 30000; + const maxRetries = overrides.maxRetries || config.get('llm.groq.maxRetries') || 1; + const generation = { ...config.get('llm.groq.generation'), ...overrides.generation }; + + return this._executeChatCompletion({ + apiKey, + baseUrl, + model, + messages, + overrides: { timeout, maxRetries, generation } + }); } _openrouterExtractResponse(response) { @@ -420,6 +481,204 @@ class LLMService { } } + // ── Groq process methods (OpenAI-compatible, reuse build helpers) ─ + + async _groqProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const messages = this._openrouterBuildImageBody(imageBuffer, mimeType, activeSkill, programmingLanguage, skillPrompt); + + const responseText = await this._groqExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('Groq image processing', startTime, { + activeSkill, + imageSize: imageBuffer.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'groq', + model: this._groqGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isImageAnalysis: true, + mimeType + } + }; + } catch (error) { + this.errorCount++; + logger.error('Groq image processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse('[image]', activeSkill); + } + } + + async _groqProcessText(text, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(15) + : []; + + const messages = this._openrouterBuildRequestBody(text, activeSkill, programmingLanguage, skillPrompt, conversationHistory); + + const responseText = await this._groqExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('Groq text processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'groq', + model: this._groqGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false + } + }; + } catch (error) { + this.errorCount++; + logger.error('Groq text processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse(text, activeSkill); + } + } + + async _groqProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage) { + if (!text || typeof text !== 'string' || text.trim().length < 2) { + logger.warn('Skipping transcription for empty or very short input'); + return { + response: '', + metadata: { provider: 'groq', skill: activeSkill, processingTime: 0, usedFallback: true, isTranscriptionResponse: true } + }; + } + + const startTime = Date.now(); + this.requestCount++; + + try { + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(10) + : []; + + const messages = this._openrouterBuildTranscriptionBody(text.trim(), activeSkill, programmingLanguage, conversationHistory); + + const responseText = await this._groqExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('Groq transcription processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'groq', + model: this._groqGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isTranscriptionResponse: true + } + }; + } catch (error) { + this.errorCount++; + logger.error('Groq transcription processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateIntelligentFallbackResponse(text, activeSkill); + } + } + + async _groqTestConnection() { + try { + const messages = [{ role: 'user', content: 'Test connection. Please respond with "OK".' }]; + const startTime = Date.now(); + const text = await this._groqExecute(messages, { maxRetries: 1, generation: { temperature: 0, maxOutputTokens: 64 } }); + const latency = Date.now() - startTime; + + logger.info('Groq connection test successful', { + response: text, + latency, + model: this._groqGetModel() + }); + + return { + success: true, + response: text, + latency, + model: this._groqGetModel() + }; + } catch (error) { + const errMsg = error.message || ''; + let friendlyError; + if (errMsg.includes('401') || errMsg.includes('unauthorized')) { + friendlyError = 'Invalid Groq API key. Get one at console.groq.com/keys.'; + } else if (errMsg.includes('429')) { + friendlyError = 'Groq rate limit exceeded. Wait a moment and try again.'; + } else if (errMsg.includes('timeout') || errMsg.includes('ENOTFOUND') || errMsg.includes('ECONNREFUSED')) { + friendlyError = 'Cannot reach Groq servers. Check your internet connection.'; + } else { + friendlyError = `Groq error: ${errMsg.substring(0, 200)}`; + } + + return { + success: false, + error: friendlyError, + errorType: 'GROQ_ERROR' + }; + } + } + async _openrouterTestConnection() { try { const messages = [{ role: 'user', content: 'Test connection. Please respond with "OK".' }]; @@ -550,7 +809,17 @@ class LLMService { } if (this.provider === 'openrouter') { - return this._openrouterProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage); + // OpenRouter path – wrap with overall timeout to avoid indefinite hangs + const timeoutMs = (config.get('llm.openrouter.timeout') || 30000) * 2; + const openRouterPromise = this._openrouterProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage); + const timeoutPromise = new Promise((_, reject) => + setTimeout(() => reject(new Error('OpenRouter image processing timed out after ' + timeoutMs + 'ms')), timeoutMs) + ); + return await Promise.race([openRouterPromise, timeoutPromise]); + } + + if (this.provider === 'groq') { + return this._groqProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage); } const startTime = Date.now(); @@ -648,6 +917,10 @@ class LLMService { return this._openrouterProcessText(text, activeSkill, sessionMemory, programmingLanguage); } + if (this.provider === 'groq') { + return this._groqProcessText(text, activeSkill, sessionMemory, programmingLanguage); + } + const startTime = Date.now(); this.requestCount++; @@ -722,6 +995,10 @@ class LLMService { return this._openrouterProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage); } + if (this.provider === 'groq') { + return this._groqProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage); + } + const startTime = Date.now(); this.requestCount++; @@ -1450,6 +1727,10 @@ Remember: Be intelligent about filtering - only provide detailed responses when return this._openrouterTestConnection(); } + if (this.provider === 'groq') { + return this._groqTestConnection(); + } + try { // First check network connectivity const networkCheck = await this.checkNetworkConnectivity(); @@ -1568,6 +1849,8 @@ Remember: Be intelligent about filtering - only provide detailed responses when updateApiKey(newApiKey) { if (this.provider === 'openrouter') { process.env.OPENROUTER_API_KEY = newApiKey; + } else if (this.provider === 'groq') { + process.env.GROQ_API_KEY = newApiKey; } else { process.env.GEMINI_API_KEY = newApiKey; } @@ -1589,6 +1872,17 @@ Remember: Be intelligent about filtering - only provide detailed responses when config: config.get('llm.openrouter') }; } + if (this.provider === 'groq') { + return { + provider: 'groq', + isInitialized: this.isInitialized, + requestCount: this.requestCount, + errorCount: this.errorCount, + successRate: this.requestCount > 0 ? ((this.requestCount - this.errorCount) / this.requestCount) * 100 : 0, + model: this._groqGetModel(), + config: config.get('llm.groq') + }; + } return { provider: 'gemini', isInitialized: this.isInitialized, @@ -1609,13 +1903,20 @@ Remember: Be intelligent about filtering - only provide detailed responses when * wins. If both fail, throw the error from the primary (SDK) method. */ async _raceGeminiMethods(request) { + // Use a generous overall timeout (2 minutes) to guarantee the race resolves/rejects + const overallTimeout = new Promise((_, reject) => + setTimeout(() => reject(new Error('Gemini race timed out after 120000 ms')), 120000) + ); try { - return await Promise.any([ - this.executeRequest(request), - this.executeAlternativeRequest(request) + return await Promise.race([ + Promise.any([ + this.executeRequest(request), + this.executeAlternativeRequest(request) + ]), + overallTimeout ]); } catch (err) { - throw err.errors?.[0] || new Error('Both Gemini request methods failed'); + throw err.errors?.[0] || err || new Error('Both Gemini request methods failed'); } } diff --git a/src/services/speech.service.js b/src/services/speech.service.js index b0c24f3e..633ccbde 100644 --- a/src/services/speech.service.js +++ b/src/services/speech.service.js @@ -394,13 +394,6 @@ try { logger.warn('Azure Speech SDK unavailable', { error: error.message }); } -let recorder = null; -try { - recorder = require('node-record-lpcm16'); -} catch (error) { - logger.warn('Local audio recorder dependency unavailable', { error: error.message }); -} - class SpeechService extends EventEmitter { constructor() { super(); @@ -423,6 +416,8 @@ class SpeechService extends EventEmitter { this.pendingFlush = false; this.audioProgram = null; this.whisperCommand = null; + this._audioCaptureWindow = null; + this.groqSttModel = null; this.initializeClient(); } @@ -433,8 +428,10 @@ class SpeechService extends EventEmitter { this.available = false; this.speechConfig = null; this.whisperCommand = null; + this.groqSttModel = null; const provider = this._getConfiguredProvider(); + console.log('[SpeechService] initializeClient() provider:', provider); this.provider = provider; if (provider === 'azure') { @@ -447,7 +444,12 @@ class SpeechService extends EventEmitter { return; } - const reason = 'Speech recognition disabled. Configure Azure or local Whisper.'; + if (provider === 'groq') { + this._initializeGroqClient(); + return; + } + + const reason = 'Speech recognition disabled. Configure Azure, Groq, or local Whisper.'; logger.warn(reason); this.emit('status', reason); } @@ -458,10 +460,6 @@ class SpeechService extends EventEmitter { throw new Error('Azure Speech SDK dependency is not installed'); } - if (!recorder || typeof recorder.record !== 'function') { - throw new Error('Local microphone recorder dependency is not installed'); - } - const subscriptionKey = this._getSetting('azureKey') || process.env.AZURE_SPEECH_KEY; const region = this._getSetting('azureRegion') || process.env.AZURE_SPEECH_REGION; @@ -507,10 +505,6 @@ class SpeechService extends EventEmitter { _initializeWhisperClient() { try { - if (!recorder || typeof recorder.record !== 'function') { - throw new Error('Local microphone recorder dependency is not installed'); - } - this.whisperCommand = this._resolveWhisperCommand(); if (!this.whisperCommand) { const reason = 'Local Whisper unavailable. Install the Whisper CLI or set WHISPER_COMMAND.'; @@ -536,7 +530,36 @@ class SpeechService extends EventEmitter { } } + _initializeGroqClient() { + try { + const apiKey = process.env.GROQ_API_KEY; + + if (!apiKey) { + const reason = 'Groq STT unavailable. Set GROQ_API_KEY to use Groq speech.'; + logger.warn(reason); + this.emit('status', reason); + return; + } + + this.groqSttModel = this._getSetting('groqSttModel') || process.env.GROQ_STT_MODEL || config.get('speech.groq.sttModel') || 'whisper-large-v3-turbo'; + + this.available = true; + logger.info('Groq STT service initialized successfully', { + model: this.groqSttModel + }); + this.emit('status', 'Groq STT ready'); + } catch (error) { + logger.error('Failed to initialize Groq STT client', { + error: error.message, + stack: error.stack + }); + this.available = false; + this.emit('status', 'Groq STT unavailable'); + } + } + startRecording() { + console.log('[SpeechService] startRecording() called, provider:', this.provider, 'available:', this.available, 'isRecording:', this.isRecording); try { if (!this.available) { const errorMsg = `Speech provider "${this.provider}" is not available`; @@ -563,6 +586,11 @@ class SpeechService extends EventEmitter { return; } + if (this.provider === 'groq') { + this._startGroqRecording(); + return; + } + throw new Error(`Unsupported speech provider: ${this.provider}`); } catch (error) { logger.error('Critical error in startRecording', { error: error.message, stack: error.stack }); @@ -671,6 +699,7 @@ class SpeechService extends EventEmitter { } _startWhisperRecording() { + console.log('[SpeechService] _startWhisperRecording() called'); this._cleanup(); this.isRecording = true; this.segmentBuffers = []; @@ -728,6 +757,11 @@ class SpeechService extends EventEmitter { return; } + if (this.provider === 'groq') { + this._finalizeGroqStop(); + return; + } + this._finalizeStop('Recording stopped'); } @@ -756,6 +790,84 @@ class SpeechService extends EventEmitter { } } + _startGroqRecording() { + this._cleanup(); + this.isRecording = true; + this.segmentBuffers = []; + this.segmentBytes = 0; + this.emit('recording-started'); + this.emit('status', 'Groq STT recording started'); + this._startMicrophoneCapture(); + + if (global.windowManager) { + global.windowManager.handleRecordingStarted(); + } + } + + async _finalizeGroqStop() { + if (!this.segmentBytes) { + this._finalizeStop('Recording stopped'); + return; + } + + const audioBuffer = Buffer.concat(this.segmentBuffers, this.segmentBytes); + this.segmentBuffers = []; + this.segmentBytes = 0; + + try { + const transcript = await this._groqTranscribe(audioBuffer); + if (transcript && transcript.trim()) { + this.emit('transcription', transcript.trim()); + } + } catch (error) { + logger.error('Groq STT transcription failed', { error: error.message }); + this.emit('error', `Groq transcription failed: ${error.message}`); + } finally { + this._finalizeStop('Recording stopped'); + } + } + + async _groqTranscribe(audioBuffer) { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) { + throw new Error('GROQ_API_KEY not configured'); + } + + const baseUrl = 'https://api.groq.com/openai/v1'; + const model = this.groqSttModel; + const wavBuffer = this._createWavBuffer(audioBuffer); + + const boundary = '----FormBoundary' + Date.now(); + const header = Buffer.from( + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="model"\r\n\r\n` + + `${model}\r\n` + + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n` + + `Content-Type: audio/wav\r\n\r\n` + ); + const footer = Buffer.from(`\r\n--${boundary}--\r\n`); + const body = Buffer.concat([header, wavBuffer, footer]); + + const response = await fetch(`${baseUrl}/audio/transcriptions`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(body.length) + }, + body + }); + + if (!response.ok) { + const errText = await response.text(); + throw new Error(`Groq STT API error ${response.status}: ${errText}`); + } + + const data = await response.json(); + return data.text || ''; + } + _finalizeStop(statusMessage) { this._cleanup(); this.emit('recording-stopped'); @@ -800,6 +912,28 @@ class SpeechService extends EventEmitter { this.recording = null; } + // Destroy the hidden audio-capture window (renderer-based capture) + if (this._audioCaptureWindow) { + try { + if (!this._audioCaptureWindow.isDestroyed()) { + this._audioCaptureWindow.webContents.send('stop-audio-capture'); + this._audioCaptureWindow.destroy(); + } + } catch (error) { + logger.error('Error cleaning up audio capture window', { error: error.message }); + } + this._audioCaptureWindow = null; + } + + // Stop renderer-based capture in the chat window + if (global.windowManager) { + try { + global.windowManager.broadcastToAllWindows('stop-renderer-capture', {}); + } catch (error) { + logger.error('Error broadcasting stop-renderer-capture', { error: error.message }); + } + } + if (this.pushStream) { try { if (typeof this.pushStream.close === 'function') { @@ -851,6 +985,11 @@ class SpeechService extends EventEmitter { return this._transcribeWhisperFile(audioFilePath); } + if (this.provider === 'groq') { + const audioBuffer = fs.readFileSync(audioFilePath); + return this._groqTranscribe(audioBuffer); + } + throw new Error('Speech service not initialized'); } @@ -894,14 +1033,33 @@ class SpeechService extends EventEmitter { }; } + if (this.provider === 'groq') { + if (!process.env.GROQ_API_KEY) { + return { success: false, message: 'GROQ_API_KEY not configured' }; + } + // Test by sending a tiny silent WAV to the transcription API + try { + const silentBuffer = Buffer.alloc(16000 * 2); // 1 second of silence (16-bit PCM, 16kHz, mono) + const result = await this._groqTranscribe(silentBuffer); + // A successful API call returns empty text or a transcription + return { success: true, message: `Groq STT connection OK (model: ${this.groqSttModel})` }; + } catch (error) { + return { success: false, message: `Groq STT connection failed: ${error.message}` }; + } + } + return { success: false, message: 'Speech service not initialized' }; } getStatus() { + const isInitialized = this.provider === 'azure' ? !!this.speechConfig + : this.provider === 'groq' ? !!process.env.GROQ_API_KEY + : !!this.whisperCommand; + return { provider: this.provider, isRecording: this.isRecording, - isInitialized: this.provider === 'azure' ? !!this.speechConfig : !!this.whisperCommand, + isInitialized, sessionDuration: this.sessionStartTime ? Date.now() - this.sessionStartTime : 0, retryCount: this.retryCount, effectiveSettings: { @@ -912,11 +1070,13 @@ class SpeechService extends EventEmitter { whisperModelDir: this._getWhisperModelDir(), whisperModel: this._getWhisperModel(), whisperLanguage: this._getWhisperLanguage(), - whisperSegmentMs: String(this._getWhisperSegmentMs()) + whisperSegmentMs: String(this._getWhisperSegmentMs()), + groqSttModel: this.groqSttModel || process.env.GROQ_STT_MODEL || '' }, config: { azure: config.get('speech.azure') || {}, whisper: config.get('speech.whisper') || {}, + groq: config.get('speech.groq') || {}, selectedProvider: this.provider } }; @@ -931,11 +1091,15 @@ class SpeechService extends EventEmitter { return !!this.whisperCommand && !!this.available; } + if (this.provider === 'groq') { + return !!process.env.GROQ_API_KEY && !!this.available; + } + return false; } updateSettings(settings = {}) { - const speechKeys = ['speechProvider', 'azureKey', 'azureRegion', 'whisperCommand', 'whisperModelDir', 'whisperModel', 'whisperLanguage', 'whisperSegmentMs']; + const speechKeys = ['speechProvider', 'azureKey', 'azureRegion', 'whisperCommand', 'whisperModelDir', 'whisperModel', 'whisperLanguage', 'whisperSegmentMs', 'groqSttModel']; let changed = false; for (const key of speechKeys) { @@ -954,8 +1118,10 @@ class SpeechService extends EventEmitter { _getConfiguredProvider() { const provider = String(this._getSetting('speechProvider') || process.env.SPEECH_PROVIDER || '').trim().toLowerCase(); + console.log('[SpeechService] _getConfiguredProvider raw provider string:', provider, 'SPEECH_PROVIDER env:', process.env.SPEECH_PROVIDER); - if (provider === 'azure' || provider === 'whisper') { + if (provider === 'azure' || provider === 'whisper' || provider === 'groq') { + console.log('[SpeechService] _getConfiguredProvider returning:', provider); return provider; } @@ -963,9 +1129,17 @@ class SpeechService extends EventEmitter { (this._getSetting('azureRegion') || process.env.AZURE_SPEECH_REGION)); if (hasAzure) { + console.log('[SpeechService] _getConfiguredProvider auto-detecting azure'); return 'azure'; } + const hasGroq = !!(process.env.GROQ_API_KEY); + if (hasGroq) { + console.log('[SpeechService] _getConfiguredProvider auto-detecting groq'); + return 'groq'; + } + + console.log('[SpeechService] _getConfiguredProvider falling back to whisper'); return 'whisper'; } @@ -1196,63 +1370,33 @@ class SpeechService extends EventEmitter { } _startMicrophoneCapture() { - if (!recorder || typeof recorder.record !== 'function') { - this.emit('error', 'Local microphone capture dependency is missing. Run npm install to restore speech recording support.'); - return; - } - - this._startMicrophoneCaptureWithFallback(['sox', 'rec', 'arecord']); - } - - _startMicrophoneCaptureWithFallback(programs) { - const queue = [...programs]; - - const tryNextProgram = () => { - const program = queue.shift(); - if (!program) { - this.emit('error', 'Could not start microphone capture with any audio program'); - return; + console.log('[SpeechService] _startMicrophoneCapture() called'); + try { + // Instead of creating a hidden BrowserWindow (which fails on unsigned macOS), + // broadcast to the chat window (already visible) to start its own getUserMedia capture. + if (global.windowManager) { + console.log('[SpeechService] Broadcasting start-renderer-capture'); + global.windowManager.broadcastToAllWindows('start-renderer-capture', {}); } - try { - this.recording = recorder.record({ - sampleRateHertz: 16000, - channels: 1, - threshold: 0, - verbose: false, - recordProgram: program, - silence: '10.0s' - }); - - const stream = this.recording.stream(); - this.audioProgram = program; - - stream.on('error', (error) => { - logger.error('Audio recording stream error', { error: error.message, program }); - if (this.recording) { - try { - this.recording.stop(); - } catch (stopError) { - logger.error('Error stopping failed recording program', { error: stopError.message }); - } - this.recording = null; - } - - if (this.isRecording) { - tryNextProgram(); - } - }); - - stream.on('data', (chunk) => { - this._handleAudioChunk(chunk); - }); - } catch (error) { - logger.error('Failed to start microphone capture program', { program, error: error.message }); - tryNextProgram(); - } - }; + this.audioProgram = 'getUserMedia'; + logger.info('Renderer-based microphone capture started via chat window'); + } catch (error) { + logger.error('Failed to start renderer-based microphone capture', { error: error.message }); + this.emit('error', `Mic capture init failed: ${error.message}`); + } + } - tryNextProgram(); + _handleRendererAudioChunk(buffer) { + // Convert ArrayBuffer (from IPC) to Buffer and feed into the existing pipeline + if (!buffer) return; + try { + const buf = Buffer.from(buffer); + console.log('[SpeechService] _handleRendererAudioChunk buffer size:', buf.length, 'provider:', this.provider, 'isRecording:', this.isRecording); + this._handleAudioChunk(buf); + } catch (error) { + logger.error('Error handling renderer audio chunk', { error: error.message }); + } } _handleAudioChunk(chunk) { @@ -1269,9 +1413,12 @@ class SpeechService extends EventEmitter { return; } - if (this.provider === 'whisper') { + if (this.provider === 'whisper' || this.provider === 'groq') { this.segmentBuffers.push(Buffer.from(chunk)); this.segmentBytes += chunk.length; + if (this.segmentBytes % 16000 === 0) { + console.log('[SpeechService] Buffered audio bytes:', this.segmentBytes); + } } } diff --git a/src/services/tts.service.js b/src/services/tts.service.js new file mode 100644 index 00000000..de0a78f7 --- /dev/null +++ b/src/services/tts.service.js @@ -0,0 +1,181 @@ +const { EventEmitter } = require('events'); +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { spawn } = require('child_process'); +const logger = require('../core/logger').createServiceLogger('TTS'); + +const GROQ_TTS_URL = 'https://api.groq.com/openai/v1/audio/speech'; + +class TtsService extends EventEmitter { + constructor() { + super(); + this._currentProcess = null; + this._enabled = false; + this._voice = 'tara'; + this._model = 'orpheus-tts-0.1-ayane'; + this._speed = 1.0; + this._initialize(); + } + + _initialize() { + try { + this._model = process.env.GROQ_TTS_MODEL || 'orpheus-tts-0.1-ayane'; + this._voice = process.env.GROQ_TTS_VOICE || 'tara'; + this._speed = parseFloat(process.env.GROQ_TTS_SPEED || '1.0') || 1.0; + this._enabled = !!process.env.GROQ_API_KEY; + logger.info(`TTS service initialized (model: ${this._model}, voice: ${this._voice})`); + } catch (error) { + logger.error('Failed to initialize TTS service', { error: error.message }); + this._enabled = false; + } + } + + isEnabled() { + return this._enabled; + } + + updateSettings(settings = {}) { + if (settings.ttsEnabled !== undefined) { + this._enabled = !!settings.ttsEnabled && !!process.env.GROQ_API_KEY; + } + if (settings.ttsVoice !== undefined) { + this._voice = settings.ttsVoice; + } + if (settings.ttsSpeed !== undefined) { + this._speed = parseFloat(settings.ttsSpeed) || 1.0; + } + if (settings.ttsModel !== undefined) { + this._model = settings.ttsModel; + } + // Re-enable check if GROQ_API_KEY might have changed + if (settings.groqKey !== undefined || settings.ttsEnabled !== undefined) { + this._enabled = !!process.env.GROQ_API_KEY && settings.ttsEnabled !== false; + } + } + + async synthesizeSpeech(text, options = {}) { + if (!this._enabled) { + throw new Error('TTS is not enabled. Configure GROQ_API_KEY and enable TTS.'); + } + + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) { + throw new Error('GROQ_API_KEY not configured'); + } + + const model = options.model || this._model; + const voice = options.voice || this._voice; + const speed = options.speed || this._speed; + + this.emit('tts-started'); + + try { + const response = await fetch(GROQ_TTS_URL, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + model, + input: text, + voice, + response_format: 'wav', + speed + }) + }); + + if (!response.ok) { + const errText = await response.text(); + throw new Error(`Groq TTS API error ${response.status}: ${errText}`); + } + + const audioBuffer = Buffer.from(await response.arrayBuffer()); + await this._playAudio(audioBuffer); + this.emit('tts-completed'); + } catch (error) { + logger.error('TTS synthesis failed', { error: error.message }); + this.emit('tts-error', error.message); + throw error; + } + } + + async _playAudio(audioBuffer) { + const ext = '.wav'; + const tempFile = path.join(os.tmpdir(), `opencluely-tts-${Date.now()}${ext}`); + + try { + fs.writeFileSync(tempFile, audioBuffer); + await this._spawnPlayer(tempFile); + } finally { + try { + fs.unlinkSync(tempFile); + } catch (_) {} + } + } + + _spawnPlayer(filePath) { + return new Promise((resolve, reject) => { + let cmd, args; + + if (process.platform === 'darwin') { + cmd = 'afplay'; + args = [filePath]; + } else if (process.platform === 'win32') { + // Windows: try using PowerShell's Media.SoundPlayer + cmd = 'powershell'; + args = ['-c', `(New-Object Media.SoundPlayer "${filePath}").PlaySync();`]; + } else { + // Linux: try ffplay, aplay, or paplay + cmd = 'ffplay'; + args = ['-nodisp', '-autoexit', filePath]; + } + + try { + const player = spawn(cmd, args, { stdio: 'ignore' }); + this._currentProcess = player; + + player.on('error', (err) => { + this._currentProcess = null; + // Fallback for Linux: try aplay + if (process.platform !== 'linux' || cmd === 'aplay') { + reject(new Error(`Audio player failed: ${err.message}`)); + return; + } + // Try aplay as fallback on Linux + const fallback = spawn('aplay', [filePath], { stdio: 'ignore' }); + this._currentProcess = fallback; + fallback.on('error', () => reject(new Error('No audio player available (tried ffplay, aplay)'))); + fallback.on('close', (code) => { + this._currentProcess = null; + code === 0 ? resolve() : reject(new Error(`aplay exited with code ${code}`)); + }); + }); + + player.on('close', (code) => { + this._currentProcess = null; + if (code === 0) { + resolve(); + } else { + reject(new Error(`Audio player exited with code ${code}`)); + } + }); + } catch (error) { + this._currentProcess = null; + reject(new Error(`Failed to spawn audio player: ${error.message}`)); + } + }); + } + + stopPlayback() { + if (this._currentProcess) { + try { + this._currentProcess.kill(); + } catch (_) {} + this._currentProcess = null; + } + } +} + +module.exports = new TtsService(); diff --git a/src/ui/audio-capture.html b/src/ui/audio-capture.html new file mode 100644 index 00000000..b4933ffc --- /dev/null +++ b/src/ui/audio-capture.html @@ -0,0 +1,75 @@ + + +Audio Capture + + + + diff --git a/src/ui/settings-window.js b/src/ui/settings-window.js index a8b1852b..94373538 100644 --- a/src/ui/settings-window.js +++ b/src/ui/settings-window.js @@ -13,6 +13,10 @@ document.addEventListener('DOMContentLoaded', () => { const whisperModelInput = document.getElementById('whisperModel'); const whisperLanguageInput = document.getElementById('whisperLanguage'); const whisperSegmentMsInput = document.getElementById('whisperSegmentMs'); + const groqSttModelInput = document.getElementById('groqSttModel'); + const ttsEnabledInput = document.getElementById('ttsEnabled'); + const ttsVoiceInput = document.getElementById('ttsVoice'); + const ttsSpeedInput = document.getElementById('ttsSpeed'); const geminiKeyInput = document.getElementById('geminiKey'); const windowGapInput = document.getElementById('windowGap'); const codingLanguageSelect = document.getElementById('codingLanguage'); @@ -23,6 +27,8 @@ document.addEventListener('DOMContentLoaded', () => { const llmProviderSelect = document.getElementById('llmProvider'); const openrouterKeyInput = document.getElementById('openrouterKey'); const openrouterModelInput = document.getElementById('openrouterModel'); + const groqKeyInput = document.getElementById('groqKey'); + const groqModelInput = document.getElementById('groqModel'); // Check if window.api exists if (!window.api) { @@ -86,11 +92,17 @@ document.addEventListener('DOMContentLoaded', () => { if (whisperModelInput) whisperModelInput.value = settings.whisperModel || ''; if (whisperLanguageInput) whisperLanguageInput.value = settings.whisperLanguage || ''; if (whisperSegmentMsInput) whisperSegmentMsInput.value = settings.whisperSegmentMs || ''; + if (groqSttModelInput) groqSttModelInput.value = settings.groqSttModel || 'whisper-large-v3-turbo'; + if (ttsEnabledInput) ttsEnabledInput.checked = settings.ttsEnabled !== false; + if (ttsVoiceInput) ttsVoiceInput.value = settings.ttsVoice || 'tara'; + if (ttsSpeedInput) ttsSpeedInput.value = settings.ttsSpeed || '1.0'; if (geminiKeyInput) geminiKeyInput.value = settings.geminiKey || ''; if (windowGapInput) windowGapInput.value = settings.windowGap || ''; if (llmProviderSelect) llmProviderSelect.value = settings.llmProvider || 'gemini'; if (openrouterKeyInput) openrouterKeyInput.value = settings.openrouterKey || ''; if (openrouterModelInput) openrouterModelInput.value = settings.openrouterModel || 'openrouter/free'; + if (groqKeyInput) groqKeyInput.value = settings.groqKey || ''; + if (groqModelInput) groqModelInput.value = settings.groqModel || 'llama-3.3-70b-versatile'; updateLLMFieldStates(); // Set C++ as default if no coding language is specified @@ -141,14 +153,18 @@ document.addEventListener('DOMContentLoaded', () => { const geminiGroup = document.getElementById('geminiLlmFields'); const openrouterGroup = document.getElementById('openrouterLlmFields'); const openrouterNote = document.getElementById('openrouterNote'); + const groqGroup = document.getElementById('groqLlmFields'); if (geminiGroup) geminiGroup.style.display = provider === 'gemini' ? '' : 'none'; if (openrouterGroup) openrouterGroup.style.display = provider === 'openrouter' ? '' : 'none'; if (openrouterNote) openrouterNote.style.display = provider === 'openrouter' ? '' : 'none'; + if (groqGroup) groqGroup.style.display = provider === 'groq' ? '' : 'none'; if (geminiKeyInput) geminiKeyInput.disabled = provider !== 'gemini'; if (openrouterKeyInput) openrouterKeyInput.disabled = provider !== 'openrouter'; if (openrouterModelInput) openrouterModelInput.disabled = provider !== 'openrouter'; + if (groqKeyInput) groqKeyInput.disabled = provider !== 'groq'; + if (groqModelInput) groqModelInput.disabled = provider !== 'groq'; }; // Save settings helper function @@ -161,6 +177,10 @@ document.addEventListener('DOMContentLoaded', () => { if (whisperModelInput) settings.whisperModel = whisperModelInput.value; if (whisperLanguageInput) settings.whisperLanguage = whisperLanguageInput.value; if (whisperSegmentMsInput) settings.whisperSegmentMs = whisperSegmentMsInput.value; + if (groqSttModelInput) settings.groqSttModel = groqSttModelInput.value; + if (ttsEnabledInput) settings.ttsEnabled = ttsEnabledInput.checked; + if (ttsVoiceInput) settings.ttsVoice = ttsVoiceInput.value; + if (ttsSpeedInput) settings.ttsSpeed = ttsSpeedInput.value; if (geminiKeyInput) settings.geminiKey = geminiKeyInput.value; if (windowGapInput) settings.windowGap = windowGapInput.value; if (codingLanguageSelect) settings.codingLanguage = codingLanguageSelect.value; @@ -168,6 +188,8 @@ document.addEventListener('DOMContentLoaded', () => { if (llmProviderSelect) settings.llmProvider = llmProviderSelect.value; if (openrouterKeyInput) settings.openrouterKey = openrouterKeyInput.value; if (openrouterModelInput) settings.openrouterModel = openrouterModelInput.value; + if (groqKeyInput) settings.groqKey = groqKeyInput.value; + if (groqModelInput) settings.groqModel = groqModelInput.value; window.api.send('save-settings', settings); }; @@ -181,6 +203,7 @@ document.addEventListener('DOMContentLoaded', () => { const azureGroup = document.getElementById('azureFields'); const whisperGroup = document.getElementById('whisperFields'); const azureNote = document.getElementById('azureFieldsNote'); + const groqSttGroup = document.getElementById('groqSttFields'); if (azureGroup) { azureGroup.style.display = provider === 'azure' ? '' : 'none'; @@ -191,6 +214,9 @@ document.addEventListener('DOMContentLoaded', () => { if (azureNote) { azureNote.style.display = provider === 'azure' ? '' : 'none'; } + if (groqSttGroup) { + groqSttGroup.style.display = provider === 'groq' ? '' : 'none'; + } // Also toggle disabled attribute for any leftover direct field refs [azureKeyInput, azureRegionInput].forEach(input => { @@ -199,6 +225,9 @@ document.addEventListener('DOMContentLoaded', () => { [whisperCommandInput, whisperModelInput, whisperLanguageInput, whisperSegmentMsInput].forEach(input => { if (input) input.disabled = provider !== 'whisper'; }); + [groqSttModelInput].forEach(input => { + if (input) input.disabled = provider !== 'groq'; + }); }; // Add event listeners for all inputs @@ -210,11 +239,17 @@ document.addEventListener('DOMContentLoaded', () => { whisperModelInput, whisperLanguageInput, whisperSegmentMsInput, + groqSttModelInput, + ttsEnabledInput, + ttsVoiceInput, + ttsSpeedInput, geminiKeyInput, windowGapInput, llmProviderSelect, openrouterKeyInput, - openrouterModelInput + openrouterModelInput, + groqKeyInput, + groqModelInput ]; inputs.forEach(input => {