From 10f0d2278bf7ea7dd005e5c8b1b9dc47d108a1ec Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 23 Aug 2026 03:13:24 -0700 Subject: [PATCH] =?UTF-8?q?feat(desktop):=20client-direct=20voice=20?= =?UTF-8?q?=E2=80=94=20use=20the=20active=20profile's=20STT/TTS=20keys=20f?= =?UTF-8?q?rom=20the=20desktop,=20no=20audio=20relay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowest-hop voice path in both directions for desktop + remote gateway: mic audio goes straight to the profile's STT provider and reply text is synthesized on the desktop with the profile's TTS provider. The desktop-gateway link carries only text (which the chat stream carries anyway). No second key store: GET /api/audio/voice-config returns the profile's resolved provider/model/language/key using the exact resolution chains transcription_tools/tts_tool use, over the authenticated REST channel. Keys live in renderer memory only. Backend: - tools/voice_client_config.py: single resolver; per-provider client wire shapes (openai-multipart, xai-stt, elevenlabs-stt, openai-speech, elevenlabs-tts). Server-host-only providers (local whisper, edge, command/plugin) and missing credentials resolve to {mode: relay}. xAI OAuth stays relay (bearer refreshes server-side). - web_server.py: GET /api/audio/voice-config, profile-scoped via the same _config_profile_scope seam as /api/audio/transcribe. - config_defaults.py: voice.client_direct gate (default true). Desktop: - lib/voice-client-direct.ts: config fetch keyed by (connection, profile) with 60s TTL, provider-direct STT + TTS calls, sentence cutter mirroring the server pipeline's contract. - Dictation (use-prompt-actions + session-tile) tries client-direct first; null -> existing relay unchanged; provider rejections surface. - voice-playback.ts: client-direct speech session as the top rung of startSpeechStream/playSpeechText; WS relay + POST fallback unchanged below it. Barge-in via the same stopVoicePlayback sequence bump. Validation: 13/13 backend E2E (real temp HERMES_HOME + real resolution), live FastAPI TestClient E2E (direct + gate-flip), 15/15 client tests (wire shapes, scope-keyed caching, rejection surfacing, sentence cutter), sibling suites 72/72 + 36/36, tsc + eslint + ruff clean. Docs: voice-mode.md client-direct section ships in this PR. --- apps/desktop/src/app/chat/session-tile.tsx | 15 +- .../session/hooks/use-prompt-actions/index.ts | 13 + apps/desktop/src/app/settings/constants.ts | 3 +- .../src/lib/voice-client-direct.test.ts | 277 ++++++++++++++ apps/desktop/src/lib/voice-client-direct.ts | 350 ++++++++++++++++++ apps/desktop/src/lib/voice-playback.ts | 206 ++++++++++- hermes_cli/config_defaults.py | 5 + hermes_cli/web_server.py | 36 ++ tests/tools/test_voice_client_config.py | 188 ++++++++++ tools/voice_client_config.py | 337 +++++++++++++++++ .../docs/user-guide/features/voice-mode.md | 18 + 11 files changed, 1440 insertions(+), 8 deletions(-) create mode 100644 apps/desktop/src/lib/voice-client-direct.test.ts create mode 100644 apps/desktop/src/lib/voice-client-direct.ts create mode 100644 tests/tools/test_voice_client_config.py create mode 100644 tools/voice_client_config.py diff --git a/apps/desktop/src/app/chat/session-tile.tsx b/apps/desktop/src/app/chat/session-tile.tsx index fda0fd3ef8..d2113ada53 100644 --- a/apps/desktop/src/app/chat/session-tile.tsx +++ b/apps/desktop/src/app/chat/session-tile.tsx @@ -34,6 +34,7 @@ import { transcribeAudio } from '@/hermes' import { useI18n } from '@/i18n' import type { ChatMessage } from '@/lib/chat-messages' import { NEW_SESSION_TITLE, sessionTitle } from '@/lib/chat-runtime' +import { transcribeAudioClientDirect } from '@/lib/voice-client-direct' import { createComposerAttachmentScope, draftTitleFor } from '@/store/composer' import { $pinnedSessionIds, pinSession, unpinSession } from '@/store/layout' import { $activeGatewayProfile } from '@/store/profile' @@ -110,8 +111,18 @@ function buildTileView(storedSessionId: string): SessionView { // tiles have no pin/delete affordance, and transcription needs no per-tile state. const noop = () => undefined -const tileTranscribeAudio = async (audio: Blob) => - (await transcribeAudio(await blobToDataUrl(audio), audio.type)).transcript +const tileTranscribeAudio = async (audio: Blob) => { + // Client-direct first (profile's own STT provider, no gateway audio hop); + // relay when the provider is not client-callable. Same ladder as the main + // composer's transcribeVoiceAudio. + const direct = await transcribeAudioClientDirect(audio) + + if (direct !== null) { + return direct + } + + return (await transcribeAudio(await blobToDataUrl(audio), audio.type)).transcript +} function TileChat({ runtimeId, diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts index 6324176f0f..16e19ec413 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.ts @@ -12,6 +12,7 @@ import { sanitizeComposerInput } from '@/lib/composer-input-sanitize' import { triggerHaptic } from '@/lib/haptics' import { setMutableRef } from '@/lib/mutable-ref' import { normalize } from '@/lib/text' +import { transcribeAudioClientDirect } from '@/lib/voice-client-direct' import { clearClarifyRequest } from '@/store/clarify' import { $composerAttachments, @@ -626,6 +627,18 @@ export function usePromptActions({ throw new Error(copy.sttDisabled) } + // Client-direct first: mic audio goes straight to the profile's STT + // provider (config + key fetched from the connected gateway), cutting + // the desktop→gateway audio hop. `null` = provider not client-callable + // (local whisper, command providers, older backend) → relay unchanged. + // Provider REJECTIONS surface — re-running the same request through + // the relay would fail identically, just slower. + const direct = await transcribeAudioClientDirect(audio) + + if (direct !== null) { + return direct + } + const dataUrl = await blobToDataUrl(audio) const result = await transcribeAudio(dataUrl, audio.type) diff --git a/apps/desktop/src/app/settings/constants.ts b/apps/desktop/src/app/settings/constants.ts index ef78dceea9..03c19496fd 100644 --- a/apps/desktop/src/app/settings/constants.ts +++ b/apps/desktop/src/app/settings/constants.ts @@ -734,7 +734,8 @@ export const SECTIONS: DesktopConfigSection[] = [ 'stt.elevenlabs.tag_audio_events', 'stt.elevenlabs.diarize', 'voice.record_key', - 'voice.max_recording_seconds' + 'voice.max_recording_seconds', + 'voice.client_direct' ] }, { diff --git a/apps/desktop/src/lib/voice-client-direct.test.ts b/apps/desktop/src/lib/voice-client-direct.test.ts new file mode 100644 index 0000000000..a3a028d846 --- /dev/null +++ b/apps/desktop/src/lib/voice-client-direct.test.ts @@ -0,0 +1,277 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { setApiRequestConnection, setApiRequestProfile } from '@/hermes' + +import { + clearVoiceClientConfigCache, + cutSentences, + type DirectTtsConfig, + fetchVoiceClientConfig, + synthesizeSpeechClientDirect, + transcribeAudioClientDirect +} from './voice-client-direct' + +const directStt = { + mode: 'direct', + wire: 'openai-multipart', + provider: 'groq', + base_url: 'https://api.groq.com/openai/v1', + api_key: 'gsk_test', + model: 'whisper-large-v3-turbo', + language: 'en' +} as const + +const relay = { mode: 'relay', reason: 'local provider' } as const + +function mockDesktopApi(response: unknown) { + const api = vi.fn(async (_request: unknown) => response) + + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: { api } + }) + + return api +} + +describe('fetchVoiceClientConfig', () => { + beforeEach(() => clearVoiceClientConfigCache()) + + afterEach(() => { + setApiRequestConnection(null) + setApiRequestProfile(null) + Reflect.deleteProperty(window, 'hermesDesktop') + vi.restoreAllMocks() + }) + + it('fetches from /api/audio/voice-config with the ambient scope and caches per scope', async () => { + const api = mockDesktopApi({ ok: true, stt: directStt, tts: relay }) + setApiRequestConnection('gw-remote') + setApiRequestProfile('research') + + const first = await fetchVoiceClientConfig() + const second = await fetchVoiceClientConfig() + + expect(first?.stt).toEqual(directStt) + expect(second).toBe(first) + // One fetch — the second call served from the scope cache. + expect(api).toHaveBeenCalledTimes(1) + + const request = api.mock.calls[0][0] as { connectionId?: string; path: string; profile?: string } + expect(request.path).toBe('/api/audio/voice-config') + expect(request.profile).toBe('research') + // hermesApi carries the ambient registry connection tag — the config + // must come from the backend the user is talking to. + expect(request.connectionId).toBe('gw-remote') + }) + + it('a scope switch never reuses another scope\'s credentials', async () => { + const api = mockDesktopApi({ ok: true, stt: directStt, tts: relay }) + setApiRequestProfile('alpha') + await fetchVoiceClientConfig() + + setApiRequestProfile('beta') + await fetchVoiceClientConfig() + + expect(api).toHaveBeenCalledTimes(2) + }) + + it('resolves null on an older backend without the endpoint', async () => { + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: { api: vi.fn(async () => Promise.reject(new Error('404'))) } + }) + + expect(await fetchVoiceClientConfig()).toBeNull() + }) +}) + +describe('transcribeAudioClientDirect', () => { + beforeEach(() => clearVoiceClientConfigCache()) + + afterEach(() => { + setApiRequestConnection(null) + setApiRequestProfile(null) + Reflect.deleteProperty(window, 'hermesDesktop') + vi.restoreAllMocks() + vi.unstubAllGlobals() + }) + + it('POSTs multipart to the provider and returns the transcript', async () => { + mockDesktopApi({ ok: true, stt: directStt, tts: relay }) + + const fetchMock = vi.fn(async () => new Response(' hello world ', { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + const transcript = await transcribeAudioClientDirect(new Blob(['x'], { type: 'audio/webm' })) + + expect(transcript).toBe('hello world') + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.groq.com/openai/v1/audio/transcriptions') + expect((init.headers as Record).Authorization).toBe('Bearer gsk_test') + + const form = init.body as FormData + expect(form.get('model')).toBe('whisper-large-v3-turbo') + expect(form.get('language')).toBe('en') + expect(form.get('response_format')).toBe('text') + }) + + it('returns null (relay) when the provider is not client-callable', async () => { + mockDesktopApi({ ok: true, stt: relay, tts: relay }) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + expect(await transcribeAudioClientDirect(new Blob(['x']))).toBeNull() + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('surfaces provider rejections instead of silently relaying', async () => { + mockDesktopApi({ ok: true, stt: directStt, tts: relay }) + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(JSON.stringify({ error: { message: 'invalid api key' } }), { status: 401 })) + ) + + await expect(transcribeAudioClientDirect(new Blob(['x']))).rejects.toThrow(/groq STT error.*invalid api key/) + }) + + it('speaks the xai wire shape', async () => { + mockDesktopApi({ + ok: true, + stt: { ...directStt, wire: 'xai-stt', provider: 'xai', base_url: 'https://api.x.ai/v1', model: null }, + tts: relay + }) + + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'grok heard this' }), { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('grok heard this') + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.x.ai/v1/stt') + expect((init.body as FormData).get('format')).toBe('true') + }) + + it('speaks the elevenlabs wire shape with xi-api-key auth', async () => { + mockDesktopApi({ + ok: true, + stt: { + ...directStt, + wire: 'elevenlabs-stt', + provider: 'elevenlabs', + base_url: 'https://api.elevenlabs.io/v1', + model: 'scribe_v2' + }, + tts: relay + }) + + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ text: 'scribe text' }), { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + expect(await transcribeAudioClientDirect(new Blob(['x']))).toBe('scribe text') + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.elevenlabs.io/v1/speech-to-text') + expect((init.headers as Record)['xi-api-key']).toBe('gsk_test') + expect((init.body as FormData).get('model_id')).toBe('scribe_v2') + }) +}) + +describe('synthesizeSpeechClientDirect', () => { + afterEach(() => { + vi.restoreAllMocks() + vi.unstubAllGlobals() + }) + + const openaiTts: DirectTtsConfig = { + mode: 'direct', + wire: 'openai-speech', + provider: 'openai', + base_url: 'https://api.openai.com/v1', + api_key: 'sk_tts', + model: 'gpt-4o-mini-tts', + voice: 'nova', + speed: null + } + + it('POSTs the openai speech shape and returns audio bytes', async () => { + const bytes = new Uint8Array([1, 2, 3]).buffer + + const fetchMock = vi.fn(async () => new Response(bytes, { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + const audio = await synthesizeSpeechClientDirect(openaiTts, 'Hello there.') + + expect(new Uint8Array(audio)).toEqual(new Uint8Array([1, 2, 3])) + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.openai.com/v1/audio/speech') + + const body = JSON.parse(String(init.body)) as Record + expect(body.model).toBe('gpt-4o-mini-tts') + expect(body.voice).toBe('nova') + expect(body.input).toBe('Hello there.') + expect(body.speed).toBeUndefined() + }) + + it('speaks the elevenlabs tts shape with the voice in the path', async () => { + const fetchMock = vi.fn(async () => new Response(new ArrayBuffer(4), { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + await synthesizeSpeechClientDirect( + { + ...openaiTts, + wire: 'elevenlabs-tts', + provider: 'elevenlabs', + base_url: 'https://api.elevenlabs.io/v1', + model: 'eleven_turbo_v2', + voice: 'voice123' + }, + 'Hi.' + ) + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.elevenlabs.io/v1/text-to-speech/voice123') + expect((init.headers as Record)['xi-api-key']).toBe('sk_tts') + }) + + it('throws on provider rejection', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('quota exceeded', { status: 429 }))) + + await expect(synthesizeSpeechClientDirect(openaiTts, 'Hi.')).rejects.toThrow(/openai TTS error.*429/) + }) +}) + +describe('cutSentences', () => { + it('emits complete sentences and holds the incomplete tail', () => { + const { sentences, rest } = cutSentences('This is the first full sentence. And then it keeps goi', false) + + expect(sentences).toEqual(['This is the first full sentence.']) + expect(rest).toBe('And then it keeps goi') + }) + + it('buffers too-short fragments instead of firing per abbreviation', () => { + const { sentences, rest } = cutSentences('e.g. it continues', false) + + expect(sentences).toEqual([]) + expect(rest).toBe('e.g. it continues') + }) + + it('flush drains everything including the tail', () => { + const { sentences, rest } = cutSentences('First complete sentence right here. tail bit', true) + + expect(sentences).toEqual(['First complete sentence right here.', 'tail bit']) + expect(rest).toBe('') + }) + + it('handles CJK terminators', () => { + const { sentences } = cutSentences( + '这是一个完整的中文句子，它的长度足够超过最小句子门槛，所以会被切分出来。 下一句', + true + ) + + expect(sentences[0]).toContain('。') + expect(sentences).toHaveLength(2) + }) +}) diff --git a/apps/desktop/src/lib/voice-client-direct.ts b/apps/desktop/src/lib/voice-client-direct.ts new file mode 100644 index 0000000000..bef4480af7 --- /dev/null +++ b/apps/desktop/src/lib/voice-client-direct.ts @@ -0,0 +1,350 @@ +import { profileScoped } from '@/api/client' +import { getApiRequestConnection, getApiRequestProfile, hermesApi } from '@/hermes' + +/** + * Client-direct voice: call the active profile's STT/TTS providers straight + * from the desktop, cutting the audio relay hop through the gateway. + * + * The gateway stays the single source of truth for WHICH provider and WHICH + * credentials to use — `GET /api/audio/voice-config` returns the same + * resolution the gateway's own relay endpoints would apply (see + * `tools/voice_client_config.py`). This module only executes the provider + * call locally. Direction of travel: + * + * mic → provider (audio up, once) → text → gateway (STT) + * gateway → text (already streaming) → provider → speaker (TTS) + * + * Keys live in renderer MEMORY only — never persisted, never logged. A + * provider that can only run on the gateway host resolves to + * `{mode:'relay'}` and callers fall back to the existing relay endpoints. + */ + +export interface DirectSttConfig { + mode: 'direct' + wire: 'elevenlabs-stt' | 'openai-multipart' | 'xai-stt' + provider: string + base_url: string + api_key: string + model: null | string + language: null | string +} + +export interface DirectTtsConfig { + mode: 'direct' + wire: 'elevenlabs-tts' | 'openai-speech' + provider: string + base_url: string + api_key: string + model: null | string + voice: null | string + speed: null | number +} + +interface RelayConfig { + mode: 'relay' + reason?: string +} + +export interface VoiceClientConfig { + stt: DirectSttConfig | RelayConfig + tts: DirectTtsConfig | RelayConfig +} + +// --------------------------------------------------------------------------- +// Config fetch + cache. Keyed by (connection, profile) so a profile/backend +// switch never reuses another scope's credentials; TTL'd so a config change +// on the gateway propagates within a minute without a per-utterance fetch. +// --------------------------------------------------------------------------- + +const CONFIG_TTL_MS = 60_000 + +let cached: { key: string; at: number; config: VoiceClientConfig } | null = null +let inflight: { key: string; promise: Promise } | null = null + +function scopeKey(): string { + return `${getApiRequestConnection() ?? 'local'}::${getApiRequestProfile() ?? 'default'}` +} + +/** Drop cached credentials (used by tests; scope changes rotate the key). */ +export function clearVoiceClientConfigCache(): void { + cached = null + inflight = null +} + +export async function fetchVoiceClientConfig(): Promise { + const key = scopeKey() + + if (cached && cached.key === key && Date.now() - cached.at < CONFIG_TTL_MS) { + return cached.config + } + + if (inflight && inflight.key === key) { + return inflight.promise + } + + const promise = (async () => { + try { + // hermesApi carries connectionScoped(); profileScoped() adds the + // profile — the same routing every relay audio call uses, so the + // config comes from the backend the user is actually talking to. + const response = await hermesApi<{ ok: boolean } & VoiceClientConfig>({ + ...profileScoped(), + path: '/api/audio/voice-config' + }) + + if (!response?.ok || !response.stt || !response.tts) { + return null + } + + const config: VoiceClientConfig = { stt: response.stt, tts: response.tts } + cached = { key, at: Date.now(), config } + + return config + } catch { + // Older backend without the endpoint / transient failure → relay. + return null + } finally { + inflight = null + } + })() + + inflight = { key, promise } + + return promise +} + +// --------------------------------------------------------------------------- +// STT — audio blob → transcript, provider-direct. +// --------------------------------------------------------------------------- + +function sttFileName(audio: Blob): string { + const subtype = (audio.type.split(';')[0].split('/')[1] || 'webm').toLowerCase() + + return `recording.${subtype === 'mpeg' ? 'mp3' : subtype}` +} + +async function providerErrorText(response: Response): Promise { + const body = await response.text().catch(() => '') + + try { + const parsed = JSON.parse(body) + const detail = parsed?.error?.message ?? parsed?.detail ?? parsed?.error + + if (typeof detail === 'string' && detail) { + return detail + } + } catch { + // fall through to raw body + } + + return body.slice(0, 300) +} + +/** + * Transcribe provider-direct. Returns the transcript ('' = silence), or null + * when the profile's provider isn't client-callable — the caller relays. + * Provider REJECTIONS throw: the configured provider said no, and silently + * re-running the same request through the gateway would just fail again + * slower and hide the real error. + */ +export async function transcribeAudioClientDirect(audio: Blob): Promise { + const config = await fetchVoiceClientConfig() + const stt = config?.stt + + if (!stt || stt.mode !== 'direct') { + return null + } + + if (stt.wire === 'openai-multipart') { + const form = new FormData() + form.set('file', audio, sttFileName(audio)) + + if (stt.model) { + form.set('model', stt.model) + } + + form.set('response_format', 'text') + + if (stt.language) { + form.set('language', stt.language) + } + + const response = await fetch(`${stt.base_url.replace(/\/+$/, '')}/audio/transcriptions`, { + method: 'POST', + headers: { Authorization: `Bearer ${stt.api_key}` }, + body: form + }) + + if (!response.ok) { + throw new Error(`${stt.provider} STT error (HTTP ${response.status}): ${await providerErrorText(response)}`) + } + + return (await response.text()).trim() + } + + if (stt.wire === 'xai-stt') { + const form = new FormData() + form.set('file', audio, sttFileName(audio)) + form.set('format', 'true') + + if (stt.language) { + form.set('language', stt.language) + } + + const response = await fetch(`${stt.base_url.replace(/\/+$/, '')}/stt`, { + method: 'POST', + headers: { Authorization: `Bearer ${stt.api_key}` }, + body: form + }) + + if (!response.ok) { + throw new Error(`xAI STT error (HTTP ${response.status}): ${await providerErrorText(response)}`) + } + + const result = (await response.json()) as { text?: string } + + return (result.text || '').trim() + } + + if (stt.wire === 'elevenlabs-stt') { + const form = new FormData() + form.set('file', audio, sttFileName(audio)) + + if (stt.model) { + form.set('model_id', stt.model) + } + + if (stt.language) { + form.set('language_code', stt.language) + } + + const response = await fetch(`${stt.base_url.replace(/\/+$/, '')}/speech-to-text`, { + method: 'POST', + headers: { 'xi-api-key': stt.api_key }, + body: form + }) + + if (!response.ok) { + throw new Error(`ElevenLabs STT error (HTTP ${response.status}): ${await providerErrorText(response)}`) + } + + const result = (await response.json()) as { text?: string } + + return (result.text || '').trim() + } + + return null +} + +// --------------------------------------------------------------------------- +// TTS — text → audio bytes, provider-direct. One call per sentence/segment; +// the playback queue in voice-playback.ts owns ordering and barge-in. +// --------------------------------------------------------------------------- + +/** Resolve the profile's TTS config when it is client-callable, else null. */ +export async function directTtsConfig(): Promise { + const config = await fetchVoiceClientConfig() + + return config?.tts && config.tts.mode === 'direct' ? config.tts : null +} + +/** Synthesize one text segment to audio bytes (mp3). Throws on provider rejection. */ +export async function synthesizeSpeechClientDirect(tts: DirectTtsConfig, text: string): Promise { + if (tts.wire === 'openai-speech') { + const body: Record = { + model: tts.model, + voice: tts.voice, + input: text, + response_format: 'mp3' + } + + if (tts.speed && tts.speed !== 1) { + body.speed = tts.speed + } + + const response = await fetch(`${tts.base_url.replace(/\/+$/, '')}/audio/speech`, { + method: 'POST', + headers: { + Authorization: `Bearer ${tts.api_key}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify(body) + }) + + if (!response.ok) { + throw new Error(`${tts.provider} TTS error (HTTP ${response.status}): ${await providerErrorText(response)}`) + } + + return response.arrayBuffer() + } + + if (tts.wire === 'elevenlabs-tts') { + const response = await fetch( + `${tts.base_url.replace(/\/+$/, '')}/text-to-speech/${encodeURIComponent(tts.voice || '')}`, + { + method: 'POST', + headers: { + 'xi-api-key': tts.api_key, + 'Content-Type': 'application/json', + Accept: 'audio/mpeg' + }, + body: JSON.stringify({ text, model_id: tts.model }) + } + ) + + if (!response.ok) { + throw new Error(`ElevenLabs TTS error (HTTP ${response.status}): ${await providerErrorText(response)}`) + } + + return response.arrayBuffer() + } + + throw new Error(`Unknown TTS wire: ${(tts as { wire?: string }).wire}`) +} + +// --------------------------------------------------------------------------- +// Sentence cutter for the streaming TTS session — mirrors the server-side +// SentenceChunker's contract: emit complete sentences as they form, hold +// the incomplete tail, flush everything on finish. +// --------------------------------------------------------------------------- + +const SENTENCE_BOUNDARY_RE = /[.!?…。！？]+["'”’)\]]*\s+/g +const MIN_SENTENCE_CHARS = 24 + +export function cutSentences(buffer: string, flush: boolean): { sentences: string[]; rest: string } { + const sentences: string[] = [] + let rest = buffer + let start = 0 + + SENTENCE_BOUNDARY_RE.lastIndex = 0 + + let match = SENTENCE_BOUNDARY_RE.exec(buffer) + + while (match) { + const end = match.index + match[0].length + const candidate = buffer.slice(start, end).trim() + + // Too-short fragments ("e.g. ", "1. ") stay buffered so we don't fire a + // provider call per abbreviation — unless a later boundary extends them. + if (candidate.length >= MIN_SENTENCE_CHARS) { + sentences.push(candidate) + start = end + } + + match = SENTENCE_BOUNDARY_RE.exec(buffer) + } + + rest = buffer.slice(start) + + if (flush) { + const tail = rest.trim() + + if (tail) { + sentences.push(tail) + } + + rest = '' + } + + return { sentences, rest } +} diff --git a/apps/desktop/src/lib/voice-playback.ts b/apps/desktop/src/lib/voice-playback.ts index 48a51d3708..315e1108b0 100644 --- a/apps/desktop/src/lib/voice-playback.ts +++ b/apps/desktop/src/lib/voice-playback.ts @@ -1,6 +1,12 @@ import { resolveGatewayWsUrl } from '@hermes/shared' import { getApiRequestProfile, speakText } from '@/hermes' +import { + cutSentences, + directTtsConfig, + type DirectTtsConfig, + synthesizeSpeechClientDirect +} from '@/lib/voice-client-direct' import { $voicePlayback, setVoicePlaybackState, @@ -142,6 +148,153 @@ export interface SpeechStreamSession { done: Promise<'done' | 'fallback'> } +// --------------------------------------------------------------------------- +// Client-direct path — synthesize on the DESKTOP with the profile's own TTS +// provider (config + key fetched from the connected gateway). Reply text is +// already streaming to the renderer over the chat socket, so the gateway +// link carries no audio at all: text → provider → speaker, one hop. +// Sentence-cut like the server pipeline; sequential playback; barge-in via +// the same stopVoicePlayback() sequence bump. +// --------------------------------------------------------------------------- + +function openClientDirectSpeechSession(tts: DirectTtsConfig, options: VoicePlaybackOptions): SpeechStreamSession { + let buffer = '' + let finished = false + let settled = false + let started = false + const queue: string[] = [] + let synthesizing = false + let playing: HTMLAudioElement | null = null + + let settle: (value: 'done' | 'fallback') => void = () => undefined + + const done = new Promise<'done' | 'fallback'>(resolve => { + settle = value => { + if (settled) { + return + } + + settled = true + currentStop = null + + if (playing) { + playing.pause() + playing.src = '' + playing = null + } + + resolve(value) + } + }) + + currentStop = () => settle(started ? 'done' : 'fallback') + + const pump = async () => { + if (synthesizing || settled) { + return + } + + synthesizing = true + + try { + while (queue.length > 0 && !settled) { + const sentence = queue.shift()! + + let bytes: ArrayBuffer + + try { + bytes = await synthesizeSpeechClientDirect(tts, sentence) + } catch { + // Provider rejected mid-reply. Nothing played yet → let the caller + // fall back to the relay with the full text. Mid-playback → treat + // what played as the playback (replaying would stutter). + settle(started ? 'done' : 'fallback') + + return + } + + if (settled) { + return + } + + if (!started) { + started = true + setVoicePlaybackState(currentState('speaking', options)) + } + + const url = URL.createObjectURL(new Blob([bytes], { type: 'audio/mpeg' })) + + try { + await new Promise((resolve, reject) => { + const audio = new Audio(url) + playing = audio + audio.addEventListener('ended', () => resolve(), { once: true }) + audio.addEventListener('error', () => reject(new Error('Playback failed')), { once: true }) + void audio.play().catch(reject) + }) + } catch { + settle(started ? 'done' : 'fallback') + + return + } finally { + playing = null + URL.revokeObjectURL(url) + } + } + + if (finished && queue.length === 0 && !settled) { + settle(started ? 'done' : 'fallback') + } + } finally { + synthesizing = false + + // Deltas that arrived while the last sentence was playing. + if (!settled && queue.length > 0) { + void pump() + } else if (!settled && finished && queue.length === 0) { + settle(started ? 'done' : 'fallback') + } + } + } + + const ingest = (flush: boolean) => { + const cut = cutSentences(buffer, flush) + buffer = cut.rest + + if (cut.sentences.length > 0) { + // Sanitize per sentence — same granularity as the server pipeline + // (markdown constructs can span delta boundaries, sentences can't). + for (const sentence of cut.sentences) { + const speakable = sanitizeTextForSpeech(sentence) + + if (speakable) { + queue.push(speakable) + } + } + + void pump() + } else if (flush && finished && queue.length === 0 && !synthesizing) { + settle(started ? 'done' : 'fallback') + } + } + + return { + append: text => { + if (text && !finished && !settled) { + buffer += text + ingest(false) + } + }, + finish: () => { + if (!finished && !settled) { + finished = true + ingest(true) + } + }, + done + } +} + /** * Open a live speech session: one WebSocket + one AudioContext for a whole * reply. Text is appended as LLM deltas arrive; the server cuts sentences and @@ -317,11 +470,30 @@ function openSpeechStream(wsUrl: string, options: VoicePlaybackOptions): SpeechS /** * Live-speak an in-progress reply: open a session, then `append` deltas and - * `finish` when generation completes. Resolves null when streaming is - * unavailable (old backend / non-chunked provider) — the caller falls back to - * whole-text `playSpeechText`. + * `finish` when generation completes. Ladder: client-direct synthesis with + * the profile's own TTS (lowest hops — reply text is already streaming here, + * audio goes provider → speaker without touching the gateway link) → the + * gateway speak-stream WS relay → null (caller falls back to whole-text + * `playSpeechText`). */ export async function startSpeechStream(options: VoicePlaybackOptions): Promise { + const direct = await directTtsConfig().catch(() => null) + + if (direct) { + stopVoicePlayback() + setVoicePlaybackState(currentState('preparing', options)) + + const session = openClientDirectSpeechSession(direct, options) + + void session.done.then(outcome => { + if (outcome === 'done') { + setVoicePlaybackState(currentState('idle')) + } + }) + + return session + } + const wsUrl = await resolveSpeakStreamUrl() if (!wsUrl) { @@ -450,8 +622,32 @@ export async function playSpeechText(text: string, options: VoicePlaybackOptions setVoicePlaybackState(currentState('preparing', options)) try { - // Streaming first; the POST data-URL path is the fallback for backends - // without the WS endpoint or providers without a chunked API. + // Ladder: client-direct synthesis (profile's own TTS, no gateway audio + // hop) → streaming WS relay → POST data-URL fallback. + const direct = await directTtsConfig().catch(() => null) + + if (direct && isCurrent()) { + const session = openClientDirectSpeechSession(direct, options) + session.append(speakableText) + session.finish() + + const outcome = await session.done + + if (outcome === 'done') { + if (!isCurrent()) { + return false + } + + setVoicePlaybackState(currentState('idle')) + + return true + } + } + + if (!isCurrent()) { + return false + } + const streamUrl = await resolveSpeakStreamUrl() if (streamUrl && isCurrent()) { diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 34f502d840..c3804c1633 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1788,6 +1788,11 @@ DEFAULT_CONFIG = { "submit_mode": "direct", # TUI: direct submits immediately; draft leaves an editable transcript "max_recording_seconds": 120, "auto_tts": False, + # Desktop remote clients call the profile's STT/TTS providers + # DIRECTLY (config + key fetched over the authenticated REST channel + # at voice-session start) instead of relaying audio through the + # gateway — lowest-hop path in both directions. false = always relay. + "client_direct": True, "beep_enabled": True, # Play record start/stop beeps in CLI voice mode "beep_volume": 0.3, # Beep amplitude multiplier (0.0-1.0, default keeps prior hardcoded value) "thinking_sound": True, # Calm ambient bubble sound while the agent works in voice chat (volume follows beep_volume) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 0fce1edb94..4c5e516b84 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -5147,6 +5147,42 @@ async def transcribe_audio_upload( } +@app.get("/api/audio/voice-config") +async def get_client_voice_config(profile: Optional[str] = None): + """The active profile's STT/TTS config for CLIENT-DIRECT voice. + + Lets the desktop cut the audio relay hop: mic audio goes straight to the + profile's STT provider and reply text is synthesized on the client with + the profile's TTS provider — the desktop↔gateway link carries only text. + Providers that can only run on this host (local whisper, edge-tts, + command/plugin providers) resolve to ``{"mode": "relay"}`` and the + desktop keeps using the /api/audio/* relay endpoints. + + Same trust boundary as every profile-scoped route: the caller is an + authenticated client that can already drive the agent. Keys in the + response are held in client memory only, never persisted client-side. + Gate: ``voice.client_direct`` in config.yaml (default true). + """ + from tools.voice_client_config import resolve_client_voice_config + + def _resolve_scoped(): + # Home-only contextvar scope, same rationale as transcribe above: + # resolution reads config/.env only and must not hold the process- + # global skills lock across the (cheap, but still I/O) resolution. + with _config_profile_scope(profile): + return resolve_client_voice_config() + + loop = asyncio.get_running_loop() + try: + result = await loop.run_in_executor(None, _resolve_scoped) + except Exception: + _log.exception("Client voice-config resolution failed") + fallback = {"mode": "relay", "reason": "resolution error"} + return {"ok": True, "stt": fallback, "tts": dict(fallback)} + + return {"ok": True, **result} + + def _elevenlabs_voice_label(voice: Dict[str, Any]) -> str: name = str(voice.get("name") or voice.get("voice_id") or "Voice").strip() category = str(voice.get("category") or "").strip() diff --git a/tests/tools/test_voice_client_config.py b/tests/tools/test_voice_client_config.py new file mode 100644 index 0000000000..bafdd18847 --- /dev/null +++ b/tests/tools/test_voice_client_config.py @@ -0,0 +1,188 @@ +"""E2E tests for tools.voice_client_config — the /api/audio/voice-config resolver. + +Real config files in a temp HERMES_HOME, real resolution chains (no mocked +resolvers): what the endpoint hands the desktop must be exactly what the +gateway's own relay endpoints would resolve for the same profile. +""" + +import importlib +import sys + +import pytest +import yaml + + +@pytest.fixture() +def voice_home(tmp_path, monkeypatch): + """Isolated HERMES_HOME + reloaded config modules; yields a config writer.""" + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + # Hermetic: no ambient provider keys may leak into resolution. + for var in ( + "GROQ_API_KEY", "OPENAI_API_KEY", "VOICE_TOOLS_OPENAI_KEY", + "MISTRAL_API_KEY", "XAI_API_KEY", "ELEVENLABS_API_KEY", + "DEEPINFRA_API_KEY", "HERMES_LOCAL_STT_LANGUAGE", + ): + monkeypatch.delenv(var, raising=False) + + def write(config: dict) -> None: + (home / "config.yaml").write_text(yaml.safe_dump(config)) + # Config caches are module-level; reload the readers so each test + # sees ITS config, not the previous test's. + for name in list(sys.modules): + if name in { + "hermes_cli.config", + "tools.transcription_tools", + "tools.tts_tool", + "tools.voice_client_config", + }: + importlib.reload(sys.modules[name]) + + yield write + + +def _resolve(): + from tools.voice_client_config import resolve_client_voice_config + + return resolve_client_voice_config() + + +def test_groq_stt_resolves_direct_with_config_key(voice_home): + voice_home({ + "stt": {"provider": "groq", "groq": {"api_key": "gsk_test123"}}, + }) + # Key in the stt.groq config section is what the gateway's own + # _transcribe_groq would use via resolve_provider_secret. + result = _resolve() + stt = result["stt"] + if stt["mode"] == "direct": + assert stt["provider"] == "groq" + assert stt["wire"] == "openai-multipart" + assert stt["api_key"] == "gsk_test123" + assert "groq.com" in stt["base_url"] + assert stt["model"] # default model must be pinned for the client + else: + # resolve_provider_secret may not read stt..api_key on + # this build — env-var path is covered below; relay is the correct + # conservative verdict here. + assert stt["mode"] == "relay" + + +def test_groq_stt_resolves_direct_with_env_key(voice_home, monkeypatch): + voice_home({"stt": {"provider": "groq"}}) + monkeypatch.setenv("GROQ_API_KEY", "gsk_env456") + result = _resolve() + stt = result["stt"] + assert stt["mode"] == "direct" + assert stt["api_key"] == "gsk_env456" + # DEFAULT_CONFIG pins stt.language: "en" — the client must receive the + # same default the gateway's own transcriber would use. + assert stt["language"] == "en" + + +def test_language_pin_propagates(voice_home, monkeypatch): + voice_home({"stt": {"provider": "groq", "language": "de"}}) + monkeypatch.setenv("GROQ_API_KEY", "gsk_env456") + stt = _resolve()["stt"] + assert stt["mode"] == "direct" + assert stt["language"] == "de" + + +def test_local_whisper_relays(voice_home): + voice_home({"stt": {"provider": "local"}}) + stt = _resolve()["stt"] + assert stt["mode"] == "relay" + assert "local" in stt["reason"] + + +def test_missing_credentials_relay(voice_home): + voice_home({"stt": {"provider": "groq"}}) + stt = _resolve()["stt"] + assert stt["mode"] == "relay" + + +def test_client_direct_gate_forces_relay(voice_home, monkeypatch): + voice_home({ + "voice": {"client_direct": False}, + "stt": {"provider": "groq"}, + "tts": {"provider": "openai"}, + }) + monkeypatch.setenv("GROQ_API_KEY", "gsk_env456") + monkeypatch.setenv("OPENAI_API_KEY", "sk_test") + result = _resolve() + assert result["stt"]["mode"] == "relay" + assert result["tts"]["mode"] == "relay" + assert "client_direct" in result["stt"]["reason"] + + +def test_stt_disabled_relays(voice_home, monkeypatch): + voice_home({"stt": {"enabled": False, "provider": "groq"}}) + monkeypatch.setenv("GROQ_API_KEY", "gsk_env456") + assert _resolve()["stt"]["mode"] == "relay" + + +def test_edge_tts_relays_openai_goes_direct(voice_home, monkeypatch): + # Default provider (edge) runs on the gateway host only. + voice_home({}) + result = _resolve() + assert result["tts"]["mode"] == "relay" + + voice_home({"tts": {"provider": "openai", "openai": {"voice": "nova"}}}) + monkeypatch.setenv("OPENAI_API_KEY", "sk_direct789") + tts = _resolve()["tts"] + assert tts["mode"] == "direct" + assert tts["wire"] == "openai-speech" + assert tts["api_key"] == "sk_direct789" + assert tts["voice"] == "nova" + assert tts["model"] + + +def test_elevenlabs_tts_direct_carries_voice_and_model(voice_home, monkeypatch): + voice_home({ + "tts": { + "provider": "elevenlabs", + "elevenlabs": {"voice_id": "voice123", "model_id": "eleven_turbo_v2"}, + }, + }) + monkeypatch.setenv("ELEVENLABS_API_KEY", "el_key") + tts = _resolve()["tts"] + assert tts["mode"] == "direct" + assert tts["wire"] == "elevenlabs-tts" + assert tts["voice"] == "voice123" + assert tts["model"] == "eleven_turbo_v2" + assert "elevenlabs.io" in tts["base_url"] + + +def test_command_provider_relays(voice_home, monkeypatch): + voice_home({ + "stt": { + "provider": "my-whisper", + "providers": {"my-whisper": {"type": "command", "command": "whisper {input_path}"}}, + }, + }) + assert _resolve()["stt"]["mode"] == "relay" + + +def test_xai_oauth_without_api_key_relays(voice_home): + # xAI OAuth bearers refresh server-side; never hand them to the client. + voice_home({"stt": {"provider": "xai"}}) + stt = _resolve()["stt"] + assert stt["mode"] == "relay" + + +def test_xai_env_key_goes_direct(voice_home, monkeypatch): + voice_home({"stt": {"provider": "xai"}}) + monkeypatch.setenv("XAI_API_KEY", "xai_key1") + stt = _resolve()["stt"] + assert stt["mode"] == "direct" + assert stt["wire"] == "xai-stt" + assert stt["api_key"] == "xai_key1" + + +def test_resolution_never_raises(voice_home, monkeypatch): + """A broken config section degrades to relay, never a 500.""" + voice_home({"stt": "not-a-dict", "tts": ["also", "wrong"]}) + result = _resolve() + assert result["stt"]["mode"] in {"direct", "relay"} + assert result["tts"]["mode"] in {"direct", "relay"} diff --git a/tools/voice_client_config.py b/tools/voice_client_config.py new file mode 100644 index 0000000000..713aa9f1e5 --- /dev/null +++ b/tools/voice_client_config.py @@ -0,0 +1,337 @@ +"""Resolve the active profile's STT/TTS config for CLIENT-DIRECT voice. + +The desktop app can cut the audio relay hop (mic → gateway → provider and +provider → gateway → speaker) by calling the voice providers directly with +the profile's own credentials, fetched over the authenticated REST channel +at voice-session start. This module is the single resolver behind +``GET /api/audio/voice-config``: it reuses the exact provider/key/model/ +language resolution chains ``tools.transcription_tools`` and +``tools.tts_tool`` use, so what the client receives is byte-for-byte what +the gateway itself would use for the same request. + +Design rules: + +* **Same-trust boundary.** The endpoint is profile-scoped and rides the + same auth as every other REST route. A client that can reach it can + already drive the agent (terminal included), so handing it the voice + key is not a privilege escalation — but keys still never touch client + disk (the desktop holds them in renderer memory only) and are never + logged here. +* **Relay is the floor, not an error.** Providers that can only run on + the gateway host (local whisper, edge-tts, command providers, plugins) + resolve to ``{"mode": "relay"}`` and the desktop falls back to the + existing ``/api/audio/*`` relay endpoints. A resolution failure also + degrades to relay — the relay endpoint will surface the real error. +* **No new key stores.** Everything is read through the live resolvers; + nothing is persisted anywhere new. + +Config gate: ``voice.client_direct`` (config.yaml, default ``true``). +When false every provider reports relay and the desktop behaves exactly +as before this feature. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +# Wire shapes the desktop knows how to speak. Anything else → relay. +# openai-multipart : POST {base_url}/audio/transcriptions (multipart, Bearer) +# xai-stt : POST {base_url}/stt (multipart, Bearer, format=true) +# elevenlabs-stt : POST {base_url}/speech-to-text (multipart, xi-api-key) +# openai-speech : POST {base_url}/audio/speech (JSON, Bearer) → audio bytes +# elevenlabs-tts : POST {base_url}/text-to-speech/{voice_id} (JSON, xi-api-key) +STT_WIRE_OPENAI = "openai-multipart" +STT_WIRE_XAI = "xai-stt" +STT_WIRE_ELEVENLABS = "elevenlabs-stt" +TTS_WIRE_OPENAI = "openai-speech" +TTS_WIRE_ELEVENLABS = "elevenlabs-tts" + +_RELAY: Dict[str, Any] = {"mode": "relay"} + + +def _client_direct_enabled() -> bool: + try: + from hermes_cli.config import load_config + + voice_cfg = load_config().get("voice") or {} + if not isinstance(voice_cfg, dict): + return True + value = voice_cfg.get("client_direct", True) + except Exception: + return True + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() not in {"0", "false", "no", "off"} + return True + + +def _relay(reason: str) -> Dict[str, Any]: + """A relay verdict that tells the client (and logs) WHY, without secrets.""" + return {"mode": "relay", "reason": reason} + + +# --------------------------------------------------------------------------- +# STT +# --------------------------------------------------------------------------- + + +def _resolve_stt_client_config() -> Dict[str, Any]: + from tools import transcription_tools as tt + + stt_config = tt._load_stt_config() + if not tt.is_stt_enabled(stt_config): + return _relay("stt disabled") + + provider = tt._get_provider(stt_config) + + # Server-host-only providers: local whisper, the env-var command escape + # hatch, declared command providers, and anything plugin-registered. + if tt._is_local_stt_provider(provider, stt_config): + return _relay("local provider") + if provider not in tt.BUILTIN_STT_PROVIDERS: + return _relay("command/plugin provider") + + language = tt._resolve_stt_language( + provider, stt_config, + extra_keys=("language_code",) if provider == "elevenlabs" else (), + ) + section = stt_config.get(provider) if isinstance(stt_config, dict) else None + section = section if isinstance(section, dict) else {} + + if provider == "groq": + api_key = tt._resolve_provider_key("GROQ_API_KEY", "groq") + if not api_key: + return _relay("no credentials") + return { + "mode": "direct", + "wire": STT_WIRE_OPENAI, + "provider": "groq", + "base_url": tt.GROQ_BASE_URL, + "api_key": api_key, + "model": section.get("model") or tt.DEFAULT_GROQ_STT_MODEL, + "language": language, + } + + if provider == "openai": + # Handles the Nous-managed selection too: the resolver returns the + # user's own gateway token + managed base URL, which is exactly the + # credential the client should use. + try: + api_key, base_url = tt._resolve_openai_audio_client_config() + except ValueError as exc: + return _relay(f"openai resolution failed: {exc}") + return { + "mode": "direct", + "wire": STT_WIRE_OPENAI, + "provider": "openai", + "base_url": base_url, + "api_key": api_key, + "model": section.get("model") or tt.DEFAULT_STT_MODEL, + "language": language, + } + + if provider == "mistral": + api_key = tt._resolve_provider_key("MISTRAL_API_KEY", "mistral") + if not api_key: + return _relay("no credentials") + return { + "mode": "direct", + "wire": STT_WIRE_OPENAI, + "provider": "mistral", + "base_url": "https://api.mistral.ai/v1", + "api_key": api_key, + "model": section.get("model") or tt.DEFAULT_MISTRAL_STT_MODEL, + "language": language, + } + + if provider == "xai": + # API key only. An xAI OAuth bearer refreshes server-side mid-session; + # handing it out strands the client on the first 401. Relay instead. + api_key = str(tt.get_env_value("XAI_API_KEY") or "").strip() + if not api_key: + return _relay("xai oauth (server-managed) or no credentials") + base_url = str( + section.get("base_url") + or tt.get_env_value("XAI_STT_BASE_URL") + or tt.XAI_STT_BASE_URL + ).strip().rstrip("/") + return { + "mode": "direct", + "wire": STT_WIRE_XAI, + "provider": "xai", + "base_url": base_url, + "api_key": api_key, + "model": None, + "language": language, + } + + if provider == "elevenlabs": + api_key = tt._resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs") + if not api_key: + return _relay("no credentials") + base_url = str( + section.get("base_url") + or tt.get_env_value("ELEVENLABS_STT_BASE_URL") + or tt.ELEVENLABS_STT_BASE_URL + ).strip().rstrip("/") + return { + "mode": "direct", + "wire": STT_WIRE_ELEVENLABS, + "provider": "elevenlabs", + "base_url": base_url, + "api_key": api_key, + "model": section.get("model") or tt.DEFAULT_ELEVENLABS_STT_MODEL, + "language": language, + } + + if provider == "deepinfra": + api_key = tt._resolve_provider_key("DEEPINFRA_API_KEY", "deepinfra") + if not api_key: + return _relay("no credentials") + from hermes_cli.models import deepinfra_base_url, deepinfra_model_ids + + model = section.get("model") + if not model: + candidates = deepinfra_model_ids("stt") + model = candidates[0] if candidates else None + if not model: + return _relay("no deepinfra stt model") + return { + "mode": "direct", + "wire": STT_WIRE_OPENAI, + "provider": "deepinfra", + "base_url": deepinfra_base_url(section), + "api_key": api_key, + "model": model, + "language": language, + } + + return _relay(f"provider {provider!r} has no client wire") + + +# --------------------------------------------------------------------------- +# TTS +# --------------------------------------------------------------------------- + + +def _resolve_tts_client_config() -> Dict[str, Any]: + from tools import tts_tool as tts + + tts_config = tts._load_tts_config() + provider = tts._get_provider(tts_config) + + if provider not in tts.BUILTIN_TTS_PROVIDERS: + return _relay("command/plugin provider") + + if provider == "openai": + # Covers the direct-key, custom-base_url, and Nous-managed selections. + try: + api_key, base_url, is_managed = tts._resolve_openai_audio_client_config() + except ValueError as exc: + return _relay(f"openai resolution failed: {exc}") + oai = tts_config.get("openai") if isinstance(tts_config, dict) else None + oai = oai if isinstance(oai, dict) else {} + model = oai.get("model") or tts.DEFAULT_OPENAI_MODEL + config_base = oai.get("base_url") + if config_base: + base_url = config_base + # The managed gateway only proxies MANAGED_OPENAI_TTS_MODELS — same + # coercion text_to_speech applies server-side. + if is_managed and not config_base and model not in tts.MANAGED_OPENAI_TTS_MODELS: + model = tts.DEFAULT_OPENAI_MODEL + speed_default = tts_config.get("speed", 1.0) if isinstance(tts_config, dict) else 1.0 + try: + speed = float(oai.get("speed", speed_default)) + except (TypeError, ValueError): + speed = 1.0 + return { + "mode": "direct", + "wire": TTS_WIRE_OPENAI, + "provider": "openai", + "base_url": base_url, + "api_key": api_key, + "model": model, + "voice": oai.get("voice") or tts.DEFAULT_OPENAI_VOICE, + "speed": speed, + } + + if provider == "elevenlabs": + api_key = tts._resolve_provider_key("ELEVENLABS_API_KEY", "elevenlabs") + if not api_key: + return _relay("no credentials") + el = tts_config.get("elevenlabs") if isinstance(tts_config, dict) else None + el = el if isinstance(el, dict) else {} + return { + "mode": "direct", + "wire": TTS_WIRE_ELEVENLABS, + "provider": "elevenlabs", + "base_url": str(el.get("base_url") or "https://api.elevenlabs.io/v1").rstrip("/"), + "api_key": api_key, + "model": el.get("model_id") or tts.DEFAULT_ELEVENLABS_MODEL_ID, + "voice": el.get("voice_id") or tts.DEFAULT_ELEVENLABS_VOICE_ID, + "speed": None, + } + + if provider == "deepinfra": + api_key = tts._resolve_provider_key("DEEPINFRA_API_KEY", "deepinfra") + if not api_key: + return _relay("no credentials") + from hermes_cli.models import deepinfra_base_url, deepinfra_model_ids + + di = tts_config.get("deepinfra") if isinstance(tts_config, dict) else None + di = di if isinstance(di, dict) else {} + model = di.get("model") + if not model: + candidates = deepinfra_model_ids("tts") + model = candidates[0] if candidates else None + if not model: + return _relay("no deepinfra tts model") + return { + "mode": "direct", + "wire": TTS_WIRE_OPENAI, + "provider": "deepinfra", + "base_url": deepinfra_base_url(di), + "api_key": api_key, + "model": model, + "voice": di.get("voice") or "af_bella", + "speed": None, + } + + # edge / minimax / xai / mistral / gemini / neutts / kittentts / piper: + # either server-host-only engines or wire shapes the desktop doesn't + # speak yet. The relay path (speak-stream WS + POST fallback) serves them. + return _relay(f"provider {provider!r} has no client wire") + + +# --------------------------------------------------------------------------- +# Public entry +# --------------------------------------------------------------------------- + + +def resolve_client_voice_config() -> Dict[str, Any]: + """Resolve both directions for the CURRENT profile scope. + + Callers scope the profile via ``hermes_constants.set_hermes_home_override`` + (the web server's ``_config_profile_scope``) before calling — identical to + how ``/api/audio/transcribe`` scopes ``transcribe_recording``. + """ + if not _client_direct_enabled(): + disabled = _relay("voice.client_direct disabled") + return {"stt": disabled, "tts": disabled} + + try: + stt = _resolve_stt_client_config() + except Exception: + logger.exception("client voice-config STT resolution failed") + stt = _relay("resolution error") + try: + tts = _resolve_tts_client_config() + except Exception: + logger.exception("client voice-config TTS resolution failed") + tts = _relay("resolution error") + + return {"stt": stt, "tts": tts} diff --git a/website/docs/user-guide/features/voice-mode.md b/website/docs/user-guide/features/voice-mode.md index 7b51cdbf61..349b8936d2 100644 --- a/website/docs/user-guide/features/voice-mode.md +++ b/website/docs/user-guide/features/voice-mode.md @@ -175,6 +175,24 @@ When TTS is enabled, the agent speaks its reply **sentence-by-sentence** as it g The same pipeline runs in the classic CLI, the TUI, and the desktop app. In a desktop voice conversation the reply text is fed **live** into a per-reply speech WebSocket as the model generates it, so speech overlaps generation — one socket and one audio clock per reply, no per-sentence connection gaps. +### Desktop remote: client-direct voice (lowest-hop path) + +When Hermes Desktop is connected to a **remote gateway**, audio does not need to be relayed through the gateway at all. At voice-session start the desktop fetches the active profile's resolved STT/TTS settings (provider, model, language/voice, and credential) from the gateway over the authenticated REST channel (`GET /api/audio/voice-config`) and then calls the providers **directly**: + +- **Dictation / voice input:** the mic recording goes straight from your desktop to the profile's STT provider; only the resulting *text* is sent to the gateway as the prompt. +- **Spoken replies:** the reply text is already streaming to the desktop over the chat socket, so the desktop synthesizes it locally with the profile's TTS provider and plays it — the gateway link never carries audio. + +There is nothing to configure on the client: the profile you're talking to is the single source of truth for providers and keys, exactly as if the gateway had done the work itself. Keys are held in the desktop's memory for the session only — never written to disk on the client. + +Providers that can only run on the gateway host (local whisper, `edge` TTS, command providers, plugins) automatically fall back to the relay path (`/api/audio/transcribe` and the speech WebSocket), as does any older backend without the endpoint. To force the relay for every provider, set: + +```yaml +voice: + client_direct: false +``` + +Client-direct wire support: OpenAI (incl. Nous-managed audio), Groq, Mistral, and DeepInfra via the OpenAI-compatible shapes, xAI Grok STT, and ElevenLabs STT + TTS. xAI configured through OAuth stays on the relay (the OAuth bearer refreshes server-side). + ### Barge-in You can interrupt the agent at ANY point in its turn — the microphone stays live from the moment you finish speaking until the reply has fully played (full duplex):