From 4e3feb8bbb0a0484e2725bb5ee8accb961540404 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:33:56 -0700 Subject: [PATCH] feat(tts): speech toggles warm up and unload local TTS engines (#100881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Desktop "Read replies aloud" / voice conversation, TUI and CLI /voice tts now hold a lease on the TTS engine. Acquiring pre-loads the configured provider (piper/kittentts model into the same LRU slot synthesis reads; lazily-installed cloud SDKs), so the first spoken reply no longer pays the model load as dead air. Releasing the last lease across surfaces unloads resident local models. - tools/tts_tool.py: warm_tts_provider / release_tts_provider / acquire_tts_lease / release_tts_lease over a _LOCAL_TTS_MODEL_CACHES registry; piper/kittentts loaders extracted so warm-up and synthesis share one resolution path. - web_server: POST /api/audio/tts-lease (profile-scoped, off-loop, failures reported in body never as HTTP errors). - tui_gateway voice.toggle + cli.py /voice tts|on|off wire the lease. - desktop: lib/tts-lease.ts (dedupe, per-lease serialization, latest intent wins) driven from useComposerVoice; setTtsLease API client. - docs: features/tts.md section. Live (real piper, isolated HERMES_HOME): first synthesis 988ms cold → 92ms after the toggle warmed the engine; release drops the model. --- apps/desktop/src/api/system.ts | 21 ++ .../chat/composer/hooks/use-composer-voice.ts | 21 ++ apps/desktop/src/hermes.ts | 1 + apps/desktop/src/lib/tts-lease.test.ts | 99 +++++++ apps/desktop/src/lib/tts-lease.ts | 76 ++++++ apps/desktop/src/types/hermes.ts | 15 ++ cli.py | 36 +++ hermes_cli/web_models.py | 11 + hermes_cli/web_server.py | 38 +++ tests/hermes_cli/test_web_server_tts_lease.py | 157 +++++++++++ tests/tools/test_tts_lifecycle_leases.py | 233 ++++++++++++++++ tools/tts_tool.py | 251 ++++++++++++++++-- tui_gateway/server.py | 35 +++ website/docs/user-guide/features/tts.md | 11 + 14 files changed, 978 insertions(+), 27 deletions(-) create mode 100644 apps/desktop/src/lib/tts-lease.test.ts create mode 100644 apps/desktop/src/lib/tts-lease.ts create mode 100644 tests/hermes_cli/test_web_server_tts_lease.py create mode 100644 tests/tools/test_tts_lifecycle_leases.py diff --git a/apps/desktop/src/api/system.ts b/apps/desktop/src/api/system.ts index 678d09e549..ba2e0bbcca 100644 --- a/apps/desktop/src/api/system.ts +++ b/apps/desktop/src/api/system.ts @@ -3,6 +3,7 @@ import type { ActionStatusResponse, AudioSpeakResponse, AudioTranscriptionResponse, + AudioTtsLeaseResponse, BackendUpdateCheckResponse, CuratorStatusResponse, DebugShareResponse, @@ -196,6 +197,26 @@ export function speakText(text: string): Promise { }) } +// Acquiring a lease pre-loads the configured TTS engine. For local engines +// that is a model load and, on a fresh install, a voice download — well past +// the default 15s Electron backend timeout. +export const AUDIO_TTS_LEASE_REQUEST_TIMEOUT_MS = 180_000 + +/** + * Tell the backend a speech-output toggle flipped so it can warm the TTS engine + * (`active: true`) or release it once no surface needs it (`active: false`). + * `lease` names the toggle — `desktop:read-aloud`, `desktop:conversation`. + */ +export function setTtsLease(lease: string, active: boolean): Promise { + return hermesApi({ + ...profileScoped(), + path: '/api/audio/tts-lease', + method: 'POST', + body: { active, lease }, + timeoutMs: AUDIO_TTS_LEASE_REQUEST_TIMEOUT_MS + }) +} + export function getElevenLabsVoices(profile?: null | string): Promise { return hermesApi({ path: '/api/audio/elevenlabs/voices', diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index ffe6c4707f..9a7db494ee 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -5,6 +5,7 @@ import { useI18n } from '@/i18n' import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages' import { triggerHaptic } from '@/lib/haptics' import { markAssistantIdSpoken, resolveSpokenReply } from '@/lib/spoken-reply' +import { CONVERSATION_LEASE, READ_ALOUD_LEASE, syncTtsLease } from '@/lib/tts-lease' import { clearWakeIndicator, syncWakeIndicatorWithVoice } from '@/lib/wake-indicator' import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' @@ -265,6 +266,26 @@ export function useComposerVoice({ useEffect(() => resumeWakeIfPaused, [resumeWakeIfPaused]) + // Speech-output toggles are TTS warm-up / release signals. Entering a voice + // conversation acquires this window's lease (pre-loads the engine so the + // first spoken reply doesn't start with dead air); ending it releases the + // lease, and the backend unloads resident local models once no surface holds + // one. Fire-and-forget — the toggle never waits on or fails from this. + useEffect(() => { + void syncTtsLease(CONVERSATION_LEASE, voiceConversationActive) + }, [voiceConversationActive]) + + useEffect(() => () => void syncTtsLease(CONVERSATION_LEASE, false), []) + + // "Read replies aloud" is the same signal, held for as long as the toggle is + // on (it mirrors voice.auto_tts, so this also warms at startup when the + // preference is already set). + const autoSpeakReplies = useStore($autoSpeakReplies) + + useEffect(() => { + void syncTtsLease(READ_ALOUD_LEASE, autoSpeakReplies) + }, [autoSpeakReplies]) + // Explicit start/end for the on-screen conversation controls (the hotkey uses // the gated toggle above). const startConversation = useCallback(() => setVoiceConversationActive(true), []) diff --git a/apps/desktop/src/hermes.ts b/apps/desktop/src/hermes.ts index 7bb69bc085..f5fb2ee7bd 100644 --- a/apps/desktop/src/hermes.ts +++ b/apps/desktop/src/hermes.ts @@ -40,6 +40,7 @@ export type { AnalyticsTotals, AudioSpeakResponse, AudioTranscriptionResponse, + AudioTtsLeaseResponse, AutomationBlueprint, AutomationBlueprintField, AuxiliaryModelsResponse, diff --git a/apps/desktop/src/lib/tts-lease.test.ts b/apps/desktop/src/lib/tts-lease.test.ts new file mode 100644 index 0000000000..166c54ac9e --- /dev/null +++ b/apps/desktop/src/lib/tts-lease.test.ts @@ -0,0 +1,99 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const setTtsLease = vi.fn(async (_lease: string, _active: boolean) => ({ ok: true })) + +vi.mock('@/hermes', () => ({ + setTtsLease: (lease: string, active: boolean) => setTtsLease(lease, active) +})) + +import { CONVERSATION_LEASE, READ_ALOUD_LEASE, resetTtsLeasesForTests, syncTtsLease } from './tts-lease' + +describe('syncTtsLease', () => { + beforeEach(() => { + resetTtsLeasesForTests() + setTtsLease.mockReset() + setTtsLease.mockImplementation(async () => ({ ok: true })) + }) + + afterEach(() => { + resetTtsLeasesForTests() + }) + + it('acquires on the first on and releases on off', async () => { + await syncTtsLease(READ_ALOUD_LEASE, true) + await syncTtsLease(READ_ALOUD_LEASE, false) + + expect(setTtsLease.mock.calls).toEqual([ + [READ_ALOUD_LEASE, true], + [READ_ALOUD_LEASE, false] + ]) + }) + + it('skips an initial off — never releases a lease it did not hold', async () => { + await syncTtsLease(CONVERSATION_LEASE, false) + + expect(setTtsLease).not.toHaveBeenCalled() + }) + + it('dedupes a repeat of the last sent state', async () => { + await syncTtsLease(READ_ALOUD_LEASE, true) + await syncTtsLease(READ_ALOUD_LEASE, true) + await syncTtsLease(READ_ALOUD_LEASE, true) + + expect(setTtsLease).toHaveBeenCalledTimes(1) + }) + + it('queues an off behind an in-flight on so the wire never sees them reordered', async () => { + let finishAcquire: () => void = () => undefined + setTtsLease.mockImplementationOnce( + () => + new Promise(resolve => { + finishAcquire = () => resolve({ ok: true }) + }) + ) + + const on = syncTtsLease(CONVERSATION_LEASE, true) + // Let the acquire actually go out (it runs on a microtask). + await Promise.resolve() + expect(setTtsLease.mock.calls).toEqual([[CONVERSATION_LEASE, true]]) + + const off = syncTtsLease(CONVERSATION_LEASE, false) + await Promise.resolve() + // Still only the acquire — the release waits for it to finish. + expect(setTtsLease).toHaveBeenCalledTimes(1) + + finishAcquire() + await Promise.all([on, off]) + + expect(setTtsLease.mock.calls).toEqual([ + [CONVERSATION_LEASE, true], + [CONVERSATION_LEASE, false] + ]) + }) + + it('coalesces a flip that reverses before its call went out — latest intent wins', async () => { + const on = syncTtsLease(CONVERSATION_LEASE, true) + const off = syncTtsLease(CONVERSATION_LEASE, false) + await Promise.all([on, off]) + + // The acquire never had a chance to go out; only the terminal state is sent + // (a release of a never-held lease is a backend no-op). + expect(setTtsLease.mock.calls).toEqual([[CONVERSATION_LEASE, false]]) + }) + + it('forgets the sent state on failure so the next flip retries', async () => { + setTtsLease.mockImplementationOnce(async () => { + throw new Error('backend not ready') + }) + + await expect(syncTtsLease(READ_ALOUD_LEASE, true)).resolves.toBeUndefined() + await syncTtsLease(READ_ALOUD_LEASE, true) + + expect(setTtsLease).toHaveBeenCalledTimes(2) + }) + + it('conversation lease is per renderer, read-aloud lease is shared', () => { + expect(CONVERSATION_LEASE).toMatch(/^desktop:conversation:[a-z0-9]+$/) + expect(READ_ALOUD_LEASE).toBe('desktop:read-aloud') + }) +}) diff --git a/apps/desktop/src/lib/tts-lease.ts b/apps/desktop/src/lib/tts-lease.ts new file mode 100644 index 0000000000..987d2765e1 --- /dev/null +++ b/apps/desktop/src/lib/tts-lease.ts @@ -0,0 +1,76 @@ +import { setTtsLease } from '@/hermes' + +// The desktop's speech-output toggles — "Read replies aloud" and voice +// conversation mode — are the user telling us TTS is about to be needed (or no +// longer is). The backend turns that into engine lifecycle: acquiring a lease +// pre-loads the configured provider (a local piper/kittentts model, a lazily +// installed SDK) so the first spoken reply starts hot instead of paying the load +// as dead air; releasing the last lease unloads resident local models. +// +// This module is the renderer's single choke point for that signal. It dedupes +// (several composers/tiles observe the same toggle), serializes per lease so a +// fast on→off→on can't be reordered on the wire, and never surfaces failures — +// warm-up is an optimization; the toggle itself must not depend on it. + +// Per-renderer id so two windows in conversation mode hold DISTINCT leases — +// window A ending its conversation must not release the engine window B is +// still speaking through. Read-aloud mirrors one config key shared by every +// window, so it deliberately uses one shared lease name. +const RENDERER_ID = Math.random().toString(36).slice(2, 10) + +export const READ_ALOUD_LEASE = 'desktop:read-aloud' +export const CONVERSATION_LEASE = `desktop:conversation:${RENDERER_ID}` + +const sent = new Map() +const inFlight = new Map>() + +/** + * Bring the backend's view of `lease` in line with `active`. Idempotent: a + * repeat of the last sent state is a no-op. The initial `false` (nothing was + * ever acquired) is also skipped — releasing a lease we never held would only + * churn the backend on app start. + */ +export function syncTtsLease(lease: string, active: boolean): Promise { + const last = sent.get(lease) + + if (last === active || (last === undefined && !active)) { + return inFlight.get(lease) ?? Promise.resolve() + } + + sent.set(lease, active) + + const previous = inFlight.get(lease) ?? Promise.resolve() + + const next = previous + .then(async () => { + // Latest intent wins: if the toggle flipped again while we were queued, + // the newer call sends its own state and this one has nothing to say. + if (sent.get(lease) !== active) { + return + } + + await setTtsLease(lease, active) + }) + .catch(() => { + // Backend not up yet / older backend without the endpoint / warm-up + // failure: forget what we "sent" so the next flip retries honestly. + if (sent.get(lease) === active) { + sent.delete(lease) + } + }) + .finally(() => { + if (inFlight.get(lease) === next) { + inFlight.delete(lease) + } + }) + + inFlight.set(lease, next) + + return next +} + +/** Test seam — forget every sent state. */ +export function resetTtsLeasesForTests() { + sent.clear() + inFlight.clear() +} diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 0f7529eb00..8c9f2a31d1 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -29,6 +29,21 @@ export interface AudioSpeakResponse { provider?: string } +/** `POST /api/audio/tts-lease` — TTS engine warm-up / release driven by speech toggles. */ +export interface AudioTtsLeaseResponse { + ok: boolean + lease: string + active: boolean + /** Live lease holders after this call (null when the backend call itself failed). */ + leases: null | number + /** Warm-up outcome: `loaded` | `cached` | `installed` | `noop` | `error`. */ + action?: string + provider?: string + /** Resident local models dropped (release path). */ + released?: number + error?: string +} + export interface ElevenLabsVoice { label: string name: string diff --git a/cli.py b/cli.py index de464f1b41..5918f0564a 100644 --- a/cli.py +++ b/cli.py @@ -15765,6 +15765,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # _voice_message_prefix property and its usage in _process_message(). tts_status = " (TTS enabled)" if self._voice_tts else "" + if self._voice_tts: + # Speech output is on from the start — warm the engine now so the + # first spoken reply doesn't pay the model load as dead air. + self._tts_lease_async(True) # Use the startup-pinned cache so the advertised shortcut always # matches the live prompt_toolkit binding — reading live config # here would drift after a mid-session config edit (Copilot @@ -15823,6 +15827,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): self._voice_tts = False self._voice_continuous = False + # Speech output is off with the mode — release the TTS engine lease so + # a resident local model (piper/kittentts) is freed once nothing else + # in this process still needs it. + self._tts_lease_async(False) + # Shut down the persistent audio stream in background if recorder is not None: def _bg_shutdown(rec=recorder): @@ -16070,6 +16079,29 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): if not owned: _cprint(f" {_DIM}Enable with /wake on{_RST}") + def _tts_lease_async(self, active: bool) -> None: + """Acquire/release this CLI's TTS engine lease in the background. + + The /voice tts toggle (and voice-mode on/off with speech output set) + is the "TTS is about to be needed / no longer needed" signal: + acquiring pre-loads the configured provider so the first reply starts + hot; releasing lets the last-holder path unload resident local models. + Never blocks the toggle and never fails it. + """ + + def _run(): + try: + from tools.tts_tool import acquire_tts_lease, release_tts_lease + + if active: + acquire_tts_lease("cli:voice-tts") + else: + release_tts_lease("cli:voice-tts") + except Exception as e: + logger.debug("voice: tts lease active=%s failed: %s", active, e) + + threading.Thread(target=_run, name="tts-lease-cli", daemon=True).start() + def _toggle_voice_tts(self): """Toggle TTS output for voice mode.""" if not self._voice_mode: @@ -16085,6 +16117,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): if not check_tts_requirements(): _cprint(f"{_DIM}Warning: No TTS provider available. Install edge-tts or set API keys.{_RST}") + # Toggle = warm-up / release signal for the TTS engine (see + # tools.tts_tool.acquire_tts_lease). + self._tts_lease_async(self._voice_tts) + _cprint(f"{_ACCENT}Voice TTS {status}.{_RST}") def _show_voice_status(self): diff --git a/hermes_cli/web_models.py b/hermes_cli/web_models.py index fa5dd37243..b03f649417 100644 --- a/hermes_cli/web_models.py +++ b/hermes_cli/web_models.py @@ -306,6 +306,17 @@ class TTSSpeakRequest(BaseModel): text: str +class TTSLeaseRequest(BaseModel): + """Body for ``POST /api/audio/tts-lease``. + + ``lease`` names the toggle/surface holding the lease (``desktop:read-aloud``, + ``desktop:conversation``); ``active`` True acquires + warms, False releases. + """ + + lease: str + active: bool = True + + # --- from web_server.py (originally lines 11549-11551) --- class OAuthSubmitBody(BaseModel): diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index aad760d01b..a2bce1d852 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -1836,6 +1836,7 @@ from hermes_cli.web_models import ( # noqa: F401 LearningNodeEdit, DebugShareRequest, TTSSpeakRequest, + TTSLeaseRequest, OAuthSubmitBody, BulkDeleteSessions, SessionImport, @@ -5665,6 +5666,43 @@ async def speak_text(payload: TTSSpeakRequest, profile: Optional[str] = None): } +@app.post("/api/audio/tts-lease") +async def tts_lease(payload: TTSLeaseRequest, profile: Optional[str] = None): + """Desktop TTS-output toggles as warm-up / release signals. + + "Read replies aloud" and voice-conversation mode are explicit "speech is + about to be needed" gestures. ``active: true`` registers the toggle as a + lease on the TTS engine and pre-loads the configured provider (local + piper/kittentts model, lazily-installed SDK) so the first spoken reply + doesn't pay the load as dead air; ``active: false`` drops the lease and, + once no surface holds one, unloads resident local models. + + Blocking work (model load, voice download) runs off the event loop. + Warm-up failures are reported in the body, never as an HTTP error — the + toggle must succeed even when the engine can't preload. + """ + lease = (payload.lease or "").strip() + if not lease: + raise HTTPException(status_code=400, detail="lease is required") + + def _apply(): + from tools.tts_tool import acquire_tts_lease, release_tts_lease + + if payload.active: + with _config_profile_scope(profile): + return acquire_tts_lease(lease) + return release_tts_lease(lease) + + try: + result = await asyncio.get_running_loop().run_in_executor(None, _apply) + except HTTPException: + raise + except Exception as exc: + _log.warning("TTS lease %s (%s) failed: %s", lease, payload.active, exc) + result = {"leases": None, "action": "error", "error": str(exc)} + return {"ok": True, "lease": lease, "active": payload.active, **result} + + def _split_text_for_speak_stream(text: str, cap: int) -> list: """Split *text* into provider-cap-sized pieces on sentence boundaries. diff --git a/tests/hermes_cli/test_web_server_tts_lease.py b/tests/hermes_cli/test_web_server_tts_lease.py new file mode 100644 index 0000000000..be32621997 --- /dev/null +++ b/tests/hermes_cli/test_web_server_tts_lease.py @@ -0,0 +1,157 @@ +"""``POST /api/audio/tts-lease`` — desktop speech toggles as TTS warm-up/release. + +The desktop's "Read replies aloud" and voice-conversation toggles call this so +the backend can pre-load the configured TTS engine when speech is about to be +needed and unload resident local models once no surface holds a lease. +""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +def isolated_profiles(tmp_path, monkeypatch, _isolate_hermes_home): + from hermes_constants import get_hermes_home + from hermes_cli import profiles + + default_home = get_hermes_home() + profiles_root = default_home / "profiles" + worker_home = profiles_root / "worker_beta" + for home in (default_home, worker_home): + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text("{}\n", encoding="utf-8") + (worker_home / ".env").write_text("", encoding="utf-8") + + monkeypatch.setattr(profiles, "_get_default_hermes_home", lambda: default_home) + monkeypatch.setattr(profiles, "_get_profiles_root", lambda: profiles_root) + return {"default": default_home, "worker_beta": worker_home} + + +@pytest.fixture +def client(monkeypatch, isolated_profiles): + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_constants import get_hermes_home + from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + c = TestClient(app) + c.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + return c + + +@pytest.fixture(autouse=True) +def _clean_leases(): + from tools import tts_tool + + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + yield + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + + +def test_active_acquires_and_warms(client, monkeypatch): + from tools import tts_tool + + warmed = [] + monkeypatch.setattr( + tts_tool, + "warm_tts_provider", + lambda cfg=None, provider=None: warmed.append(1) or {"provider": "piper", "warmed": True, "action": "loaded"}, + ) + + resp = client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["lease"] == "desktop:read-aloud" + assert body["active"] is True + assert body["leases"] == 1 + assert body["action"] == "loaded" + assert warmed == [1] + assert tts_tool.tts_lease_holders() == ["desktop:read-aloud"] + + +def test_inactive_releases_and_unloads_when_last(client, monkeypatch): + from tools import tts_tool + + monkeypatch.setattr(tts_tool, "warm_tts_provider", lambda cfg=None, provider=None: {"action": "noop", "warmed": False, "provider": "piper"}) + client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": True}) + client.post("/api/audio/tts-lease", json={"lease": "desktop:conversation:abc", "active": True}) + tts_tool._piper_voice_cache["voice"] = object() + + first = client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": False}).json() + assert first["leases"] == 1 + assert first["released"] == 0 + assert len(tts_tool._piper_voice_cache) == 1 + + last = client.post("/api/audio/tts-lease", json={"lease": "desktop:conversation:abc", "active": False}).json() + assert last["leases"] == 0 + assert last["released"] == 1 + assert tts_tool._piper_voice_cache == {} + + +def test_warm_failure_is_reported_not_an_http_error(client, monkeypatch): + from tools import tts_tool + + def _boom(cfg=None, provider=None): + raise RuntimeError("engine exploded") + + monkeypatch.setattr(tts_tool, "warm_tts_provider", _boom) + resp = client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["action"] == "error" + assert "engine exploded" in body["error"] + + +def test_blank_lease_rejected(client): + resp = client.post("/api/audio/tts-lease", json={"lease": " ", "active": True}) + assert resp.status_code == 400 + + +def test_active_default_true(client, monkeypatch): + from tools import tts_tool + + monkeypatch.setattr(tts_tool, "warm_tts_provider", lambda cfg=None, provider=None: {"action": "noop", "warmed": False, "provider": "x"}) + resp = client.post("/api/audio/tts-lease", json={"lease": "tui:x"}) + assert resp.json()["active"] is True + assert tts_tool.tts_lease_holders() == ["tui:x"] + + +def test_acquire_resolves_provider_inside_target_profile(client, isolated_profiles, monkeypatch): + """Warm-up must read the REQUESTING profile's tts config, like /api/audio/speak.""" + import yaml + from tools import tts_tool + + (isolated_profiles["worker_beta"] / "config.yaml").write_text( + yaml.safe_dump({"tts": {"provider": "kittentts"}}), encoding="utf-8" + ) + seen = {} + + def _fake_warm(cfg=None, provider=None): + from hermes_constants import get_hermes_home + + seen["home"] = str(get_hermes_home()) + seen["provider"] = tts_tool._get_provider(tts_tool._load_tts_config()) + return {"action": "noop", "warmed": False, "provider": seen["provider"]} + + monkeypatch.setattr(tts_tool, "warm_tts_provider", _fake_warm) + resp = client.post("/api/audio/tts-lease?profile=worker_beta", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 200 + assert seen["home"] == str(isolated_profiles["worker_beta"]) + assert seen["provider"] == "kittentts" + + +def test_unknown_profile_404(client): + resp = client.post("/api/audio/tts-lease?profile=ghost", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 404 diff --git a/tests/tools/test_tts_lifecycle_leases.py b/tests/tools/test_tts_lifecycle_leases.py new file mode 100644 index 0000000000..e84478dbff --- /dev/null +++ b/tests/tools/test_tts_lifecycle_leases.py @@ -0,0 +1,233 @@ +"""TTS engine lifecycle driven by speech-output toggles (issue #100881). + +Local engines load lazily on first synthesis, so the first spoken reply after +"read replies aloud" / voice conversation turns on pays the model load as dead +air. The toggles now hold *leases*: acquiring warms the configured provider +into the SAME cache slot synthesis reads; releasing the last lease unloads +resident local models. +""" + +from __future__ import annotations + +import pytest + +from tools import tts_tool + + +@pytest.fixture(autouse=True) +def _clean_lifecycle(monkeypatch): + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + yield + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + + +class _FakePiperVoice: + loads = 0 + synthesized: list = [] + + @classmethod + def load(cls, model_path, use_cuda=False): + cls.loads += 1 + inst = cls() + inst.model_path = model_path + return inst + + def synthesize_wav(self, text, wav_file, syn_config=None): + type(self).synthesized.append(text) + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 160) + + +@pytest.fixture +def fake_piper(monkeypatch, tmp_path): + _FakePiperVoice.loads = 0 + _FakePiperVoice.synthesized = [] + monkeypatch.setattr(tts_tool, "_import_piper", lambda: _FakePiperVoice) + # Pretend the voice is already on disk so no download subprocess runs. + voices_dir = tmp_path / "voices" + voices_dir.mkdir() + (voices_dir / "en_US-test-medium.onnx").write_bytes(b"onnx") + (voices_dir / "en_US-test-medium.onnx.json").write_text("{}") + cfg = {"provider": "piper", "piper": {"voice": "en_US-test-medium", "voices_dir": str(voices_dir)}} + monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: cfg) + return cfg + + +# -------------------------------------------------------------------------- +# warm_tts_provider: warm-up populates the exact slot synthesis reads +# -------------------------------------------------------------------------- + + +def test_warm_loads_piper_into_synthesis_cache(fake_piper, tmp_path): + result = tts_tool.warm_tts_provider(fake_piper) + + assert result["warmed"] is True + assert result["action"] == "loaded" + assert result["provider"] == "piper" + assert _FakePiperVoice.loads == 1 + assert len(tts_tool._piper_voice_cache) == 1 + + # The load that would have happened on the first reply is already done: + # synthesis reuses the warmed instance without loading again. + out = tts_tool._generate_piper_tts("hello", str(tmp_path / "out.wav"), fake_piper) + assert out.endswith(".wav") + assert _FakePiperVoice.loads == 1 + assert _FakePiperVoice.synthesized == ["hello"] + + +def test_warm_twice_is_a_cache_hit(fake_piper): + tts_tool.warm_tts_provider(fake_piper) + second = tts_tool.warm_tts_provider(fake_piper) + + assert second["action"] == "cached" + assert _FakePiperVoice.loads == 1 + + +def test_warm_reads_configured_provider_when_none_given(fake_piper): + result = tts_tool.warm_tts_provider() + assert result["provider"] == "piper" + assert result["action"] == "loaded" + + +def test_warm_never_raises_on_engine_failure(monkeypatch): + def _boom(): + raise ImportError("No module named 'piper'") + + monkeypatch.setattr(tts_tool, "_import_piper", _boom) + result = tts_tool.warm_tts_provider({"provider": "piper"}) + + assert result["warmed"] is False + assert result["action"] == "error" + assert "piper" in result["error"] + assert tts_tool._piper_voice_cache == {} + + +def test_warm_is_noop_for_cloud_provider_without_lazy_sdk(monkeypatch): + result = tts_tool.warm_tts_provider({"provider": "openai"}) + assert result == {"provider": "openai", "warmed": False, "action": "noop"} + + +def test_warm_lazy_sdk_provider_reports_cached_when_installed(monkeypatch): + import types + + fake = types.SimpleNamespace( + is_available=lambda feature: feature == "tts.edge", + ensure=lambda *a, **k: pytest.fail("ensure must not run when the SDK is present"), + ) + monkeypatch.setitem(__import__("sys").modules, "tools.lazy_deps", fake) + result = tts_tool.warm_tts_provider({"provider": "edge"}) + assert result["warmed"] is True + assert result["action"] == "cached" + + +def test_warm_lazy_sdk_provider_installs_when_missing(monkeypatch): + import types + + calls = [] + fake = types.SimpleNamespace( + is_available=lambda feature: False, + ensure=lambda feature, prompt: calls.append((feature, prompt)), + ) + monkeypatch.setitem(__import__("sys").modules, "tools.lazy_deps", fake) + result = tts_tool.warm_tts_provider({"provider": "edge"}) + assert result["action"] == "installed" + assert calls == [("tts.edge", False)] + + +# -------------------------------------------------------------------------- +# release_tts_provider +# -------------------------------------------------------------------------- + + +def test_release_drops_every_local_cache(fake_piper): + tts_tool.warm_tts_provider(fake_piper) + tts_tool._kittentts_model_cache["m"] = object() + + assert tts_tool.release_tts_provider() == {"released": 2} + assert tts_tool._piper_voice_cache == {} + assert tts_tool._kittentts_model_cache == {} + + +def test_release_scoped_to_one_provider(fake_piper): + tts_tool.warm_tts_provider(fake_piper) + tts_tool._kittentts_model_cache["m"] = object() + + assert tts_tool.release_tts_provider("kittentts") == {"released": 1} + assert len(tts_tool._piper_voice_cache) == 1 + + +def test_release_with_nothing_resident_is_zero(): + assert tts_tool.release_tts_provider() == {"released": 0} + + +# -------------------------------------------------------------------------- +# Leases: warm on acquire, unload only when the LAST holder releases +# -------------------------------------------------------------------------- + + +def test_acquire_warms_and_counts(fake_piper): + result = tts_tool.acquire_tts_lease("desktop:read-aloud") + assert result["leases"] == 1 + assert result["action"] == "loaded" + assert tts_tool.tts_lease_holders() == ["desktop:read-aloud"] + + +def test_last_release_unloads_but_earlier_release_does_not(fake_piper): + tts_tool.acquire_tts_lease("desktop:read-aloud") + tts_tool.acquire_tts_lease("tui:voice-tts") + assert len(tts_tool._piper_voice_cache) == 1 + + # One surface turning speech off must not pull the model from under the + # other surface that still speaks through this process. + first = tts_tool.release_tts_lease("desktop:read-aloud") + assert first == {"leases": 1, "released": 0} + assert len(tts_tool._piper_voice_cache) == 1 + + last = tts_tool.release_tts_lease("tui:voice-tts") + assert last == {"leases": 0, "released": 1} + assert tts_tool._piper_voice_cache == {} + + +def test_reacquire_is_idempotent_and_reheals_cache(fake_piper): + tts_tool.acquire_tts_lease("cli:voice-tts") + tts_tool.release_tts_provider() # something else dropped the model + result = tts_tool.acquire_tts_lease("cli:voice-tts") + + assert result["leases"] == 1 + assert result["action"] == "loaded" + assert _FakePiperVoice.loads == 2 + + +def test_release_unknown_lease_is_noop(fake_piper): + tts_tool.acquire_tts_lease("a") + assert tts_tool.release_tts_lease("never-acquired") == {"leases": 1, "released": 0} + assert len(tts_tool._piper_voice_cache) == 1 + + +def test_acquire_failure_still_registers_lease(monkeypatch): + def _boom(): + raise RuntimeError("engine missing") + + monkeypatch.setattr(tts_tool, "_import_piper", _boom) + result = tts_tool.acquire_tts_lease("desktop:conversation", {"provider": "piper"}) + assert result["action"] == "error" + assert result["leases"] == 1 + assert tts_tool.tts_lease_holders() == ["desktop:conversation"] + + +# -------------------------------------------------------------------------- +# Registry invariant: every local engine cache is release-able +# -------------------------------------------------------------------------- + + +def test_every_local_warmer_has_a_registered_cache(): + warmers = tts_tool._local_tts_warmers() + assert set(warmers) == set(tts_tool._LOCAL_TTS_MODEL_CACHES) + assert tts_tool._LOCAL_TTS_MODEL_CACHES["piper"] is tts_tool._piper_voice_cache + assert tts_tool._LOCAL_TTS_MODEL_CACHES["kittentts"] is tts_tool._kittentts_model_cache diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 3aa4875922..60d182e5f9 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -2893,10 +2893,182 @@ def _tts_cache_get_or_load(cache: Dict[str, Any], key: str, load: Callable[[], A return value +# =========================================================================== +# Local-engine lifecycle: warm-up / release driven by TTS-output toggles +# =========================================================================== +# +# Local engines (Piper, KittenTTS) load their model lazily on the first +# synthesis call, so the first spoken reply after a user turns on "read +# replies aloud" / a voice conversation pays the whole load (plus a voice +# download on a fresh install) as dead air before the first word. And once +# loaded, the model stays resident for the process lifetime even after every +# TTS-output toggle is off again. +# +# The toggles ARE the intent signal. Every surface that flips speech output +# on holds a *lease* here (warming the configured engine as a side effect); +# flipping it off releases the lease, and when the last lease is gone the +# local model caches are dropped. Lease-counting instead of a bare +# on/off keeps one surface's "off" from unloading a model another surface +# (TUI /voice tts, desktop read-aloud, desktop conversation) still needs — +# they share this process's caches. +# +# Cloud providers have no resident model; warming them is a no-op beyond +# making sure the lazily-installed SDK is importable (edge-tts), which is +# also first-use latency users see as silence. + +# Provider name → local model cache it populates. The single registry both +# warm_tts_provider() and the release path consult — a new local engine adds +# one row here (at its cache declaration) plus a loader in +# _local_tts_warmers() and gets warm/release for free. +_LOCAL_TTS_MODEL_CACHES: Dict[str, Dict[str, Any]] = {} + + +def _local_tts_warmers() -> Dict[str, Callable[[Dict[str, Any]], Any]]: + # Resolved lazily: the loader functions are defined later in this module. + return { + "piper": lambda cfg: _load_piper_voice_for_config(cfg)[0], + "kittentts": lambda cfg: _load_kittentts_model_for_config(cfg)[0], + } + + +def _lazy_sdk_feature_for_provider(provider: str) -> Optional[str]: + """tools.lazy_deps feature key for providers whose SDK installs on first use.""" + return { + "edge": "tts.edge", + "elevenlabs": "tts.elevenlabs", + "mistral": "tts.mistral", + }.get(provider) + + +_tts_lease_lock = threading.Lock() +_tts_leases: set = set() + + +def warm_tts_provider( + tts_config: Optional[Dict[str, Any]] = None, + provider: Optional[str] = None, +) -> Dict[str, Any]: + """Pre-load the configured TTS provider so the next synthesis starts hot. + + * Local engines (Piper, KittenTTS): resolve the configured voice/model + exactly as synthesis would (including first-use voice download) and + load it into the same LRU cache slot synthesis reads. + * Lazily-installed cloud SDKs (edge-tts, ElevenLabs, Mistral): make sure + the SDK is importable, installing it if lazy installs are allowed. + * Everything else: nothing to warm — reported as ``action: "noop"``. + + Never raises; the result dict carries ``warmed`` / ``action`` / ``error`` + so callers on a toggle path can log and move on. Blocking — callers on a + UI thread should run it in the background. + """ + if tts_config is None: + tts_config = _load_tts_config() + name = (provider or _get_provider(tts_config) or "").lower().strip() + result: Dict[str, Any] = {"provider": name, "warmed": False, "action": "noop"} + + warmer = _local_tts_warmers().get(name) + if warmer is not None: + cache = _LOCAL_TTS_MODEL_CACHES.get(name) + before = len(cache) if cache is not None else 0 + started = time.monotonic() + try: + warmer(tts_config) + except Exception as exc: # engine missing, download failed, bad voice… + logger.warning("[TTS] warm-up for %s failed: %s", name, exc) + result.update(action="error", error=str(exc)) + return result + after = len(cache) if cache is not None else 0 + result.update( + warmed=True, + action="loaded" if after > before else "cached", + elapsed_ms=int((time.monotonic() - started) * 1000), + ) + logger.info("[TTS] warm-up %s: %s in %dms", name, result["action"], result["elapsed_ms"]) + return result + + feature = _lazy_sdk_feature_for_provider(name) + if feature is not None: + try: + from tools.lazy_deps import ensure, is_available + + if is_available(feature): + result.update(warmed=True, action="cached") + else: + ensure(feature, prompt=False) + result.update(warmed=True, action="installed") + except Exception as exc: + logger.debug("[TTS] SDK warm-up for %s skipped: %s", name, exc) + result.update(action="error", error=str(exc)) + return result + + +def release_tts_provider(provider: Optional[str] = None) -> Dict[str, Any]: + """Drop resident local TTS models so their memory is returned. + + With ``provider`` given, only that engine's cache is cleared; otherwise + every local engine cache is. Cloud providers hold nothing to release. + Returns ``{"released": }``. The next + synthesis simply reloads (or a warm-up does it ahead of time). + """ + name = (provider or "").lower().strip() + released = 0 + for cache_name, cache in _LOCAL_TTS_MODEL_CACHES.items(): + if name and cache_name != name: + continue + released += len(cache) + cache.clear() + if released: + logger.info("[TTS] released %d resident local model(s)", released) + return {"released": released} + + +def acquire_tts_lease(lease: str, tts_config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Register ``lease`` as a live TTS-output consumer and warm the provider. + + ``lease`` names the surface/toggle (e.g. ``"desktop:read-aloud"``, + ``"tui:voice-tts"``). Re-acquiring an existing lease is idempotent (still + re-warms — cheap on a cache hit, and heals a cache cleared elsewhere). + """ + with _tts_lease_lock: + _tts_leases.add(lease) + holders = len(_tts_leases) + result = warm_tts_provider(tts_config) + result["leases"] = holders + return result + + +def release_tts_lease(lease: str) -> Dict[str, Any]: + """Drop ``lease``; when it was the last one, unload resident local models. + + Releasing a lease that was never acquired is a no-op (still reports the + live holder count) so surfaces can call it unconditionally on their + "off" path. + """ + with _tts_lease_lock: + _tts_leases.discard(lease) + holders = len(_tts_leases) + result: Dict[str, Any] = {"leases": holders, "released": 0} + if holders == 0: + result["released"] = release_tts_provider()["released"] + return result + + +def tts_lease_holders() -> List[str]: + """Snapshot of live lease names (diagnostics / tests).""" + with _tts_lease_lock: + return sorted(_tts_leases) + + +def _reset_tts_leases_for_tests() -> None: + with _tts_lease_lock: + _tts_leases.clear() + + # Module-level cache for Piper voice instances. Voices are keyed on their # absolute .onnx model path so switching voices doesn't invalidate older # cached voices. _piper_voice_cache: Dict[str, Any] = {} +_LOCAL_TTS_MODEL_CACHES["piper"] = _piper_voice_cache def _check_piper_available() -> bool: @@ -2973,15 +3145,16 @@ def _resolve_piper_voice_path(voice: str, download_dir: Path) -> str: return str(cached) -def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: - """Generate speech using the local Piper engine. +def _load_piper_voice_for_config(tts_config: Dict[str, Any]) -> Tuple[Any, Dict[str, Any]]: + """Resolve + load (or fetch from cache) the Piper voice ``tts_config`` selects. - Loads the voice model once per process (cached by absolute path) and - writes a WAV file. Caller is responsible for converting to MP3/Opus - via ffmpeg when a different output format is required. + Shared by synthesis and :func:`warm_tts_provider` so a warm-up populates + exactly the cache slot the next synthesis call will hit — same voice + resolution, same download-on-first-use, same cache key. + + Returns ``(voice, piper_config)``. """ PiperVoice = _import_piper() - import wave piper_config = tts_config.get("piper") or {} if isinstance(tts_config, dict) else {} voice_name = piper_config.get("voice") or DEFAULT_PIPER_VOICE @@ -2991,15 +3164,6 @@ def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) model_path = _resolve_piper_voice_path(voice_name, download_dir) - # Tolerant speaker_id parse: drop bad input (non-int strings, lists, dicts) - # to 0 (Piper's own default). Booleans are rejected outright — True/False - # would silently coerce to 1/0 and hide a config mistake. - _raw_speaker = piper_config.get("speaker_id", 0) - if isinstance(_raw_speaker, bool) or not isinstance(_raw_speaker, int): - speaker_id = 0 - else: - speaker_id = _raw_speaker - # speaker_id is applied per-call via syn_config.speaker_id — the same # PiperVoice instance serves all speakers, so it stays out of the cache # key. Multi-speaker workflows share one model load. @@ -3012,6 +3176,28 @@ def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) return v voice = _tts_cache_get_or_load(_piper_voice_cache, cache_key, _load_piper_voice) + return voice, piper_config + + +def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: + """Generate speech using the local Piper engine. + + Loads the voice model once per process (cached by absolute path) and + writes a WAV file. Caller is responsible for converting to MP3/Opus + via ffmpeg when a different output format is required. + """ + import wave + + voice, piper_config = _load_piper_voice_for_config(tts_config) + + # Tolerant speaker_id parse: drop bad input (non-int strings, lists, dicts) + # to 0 (Piper's own default). Booleans are rejected outright — True/False + # would silently coerce to 1/0 and hide a config mistake. + _raw_speaker = piper_config.get("speaker_id", 0) + if isinstance(_raw_speaker, bool) or not isinstance(_raw_speaker, int): + speaker_id = 0 + else: + speaker_id = _raw_speaker # Optional synthesis knobs — only pass a SynthesisConfig when at least # one advanced knob is configured, so we don't depend on a newer Piper @@ -3079,6 +3265,28 @@ def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) # Module-level cache for KittenTTS model instance _kittentts_model_cache: Dict[str, Any] = {} +_LOCAL_TTS_MODEL_CACHES["kittentts"] = _kittentts_model_cache + + +def _load_kittentts_model_for_config(tts_config: Dict[str, Any]) -> Tuple[Any, Dict[str, Any]]: + """Load (or fetch from cache) the KittenTTS model ``tts_config`` selects. + + Shared by synthesis and :func:`warm_tts_provider` — same model name, + same cache key. Returns ``(model, kittentts_config)``. + """ + KittenTTS = _import_kittentts() + kt_config = tts_config.get("kittentts", {}) if isinstance(tts_config, dict) else {} + kt_config = kt_config or {} + model_name = kt_config.get("model", DEFAULT_KITTENTTS_MODEL) + + def _load_kittentts_model(): + logger.info("[KittenTTS] Loading model: %s", model_name) + m = KittenTTS(model_name) + logger.info("[KittenTTS] Model loaded successfully") + return m + + model = _tts_cache_get_or_load(_kittentts_model_cache, model_name, _load_kittentts_model) + return model, kt_config def _generate_kittentts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: @@ -3095,22 +3303,11 @@ def _generate_kittentts(text: str, output_path: str, tts_config: Dict[str, Any]) Returns: Path to the saved audio file. """ - KittenTTS = _import_kittentts() - kt_config = tts_config.get("kittentts", {}) - model_name = kt_config.get("model", DEFAULT_KITTENTTS_MODEL) + model, kt_config = _load_kittentts_model_for_config(tts_config) voice = kt_config.get("voice", DEFAULT_KITTENTTS_VOICE) speed = kt_config.get("speed", 1.0) clean_text = kt_config.get("clean_text", True) - # Use cached model instance if available - def _load_kittentts_model(): - logger.info("[KittenTTS] Loading model: %s", model_name) - m = KittenTTS(model_name) - logger.info("[KittenTTS] Model loaded successfully") - return m - - model = _tts_cache_get_or_load(_kittentts_model_cache, model_name, _load_kittentts_model) - # Generate audio (returns numpy array at 24kHz) audio = model.generate(text, voice=voice, speed=speed, clean_text=clean_text) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 470a7b19b8..c25266436e 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -16702,6 +16702,30 @@ def _voice_tts_enabled() -> bool: return os.environ.get("HERMES_VOICE_TTS", "").strip() == "1" +def _tts_lease_async(lease: str, active: bool) -> None: + """Acquire/release a TTS engine lease off the RPC thread. + + Speech-output toggles are the signal that TTS is about to be needed (or + no longer is). Acquiring warms the configured provider — for local + engines that is a model load, possibly a voice download — so it must not + block the toggle's RPC reply. Release is cheap but rides the same thread + for symmetry. Best-effort: a failure here never affects the toggle. + """ + + def _run(): + try: + from tools.tts_tool import acquire_tts_lease, release_tts_lease + + if active: + acquire_tts_lease(lease) + else: + release_tts_lease(lease) + except Exception as e: + logger.debug("voice: tts lease %s active=%s failed: %s", lease, active, e) + + threading.Thread(target=_run, name=f"tts-lease-{lease}", daemon=True).start() + + def _any_session_running() -> bool: """True while any session's agent turn is in flight. @@ -17551,6 +17575,12 @@ def _(rid, params: dict) -> dict: except Exception: stop_hint = "" + # Voice mode with speech output already on (voice.auto_tts / + # prior /voice tts) means replies will be spoken — warm the + # engine now rather than on the first reply. + if _voice_tts_enabled(): + _tts_lease_async("tui:voice-tts", True) + if not enabled: # Disabling the mode must tear the continuous loop down; the # loop holds the microphone and would otherwise keep running. @@ -17567,6 +17597,7 @@ def _(rid, params: dict) -> dict: # and silence any in-flight streaming speech. os.environ["HERMES_VOICE_TTS"] = "0" _tts_stream_stop(user_barge=False) + _tts_lease_async("tui:voice-tts", False) return _ok( rid, @@ -17586,6 +17617,10 @@ def _(rid, params: dict) -> dict: os.environ["HERMES_VOICE_TTS"] = "1" if new_value else "0" if not new_value: _tts_stream_stop(user_barge=False) + # The TTS toggle is the "speech is about to be needed" signal: on → + # pre-load the configured engine so the first reply starts hot; off → + # release the lease (last holder gone = resident local model freed). + _tts_lease_async("tui:voice-tts", new_value) # Include ``record_key`` on every branch so a /voice tts toggle # doesn't reset the TUI's cached shortcut to the default when a # user has a custom binding configured (Copilot review, round 2 diff --git a/website/docs/user-guide/features/tts.md b/website/docs/user-guide/features/tts.md index 3fbfce34b6..aacc31e3bf 100644 --- a/website/docs/user-guide/features/tts.md +++ b/website/docs/user-guide/features/tts.md @@ -256,6 +256,17 @@ tts: **Advanced knobs** (`tts.piper.length_scale` / `noise_scale` / `noise_w_scale` / `volume` / `normalize_audio`, `use_cuda`) correspond 1:1 to Piper's `SynthesisConfig`. They're ignored on older `piper-tts` versions. +### Warm-up and unload via speech toggles (local engines) + +Local engines (Piper, KittenTTS) load their model lazily, so without help the *first* spoken reply after you turn speech on pays the whole model load — and on a fresh install the voice download — as silence before the first word. Hermes treats the speech-output toggles as the signal that TTS is about to be needed: + +- **Desktop** — turning on **Read replies aloud**, or starting a **voice conversation**, pre-loads the configured engine in the background right away. Turning both off again unloads the resident model (a Piper voice is tens of MB; KittenTTS up to ~80MB) so it isn't parked in RAM for nothing. +- **CLI / TUI** — `/voice tts` (and `/voice on` when `voice.auto_tts` is set) do the same; `/voice off` releases. + +Each toggle holds a *lease* on the engine; the model is only unloaded when the last lease across surfaces is released, so switching off read-aloud in one Desktop window never pulls the voice out from under a conversation running in another. For cloud providers there is no model to hold — the toggle only makes sure a lazily-installed SDK (edge-tts, ElevenLabs, Mistral) is present. Warm-up is best-effort: if the engine can't load, the toggle still succeeds and the first reply falls back to loading on demand as before. + +The Desktop calls `POST /api/audio/tts-lease` with `{"lease": "", "active": true|false}`; other frontends can use the same endpoint. + ### Custom command providers If a TTS engine you want isn't natively supported (VoxCPM, MLX-Kokoro, XTTS CLI, a voice-cloning script, anything else that exposes a CLI), you can wire it in as a **command-type provider** without writing any Python. Hermes writes the input text to a temp UTF-8 file, runs your shell command, and reads the audio file the command produced.