#!/usr/bin/env python3 """ Text-to-Speech Tool Module Built-in TTS providers: - Edge TTS (default, free, no API key): Microsoft Edge neural voices - ElevenLabs (premium): High-quality voices, needs ELEVENLABS_API_KEY - OpenAI TTS: Good quality, needs OPENAI_API_KEY - MiniMax TTS: High-quality with voice cloning, needs the selected region's key - Mistral (Voxtral TTS): Multilingual, native Opus, needs MISTRAL_API_KEY - Google Gemini TTS: Controllable, 30 prebuilt voices, needs GEMINI_API_KEY - xAI TTS: Grok voices, uses xAI Grok OAuth credentials or XAI_API_KEY - NeuTTS (local, free, no API key): On-device TTS via neutts - KittenTTS (local, free, no API key): On-device 25MB model - Piper (local, free, no API key): OHF-Voice/piper1-gpl neural VITS, 44 languages Custom command providers: any number of named ``type: command`` providers under ``tts.providers.`` in ``~/.hermes/config.yaml``; Hermes writes the text to a temp file and runs the shell template (see the Local Command section of ``website/docs/user-guide/features/tts.md``). Output: Opus (.ogg) for voice-bubble platforms (Telegram etc.), MP3 elsewhere. Configuration lives under the ``tts:`` key; the user chooses provider/voice, the model just sends text. Module layout: this file owns config resolution, the command/plugin provider layers, the OpenAI/DeepInfra backends (managed-gateway aware), provider dispatch, the lifecycle leases and the tool registration. Sibling modules: ``tts_tool_providers`` (cloud backends), ``tts_tool_local`` (on-device engines + model caches), ``tts_tool_delivery`` (chunking / ffmpeg / packing), ``tts_tool_speaker`` (streaming speaker pipeline). Their names are re-imported here so ``tools.tts_tool.`` keeps resolving. Usage: from tools.tts_tool import text_to_speech_tool, check_tts_requirements result = text_to_speech_tool(text="Hello world") """ import asyncio import datetime import importlib.util import json import logging import os import re import subprocess import tempfile import threading import time import uuid from pathlib import Path from typing import Callable, Dict, Any, List, Optional from urllib.parse import urljoin from hermes_constants import display_hermes_home logger = logging.getLogger(__name__) def get_env_value(name, default=None): """Read env values through the live config module. Resolved at call time: tests monkeypatch/restore ``hermes_cli.config.get_env_value`` and must not leave TTS holding a stale function for the rest of the process. """ try: from hermes_cli.config import get_env_value as _get_env_value except ImportError: return os.getenv(name, default) value = _get_env_value(name) return default if value is None else value def _resolve_provider_key(env_var: str, provider_id: str) -> str: """Resolve a TTS provider API key via the shared voice-key resolver. ``tools.tool_backend_helpers.resolve_provider_secret`` is the single owner of STT/TTS key resolution (config > env/.env > credential pool). Resolved at call time so tests that reload the helpers module see the live function. """ try: from tools.tool_backend_helpers import resolve_provider_secret except ImportError: # pragma: no cover — helpers are in-repo return str(get_env_value(env_var) or "").strip() return resolve_provider_secret(env_var, provider_id, env_getter=get_env_value) from tools.managed_tool_gateway import resolve_managed_tool_gateway from tools.tts_command_provider import ( command_env_passthrough as _command_provider_env_passthrough, render_command_template as _render_command_tts_template, run_command_provider as _run_command_tts, shell_quote_context as _shell_quote_context, # noqa: F401 — tests import via this module ) from tools.tool_backend_helpers import ( NOUS_MANAGED_PROVIDER, managed_nous_tools_enabled, nous_tool_gateway_unavailable_message, read_selection, resolve_openai_audio_api_key, selection_error, ) from tools.tts_tool_delivery import ( # noqa: F401 — historical names re-exported FALLBACK_MAX_TEXT_LENGTH, AudioDeliveryProfile, _build_audio_delivery_files, _concat_audio_files, _convert_to_opus, _has_ffmpeg, _pack_audio_files_for_delivery, _repair_ogg_container, _resolve_audio_delivery_profile, _sniff_audio_container, _split_oversized_sentence, _split_text_for_tts, _wrap_pcm_as_wav, ) from tools.tts_tool_providers import ( # noqa: F401 — historical names re-exported DEFAULT_ELEVENLABS_MODEL_ID, DEFAULT_ELEVENLABS_STREAMING_MODEL_ID, DEFAULT_ELEVENLABS_VOICE_ID, DEFAULT_GEMINI_TTS_BASE_URL, DEFAULT_GEMINI_TTS_MODEL, DEFAULT_GEMINI_TTS_VOICE, DEFAULT_MINIMAX_BASE_URL, DEFAULT_MINIMAX_CN_BASE_URL, DEFAULT_XAI_BASE_URL, DEFAULT_XAI_VOICE_ID, TTS_RESPONSE_BODY_LIMIT_BYTES, _XAI_FIRST_SENTENCE_RE, _XAI_INLINE_SPEECH_TAGS, _XAI_WRAPPING_SPEECH_TAGS, _apply_xai_auto_speech_tags, _elevenlabs_environment_kwargs, _generate_edge_tts, _generate_elevenlabs, _generate_gemini_tts, _generate_minimax_tts, _generate_mistral_tts, _generate_xai_tts, _read_tts_response_bytes, _resolve_minimax_tts_runtime, _tts_response_format_from_path, ) from tools.tts_tool_local import ( # noqa: F401 — historical names re-exported DEFAULT_PIPER_VOICE, _LOCAL_TTS_MODEL_CACHES, _TTS_MODEL_CACHE_MAX, _generate_kittentts, _generate_neutts, _generate_piper_tts, _kittentts_model_cache, _load_kittentts_model_for_config, _load_piper_voice_for_config, _piper_voice_cache, _resolve_piper_voice_path, _tts_cache_get_or_load, ) from tools.tts_tool_speaker import ( # noqa: F401 — historical names re-exported stream_tts_to_speaker, ) # --------------------------------------------------------------------------- # Lazy imports -- providers are imported only when actually used to avoid # crashing in headless environments (SSH, Docker, WSL, no PortAudio). # --------------------------------------------------------------------------- def _lazy_ensure(feature: str) -> None: """Best-effort ``tools.lazy_deps.ensure`` so an SDK installs on first use. Users who enabled a provider by editing config.yaml never ran the post-setup hook. Any failure (lazy_deps missing, install refused) falls through so the raw import below still raises a clean ImportError. """ try: from tools.lazy_deps import ensure ensure(feature, prompt=False) except Exception: pass def _import_edge_tts(): """Lazy import edge_tts. Returns the module or raises ImportError.""" _lazy_ensure("tts.edge") import edge_tts return edge_tts def _import_elevenlabs(): """Lazy import the ElevenLabs client class or raise ImportError.""" _lazy_ensure("tts.elevenlabs") from elevenlabs.client import ElevenLabs return ElevenLabs def _import_openai_client(): from openai import OpenAI as OpenAIClient return OpenAIClient def _import_mistral_client(): """Lazy import the Mistral client class or raise ImportError.""" _lazy_ensure("tts.mistral") from mistralai.client import Mistral return Mistral def _import_sounddevice(): """Raises ImportError/OSError when PortAudio is unavailable.""" import sounddevice as sd return sd def _import_kittentts(): from kittentts import KittenTTS return KittenTTS def _import_piper(): """``pip install piper-tts`` ships cross-platform wheels with embedded espeak-ng.""" from piper import PiperVoice return PiperVoice def _package_installed(name: str) -> bool: try: return importlib.util.find_spec(name) is not None except Exception: return False def _check_neutts_available() -> bool: return _package_installed("neutts") def _check_kittentts_available() -> bool: return _package_installed("kittentts") def _check_piper_available() -> bool: return _package_installed("piper") # =========================================================================== # Defaults # =========================================================================== DEFAULT_PROVIDER = "edge" DEFAULT_OPENAI_MODEL = "gpt-4o-mini-tts" # The managed OpenAI audio gateway (Nous portal proxy) only proxies these # speech models; anything else is 400 "Unsupported managed OpenAI speech model". MANAGED_OPENAI_TTS_MODELS = frozenset({"gpt-4o-mini-tts"}) DEFAULT_OPENAI_VOICE = "alloy" DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" # DeepInfra base URL is resolved via hermes_cli.models.deepinfra_base_url (shared). DEFAULT_DEEPINFRA_TTS_VOICE = "default" def _get_default_output_dir() -> str: from hermes_constants import get_hermes_dir return str(get_hermes_dir("cache/audio", "audio_cache")) DEFAULT_OUTPUT_DIR = _get_default_output_dir() _DEFAULT_OUTPUT_DIR_AT_IMPORT = DEFAULT_OUTPUT_DIR def _default_output_dir() -> str: """Return the active profile's audio output dir at call time. Long-lived multi-profile runtimes (dashboard, TUI/Desktop backend, cron) import this module once and later switch profiles via ``set_hermes_home_override()``; a frozen constant would keep writing into the launch profile's cache. ``DEFAULT_OUTPUT_DIR`` stays as a module attribute for tests/patchers and wins whenever it has been patched. """ configured = DEFAULT_OUTPUT_DIR if configured != _DEFAULT_OUTPUT_DIR_AT_IMPORT: return configured return _get_default_output_dir() # Per-provider input-character caps (from official provider docs); override # via ``tts..max_text_length``. PROVIDER_MAX_TEXT_LENGTH: Dict[str, int] = { "edge": 5000, # edge-tts practical sync limit "openai": 4096, # https://platform.openai.com/docs/guides/text-to-speech "xai": 15000, # https://docs.x.ai/developers/model-capabilities/audio/text-to-speech "minimax": 10000, # https://platform.minimax.io/docs/api-reference/speech-t2a-http (sync) "mistral": 4000, # conservative; no published per-request cap "gemini": 32000, # 32k-token context window; char cap is conservative "elevenlabs": 10000, # fallback when model-aware lookup can't resolve (multilingual_v2) "neutts": 2000, # local model, quality falls off on long text "kittentts": 2000, # local 25MB model "piper": 5000, # local VITS model, phoneme-based; practical cap } # ElevenLabs caps vary by model_id. https://elevenlabs.io/docs/overview/models ELEVENLABS_MODEL_MAX_TEXT_LENGTH: Dict[str, int] = { "eleven_v3": 5000, "eleven_ttv_v3": 5000, "eleven_multilingual_v2": 10000, "eleven_multilingual_v1": 10000, "eleven_english_sts_v2": 10000, "eleven_english_sts_v1": 10000, "eleven_flash_v2": 30000, "eleven_flash_v2_5": 40000, } # Back-compat alias. Prefer ``_resolve_max_text_length()`` for new code. MAX_TEXT_LENGTH = FALLBACK_MAX_TEXT_LENGTH def _positive_int_override(value: Any) -> Optional[int]: """A user ``max_text_length`` override, or None when absent/bool/non-positive.""" if isinstance(value, bool) or not isinstance(value, int) or value <= 0: return None return value def _resolve_max_text_length( provider: Optional[str], tts_config: Optional[Dict[str, Any]] = None, ) -> int: """Return the input-character cap for *provider*. Order: ``tts..max_text_length`` > ElevenLabs model table > ``PROVIDER_MAX_TEXT_LENGTH`` > command provider's own ``max_text_length`` (else ``DEFAULT_COMMAND_TTS_MAX_TEXT_LENGTH``) > ``FALLBACK_MAX_TEXT_LENGTH``. Non-positive / non-int overrides fall through so a broken config can't disable truncation. """ if not provider: return FALLBACK_MAX_TEXT_LENGTH key = provider.lower().strip() cfg = tts_config or {} prov_cfg = cfg.get(key) if isinstance(cfg.get(key), dict) else {} override = _positive_int_override(prov_cfg.get("max_text_length") if prov_cfg else None) if override: return override if key == "elevenlabs": model_id = (prov_cfg or {}).get("model_id") or DEFAULT_ELEVENLABS_MODEL_ID mapped = ELEVENLABS_MODEL_MAX_TEXT_LENGTH.get(str(model_id).strip()) if mapped: return mapped if key in PROVIDER_MAX_TEXT_LENGTH: return PROVIDER_MAX_TEXT_LENGTH[key] if key not in BUILTIN_TTS_PROVIDERS: named = _get_named_provider_config(cfg, key) if _is_command_provider_config(named): return _positive_int_override(named.get("max_text_length")) or DEFAULT_COMMAND_TTS_MAX_TEXT_LENGTH return FALLBACK_MAX_TEXT_LENGTH # =========================================================================== # Config loader -- reads tts: section from ~/.hermes/config.yaml # =========================================================================== def _load_tts_config() -> Dict[str, Any]: """Return the ``tts`` config section ({} when unavailable).""" try: from hermes_cli.config import load_config config = load_config() return config.get("tts") or {} except ImportError: logger.debug("hermes_cli.config not available, using default TTS config") return {} except Exception as e: logger.warning("Failed to load TTS config: %s", e, exc_info=True) return {} def _get_provider(tts_config: Dict[str, Any]) -> str: """The explicitly configured TTS provider, or the free default. Inference credentials do not imply consent to paid speech generation: cloud TTS is opt-in via ``tts.provider``. The managed selection (``tts.provider: nous``) is serviced by the OpenAI implementation, routed through the managed openai-audio gateway by ``_resolve_openai_audio_client_config``. """ provider = (tts_config.get("provider") or DEFAULT_PROVIDER).lower().strip() if provider == NOUS_MANAGED_PROVIDER: return "openai" return provider # =========================================================================== # Custom command providers (type: command under tts.providers.) # =========================================================================== # # Config shape:: # # tts: # provider: piper-en # providers: # piper-en: # type: command # command: "piper -m ~/model.onnx -f {output_path} < {input_path}" # output_format: wav # # Placeholders: ``{input_path}``, ``{text_path}`` (alias), ``{output_path}``, # ``{format}``, ``{voice}``, ``{model}``, ``{speed}``; ``{{``/``}}`` for literal # braces. Values are shell-quoted for their surrounding quote context. Built-in # provider names always win over a same-named entry under ``tts.providers``. # Any ``tts.provider`` value NOT in this set refers to ``tts.providers.``. BUILTIN_TTS_PROVIDERS = frozenset({ "edge", "elevenlabs", "openai", "minimax", "xai", "mistral", "gemini", "neutts", "kittentts", "piper", "deepinfra", }) DEFAULT_COMMAND_TTS_TIMEOUT_SECONDS = 120 DEFAULT_COMMAND_TTS_OUTPUT_FORMAT = "mp3" COMMAND_TTS_OUTPUT_FORMATS = frozenset( {"mp3", "wav", "ogg", "flac", "m4a", "aac", "amr", "opus"} ) DEFAULT_COMMAND_TTS_MAX_TEXT_LENGTH = 5000 # Platforms whose native voice-bubble delivery requires Ogg/Opus audio # (MP3 renders as a broken attachment there). OPUS_VOICE_PLATFORMS = frozenset({ "telegram", "matrix", "feishu", "whatsapp", "signal", }) # Built-ins that emit Opus natively when asked for .ogg (no ffmpeg needed). _NATIVE_OPUS_PROVIDERS = frozenset({"openai", "elevenlabs", "mistral", "gemini"}) # Built-ins whose native output (MP3/WAV) needs ffmpeg for voice-bubble delivery. _FFMPEG_OPUS_PROVIDERS = frozenset({"edge", "neutts", "minimax", "xai", "kittentts", "piper"}) def _get_provider_section(tts_config: Dict[str, Any], name: str) -> Dict[str, Any]: """Return a provider config block if it's a dict, else an empty dict.""" if not isinstance(tts_config, dict): return {} section = tts_config.get(name) return section if isinstance(section, dict) else {} def _get_named_provider_config( tts_config: Dict[str, Any], name: str, ) -> Dict[str, Any]: """Config dict for a user-declared provider, or {}. ``tts.providers.`` is canonical; ``tts.`` is accepted as back-compat only for non-built-in names (so a user's ``tts.openai`` block still means the OpenAI provider, not a custom command). """ providers = _get_provider_section(tts_config, "providers") section = providers.get(name) if isinstance(providers, dict) else None if isinstance(section, dict): return section if name.lower() not in BUILTIN_TTS_PROVIDERS: legacy = _get_provider_section(tts_config, name) if legacy: return legacy return {} def _is_command_provider_config(config: Dict[str, Any]) -> bool: """True when *config* declares a command-type provider (has a non-empty ``command``).""" if not isinstance(config, dict): return False ptype = str(config.get("type") or "").strip().lower() if ptype and ptype != "command": return False command = config.get("command") return isinstance(command, str) and bool(command.strip()) def _resolve_command_provider_config( provider: str, tts_config: Dict[str, Any], ) -> Optional[Dict[str, Any]]: """The provider config when *provider* is a user-declared command provider. None for built-in names (native handlers win), unknown names, or non-command types. """ if not provider: return None key = provider.lower().strip() if key in BUILTIN_TTS_PROVIDERS: return None config = _get_named_provider_config(tts_config, key) if _is_command_provider_config(config): return config return None def _dispatch_to_plugin_provider( text: str, output_path: str, provider: str, tts_config: Dict[str, Any], ) -> Optional[str]: """Route to a plugin-registered TTS provider; None means "fall through". Invariants enforced here even though the caller checks them too, so a caller refactor can't silently break them: 1. Built-in names never reach the plugin registry. 2. A same-named ``type: command`` provider wins over a plugin. 3. Dispatch fires only for a registered :class:`TTSProvider` whose name equals the configured value; unknown names return None. Plugin exceptions propagate — the outer ``text_to_speech_tool`` converts them to the standard error envelope. """ if not provider: return None key = provider.lower().strip() if key in BUILTIN_TTS_PROVIDERS: return None if _is_command_provider_config(_get_named_provider_config(tts_config, key)): return None try: from agent.tts_registry import get_provider from hermes_cli.plugins import _ensure_plugins_discovered _ensure_plugins_discovered() plugin_provider = get_provider(key) if plugin_provider is None: # Long-lived sessions may have discovered plugins before this one # was installed/enabled; retry once with a forced refresh. _ensure_plugins_discovered(force=True) plugin_provider = get_provider(key) except Exception as exc: # noqa: BLE001 — discovery failure is non-fatal logger.debug("tts plugin dispatch skipped (discovery failed): %s", exc) return None if plugin_provider is None: return None # voice/model/speed/format are optional per the TTSProvider.synthesize # contract; providers fall back to their own defaults on None. cfg = tts_config if isinstance(tts_config, dict) else {} voice = cfg.get("voice") model = cfg.get("model") speed = cfg.get("speed") fmt = cfg.get("output_format", DEFAULT_COMMAND_TTS_OUTPUT_FORMAT) logger.info("Generating speech with plugin TTS provider '%s'...", key) written = plugin_provider.synthesize( text, output_path, voice=voice if isinstance(voice, str) and voice else None, model=model if isinstance(model, str) and model else None, speed=float(speed) if isinstance(speed, (int, float)) else None, format=str(fmt).lower() if fmt else "mp3", ) # Contract: returns the (possibly rewritten) output path; tolerate None. return written if isinstance(written, str) and written else output_path def _plugin_provider_is_voice_compatible(provider: str) -> bool: """True when the registered plugin provider opts into voice-bubble delivery. Any registry/property failure means False (safe default, like command providers). """ if not provider: return False key = provider.lower().strip() if key in BUILTIN_TTS_PROVIDERS: return False try: from agent.tts_registry import get_provider plugin_provider = get_provider(key) if plugin_provider is None: return False return bool(plugin_provider.voice_compatible) except Exception as exc: # noqa: BLE001 logger.debug("tts plugin voice_compatible check failed for '%s': %s", key, exc) return False def _iter_command_providers(tts_config: Dict[str, Any]): """Yield (name, config) pairs for every declared command-type provider.""" if not isinstance(tts_config, dict): return providers = _get_provider_section(tts_config, "providers") for name, cfg in (providers or {}).items(): if ( isinstance(name, str) and name.lower() not in BUILTIN_TTS_PROVIDERS and _is_command_provider_config(cfg) ): yield name, cfg def _get_command_tts_timeout(config: Dict[str, Any]) -> float: """Timeout in seconds; invalid or non-positive values fall back to the default.""" raw = config.get("timeout", config.get("timeout_seconds", DEFAULT_COMMAND_TTS_TIMEOUT_SECONDS)) try: value = float(raw) except (TypeError, ValueError): return float(DEFAULT_COMMAND_TTS_TIMEOUT_SECONDS) if value <= 0: return float(DEFAULT_COMMAND_TTS_TIMEOUT_SECONDS) return value def _get_command_tts_output_format( config: Dict[str, Any], output_path: Optional[str] = None, ) -> str: """Validated output format: the output path's suffix wins, then ``format``/``output_format``.""" if output_path: suffix = Path(output_path).suffix.lower().strip().lstrip(".") if suffix in COMMAND_TTS_OUTPUT_FORMATS: return suffix raw = ( config.get("format") or config.get("output_format") or DEFAULT_COMMAND_TTS_OUTPUT_FORMAT ) fmt = str(raw).lower().strip().lstrip(".") return fmt if fmt in COMMAND_TTS_OUTPUT_FORMATS else DEFAULT_COMMAND_TTS_OUTPUT_FORMAT def _is_command_tts_voice_compatible(config: Dict[str, Any]) -> bool: """True only when the user explicitly opted in to voice delivery.""" value = config.get("voice_compatible", False) if isinstance(value, str): return value.strip().lower() in {"1", "true", "yes", "on"} return bool(value) def _configured_command_tts_output_path(path: Path, config: Dict[str, Any]) -> Path: """Return an output path whose extension matches the provider's output_format.""" fmt = _get_command_tts_output_format(config) return path.with_suffix(f".{fmt}") def _generate_command_tts( text: str, output_path: str, provider_name: str, config: Dict[str, Any], tts_config: Dict[str, Any], ) -> str: """Generate speech by running a user-configured shell command. Returns the absolute path of the audio file the command wrote. Raises ``ValueError`` for invalid provider config and ``RuntimeError`` for timeouts / non-zero exits / empty output. """ command_template = str(config.get("command") or "").strip() if not command_template: raise ValueError( f"tts.providers.{provider_name}.command is not configured" ) output = Path(output_path).expanduser() output.parent.mkdir(parents=True, exist_ok=True) if output.exists(): output.unlink() timeout = _get_command_tts_timeout(config) output_format = _get_command_tts_output_format(config, str(output)) speed = config.get("speed", tts_config.get("speed", "")) with tempfile.TemporaryDirectory() as tmpdir: text_path = Path(tmpdir) / "input.txt" text_path.write_text(text, encoding="utf-8") placeholders = { "input_path": str(text_path), "text_path": str(text_path), "output_path": str(output), "format": output_format, "voice": str(config.get("voice", "")), "model": str(config.get("model", "")), "speed": str(speed), } command = _render_command_tts_template(command_template, placeholders) try: _run_command_tts( command, timeout, env_passthrough=_command_provider_env_passthrough(config), ) except subprocess.TimeoutExpired as exc: raise RuntimeError( f"TTS provider '{provider_name}' timed out after {timeout:g}s" ) from exc except subprocess.CalledProcessError as exc: detail_parts = [] if exc.stderr: detail_parts.append(f"stderr: {exc.stderr.strip()}") if exc.stdout: detail_parts.append(f"stdout: {exc.stdout.strip()}") detail = "; ".join(detail_parts) or "no command output" raise RuntimeError( f"TTS provider '{provider_name}' exited with code " f"{exc.returncode}: {detail}" ) from exc if not output.exists() or output.stat().st_size <= 0: raise RuntimeError( f"TTS provider '{provider_name}' produced no output at {output}" ) return str(output) def _has_any_command_tts_provider(tts_config: Optional[Dict[str, Any]] = None) -> bool: """Return True when any command-type TTS provider is configured.""" if tts_config is None: tts_config = _load_tts_config() for _name, _cfg in _iter_command_providers(tts_config): return True return False # =========================================================================== # Provider: OpenAI TTS (also every OpenAI-compatible endpoint — DeepInfra # delegates here). Kept in the origin module: it shares the managed-gateway # selection logic below. # =========================================================================== def _generate_openai_tts( text: str, output_path: str, tts_config: Dict[str, Any], *, api_key: Optional[str] = None, base_url: Optional[str] = None, model: Optional[str] = None, voice: Optional[str] = None, speed: Optional[float] = None, instructions: Optional[str] = None, ) -> str: """Generate audio via the OpenAI ``audio.speech.create`` SDK shape. Explicit kwargs let OpenAI-compatible backends (DeepInfra) pass their own credentials/model/voice and skip ``_resolve_openai_audio_client_config`` (the managed-gateway path). When None: ``api_key`` comes from the OpenAI auth chain, ``base_url`` from ``tts.openai.base_url`` then the auth-chain fallback then the OpenAI default, model/voice/speed from ``tts.openai`` (speed falling back to global ``tts.speed``). ``instructions`` is forwarded only when truthy so ``tts-1`` and strict OpenAI-compatible servers that reject unknown kwargs are unaffected. """ fallback_base: Optional[str] = None is_managed = False explicit_base_url = base_url is not None if api_key is None: api_key, fallback_base, is_managed = _resolve_openai_audio_client_config() # ``tts.openai: null`` in YAML yields None — coalesce so .get() is safe. oai_config = (tts_config.get("openai") if isinstance(tts_config, dict) else None) or {} if model is None: model = oai_config.get("model", DEFAULT_OPENAI_MODEL) if voice is None: voice = oai_config.get("voice", DEFAULT_OPENAI_VOICE) config_base_url = oai_config.get("base_url") if base_url is None: # Config override beats the auth-chain fallback; an explicit arg # (DeepInfra) skipped this block and always wins. base_url = config_base_url or fallback_base or DEFAULT_OPENAI_BASE_URL if speed is None: speed_default = tts_config.get("speed", 1.0) if isinstance(tts_config, dict) else 1.0 speed = float(oai_config.get("speed", speed_default)) language = oai_config.get("language") # The managed gateway only proxies MANAGED_OPENAI_TTS_MODELS; coerce a # direct-OpenAI model (e.g. "tts-1-hd") unless the user redirected # base_url to their own endpoint. if ( is_managed and not explicit_base_url and not config_base_url and model not in MANAGED_OPENAI_TTS_MODELS ): logger.warning( "TTS: managed OpenAI audio gateway does not support model %r; " "falling back to %s. Set VOICE_TOOLS_OPENAI_KEY or OPENAI_API_KEY " "to use %r directly.", model, DEFAULT_OPENAI_MODEL, model, ) model = DEFAULT_OPENAI_MODEL response_format = _tts_response_format_from_path(output_path) OpenAIClient = _import_openai_client() client = OpenAIClient(api_key=api_key, base_url=base_url) try: create_kwargs: Dict[str, Any] = { "model": model, "voice": voice, "input": text, "response_format": response_format, "extra_headers": {"x-idempotency-key": str(uuid.uuid4())}, } if speed != 1.0: create_kwargs["speed"] = max(0.25, min(4.0, speed)) if instructions: create_kwargs["instructions"] = instructions if language: create_kwargs["extra_body"] = {"lang_code": language} response = client.audio.speech.create(**create_kwargs) response.stream_to_file(output_path) return output_path finally: close = getattr(client, "close", None) if callable(close): close() def _generate_deepinfra_tts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: """Resolve DeepInfra credentials/model, then delegate to the OpenAI handler. DeepInfra's audio endpoint is OpenAI-compatible. Model ids come live from the shared ``hermes_cli.models`` catalog helpers (no hardcoded ids, so retired models disappear without a patch). """ api_key = _resolve_provider_key("DEEPINFRA_API_KEY", "deepinfra") if not api_key: raise ValueError( "DEEPINFRA_API_KEY not set. Run `hermes setup` to configure, " "or set the env var directly." ) # ``tts.deepinfra: null`` yields None (no DEFAULT_CONFIG block to merge over). di_config = tts_config.get("deepinfra") if isinstance(tts_config, dict) else None if not isinstance(di_config, dict): di_config = {} from hermes_cli.models import deepinfra_base_url, deepinfra_model_ids model = di_config.get("model") if not isinstance(model, str) or not model.strip(): candidates = deepinfra_model_ids("tts") if not candidates: raise ValueError( "No DeepInfra TTS model available. Pin one in config.yaml " "under tts.deepinfra.model, or check connectivity to " "api.deepinfra.com so the live catalog can be fetched." ) model = candidates[0] return _generate_openai_tts( text, output_path, tts_config, api_key=api_key, base_url=deepinfra_base_url(di_config), model=model, voice=di_config.get("voice", DEFAULT_DEEPINFRA_TTS_VOICE), speed=float(di_config.get("speed", tts_config.get("speed", 1.0))), ) # =========================================================================== # Local-engine lifecycle: warm-up / release driven by TTS-output toggles # =========================================================================== # # Local engines load their model lazily on first synthesis, so the first spoken # reply after a user turns speech output on pays the whole load as dead air, # and the model then stays resident forever. The toggles ARE the intent # signal: every surface that flips speech output on holds a *lease* here # (warming the configured engine); when the last lease is released the local # model caches are dropped. Lease-counting keeps one surface's "off" from # unloading a model another surface in this process still needs. Cloud # providers have nothing resident; warming them only ensures the lazily # installed SDK is importable. def _local_tts_warmers() -> Dict[str, Callable[[Dict[str, Any]], Any]]: """Provider name → loader populating that engine's cache slot (same key synthesis uses).""" return { "piper": lambda cfg: _load_piper_voice_for_config(cfg)[0], "kittentts": lambda cfg: _load_kittentts_model_for_config(cfg)[0], } def _lazy_sdk_feature_for_provider(provider: str) -> Optional[str]: """tools.lazy_deps feature key for providers whose SDK installs on first use.""" return { "edge": "tts.edge", "elevenlabs": "tts.elevenlabs", "mistral": "tts.mistral", }.get(provider) _tts_lease_lock = threading.Lock() _tts_leases: set = set() def _signal_user_tts_provider(name: str, tts_config: Dict[str, Any], hook: str) -> Optional[str]: """Forward a lease ``hook`` (``"warm"`` / ``"release"``) to a user-declared provider. Command providers run their optional ``warm_command`` / ``release_command`` (same template/env/timeout rules as ``command``; output discarded) on a background thread so a toggle never waits on a model server. Plugin providers get :meth:`TTSProvider.warm` / :meth:`TTSProvider.release`. Best-effort: failures are logged at debug. Returns the action taken. """ if not name or name in BUILTIN_TTS_PROVIDERS: return None cfg = _get_named_provider_config(tts_config, name) try: if _is_command_provider_config(cfg): template = str(cfg.get(f"{hook}_command") or "").strip() if not template: return None command = _render_command_tts_template(template, { "voice": str(cfg.get("voice", "")), "model": str(cfg.get("model", "")), "speed": str(cfg.get("speed", tts_config.get("speed", ""))), }) def _run() -> None: try: _run_command_tts(command, _get_command_tts_timeout(cfg), env_passthrough=_command_provider_env_passthrough(cfg)) except Exception as exc: # noqa: BLE001 — best-effort hook logger.debug("[TTS] %s_command for %s failed: %s", hook, name, exc) threading.Thread(target=_run, name=f"tts-{hook}-{name}", daemon=True).start() return hook from agent.tts_registry import get_provider from hermes_cli.plugins import _ensure_plugins_discovered _ensure_plugins_discovered() plugin_provider = get_provider(name) if plugin_provider is None: return None getattr(plugin_provider, hook)() return hook except Exception as exc: # noqa: BLE001 — best-effort hook logger.debug("[TTS] %s hook for %s failed: %s", hook, name, exc) return "error" def warm_tts_provider( tts_config: Optional[Dict[str, Any]] = None, provider: Optional[str] = None, ) -> Dict[str, Any]: """Pre-load the configured TTS provider so the next synthesis starts hot. Local engines load their voice/model into the same LRU slot synthesis reads (including first-use download); lazily-installed cloud SDKs are made importable; user-declared providers get ``warm_command`` / :meth:`TTSProvider.warm`; everything else is ``action: "noop"``. Never raises — the result dict carries ``warmed`` / ``action`` / ``error``. Blocking; UI threads should run it in the background. """ if tts_config is None: tts_config = _load_tts_config() name = (provider or _get_provider(tts_config) or "").lower().strip() result: Dict[str, Any] = {"provider": name, "warmed": False, "action": "noop"} warmer = _local_tts_warmers().get(name) if warmer is not None: cache = _LOCAL_TTS_MODEL_CACHES.get(name) before = len(cache) if cache is not None else 0 started = time.monotonic() try: warmer(tts_config) except Exception as exc: # engine missing, download failed, bad voice… logger.warning("[TTS] warm-up for %s failed: %s", name, exc) result.update(action="error", error=str(exc)) return result after = len(cache) if cache is not None else 0 result.update( warmed=True, action="loaded" if after > before else "cached", elapsed_ms=int((time.monotonic() - started) * 1000), ) logger.info("[TTS] warm-up %s: %s in %dms", name, result["action"], result["elapsed_ms"]) return result signalled = _signal_user_tts_provider(name, tts_config, "warm") if signalled is not None: result.update(warmed=signalled != "error", action="warmed" if signalled != "error" else "error") return result feature = _lazy_sdk_feature_for_provider(name) if feature is not None: try: from tools.lazy_deps import ensure, is_available if is_available(feature): result.update(warmed=True, action="cached") else: ensure(feature, prompt=False) result.update(warmed=True, action="installed") except Exception as exc: logger.debug("[TTS] SDK warm-up for %s skipped: %s", name, exc) result.update(action="error", error=str(exc)) return result def release_tts_provider(provider: Optional[str] = None) -> Dict[str, Any]: """Drop resident local TTS models so their memory is returned. With ``provider`` given only that engine's cache is cleared; otherwise every local cache is, and the configured user-declared provider is signalled (plugin ``release()`` / command ``release_command``). Returns ``{"released": }``. """ name = (provider or "").lower().strip() if not name: tts_config = _load_tts_config() _signal_user_tts_provider(_get_provider(tts_config), tts_config, "release") released = 0 for cache_name, cache in _LOCAL_TTS_MODEL_CACHES.items(): if name and cache_name != name: continue released += len(cache) cache.clear() if released: logger.info("[TTS] released %d resident local model(s)", released) return {"released": released} def acquire_tts_lease(lease: str, tts_config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Register ``lease`` (e.g. ``"desktop:read-aloud"``) as a live consumer and warm the provider. Re-acquiring is idempotent but still re-warms (cheap on a cache hit, and heals a cache cleared elsewhere). """ with _tts_lease_lock: _tts_leases.add(lease) holders = len(_tts_leases) result = warm_tts_provider(tts_config) result["leases"] = holders return result def release_tts_lease(lease: str) -> Dict[str, Any]: """Drop ``lease``; when it was the last one, unload resident local models. Releasing a never-acquired lease is a no-op (still reports the holder count) so surfaces can call it unconditionally on their "off" path. """ with _tts_lease_lock: _tts_leases.discard(lease) holders = len(_tts_leases) result: Dict[str, Any] = {"leases": holders, "released": 0} if holders == 0: result["released"] = release_tts_provider()["released"] return result def tts_lease_holders() -> List[str]: """Snapshot of live lease names (diagnostics / tests).""" with _tts_lease_lock: return sorted(_tts_leases) def _reset_tts_leases_for_tests() -> None: with _tts_lease_lock: _tts_leases.clear() # =========================================================================== # Built-in provider dispatch # =========================================================================== # provider -> (importer-name or None, "package missing" error, log line, # generator-name). Names are looked up in module globals at call time so # tests that monkeypatch ``tools.tts_tool._import_x`` / ``_generate_x`` apply. _BUILTIN_DISPATCH: Dict[str, tuple] = { "elevenlabs": ( "_import_elevenlabs", "ElevenLabs provider selected but 'elevenlabs' package not installed. Run: pip install elevenlabs", "Generating speech with ElevenLabs...", "_generate_elevenlabs", ), "openai": ( "_import_openai_client", "OpenAI provider selected but 'openai' package not installed.", "Generating speech with OpenAI TTS...", "_generate_openai_tts", ), "deepinfra": ( "_import_openai_client", "DeepInfra TTS uses the 'openai' SDK but it isn't installed.", "Generating speech with DeepInfra TTS...", "_generate_deepinfra_tts", ), "minimax": (None, None, "Generating speech with MiniMax TTS...", "_generate_minimax_tts"), "xai": (None, None, "Generating speech with xAI TTS...", "_generate_xai_tts"), "mistral": ( "_import_mistral_client", "Mistral provider selected but 'mistralai' package not installed. " "Run `hermes setup` to install Mistral support.", "Generating speech with Mistral Voxtral TTS...", "_generate_mistral_tts", ), "gemini": (None, None, "Generating speech with Google Gemini TTS...", "_generate_gemini_tts"), "kittentts": ( "_import_kittentts", "KittenTTS provider selected but 'kittentts' package not installed. " "Run 'hermes setup tts' and choose KittenTTS, or install manually: " "pip install https://github.com/KittenML/KittenTTS/releases/download/0.8.1/kittentts-0.8.1-py3-none-any.whl", "Generating speech with KittenTTS (local, ~25MB)...", "_generate_kittentts", ), "piper": ( "_import_piper", "Piper provider selected but 'piper-tts' package not installed. " "Run 'hermes tools' and select Piper under TTS, or install manually: " "pip install piper-tts", "Generating speech with Piper (local)...", "_generate_piper_tts", ), } _NEUTTS_MISSING_ERROR = ( "NeuTTS provider selected but neutts is not installed. " "Run hermes setup and choose NeuTTS, or install espeak-ng and run python -m pip install -U neutts[all]." ) def _error_json(message: str) -> str: return json.dumps({"success": False, "error": message}, ensure_ascii=False) def _run_edge_tts(text: str, file_str: str, tts_config: Dict[str, Any]) -> None: """Run the async Edge generator from sync code (worker thread; direct run if that fails).""" try: import concurrent.futures with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: pool.submit( lambda: asyncio.run(_generate_edge_tts(text, file_str, tts_config)) ).result(timeout=60) except RuntimeError: asyncio.run(_generate_edge_tts(text, file_str, tts_config)) def _select_builtin_engine(provider: str) -> tuple: """Check a built-in provider's SDK. Returns ``(engine, None)`` or ``(provider, error_json)``. Unknown names take the Edge default; when edge-tts is missing, NeuTTS is the local fallback (``engine`` then differs from ``provider``). """ entry = _BUILTIN_DISPATCH.get(provider) if entry is not None: importer_name, missing_error = entry[0], entry[1] if importer_name is not None and not _importable(globals()[importer_name]): return provider, _error_json(missing_error) return provider, None if provider == "neutts": if not _check_neutts_available(): return provider, _error_json(_NEUTTS_MISSING_ERROR) logger.info("Generating speech with NeuTTS (local)...") return provider, None if _importable(_import_edge_tts): return provider, None # Edge default; the reported provider stays as configured if _check_neutts_available(): logger.info("Edge TTS not available, falling back to NeuTTS (local)...") return "neutts", None return provider, _error_json( "No TTS provider available. Install edge-tts (pip install edge-tts) " "or set up NeuTTS for local synthesis." ) def _synthesize_builtin(engine: str, text: str, file_str: str, tts_config: Dict[str, Any], instructions: Optional[str]) -> None: """Run the already-selected built-in *engine* (the caller logs the engine-selection line).""" entry = _BUILTIN_DISPATCH.get(engine) if entry is not None: logger.info(entry[2]) if engine == "openai": _generate_openai_tts(text, file_str, tts_config, instructions=instructions) else: globals()[entry[3]](text, file_str, tts_config) elif engine == "neutts": _generate_neutts(text, file_str, tts_config) else: logger.info("Generating speech with Edge TTS...") _run_edge_tts(text, file_str, tts_config) def _finalize_voice_delivery( file_str: str, provider: str, command_provider_config: Optional[Dict[str, Any]], want_opus: bool, ) -> tuple: """Decide voice-bubble eligibility and Opus-convert when needed. Command and plugin providers are documents by default and opt in via ``voice_compatible``; native-Opus built-ins are voice-compatible when the platform wants Opus and they wrote .ogg; MP3/WAV built-ins are converted with ffmpeg only when the platform needs Opus. Returns ``(path, voice_compatible)``. """ voice_compatible = False if command_provider_config is not None: opted_in = _is_command_tts_voice_compatible(command_provider_config) elif provider not in BUILTIN_TTS_PROVIDERS: opted_in = _plugin_provider_is_voice_compatible(provider) elif want_opus and provider in _FFMPEG_OPUS_PROVIDERS and not file_str.endswith(".ogg"): opus_path = _convert_to_opus(file_str) if opus_path: return opus_path, True return file_str, False elif provider in _NATIVE_OPUS_PROVIDERS: return file_str, want_opus and file_str.endswith(".ogg") else: return file_str, False if opted_in: if not file_str.endswith(".ogg"): opus_path = _convert_to_opus(file_str) if opus_path: file_str = opus_path voice_compatible = file_str.endswith(".ogg") return file_str, voice_compatible # =========================================================================== # Main tool function # =========================================================================== def _apply_call_overrides(tts_config: Dict[str, Any], speed: Optional[float], provider: Optional[str]): """Apply per-call ``speed`` (clamped, on a shallow copy) and resolve the provider name.""" if speed is not None: clamped = max(0.25, min(4.0, float(speed))) tts_config = dict(tts_config) # shallow copy to avoid mutating the cache tts_config["speed"] = clamped provider = provider.lower().strip() if provider else _get_provider(tts_config) return tts_config, provider def _session_platform() -> tuple: """``(platform, wants_opus)`` — platforms delivering voice bubbles only as Ogg/Opus want Opus.""" from gateway.session_context import get_session_env platform = get_session_env("HERMES_SESSION_PLATFORM", "").lower() return platform, platform in OPUS_VOICE_PLATFORMS def _resolve_output_base( output_path: Optional[str], provider: str, command_provider_config: Optional[Dict[str, Any]], want_opus: bool, ) -> tuple: """Pick the output file. Returns ``(Path, None)`` or ``(None, error_json)``. A caller-supplied path is rejected on ``..`` traversal (bug or prompt-injection; an absolute path is fine) and on protected credential/ system locations. Command providers get their configured extension. Default: ``