diff --git a/tools/transcription_audio.py b/tools/transcription_audio.py index 07e45a6734..15413f370a 100644 --- a/tools/transcription_audio.py +++ b/tools/transcription_audio.py @@ -2,10 +2,9 @@ Binary discovery, the shared ffmpeg m4a encode (transcode + silence trim), source/format validation, WeChat .silk decoding, CAF conversion and the -best-effort cloud pre-upload silence trim. - -Split out of ``tools/transcription_tools.py``, which re-imports every name (patch -surface) and is imported lazily here so origin patches still intercept. +best-effort cloud pre-upload silence trim. Every name is re-imported by +``tools/transcription_tools.py`` (patch surface), which is imported lazily here +so origin patches still intercept. """ from __future__ import annotations @@ -61,17 +60,11 @@ def _run_quiet(command: list, *, timeout: float, env: Optional[dict] = None) -> # Shared encode profile for every STT-bound m4a (transcode and silence-trim): # 16 kHz mono 32 kbps AAC, faststart. One owner so codec/bitrate never drift. -_STT_M4A_ENCODE_ARGS = ( - "-vn", "-ac", "1", "-ar", "16000", - "-c:a", "aac", "-b:a", "32k", "-movflags", "+faststart", -) +_STT_M4A_ENCODE_ARGS = ("-vn", "-ac", "1", "-ar", "16000", "-c:a", "aac", "-b:a", "32k", "-movflags", "+faststart") def _run_ffmpeg_stt_encode(ffmpeg: str, input_path: str, output_path: str, *, audio_filter: Optional[str] = None) -> None: - """Run the shared STT m4a encode, optionally with an ``-af`` filter. - - Raises on failure — callers own the error semantics (transcode reports, trim swallows). - """ + """Run the shared STT m4a encode, optionally with an ``-af`` filter. Raises on failure; callers own the semantics.""" command = [ffmpeg, "-y", "-i", input_path] if audio_filter: command += ["-af", audio_filter] @@ -80,11 +73,10 @@ def _run_ffmpeg_stt_encode(ffmpeg: str, input_path: str, output_path: str, *, au def _transcode_audio_for_stt(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]: - """Transcode to a compact 16 kHz mono AAC/m4a for STT upload. + """Transcode to a compact 16 kHz mono AAC/m4a for STT upload; ``(converted_path, None)`` or ``(None, error)``. Newer OpenAI models reject containers ``whisper-1`` accepted (notably Ogg/Opus voice notes) and gateway downloads may carry a misleading extension. - Returns ``(converted_path, None)`` or ``(None, error)``. """ from tools.transcription_tools import _find_ffmpeg_binary, _run_ffmpeg_stt_encode ffmpeg = _find_ffmpeg_binary() @@ -110,16 +102,13 @@ def _validate_audio_file_size(audio_path: Path, *, enforce_size_limit: bool = Tr except OSError as e: return _error_result(f"Failed to access file: {e}") if enforce_size_limit and file_size > MAX_FILE_SIZE: - return _error_result( - f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)" - ) + return _error_result(f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)") return None def _validate_audio_source_file(file_path: str, *, enforce_size_limit: bool = True) -> Optional[Dict[str, Any]]: """Validate source path safety (and optionally size) before any decoder runs.""" audio_path = Path(file_path) - if os.path.islink(audio_path): return _error_result(f"Path is a symbolic link: {file_path}") if not audio_path.exists(): @@ -137,9 +126,7 @@ def _validate_audio_file(file_path: str, *, enforce_size_limit: bool = True) -> suffix = Path(file_path).suffix if suffix.lower() not in SUPPORTED_FORMATS: - return _error_result( - f"Unsupported format: {suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}" - ) + return _error_result(f"Unsupported format: {suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}") return None @@ -150,8 +137,7 @@ def _prepare_audio_for_transcription(file_path: str) -> tuple[Optional[str], Opt if audio_path.suffix.lower() != ".silk": return file_path, None, None if not _HAS_PILK: - # pilk is a tiny silk-v3 codec binding — lazy-install on first .silk - # voice note instead of bloating the base install. + # pilk is a tiny silk-v3 codec binding — lazy-installed on first .silk voice note. _lazy_ensure_quietly("stt.silk") if not _safe_find_spec("pilk"): return None, None, _error_result( @@ -221,23 +207,22 @@ def _convert_caf_to_wav(file_path: str) -> Optional[str]: # ---- Cloud pre-upload silence trim -------------------------------------- # -# Local faster-whisper gets Silero VAD; cloud providers get the raw file, so -# every second of silence is paid for twice (upload + per-minute billing) and -# cloud Whisper hallucinates on it. Before uploading to a built-in cloud -# provider we collapse long pauses with ffmpeg's silenceremove, keeping -# ``stt.cloud_trim_keep_ms`` of each pause so word boundaries survive. -# Purely best-effort — ANY of these uploads the original untouched: -# ``stt.cloud_trim_silence: false``, ffmpeg/ffprobe missing, trim failure or -# timeout, a ~empty result (the provider, not a dB heuristic, decides "no -# speech"), or <10% saving. Command-type and plugin providers are NOT trimmed: -# they may wrap local CLIs that want the original bytes. +# Local faster-whisper gets Silero VAD; cloud providers get the raw file, so silence +# is paid for twice (upload + per-minute billing) and cloud Whisper hallucinates on +# it. Before uploading to a built-in cloud provider we collapse long pauses with +# ffmpeg's silenceremove, keeping ``stt.cloud_trim_keep_ms`` of each pause so word +# boundaries survive. Purely best-effort — ANY of these uploads the original: +# ``stt.cloud_trim_silence: false``, ffmpeg/ffprobe missing, trim failure/timeout, a +# ~empty result (the provider, not a dB heuristic, decides "no speech"), or <10% +# saving. Command-type and plugin providers are NOT trimmed: they may wrap local +# CLIs that want the original bytes. _CLOUD_TRIM_THRESHOLD_DB_DEFAULT = -40 # audio below this level counts as silence _CLOUD_TRIM_KEEP_MS_DEFAULT = 300 # how much of each pause survives the trim _CLOUD_TRIM_MIN_SAVING = 0.10 # use the trimmed file only when >=10% shorter _CLOUD_TRIM_MIN_RESULT_SECONDS = 0.3 # all-silence guard floor: never upload ~empty audio -# Below this the trim can't pay for itself (several providers bill a 10s -# minimum per request) and the encode would sit on the synchronous voice-note path. +# Below this the trim can't pay for itself (several providers bill a 10s minimum) +# and the encode would sit on the synchronous voice-note path. _CLOUD_TRIM_MIN_INPUT_SECONDS = 12.0 @@ -252,10 +237,7 @@ def _probe_audio_duration(file_path: str) -> Optional[float]: if not ffprobe: return None command = [ - ffprobe, "-v", "error", - "-show_entries", "format=duration", - "-of", "default=noprint_wrappers=1:nokey=1", - file_path, + ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", file_path, ] try: return float(_run_quiet(command, timeout=30).stdout.strip()) @@ -274,11 +256,9 @@ def _cloud_trim_settings(stt_config: Dict[str, Any]) -> tuple[bool, int, int]: def _trim_silence_for_cloud_stt(file_path: str, stt_config: Dict[str, Any]) -> Optional[str]: - """Return a silence-trimmed copy of *file_path* for cloud upload, or None. + """Return a silence-trimmed copy of *file_path* for cloud upload, or None (= upload the original). - ``None`` always means "upload the original" (disabled, tools missing, clip - too short, trim failed, mostly silence, or not enough saving). On success - the caller owns deleting the returned file's parent directory. + On success the caller owns deleting the returned file's parent directory. """ from tools.transcription_tools import _find_ffmpeg_binary, _probe_audio_duration, _run_ffmpeg_stt_encode enabled, threshold_db, keep_ms = _cloud_trim_settings(stt_config) @@ -301,8 +281,7 @@ def _trim_silence_for_cloud_stt(file_path: str, stt_config: Dict[str, Any]) -> O return None keep_seconds = keep_ms / 1000.0 - # start_periods=1 strips leading silence; stop_periods=-1 collapses every - # interior/trailing silence, keeping ``keep_seconds`` of each pause. + # start_periods=1 strips leading silence; stop_periods=-1 collapses every interior/trailing silence. filter_expr = ( f"silenceremove=" f"start_periods=1:start_threshold={threshold_db}dB:start_silence={keep_seconds}:" @@ -310,8 +289,7 @@ def _trim_silence_for_cloud_stt(file_path: str, stt_config: Dict[str, Any]) -> O ) work_dir = tempfile.mkdtemp(prefix="hermes-stt-trim-") trimmed_path = os.path.join(work_dir, f"{Path(file_path).stem or 'audio'}-trimmed.m4a") - # Scale the all-silence guard with keep_ms: an output consisting solely - # of kept pause must never be uploaded as "speech". + # Scale the all-silence guard with keep_ms: output that is solely kept pause must never upload as "speech". min_result_seconds = max(_CLOUD_TRIM_MIN_RESULT_SECONDS, 2 * keep_seconds) keep_result = False try: diff --git a/tools/transcription_cloud.py b/tools/transcription_cloud.py index c81f008a6d..a1d20f9d51 100644 --- a/tools/transcription_cloud.py +++ b/tools/transcription_cloud.py @@ -1,11 +1,10 @@ """Cloud STT providers. -OpenAI-SDK-shaped backends (groq, openai, deepinfra), Mistral Voxtral, the REST -multipart backends (xAI, ElevenLabs), and OpenAI audio credential resolution -(config > keyless local server > env > managed Nous gateway). - -Split out of ``tools/transcription_tools.py``, which re-imports every name (patch -surface) and is imported lazily here so origin patches still intercept. +OpenAI-SDK-shaped backends (groq, openai, deepinfra), Mistral Voxtral, REST multipart +backends (xAI, ElevenLabs), and OpenAI audio credential resolution (config > keyless +local server > env > managed Nous gateway). Every name is re-imported by +``tools/transcription_tools.py`` (patch surface), imported lazily here so origin +patches still intercept. """ from __future__ import annotations @@ -14,7 +13,7 @@ import logging import re import tempfile from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional from urllib.parse import urljoin from utils import is_truthy_value @@ -35,25 +34,17 @@ def _has_xai_stt_credentials() -> bool: return bool(resolve_xai_http_credentials().get("api_key")) -def _close_client(client: Any) -> None: - close = getattr(client, "close", None) - if callable(close): - close() - - def _with_openai_client(api_key: str, base_url: Optional[str], file_path: str, log_label: str, body): - """Run ``body(client)`` against a fresh OpenAI SDK client (30s timeout, no SDK retries). - - Always closes the client; any exception maps to the shared envelope via - :func:`_openai_sdk_failure`. - """ + """Run ``body(client)`` on a fresh OpenAI SDK client (30s timeout, no retries); always closed, errors -> envelope.""" try: from openai import OpenAI client = OpenAI(api_key=api_key, base_url=base_url, timeout=30, max_retries=0) try: return body(client) finally: - _close_client(client) + close = getattr(client, "close", None) + if callable(close): + close() except Exception as e: return _openai_sdk_failure(e, file_path, log_label) @@ -69,8 +60,8 @@ def _cloud_failure(exc: BaseException, file_path: str, label: str, detail: Optio def _openai_sdk_failure(exc: BaseException, file_path: str, log_label: str) -> Dict[str, Any]: """Map an OpenAI-SDK-shaped exception to the shared error envelope. - Order matters: APIConnectionError is checked before APITimeoutError (its - subclass) so timeouts report as connection errors, as they always have. + APIConnectionError is checked before APITimeoutError (its subclass) so timeouts + report as connection errors, as they always have. """ try: from openai import APIError, APIConnectionError, APITimeoutError @@ -78,33 +69,31 @@ def _openai_sdk_failure(exc: BaseException, file_path: str, log_label: str) -> D APIError = APIConnectionError = APITimeoutError = () if isinstance(exc, PermissionError): return _error_result(f"Permission denied: {file_path}") - if isinstance(exc, APIConnectionError): - return _error_result(f"Connection error: {exc}") - if isinstance(exc, APITimeoutError): - return _error_result(f"Request timeout: {exc}") - if isinstance(exc, APIError): - return _error_result(f"API error: {exc}") + for cls, label in ((APIConnectionError, "Connection error"), (APITimeoutError, "Request timeout"), (APIError, "API error")): + if isinstance(exc, cls): + return _error_result(f"{label}: {exc}") logger.error("%s transcription failed: %s", log_label, exc, exc_info=True) return _error_result(f"Transcription failed: {exc}") -def _transcribe_groq( - file_path: str, - model_name: str, - *, - language: Optional[str] = None, - prompt: Optional[str] = None, -) -> Dict[str, Any]: - """Transcribe using Groq Whisper API (free tier available). +def _sdk_prompt_kwargs(language: Optional[str], prompt: Optional[str]) -> Dict[str, Any]: + """``language``/``prompt`` create-kwargs, each only when set so the bare request stays byte-identical.""" + kwargs: Dict[str, Any] = {} + if language: + kwargs["language"] = language + if prompt: + kwargs["prompt"] = prompt + return kwargs - Language: hook override > ``stt.groq.language`` > ``stt.language`` > env; - otherwise Groq auto-detects. - """ + +def _transcribe_groq( + file_path: str, model_name: str, *, language: Optional[str] = None, prompt: Optional[str] = None +) -> Dict[str, Any]: + """Transcribe via the Groq Whisper API; language: hook > ``stt.groq.language`` > ``stt.language`` > env > auto.""" from tools.transcription_tools import _HAS_OPENAI, _resolve_provider_key, _resolve_stt_language api_key = _resolve_provider_key("GROQ_API_KEY", "groq") if not api_key: return _error_result("GROQ_API_KEY not set") - if not _HAS_OPENAI: return _error_result("openai package not installed") @@ -116,45 +105,26 @@ def _transcribe_groq( language = language or _resolve_stt_language("groq") def _run(client): - create_kwargs = { - "model": model_name, - "response_format": "text", - } - if language: - create_kwargs["language"] = language - if prompt: - # Only sent when set so the no-hook, no-config request stays byte-identical. - create_kwargs["prompt"] = prompt + create_kwargs = {"model": model_name, "response_format": "text", **_sdk_prompt_kwargs(language, prompt)} with open(file_path, "rb") as audio_file: - transcription = client.audio.transcriptions.create( - file=audio_file, - **create_kwargs, - ) - + transcription = client.audio.transcriptions.create(file=audio_file, **create_kwargs) transcript_text = str(transcription).strip() logger.info("Transcribed %s via Groq API (%s, lang=%s, %d chars)", Path(file_path).name, model_name, language or "auto", len(transcript_text)) - return _ok_result(transcript_text, "groq") return _with_openai_client(api_key, GROQ_BASE_URL, file_path, "Groq", _run) def _transcribe_openai( - file_path: str, - model_name: str, - *, - api_key: Optional[str] = None, - base_url: Optional[str] = None, - provider_label: str = "openai", - language: Optional[str] = None, + file_path: str, model_name: str, *, api_key: Optional[str] = None, + base_url: Optional[str] = None, provider_label: str = "openai", language: Optional[str] = None, prompt: Optional[str] = None, ) -> Dict[str, Any]: """Transcribe via the OpenAI ``audio.transcriptions.create`` SDK shape. - Shared backend for every OpenAI-compatible STT endpoint (DeepInfra etc.): - callers pass explicit ``api_key``/``base_url`` to skip the OpenAI-only auth - chain and a ``provider_label`` for the response's ``provider``. + Shared by every OpenAI-compatible endpoint (DeepInfra etc.): explicit ``api_key``/ + ``base_url`` skip the OpenAI-only auth chain; ``provider_label`` names the response's provider. """ from tools.transcription_tools import _HAS_OPENAI, _resolve_openai_audio_client_config, _resolve_stt_language if api_key is None: @@ -170,8 +140,7 @@ def _transcribe_openai( if not _HAS_OPENAI: return _error_result("openai package not installed") - # Auto-correct a Groq-only model on the native OpenAI path only — - # third-party endpoints may legitimately serve a whisper-large-v3 variant. + # Auto-correct a Groq-only model on the native OpenAI path only (third-party endpoints may serve it). if provider_label == "openai" and model_name in GROQ_MODELS: logger.info("Model %s not available on OpenAI, using %s", model_name, DEFAULT_STT_MODEL) model_name = DEFAULT_STT_MODEL @@ -188,14 +157,12 @@ def _transcribe_openai( } if language: if model_name == "gpt-transcribe": - # gpt-transcribe replaces ``language`` with a ``languages`` - # list and rejects requests sending the legacy field. + # gpt-transcribe takes a ``languages`` list and rejects the legacy field. create_kwargs["extra_body"] = {"languages": [language]} else: create_kwargs["language"] = language logger.debug("Using language hint '%s' for OpenAI STT", language) - if prompt: - # Only sent when set so the no-hook, no-config request stays byte-identical. + if prompt: # only when set so the bare request stays byte-identical create_kwargs["prompt"] = prompt return client.audio.transcriptions.create(**create_kwargs) @@ -206,8 +173,7 @@ def _transcribe_openai( message = str(exc).lower() if not any(k in message for k in ("unsupported", "corrupted", "invalid file")): raise - # Newer models reject some containers whisper-1 accepted - # (notably Ogg/Opus voice notes): transcode to m4a, retry once. + # Newer models reject containers whisper-1 accepted (Ogg/Opus voice notes): transcode, retry once. converted_path, transcode_error = _transcode_audio_for_stt(file_path, work_dir) if transcode_error: return _error_result(transcode_error) @@ -222,18 +188,13 @@ def _transcribe_openai( "Transcribed %s via %s (%s, %d chars)", Path(file_path).name, provider_label, model_name, len(transcript_text), ) - return _ok_result(transcript_text, provider_label) return _with_openai_client(api_key, base_url, file_path, provider_label, _run) def _transcribe_mistral( - file_path: str, - model_name: str, - *, - language: Optional[str] = None, - prompt: Optional[str] = None, + file_path: str, model_name: str, *, language: Optional[str] = None, prompt: Optional[str] = None ) -> Dict[str, Any]: """Transcribe with the ``mistralai`` SDK (``/v1/audio/transcriptions``); requires ``MISTRAL_API_KEY``.""" from tools.transcription_tools import _resolve_provider_key, _resolve_stt_language @@ -247,17 +208,13 @@ def _transcribe_mistral( with Mistral(api_key=api_key) as client: with open(file_path, "rb") as audio_file: + # Language: hook override > stt.mistral.language > stt.language > env > auto. + language = language or _resolve_stt_language("mistral") complete_kwargs: Dict[str, Any] = { "model": model_name, "file": {"content": audio_file, "file_name": Path(file_path).name}, + **_sdk_prompt_kwargs(language, prompt), } - # Language: hook override > stt.mistral.language > stt.language > env > auto. - language = language or _resolve_stt_language("mistral") - if language: - complete_kwargs["language"] = language - if prompt: - # Only sent when set so the no-hook, no-config request stays byte-identical. - complete_kwargs["prompt"] = prompt result = client.audio.transcriptions.complete(**complete_kwargs) transcript_text = _extract_transcript_text(result) @@ -271,6 +228,9 @@ def _transcribe_mistral( return _cloud_failure(e, file_path, "Mistral transcription", type(e).__name__) +# ---- REST multipart backends (xAI, ElevenLabs) ---------------------------- + + def _post_audio_multipart(url: str, headers: Dict[str, str], file_path: str, data: Dict[str, str]): import requests @@ -281,22 +241,17 @@ def _post_audio_multipart(url: str, headers: Dict[str, str], file_path: str, dat ) -def _http_error_detail(response, extract) -> str: - """``extract(json_body)`` -> detail string, falling back to the first 300 chars of the body.""" - try: - return extract(response.json()) or response.text[:300] - except Exception: - return response.text[:300] - - def _rest_transcript(response, label: str, extract_detail, extract_text): - """Turn a multipart STT response into ``(text, body, None)`` or ``(None, None, error_envelope)``. + """Multipart STT response -> ``(text, body, None)`` or ``(None, None, error_envelope)``. - Non-200 -> ``"