feat: add STT voice transcription for all messaging channels (#28)
* feat: add STT voice transcription for all channels Automatically transcribes audio/voice messages (Telegram, WeChat, Slack, etc.) into text before the agent sees them. Enabled via config, off by default. Changes: - EvoScientist/stt.py: new STT engine using faster-whisper with lazy model loading and per-language model selection (zh/en/auto) - EvoScientist/channels/base.py: hook in _enqueue_raw() to transcribe audio files and prepend transcript to message text; removes the raw [voice: ...] annotation after successful transcription so the agent does not attempt further audio processing - EvoScientist/config/settings.py: stt_enabled (default False), stt_language (default "auto") - pyproject.toml: optional [stt] dependency group (faster-whisper>=1.0) - tests/test_stt.py: unit tests covering all backends and channel integration Usage: pip install 'EvoScientist[stt]' EvoSci config set stt_enabled true EvoSci config set stt_language zh # zh / en / auto Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix: remove unused imports (ruff F401) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix: address PR #28 reviewer feedback Changes per SemiGlassFace review (CHANGES_REQUESTED): 1. Cache config at channel __init__ — no longer calls load_config() on every incoming message; STT settings stored as instance attributes (_stt_enabled, _stt_language, _stt_model, _stt_device, _stt_compute_type) set once during Channel.__init__(). 2. Replace deprecated asyncio.get_event_loop() with get_running_loop() to avoid DeprecationWarning on Python 3.12+. 3. Annotation removal now uses exact path matching instead of substring search — checks fp == a or a.endswith(f": {fp}]") so only the correct annotation is removed after transcription. 4. Expose stt_model, stt_device, stt_compute_type as config fields so users can override the HuggingFace model id, inference device, and quantisation without touching code. transcribe_file() forwards all three to the engine. Also: _engines dict replaced with single _engine + _engine_key tuple (model_id, device, compute_type) — reuses cached model unless settings change, simpler than a dict. Tests: 19 STT-specific tests all pass; total 1105 tests green, ruff clean. * fix: resolve ruff lint errors (UP037, I001, PT006) * style: apply ruff format --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Jiao Huifeng
@@ -281,6 +281,16 @@ class Channel(ChannelPlugin, ABC):
|
||||
|
||||
self.config = config
|
||||
|
||||
# Cache STT config at startup to avoid loading it on every message
|
||||
from ..config.settings import load_config as _load_cfg
|
||||
|
||||
_global = _load_cfg()
|
||||
self._stt_enabled: bool = _global.stt_enabled
|
||||
self._stt_language: str = _global.stt_language
|
||||
self._stt_model: str = _global.stt_model
|
||||
self._stt_device: str = _global.stt_device
|
||||
self._stt_compute_type: str = _global.stt_compute_type
|
||||
|
||||
# Auto-configure formatter from capabilities
|
||||
self._formatter = UnifiedFormatter.for_channel(self.capabilities.format_type)
|
||||
self._queue: asyncio.Queue[InboundMessage] = asyncio.Queue(
|
||||
@@ -934,7 +944,41 @@ class Channel(ChannelPlugin, ABC):
|
||||
InboundMessage, and put it on the queue.
|
||||
|
||||
Convenience method for subclass ``_on_message`` handlers.
|
||||
If STT is enabled and the message contains audio files, each audio
|
||||
file is transcribed and the result is prepended to ``raw.text``.
|
||||
"""
|
||||
if raw.media_files and self._stt_enabled:
|
||||
from ..stt import is_audio_file, transcribe_file
|
||||
|
||||
transcripts: list[str] = []
|
||||
transcribed_files: set[str] = set()
|
||||
for fp in raw.media_files:
|
||||
if is_audio_file(fp):
|
||||
text = await transcribe_file(
|
||||
fp,
|
||||
language=self._stt_language,
|
||||
model=self._stt_model,
|
||||
device=self._stt_device,
|
||||
compute_type=self._stt_compute_type,
|
||||
)
|
||||
if text:
|
||||
transcripts.append(text)
|
||||
transcribed_files.add(fp)
|
||||
_logger.info(f"[STT] {self.name}: {fp} → {text[:80]}...")
|
||||
if transcripts:
|
||||
prefix = "\n".join(transcripts)
|
||||
raw.text = (prefix + "\n" + raw.text).strip() if raw.text else prefix
|
||||
# Remove annotations for transcribed files (exact path match)
|
||||
# so the agent does not attempt to process the audio file itself
|
||||
raw.content_annotations = [
|
||||
a
|
||||
for a in raw.content_annotations
|
||||
if not any(
|
||||
fp == a or a.endswith(f": {fp}]") or a == f"[voice: {fp}]"
|
||||
for fp in transcribed_files
|
||||
)
|
||||
]
|
||||
|
||||
msg = await self._build_inbound_async(raw)
|
||||
if msg is None:
|
||||
return
|
||||
|
||||
@@ -199,6 +199,13 @@ class EvoScientistConfig:
|
||||
# ccproxy
|
||||
ccproxy_port: int = 8000
|
||||
|
||||
# STT (Speech-to-Text) Settings
|
||||
stt_enabled: bool = False
|
||||
stt_language: str = "auto" # "auto" | "zh" | "en"
|
||||
stt_model: str = "" # override model id; empty = auto-select by language
|
||||
stt_device: str = "cpu" # "cpu" | "cuda"
|
||||
stt_compute_type: str = "int8" # "int8" | "float16" | "float32"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Config file operations
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Speech-to-text transcription.
|
||||
|
||||
Default language → model mapping (all via faster-whisper):
|
||||
"zh" → Systran/faster-whisper-small (language=zh, ~250MB)
|
||||
"en" → Systran/faster-whisper-small.en (~250MB, en-only)
|
||||
"auto" → Systran/faster-whisper-small (~250MB, multilingual auto-detect)
|
||||
|
||||
All three parameters can be overridden via config:
|
||||
stt_model — any HuggingFace faster-whisper model id
|
||||
stt_device — "cpu" (default) or "cuda"
|
||||
stt_compute_type — "int8" (default), "float16", "float32", etc.
|
||||
|
||||
Install:
|
||||
pip install 'EvoScientist[stt]'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_AUDIO_EXTS = frozenset(
|
||||
{
|
||||
".ogg",
|
||||
".mp3",
|
||||
".m4a",
|
||||
".wav",
|
||||
".flac",
|
||||
".opus",
|
||||
".weba",
|
||||
".webm",
|
||||
}
|
||||
)
|
||||
|
||||
# Default language → HuggingFace model id
|
||||
STT_MODELS: dict[str, str] = {
|
||||
"zh": "Systran/faster-whisper-small",
|
||||
"en": "Systran/faster-whisper-small.en",
|
||||
"auto": "Systran/faster-whisper-small",
|
||||
}
|
||||
|
||||
# Single cached engine — re-created only when settings change
|
||||
_engine: _WhisperEngine | None = None
|
||||
_engine_key: tuple[str, str, str] | None = None # (model_id, device, compute_type)
|
||||
|
||||
|
||||
# ── Engine ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _WhisperEngine:
|
||||
"""faster-whisper transcription engine."""
|
||||
|
||||
def __init__(self, model_id: str, device: str, compute_type: str) -> None:
|
||||
from faster_whisper import WhisperModel # type: ignore[import]
|
||||
|
||||
self._model = WhisperModel(model_id, device=device, compute_type=compute_type)
|
||||
|
||||
def transcribe(self, file_path: str, language: str | None) -> str:
|
||||
segments, _ = self._model.transcribe(
|
||||
file_path,
|
||||
language=language,
|
||||
beam_size=5,
|
||||
vad_filter=True,
|
||||
vad_parameters={"min_silence_duration_ms": 500},
|
||||
)
|
||||
# Skip segments where the model is not confident there is real speech
|
||||
parts = [s.text.strip() for s in segments if s.no_speech_prob < 0.6]
|
||||
return " ".join(parts).strip()
|
||||
|
||||
|
||||
def _get_engine(
|
||||
model_id: str,
|
||||
device: str = "cpu",
|
||||
compute_type: str = "int8",
|
||||
) -> _WhisperEngine:
|
||||
global _engine, _engine_key
|
||||
key = (model_id, device, compute_type)
|
||||
if _engine is None or _engine_key != key:
|
||||
logger.info(
|
||||
f"[STT] Loading model '{model_id}' device={device} compute={compute_type}"
|
||||
)
|
||||
_engine = _WhisperEngine(model_id, device, compute_type)
|
||||
_engine_key = key
|
||||
return _engine
|
||||
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_audio_file(file_path: str) -> bool:
|
||||
"""Return True if *file_path* has an audio extension."""
|
||||
return Path(file_path).suffix.lower() in _AUDIO_EXTS
|
||||
|
||||
|
||||
async def transcribe_file(
|
||||
file_path: str,
|
||||
language: str = "auto",
|
||||
model: str = "",
|
||||
device: str = "cpu",
|
||||
compute_type: str = "int8",
|
||||
) -> str | None:
|
||||
"""Transcribe an audio file asynchronously.
|
||||
|
||||
Args:
|
||||
file_path: Path to the audio file.
|
||||
language: Language hint — ``"zh"``, ``"en"``, or ``"auto"`` (default).
|
||||
model: Override the HuggingFace model id. Empty = use STT_MODELS mapping.
|
||||
device: Inference device — ``"cpu"`` (default) or ``"cuda"``.
|
||||
compute_type: Quantisation — ``"int8"`` (default), ``"float16"``, etc.
|
||||
|
||||
Returns the transcript string, or ``None`` on error / silence.
|
||||
"""
|
||||
if not is_audio_file(file_path):
|
||||
return None
|
||||
try:
|
||||
model_id = model or STT_MODELS.get(language, STT_MODELS["auto"])
|
||||
lang: str | None = None if language == "auto" else language
|
||||
engine = _get_engine(model_id, device, compute_type)
|
||||
loop = asyncio.get_running_loop()
|
||||
result: str = await loop.run_in_executor(
|
||||
None, engine.transcribe, file_path, lang
|
||||
)
|
||||
return result or None
|
||||
except ImportError as e:
|
||||
logger.warning(
|
||||
f"[STT] Missing dependency: {e}. "
|
||||
"Install with: pip install 'EvoScientist[stt]'"
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"[STT] Transcription failed for {file_path}: {e}")
|
||||
return None
|
||||
Reference in New Issue
Block a user