Teknium
aed6720dab
run.py: - built-in adapter creation: 9-branch if/elif -> _BUILTIN_ADAPTERS table - idle slash-command routing: 35 `if canonical == ...` branches -> _gateway_idle_command_handlers() - shared helpers: _send_command_ack (4 sites), _command_origin_for_source (2), _session_entry_for_manager (goal/heartbeat), _toggle_adapter_auto_tts_set (2), _load_env_or_agent_cfg_timeout (2), _float_env reuse (2), _resolve_session_key_or_none (3), _running_agent_ids (4), _schedule_rename_from_title_thread (2), _write_runtime_status_quiet (5), _AUTO_RESET_CONTEXT_NOTES/_auto_reset_reason_text - ruff SIM102/SIM103/SIM105/SIM108/SIM118 + F401 across gateway/ (semantics re-reviewed; sqlite Row `.keys()` and side-effecting assignments kept) - two hand-reviewed comment/docstring compaction passes (AST-identical, rationale kept) slash_commands.py: - /model: typed path and picker callback shared one 200-line commit block -> _perform_model_switch + _commit_model_switch - comment/docstring compaction (AST-identical)
313 lines
12 KiB
Python
313 lines
12 KiB
Python
"""Gateway streaming-TTS consumer — LLM deltas to adapter PCM audio sink.
|
|
|
|
Bridges the synchronous agent ``stream_delta_callback`` (worker thread) to a
|
|
voice-capable adapter's streaming-audio contract so playback begins while the
|
|
LLM is still generating.
|
|
|
|
Lifecycle::
|
|
|
|
consumer = StreamingTTSConsumer(adapter, chat_id, tts_config, loop, metadata)
|
|
agent.stream_delta_callback = consumer.on_delta # sync, non-blocking
|
|
... agent runs in executor ...
|
|
consumer.finish() # signal end-of-text
|
|
success = await consumer.wait_complete(timeout=10)
|
|
if consumer.suppress_whole_file: ... # skip whole-file auto-TTS
|
|
consumer.abort("cancelled") # idempotent cancellation
|
|
|
|
``on_delta`` never blocks: it feeds a ``SentenceChunker`` and queues clauses on a
|
|
thread-safe ``queue.Queue``; the ``_run`` task on the gateway loop drains it,
|
|
synthesises via a ``StreamingTTSProvider`` and writes PCM to the adapter. State
|
|
is per instance (concurrent chats cannot cross-contaminate); abort is idempotent
|
|
and late chunks are dropped. Outcome contract: full success -> ``completed``;
|
|
failure before any audible output -> ``suppress_whole_file=False`` (gateway falls
|
|
back to whole-file TTS); failure after partial audio -> ``partial`` and
|
|
``suppress_whole_file=True`` (never replay the response from the beginning).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import queue
|
|
import threading
|
|
from typing import Any, Dict, Optional
|
|
|
|
from gateway.platforms.base import AudioFormat, StreamingTTSHandle
|
|
import contextlib
|
|
|
|
logger = logging.getLogger("gateway.streaming_tts_consumer")
|
|
|
|
_ABORT = object()
|
|
_DONE = object()
|
|
|
|
|
|
class StreamingTTSConsumer:
|
|
"""Consumes LLM text deltas and produces streaming PCM audio for an adapter."""
|
|
|
|
def __init__(
|
|
self,
|
|
adapter: Any,
|
|
chat_id: str,
|
|
tts_config: Dict[str, Any],
|
|
loop: asyncio.AbstractEventLoop,
|
|
*,
|
|
metadata: Optional[Dict[str, Any]] = None,
|
|
audio_format: Optional[AudioFormat] = None,
|
|
) -> None:
|
|
from tools.tts_streaming import SentenceChunker, resolve_streaming_provider
|
|
|
|
self._adapter = adapter
|
|
self._chat_id = chat_id
|
|
self._loop = loop
|
|
self._metadata = metadata
|
|
|
|
# Resolved once; None => inactive, gateway falls back to whole-file TTS.
|
|
self._streamer = resolve_streaming_provider(tts_config)
|
|
self._chunker = SentenceChunker()
|
|
|
|
if self._streamer is not None:
|
|
self._audio_format = AudioFormat(
|
|
sample_rate=int(getattr(self._streamer, "sample_rate", AudioFormat.sample_rate)),
|
|
channels=int(getattr(self._streamer, "channels", AudioFormat.channels)),
|
|
sample_width=int(getattr(self._streamer, "sample_width", AudioFormat.sample_width)),
|
|
)
|
|
else:
|
|
self._audio_format = audio_format or AudioFormat()
|
|
|
|
# Thread-safe queue of completed clauses plus the _DONE/_ABORT sentinels.
|
|
self._queue: "queue.Queue[Any]" = queue.Queue(maxsize=256)
|
|
|
|
self._handle: Optional[StreamingTTSHandle] = None
|
|
self._completed = False
|
|
self._partial = False
|
|
self._aborted = False
|
|
self._finished = False
|
|
self._dropped = False
|
|
self._suppress_whole_file = False
|
|
self._task: Optional[asyncio.Task] = None
|
|
self._lock = threading.Lock()
|
|
self._strip_markdown = None # lazily imported to avoid import cycles
|
|
|
|
@property
|
|
def active(self) -> bool: # usable streaming provider resolved
|
|
return self._streamer is not None
|
|
|
|
@property
|
|
def completed(self) -> bool: # streaming audio fully delivered
|
|
return self._completed
|
|
|
|
@property
|
|
def partial(self) -> bool: # some audio was audible before a failure/drop
|
|
return self._partial
|
|
|
|
@property
|
|
def audible(self) -> bool: # first PCM chunk has been written
|
|
return bool(self._handle and self._handle.audible)
|
|
|
|
@property
|
|
def dropped(self) -> bool: # queue saturation dropped at least one clause
|
|
return self._dropped
|
|
|
|
@property
|
|
def suppress_whole_file(self) -> bool: # gateway should skip whole-file TTS fallback
|
|
return self._suppress_whole_file
|
|
|
|
@property
|
|
def done(self) -> bool: # async drain task has terminated
|
|
return self._task is not None and self._task.done()
|
|
|
|
def on_delta(self, text: str) -> None:
|
|
"""Receive a text delta from the agent. Non-blocking."""
|
|
if self._aborted or not self.active or self._finished:
|
|
return
|
|
try:
|
|
for clause in self._chunker.feed(text):
|
|
self._queue.put_nowait(clause)
|
|
except queue.Full:
|
|
self._dropped = True
|
|
logger.debug("streaming TTS queue full, dropping clause")
|
|
except Exception:
|
|
logger.debug("streaming TTS on_delta error", exc_info=True)
|
|
|
|
def finish(self) -> None:
|
|
"""Signal end-of-text, flush the chunker tail, then enqueue ``_DONE``.
|
|
|
|
The sentinel follows all flushed clauses so the drain loop has a
|
|
deterministic termination that cannot race a late ``on_delta``.
|
|
"""
|
|
if self._finished:
|
|
return
|
|
self._finished = True
|
|
if self._aborted or not self.active:
|
|
return
|
|
try:
|
|
for clause in self._chunker.flush():
|
|
self._queue.put_nowait(clause)
|
|
except queue.Full:
|
|
self._dropped = True
|
|
logger.debug("streaming TTS queue full while flushing tail")
|
|
except Exception:
|
|
pass
|
|
# The load-bearing _DONE sentinel must never be lost: evict a clause if full.
|
|
while True:
|
|
try:
|
|
self._queue.put_nowait(_DONE)
|
|
return
|
|
except queue.Full:
|
|
try:
|
|
self._queue.get_nowait()
|
|
self._dropped = True
|
|
except queue.Empty:
|
|
continue
|
|
|
|
def start(self) -> asyncio.Task:
|
|
"""Create (once) and return the async drain task on the gateway loop."""
|
|
if self._task is None:
|
|
self._task = self._loop.create_task(self._run())
|
|
return self._task
|
|
|
|
def _settle(self, *, failed: bool) -> None:
|
|
"""Set the outcome flags from what was audible.
|
|
|
|
Never report completion after a failure or a dropped clause; keep
|
|
suppression whenever audio was audible so the gateway does not replay
|
|
the response from the beginning.
|
|
"""
|
|
audible = self._handle.audible
|
|
degraded = failed or self._dropped
|
|
self._completed = audible and not degraded
|
|
if audible and degraded:
|
|
self._partial = True
|
|
self._suppress_whole_file = audible
|
|
|
|
async def _run(self) -> None:
|
|
"""Drain clauses from the queue, synthesise, and write to the adapter."""
|
|
if not self.active:
|
|
return
|
|
if not self._adapter.supports_streaming_tts(self._chat_id, self._audio_format):
|
|
logger.debug("adapter %s does not support streaming TTS", getattr(self._adapter, "name", "?"))
|
|
return
|
|
try:
|
|
self._handle = await self._adapter.begin_streaming_tts(
|
|
self._chat_id, self._audio_format, metadata=self._metadata,
|
|
)
|
|
except Exception as exc:
|
|
logger.debug("begin_streaming_tts failed: %s", exc)
|
|
self._handle = None
|
|
return
|
|
if self._handle is None:
|
|
return
|
|
|
|
self._suppress_whole_file = False
|
|
try:
|
|
while not self._aborted:
|
|
try:
|
|
item = await asyncio.to_thread(self._queue.get, True, 0.1)
|
|
except queue.Empty:
|
|
continue
|
|
if item is _ABORT or item is _DONE or self._aborted:
|
|
break
|
|
if not isinstance(item, str):
|
|
continue
|
|
try:
|
|
await self._synthesise_and_write(item)
|
|
except Exception as exc:
|
|
logger.warning("streaming TTS clause failed: %s", exc)
|
|
self._settle(failed=True)
|
|
await self._safe_abort(str(exc))
|
|
return
|
|
|
|
if not self._aborted and self._handle is not None:
|
|
try:
|
|
await self._adapter.finish_streaming_tts(self._handle, interrupted=self._aborted)
|
|
except Exception as exc:
|
|
logger.debug("finish_streaming_tts error: %s", exc)
|
|
self._settle(failed=True)
|
|
await self._safe_abort("finish_streaming_tts failed")
|
|
else:
|
|
self._settle(failed=False)
|
|
except Exception as exc:
|
|
logger.warning("streaming TTS consumer error: %s", exc)
|
|
await self._safe_abort(str(exc))
|
|
finally:
|
|
try:
|
|
while not self._queue.empty():
|
|
self._queue.get_nowait()
|
|
except Exception:
|
|
pass
|
|
|
|
async def _synthesise_and_write(self, clause: str) -> None:
|
|
"""Synthesise one clause via the streamer and write PCM chunks."""
|
|
if self._handle is None or self._handle.aborted or self._streamer is None:
|
|
return
|
|
cleaned = self._strip_markdown_for_tts(clause)
|
|
if not cleaned.strip():
|
|
return
|
|
iterator = iter(self._streamer.stream(cleaned))
|
|
while True:
|
|
# next() runs in a thread so a blocking provider never stalls the loop.
|
|
chunk = await asyncio.to_thread(next, iterator, _DONE)
|
|
if chunk is _DONE:
|
|
return
|
|
if self._aborted or self._handle.aborted:
|
|
return
|
|
if not chunk:
|
|
continue
|
|
was_audible = self._handle.audible
|
|
await self._adapter.write_streaming_tts(self._handle, chunk)
|
|
if not was_audible:
|
|
self._handle.audible = True
|
|
self._suppress_whole_file = True
|
|
|
|
def _strip_markdown_for_tts(self, text: str) -> str:
|
|
"""Lazy-import and apply the TTS markdown stripper."""
|
|
if self._strip_markdown is None:
|
|
try:
|
|
from tools.tts_tool import _strip_markdown_for_tts as _strip
|
|
self._strip_markdown = _strip
|
|
except ImportError:
|
|
self._strip_markdown = lambda t: t # noqa: E731
|
|
return self._strip_markdown(text).strip()
|
|
|
|
async def _safe_abort(self, reason: str) -> None:
|
|
"""Abort the adapter stream, swallowing errors (idempotent)."""
|
|
if self._handle is None:
|
|
return
|
|
try:
|
|
await self._adapter.abort_streaming_tts(self._handle, error=reason)
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
if self._handle:
|
|
self._handle.aborted = True
|
|
|
|
def abort(self, reason: str = "cancelled") -> None:
|
|
"""Idempotent cancellation from any thread."""
|
|
with self._lock:
|
|
if self._aborted:
|
|
return
|
|
self._aborted = True
|
|
# The _ABORT sentinel is load-bearing and must reach the queue even when
|
|
# the bounded queue is full: evict an item to make room.
|
|
for _attempt in range(3):
|
|
try:
|
|
self._queue.put_nowait(_ABORT)
|
|
break
|
|
except queue.Full:
|
|
try:
|
|
self._queue.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
else:
|
|
logger.debug("streaming TTS _ABORT sentinel could not be enqueued")
|
|
if self._handle is not None and not self._handle.aborted:
|
|
with contextlib.suppress(Exception):
|
|
self._loop.call_soon_threadsafe(asyncio.create_task, self._safe_abort(reason))
|
|
|
|
async def wait_complete(self, timeout: float = 10.0) -> bool:
|
|
"""Wait for the drain task to finish. Returns True only on full success."""
|
|
if self._task is not None:
|
|
with contextlib.suppress(asyncio.CancelledError, Exception):
|
|
await asyncio.wait_for(asyncio.shield(self._task), timeout=timeout)
|
|
return self._completed
|