diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 236be550ea..99637839ac 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -758,6 +758,17 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str: # message representation stays consistent ("system" everywhere). DEVELOPER_ROLE_MODELS = ("gpt-5", "codex") +_LOCAL_CRON_DELIVERY_NOTE = ( + "Cron jobs scheduled from this session are LOCAL-ONLY: their output " + "is saved (viewable via cronjob action='list') but is NOT delivered " + "back into this session — there is no live-delivery channel here. " + "If the user wants to be notified when a job runs, the job's " + "`deliver` must target a gateway-connected messaging platform " + "(e.g. deliver='telegram' or 'all'). Do not promise that a " + "deliver='origin' or default-deliver cron job will message them " + "in this session." +) + PLATFORM_HINTS = { "whatsapp": ( "You are on a text messaging communication platform, WhatsApp. " @@ -799,12 +810,19 @@ PLATFORM_HINTS = { "Prefer bullet lists and labeled key:value pairs for structured data. " "You can send media files natively: to deliver a file to the user, " "include MEDIA:/absolute/path/to/file in your response. Images " - "(.png, .jpg, .webp) appear as photos, audio (.ogg) sends as voice " - "bubbles, and videos (.mp4) play inline. You can also include image " + "(.png, .jpg, .webp) appear as photos and videos (.mp4) play inline. " + "Audio: put [[audio_as_voice]] on its own line in the same response " + "to send ANY audio file as a native voice bubble (non-Opus formats " + "are transcoded automatically); without the directive, .mp3/.m4a " + "arrive as playable audio files and other formats as documents. " + "You can also include image " "URLs in markdown format ![alt](url) and they will be sent as native photos." ), "discord": ( "You are in a Discord server or group chat communicating with your user. " + "Discord renders standard markdown natively (bold, italic, code " + "blocks, links); tables are NOT supported — use bullet lists or " + "labeled lines. " "You can send media files natively: include MEDIA:/absolute/path/to/file " "in your response. Images (.png, .jpg, .webp) are sent as photo " "attachments, audio as file attachments. You can also include image URLs " @@ -812,6 +830,9 @@ PLATFORM_HINTS = { ), "slack": ( "You are in a Slack workspace communicating with your user. " + "Standard markdown is auto-converted to Slack formatting (bold, " + "headers, links, code); tables are NOT supported — use bullet lists " + "or labeled lines. " "You can send media files natively: include MEDIA:/absolute/path/to/file " "in your response. Images (.png, .jpg, .webp) are uploaded as photo " "attachments, audio as file attachments. You can also include image URLs " @@ -846,31 +867,25 @@ PLATFORM_HINTS = { "destination — put the primary content directly in your response." ), "cli": ( - "You are a CLI AI Agent. Try not to use markdown but simple text " - "renderable inside a terminal. " - "File delivery: there is no attachment channel — the user reads your " - "response directly in their terminal. Do NOT emit MEDIA:/path tags " - "(those are only intercepted on messaging platforms like Telegram, " - "Discord, Slack, etc.; on the CLI they render as literal text). " - "When referring to a file you created or changed, just state its " - "absolute path in plain text; the user can open it from there. " - "Cron jobs scheduled from this session are LOCAL-ONLY: their output is " - "saved (viewable via cronjob action='list') but is NOT delivered back " - "into this terminal — there is no live-delivery channel here. If the " - "user wants to be notified when a job runs, the job's `deliver` must " - "target a gateway-connected messaging platform (e.g. deliver='telegram' " - "or 'all'). Do not promise the user that a deliver='origin' or " - "default-deliver cron job will message them in this session." + # Maintainer-verified 2026-08-29 (live screenshot): the CLI prints + # raw text — markdown control characters render literally. + "You are in a plain terminal (CLI). Markdown does NOT render — " + "asterisks, headers, and fences appear as literal characters, so " + "write plain text (indentation and blank lines are your only " + "layout tools). Files: there is no attachment channel and " + "MEDIA:/path tags are NOT intercepted here (they print as " + "literal text) — deliver a file by stating its absolute path or " + "URL in plain text; the user opens it themselves. " + + _LOCAL_CRON_DELIVERY_NOTE ), "tui": ( - "You are running in the Hermes terminal UI (TUI). " - "Cron jobs scheduled from this session are LOCAL-ONLY: their output is " - "saved (viewable via cronjob action='list') but is NOT delivered back " - "into this TUI session — there is no live-delivery channel here. If the " - "user wants to be notified when a job runs, the job's `deliver` must " - "target a gateway-connected messaging platform (e.g. deliver='telegram' " - "or 'all'). Do not promise the user that a deliver='origin' or " - "default-deliver cron job will message them in this session." + # Same file-delivery reality as the CLI (maintainer-confirmed): + # no MEDIA: interception in tui/ — tags would print literally. + "You are in the Hermes terminal UI (TUI). Files: there is no " + "attachment channel and MEDIA:/path tags are NOT intercepted " + "here (they print as literal text) — deliver a file by stating " + "its absolute path or URL in plain text. " + + _LOCAL_CRON_DELIVERY_NOTE ), "desktop": ( # Dieted (#95681, maintainer-directed) after a live premise battery @@ -954,7 +969,9 @@ PLATFORM_HINTS = { "links are supported. " "You can send media files natively: include MEDIA:/absolute/path/to/file " "in your response. Images (.jpg, .png, .webp) are uploaded and displayed " - "inline, audio files as voice messages, and other files as attachments." + "inline, audio files as native voice messages (non-Opus formats are " + "transcoded automatically; without ffmpeg they fall back to file " + "attachments), and other files as attachments." ), "weixin": ( "You are on Weixin/WeChat. Markdown formatting is supported, so you may use it when " @@ -1018,18 +1035,15 @@ PLATFORM_HINTS = { "a raw host filesystem path. For those cases, state the plain file path " "in your response text instead of a MEDIA: tag." ), - "webui": ( - "You are in the Hermes WebUI, a browser-based chat interface. " - "Full Markdown rendering is supported — headings, bold, italic, code " - "blocks, tables, math (LaTeX), and Mermaid diagrams all render natively. " - "To display local or remote media/files inline, include " - "MEDIA:/absolute/path/to/file or MEDIA:https://... in your response. " - "Local file paths must be absolute. Images, audio (with playback speed " - "controls), video, PDFs, HTML, CSV, diffs/patches, and Excalidraw files " - "render as rich previews. Do not use Markdown image syntax like " - "![alt](/path) for local files; local paths are not served that way. " - "Use MEDIA:/absolute/path instead." - ), + # NOTE: a "webui" hint lived here until 2026-08-29. It was a ghost + # (verified in the all-platform hint audit, PR #97873): no code path + # constructs platform="webui" — the dashboard chat resolves to + # 'desktop' or 'tui' (tui_gateway/server.py:_resolve_session_platform), + # and the browser chat tab is an xterm.js PTY hosting the TUI, not an + # HTML chat renderer. Its content (tables/LaTeX/Mermaid, MEDIA: rich + # previews incl. Excalidraw) described a renderer that does not exist + # anywhere in web/. If a real WebUI chat surface ships, write a hint + # from its actual renderer — do not resurrect this text. } # Telegram rich-messages extension — only injected when the user has opted in diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index 94c4f8a3dc..d05c5265e7 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -62,6 +62,45 @@ _AUDIO_EXTS = frozenset(_AUDIO_MIME_TYPES) # delivered as a regular document. _TELEGRAM_AUDIO_ATTACHMENT_EXTS = frozenset({'.mp3', '.m4a'}) _TELEGRAM_VOICE_EXTS = frozenset({'.ogg', '.opus'}) + + +def transcode_to_ogg_opus(path: str, *, bitrate: str = "32k") -> "str | None": + """Best-effort ffmpeg transcode of any audio file to Ogg/Opus (voip-tuned). + + The shared engine behind native voice-bubble delivery for platforms whose + voice channel only accepts Opus/OGG (Telegram sendVoice, Feishu opus + audio, Matrix MSC3245, WhatsApp voice notes). Returns the path of a NEW + temp ``.ogg`` file (caller owns cleanup), or ``None`` when ffmpeg is + missing or the conversion fails — callers keep their previous fallback + (document/attachment delivery). Blocking; call via ``asyncio.to_thread`` + from async code. + """ + import shutil as _shutil + import subprocess as _subprocess + import tempfile as _tempfile + + ffmpeg = _shutil.which("ffmpeg") + if not ffmpeg: + return None + + fd, ogg_path = _tempfile.mkstemp(prefix="voice_transcode_", suffix=".ogg") + os.close(fd) + try: + result = _subprocess.run( + [ffmpeg, "-v", "error", "-y", "-i", str(path), + "-acodec", "libopus", "-ac", "1", "-b:a", bitrate, "-vbr", "on", + "-application", "voip", "-compression_level", "10", ogg_path], + capture_output=True, timeout=60, stdin=_subprocess.DEVNULL, + ) + if result.returncode == 0 and os.path.getsize(ogg_path) > 0: + return ogg_path + except Exception: + logger.debug("voice transcode to Ogg/Opus failed for %s", path, exc_info=True) + try: + os.unlink(ogg_path) + except OSError: + pass + return None _POST_DELIVERY_CALLBACK_TIMEOUT_SECONDS = 30.0 # Delivery-time history is best-effort dedup metadata, not canonical state. # Keep this comfortably below the Discord heartbeat watchdog window and fail @@ -184,6 +223,12 @@ def should_send_media_as_audio(platform, ext: str, is_voice: bool = False) -> bo if normalized_ext not in _AUDIO_EXTS: return False if _platform_name(platform) == "telegram": + if is_voice: + # Explicit [[audio_as_voice]] intent: ANY audio format routes to + # the voice sender — the adapter transcodes non-Opus input to + # Ogg/Opus on the fly (transcode_to_ogg_opus), so the intent no + # longer dead-ends into document delivery for .mp3/.wav/etc. + return True if normalized_ext in _TELEGRAM_VOICE_EXTS: return is_voice return normalized_ext in _TELEGRAM_AUDIO_ATTACHMENT_EXTS @@ -6881,6 +6926,7 @@ class BasePlatformAdapter(ABC): chat_id=event.source.chat_id, audio_path=media_path, metadata=_final_thread_metadata, + is_voice=is_voice, ) elif ext in _VIDEO_EXTS: logger.info( diff --git a/gateway/run.py b/gateway/run.py index 5887ffe0ba..8ce292a548 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -23422,6 +23422,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew chat_id=event.source.chat_id, audio_path=media_path, metadata=_thread_meta, + is_voice=is_voice, ) elif ext in _VIDEO_EXTS: await adapter.send_video( @@ -23755,6 +23756,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew chat_id=source.chat_id, audio_path=media_path, metadata=_thread_metadata, + is_voice=_is_voice, ) elif _ext in _VIDEO_EXTS: await adapter.send_video( diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py index 42a70e79fb..be20f1a3df 100644 --- a/plugins/platforms/feishu/adapter.py +++ b/plugins/platforms/feishu/adapter.py @@ -2247,15 +2247,36 @@ class FeishuAdapter(BasePlatformAdapter): metadata: Optional[Dict[str, Any]] = None, **kwargs, ) -> SendResult: - """Send audio to Feishu as a file attachment plus optional caption.""" - return await self._send_uploaded_file_message( - chat_id=chat_id, - file_path=audio_path, - reply_to=reply_to, - metadata=metadata, - caption=caption, - outbound_message_type="audio", - ) + """Send audio to Feishu as a native voice message (opus) or file. + + Feishu's voice channel only accepts Opus (msg_type='audio' with an + opus upload). Non-opus audio (mp3/wav/flac/...) is transcoded on the + fly via the shared ffmpeg engine so audio actually arrives as a + playable voice message; when ffmpeg is unavailable the original + file is sent as a file attachment (previous behavior). + """ + transcoded_path: Optional[str] = None + ext = Path(audio_path).suffix.lower() + if ext not in _FEISHU_OPUS_UPLOAD_EXTENSIONS: + from gateway.platforms.base import transcode_to_ogg_opus + transcoded_path = await asyncio.to_thread(transcode_to_ogg_opus, audio_path) + if transcoded_path: + audio_path = transcoded_path + try: + return await self._send_uploaded_file_message( + chat_id=chat_id, + file_path=audio_path, + reply_to=reply_to, + metadata=metadata, + caption=caption, + outbound_message_type="audio", + ) + finally: + if transcoded_path: + try: + os.unlink(transcoded_path) + except OSError: + pass async def send_document( self, diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index a7cd3b5a85..345a172fb9 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -7728,10 +7728,34 @@ class TelegramAdapter(BasePlatformAdapter): """Send audio as a native Telegram voice message or audio file.""" if not self._bot: return SendResult(success=False, error="Not connected") - + + _transcoded_voice_path: Optional[str] = None try: if not os.path.exists(audio_path): return SendResult(success=False, error=self._missing_media_path_error("Audio", audio_path)) + + # Telegram sendVoice only accepts Ogg/Opus. When the caller + # explicitly asked for a voice bubble ([[audio_as_voice]] → + # is_voice=True in kwargs), transcode any other audio format + # (mp3/wav/flac/...) to Ogg/Opus on the fly via the shared + # ffmpeg engine — previously that intent dead-ended into + # document delivery. Without the explicit intent, extension + # behavior is unchanged (.mp3/.m4a → sendAudio; .ogg → here + # only when flagged; others → document fallback below). + _voice_ext = os.path.splitext(audio_path)[1].lower() + if kwargs.get("is_voice") and _voice_ext not in (".ogg", ".opus"): + from gateway.platforms.base import transcode_to_ogg_opus + _transcoded_voice_path = await asyncio.to_thread( + transcode_to_ogg_opus, audio_path + ) + if _transcoded_voice_path: + audio_path = _transcoded_voice_path + else: + logger.warning( + "[%s] voice transcode unavailable for %s — sending " + "original format (install ffmpeg for voice bubbles)", + self.name, os.path.basename(audio_path), + ) # Compute duration locally — Telegram drops it for long clips # (~5 min+), which then show 0:00 in the player. @@ -7866,6 +7890,12 @@ class TelegramAdapter(BasePlatformAdapter): exc_info=True, ) return await super().send_voice(chat_id, audio_path, caption, reply_to, metadata=metadata) + finally: + if _transcoded_voice_path: + try: + os.unlink(_transcoded_voice_path) + except OSError: + pass async def send_multiple_images( self, diff --git a/tests/gateway/test_tts_media_routing.py b/tests/gateway/test_tts_media_routing.py index d5814bda7a..006d654a91 100644 --- a/tests/gateway/test_tts_media_routing.py +++ b/tests/gateway/test_tts_media_routing.py @@ -82,6 +82,7 @@ async def test_base_adapter_routes_voice_tagged_telegram_ogg_media_tag_to_voice_ chat_id="chat-1", audio_path=str(media_file), metadata={"notify": True}, + is_voice=True, ) adapter.send_document.assert_not_awaited() diff --git a/tests/gateway/test_voice_transcode.py b/tests/gateway/test_voice_transcode.py new file mode 100644 index 0000000000..c7d8507ecb --- /dev/null +++ b/tests/gateway/test_voice_transcode.py @@ -0,0 +1,129 @@ +"""Voice transcode arc (#97873): [[audio_as_voice]] works for every audio format. + +Found by a maintainer-directed all-platform hint verification: the telegram +hint claimed '.ogg sends as voice bubbles' but bare MEDIA:.ogg (no directive) +ships as a document, and mp3+directive DEAD-ENDED into document delivery. +Fix: shared transcode_to_ogg_opus in base.py; telegram/feishu send_voice +transcode non-opus input; should_send_media_as_audio honors explicit +is_voice for any audio ext on telegram; is_voice threaded through the three +dispatch call sites. +""" +import asyncio +import os +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from gateway.platforms.base import should_send_media_as_audio, transcode_to_ogg_opus + + +class TestVoiceRouting(unittest.TestCase): + def test_telegram_explicit_voice_any_audio_ext(self): + """[[audio_as_voice]] intent routes EVERY audio format to send_voice.""" + for ext in (".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus"): + self.assertTrue( + should_send_media_as_audio("telegram", ext, is_voice=True), + f"{ext} with is_voice=True must route to the voice sender", + ) + + def test_telegram_without_intent_unchanged(self): + """No directive: .mp3/.m4a -> sendAudio; .ogg NOT voice; .wav -> document.""" + self.assertTrue(should_send_media_as_audio("telegram", ".mp3", is_voice=False)) + self.assertTrue(should_send_media_as_audio("telegram", ".m4a", is_voice=False)) + self.assertFalse(should_send_media_as_audio("telegram", ".ogg", is_voice=False)) + self.assertFalse(should_send_media_as_audio("telegram", ".wav", is_voice=False)) + + def test_other_platforms_unchanged(self): + self.assertTrue(should_send_media_as_audio("feishu", ".mp3", is_voice=False)) + self.assertFalse(should_send_media_as_audio("feishu", ".qzx7", is_voice=False)) + + +class TestTranscodeEngine(unittest.TestCase): + def test_missing_ffmpeg_returns_none(self): + with patch("shutil.which", return_value=None): + self.assertIsNone(transcode_to_ogg_opus("/tmp/x.mp3")) + + def test_transcode_failure_cleans_up(self): + """A failing ffmpeg leaves no orphan temp file behind.""" + import subprocess + + fake = subprocess.CompletedProcess(args=[], returncode=1, stdout=b"", stderr=b"boom") + with patch("shutil.which", return_value="ffmpeg"), \ + patch("subprocess.run", return_value=fake): + before = set(os.listdir(tempfile.gettempdir())) + self.assertIsNone(transcode_to_ogg_opus("/tmp/x.mp3")) + leaked = [f for f in os.listdir(tempfile.gettempdir()) + if f.startswith("voice_transcode_") and f not in before] + self.assertEqual(leaked, []) + + +class TestTelegramSendVoiceTranscode(unittest.TestCase): + def test_mp3_with_intent_is_transcoded_and_sent_as_voice(self): + """mp3 + is_voice=True: adapter transcodes, calls bot.send_voice, cleans up.""" + from plugins.platforms.telegram.adapter import TelegramAdapter + + adapter = TelegramAdapter.__new__(TelegramAdapter) + + sent = {} + + class _Msg: + message_id = 42 + + async def fake_retry(send_fn, kwargs_dict, *a, **kw): + sent["fn"] = getattr(send_fn, "__name__", str(send_fn)) + sent["kwargs"] = kwargs_dict + return _Msg() + + class _Bot: + async def send_voice(self, **kw): # identity only + raise AssertionError("should go through retry helper") + + adapter._bot = _Bot() + adapter._reply_to_mode = None + # 'name' is a read-only property on the adapter; the send path only + # uses it for log strings, so bypass via the instance dict shim. + object.__setattr__(adapter, "_name", "telegram-test") + adapter._send_with_dm_topic_reply_anchor_retry = fake_retry + adapter._metadata_thread_id = lambda *_: None + adapter._reply_to_message_id_for_send = lambda *a, **k: None + adapter._thread_kwargs_for_send = lambda *a, **k: {} + adapter._notification_kwargs = lambda *_: {} + adapter._missing_media_path_error = lambda kind, p: f"missing {p}" + + fd, mp3 = tempfile.mkstemp(suffix=".mp3") + os.write(fd, b"ID3fakebytes") + os.close(fd) + + fd2, fake_ogg = tempfile.mkstemp(suffix=".ogg") + os.write(fd2, b"OggSfakeopus") + os.close(fd2) + + transcode_calls = [] + + def fake_transcode(path, **kw): + transcode_calls.append(path) + return fake_ogg + + try: + with patch("gateway.platforms.base.transcode_to_ogg_opus", side_effect=fake_transcode), \ + patch("plugins.platforms.telegram.adapter._probe_voice_duration_seconds", return_value=1): + result = asyncio.run( + adapter.send_voice("123", mp3, is_voice=True) + ) + self.assertTrue(result.success, result.error) + self.assertEqual(transcode_calls, [mp3]) + self.assertIn("voice", sent["kwargs"]) # routed to send_voice branch + self.assertFalse(os.path.exists(fake_ogg), "transcoded temp must be cleaned up") + finally: + for p in (mp3, fake_ogg): + try: + os.unlink(p) + except OSError: + pass + + +if __name__ == "__main__": + unittest.main()