ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

fix(prompt)+feat(gateway): platform-hint truth pass + universal voice-bubble transcode (all 22 hints source-verified) (#97873)

Browse Source 
* fix(prompt): platform-hint truth pass — CLI/TUI file-delivery reality (paths/URLs only, MEDIA: prints literally), CLI no-markdown verified live, Slack/Discord markdown+tables truth, shared local-cron constant

* feat(gateway): universal voice-bubble delivery — shared transcode_to_ogg_opus; telegram [[audio_as_voice]] any-format; feishu native voice; hints to new truth

* chore: delete the webui ghost hint (tombstone comment, audit-verified); sync send_voice signature pin in tts routing test
This commit is contained in:
Teknium
2026-08-29 05:57:13 -07:00
committed by GitHub
parent b1ff8722a5
commit ccc367dce0
7 changed files with 291 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/prompt_builder.py
+52 -38
View File
@@ -758,6 +758,17 @@ def hud_surface_note(valid_tool_names: "set[str] | None" = None) -> str:
# message representation stays consistent ("system" everywhere).
DEVELOPER_ROLE_MODELS = ("gpt-5", "codex")
_LOCAL_CRON_DELIVERY_NOTE = (
"Cron jobs scheduled from this session are LOCAL-ONLY: their output "
"is saved (viewable via cronjob action='list') but is NOT delivered "
"back into this session — there is no live-delivery channel here. "
"If the user wants to be notified when a job runs, the job's "
"`deliver` must target a gateway-connected messaging platform "
"(e.g. deliver='telegram' or 'all'). Do not promise that a "
"deliver='origin' or default-deliver cron job will message them "
"in this session."
)
PLATFORM_HINTS = {
"whatsapp": (
"You are on a text messaging communication platform, WhatsApp. "
@@ -799,12 +810,19 @@ PLATFORM_HINTS = {
"Prefer bullet lists and labeled key:value pairs for structured data. "
"You can send media files natively: to deliver a file to the user, "
"include MEDIA:/absolute/path/to/file in your response. Images "
"(.png, .jpg, .webp) appear as photos, audio (.ogg) sends as voice "
"bubbles, and videos (.mp4) play inline. You can also include image "
"(.png, .jpg, .webp) appear as photos and videos (.mp4) play inline. "
"Audio: put [[audio_as_voice]] on its own line in the same response "
"to send ANY audio file as a native voice bubble (non-Opus formats "
"are transcoded automatically); without the directive, .mp3/.m4a "
"arrive as playable audio files and other formats as documents. "
"You can also include image "
"URLs in markdown format ![alt](url) and they will be sent as native photos."
),
"discord": (
"You are in a Discord server or group chat communicating with your user. "
"Discord renders standard markdown natively (bold, italic, code "
"blocks, links); tables are NOT supported — use bullet lists or "
"labeled lines. "
"You can send media files natively: include MEDIA:/absolute/path/to/file "
"in your response. Images (.png, .jpg, .webp) are sent as photo "
"attachments, audio as file attachments. You can also include image URLs "
@@ -812,6 +830,9 @@ PLATFORM_HINTS = {
),
"slack": (
"You are in a Slack workspace communicating with your user. "
"Standard markdown is auto-converted to Slack formatting (bold, "
"headers, links, code); tables are NOT supported — use bullet lists "
"or labeled lines. "
"You can send media files natively: include MEDIA:/absolute/path/to/file "
"in your response. Images (.png, .jpg, .webp) are uploaded as photo "
"attachments, audio as file attachments. You can also include image URLs "
@@ -846,31 +867,25 @@ PLATFORM_HINTS = {
"destination — put the primary content directly in your response."
),
"cli": (
"You are a CLI AI Agent. Try not to use markdown but simple text "
"renderable inside a terminal. "
"File delivery: there is no attachment channel — the user reads your "
"response directly in their terminal. Do NOT emit MEDIA:/path tags "
"(those are only intercepted on messaging platforms like Telegram, "
"Discord, Slack, etc.; on the CLI they render as literal text). "
"When referring to a file you created or changed, just state its "
"absolute path in plain text; the user can open it from there. "
"Cron jobs scheduled from this session are LOCAL-ONLY: their output is "
"saved (viewable via cronjob action='list') but is NOT delivered back "
"into this terminal — there is no live-delivery channel here. If the "
"user wants to be notified when a job runs, the job's `deliver` must "
"target a gateway-connected messaging platform (e.g. deliver='telegram' "
"or 'all'). Do not promise the user that a deliver='origin' or "
"default-deliver cron job will message them in this session."
# Maintainer-verified 2026-08-29 (live screenshot): the CLI prints
# raw text — markdown control characters render literally.
"You are in a plain terminal (CLI). Markdown does NOT render — "
"asterisks, headers, and fences appear as literal characters, so "
"write plain text (indentation and blank lines are your only "
"layout tools). Files: there is no attachment channel and "
"MEDIA:/path tags are NOT intercepted here (they print as "
"literal text) — deliver a file by stating its absolute path or "
"URL in plain text; the user opens it themselves. "
+ _LOCAL_CRON_DELIVERY_NOTE
),
"tui": (
"You are running in the Hermes terminal UI (TUI). "
"Cron jobs scheduled from this session are LOCAL-ONLY: their output is "
"saved (viewable via cronjob action='list') but is NOT delivered back "
"into this TUI session — there is no live-delivery channel here. If the "
"user wants to be notified when a job runs, the job's `deliver` must "
"target a gateway-connected messaging platform (e.g. deliver='telegram' "
"or 'all'). Do not promise the user that a deliver='origin' or "
"default-deliver cron job will message them in this session."
# Same file-delivery reality as the CLI (maintainer-confirmed):
# no MEDIA: interception in tui/ — tags would print literally.
"You are in the Hermes terminal UI (TUI). Files: there is no "
"attachment channel and MEDIA:/path tags are NOT intercepted "
"here (they print as literal text) — deliver a file by stating "
"its absolute path or URL in plain text. "
+ _LOCAL_CRON_DELIVERY_NOTE
),
"desktop": (
# Dieted (#95681, maintainer-directed) after a live premise battery
@@ -954,7 +969,9 @@ PLATFORM_HINTS = {
"links are supported. "
"You can send media files natively: include MEDIA:/absolute/path/to/file "
"in your response. Images (.jpg, .png, .webp) are uploaded and displayed "
"inline, audio files as voice messages, and other files as attachments."
"inline, audio files as native voice messages (non-Opus formats are "
"transcoded automatically; without ffmpeg they fall back to file "
"attachments), and other files as attachments."
),
"weixin": (
"You are on Weixin/WeChat. Markdown formatting is supported, so you may use it when "
@@ -1018,18 +1035,15 @@ PLATFORM_HINTS = {
"a raw host filesystem path. For those cases, state the plain file path "
"in your response text instead of a MEDIA: tag."
),
"webui": (
"You are in the Hermes WebUI, a browser-based chat interface. "
"Full Markdown rendering is supported — headings, bold, italic, code "
"blocks, tables, math (LaTeX), and Mermaid diagrams all render natively. "
"To display local or remote media/files inline, include "
"MEDIA:/absolute/path/to/file or MEDIA:https://... in your response. "
"Local file paths must be absolute. Images, audio (with playback speed "
"controls), video, PDFs, HTML, CSV, diffs/patches, and Excalidraw files "
"render as rich previews. Do not use Markdown image syntax like "
"![alt](/path) for local files; local paths are not served that way. "
"Use MEDIA:/absolute/path instead."
),
# NOTE: a "webui" hint lived here until 2026-08-29. It was a ghost
# (verified in the all-platform hint audit, PR #97873): no code path
# constructs platform="webui" — the dashboard chat resolves to
# 'desktop' or 'tui' (tui_gateway/server.py:_resolve_session_platform),
# and the browser chat tab is an xterm.js PTY hosting the TUI, not an
# HTML chat renderer. Its content (tables/LaTeX/Mermaid, MEDIA: rich
# previews incl. Excalidraw) described a renderer that does not exist
# anywhere in web/. If a real WebUI chat surface ships, write a hint
# from its actual renderer — do not resurrect this text.
}
# Telegram rich-messages extension — only injected when the user has opted in
gateway/platforms/base.py
+46
View File
@@ -62,6 +62,45 @@ _AUDIO_EXTS = frozenset(_AUDIO_MIME_TYPES)
# delivered as a regular document.
_TELEGRAM_AUDIO_ATTACHMENT_EXTS = frozenset({'.mp3', '.m4a'})
_TELEGRAM_VOICE_EXTS = frozenset({'.ogg', '.opus'})
def transcode_to_ogg_opus(path: str, *, bitrate: str = "32k") -> "str | None":
"""Best-effort ffmpeg transcode of any audio file to Ogg/Opus (voip-tuned).
The shared engine behind native voice-bubble delivery for platforms whose
voice channel only accepts Opus/OGG (Telegram sendVoice, Feishu opus
audio, Matrix MSC3245, WhatsApp voice notes). Returns the path of a NEW
temp ``.ogg`` file (caller owns cleanup), or ``None`` when ffmpeg is
missing or the conversion fails — callers keep their previous fallback
(document/attachment delivery). Blocking; call via ``asyncio.to_thread``
from async code.
"""
import shutil as _shutil
import subprocess as _subprocess
import tempfile as _tempfile
ffmpeg = _shutil.which("ffmpeg")
if not ffmpeg:
return None
fd, ogg_path = _tempfile.mkstemp(prefix="voice_transcode_", suffix=".ogg")
os.close(fd)
try:
result = _subprocess.run(
[ffmpeg, "-v", "error", "-y", "-i", str(path),
"-acodec", "libopus", "-ac", "1", "-b:a", bitrate, "-vbr", "on",
"-application", "voip", "-compression_level", "10", ogg_path],
capture_output=True, timeout=60, stdin=_subprocess.DEVNULL,
)
if result.returncode == 0 and os.path.getsize(ogg_path) > 0:
return ogg_path
except Exception:
logger.debug("voice transcode to Ogg/Opus failed for %s", path, exc_info=True)
try:
os.unlink(ogg_path)
except OSError:
pass
return None
_POST_DELIVERY_CALLBACK_TIMEOUT_SECONDS = 30.0
# Delivery-time history is best-effort dedup metadata, not canonical state.
# Keep this comfortably below the Discord heartbeat watchdog window and fail
@@ -184,6 +223,12 @@ def should_send_media_as_audio(platform, ext: str, is_voice: bool = False) -> bo
if normalized_ext not in _AUDIO_EXTS:
return False
if _platform_name(platform) == "telegram":
if is_voice:
# Explicit [[audio_as_voice]] intent: ANY audio format routes to
# the voice sender — the adapter transcodes non-Opus input to
# Ogg/Opus on the fly (transcode_to_ogg_opus), so the intent no
# longer dead-ends into document delivery for .mp3/.wav/etc.
return True
if normalized_ext in _TELEGRAM_VOICE_EXTS:
return is_voice
return normalized_ext in _TELEGRAM_AUDIO_ATTACHMENT_EXTS
@@ -6881,6 +6926,7 @@ class BasePlatformAdapter(ABC):
chat_id=event.source.chat_id,
audio_path=media_path,
metadata=_final_thread_metadata,
is_voice=is_voice,
)
elif ext in _VIDEO_EXTS:
logger.info(
gateway/run.py
+2
View File
@@ -23422,6 +23422,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
chat_id=event.source.chat_id,
audio_path=media_path,
metadata=_thread_meta,
is_voice=is_voice,
)
elif ext in _VIDEO_EXTS:
await adapter.send_video(
@@ -23755,6 +23756,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
chat_id=source.chat_id,
audio_path=media_path,
metadata=_thread_metadata,
is_voice=_is_voice,
)
elif _ext in _VIDEO_EXTS:
await adapter.send_video(
plugins/platforms/feishu/adapter.py
+30 -9
View File
@@ -2247,15 +2247,36 @@ class FeishuAdapter(BasePlatformAdapter):
metadata: Optional[Dict[str, Any]] = None,
**kwargs,
) -> SendResult:
"""Send audio to Feishu as a file attachment plus optional caption."""
return await self._send_uploaded_file_message(
chat_id=chat_id,
file_path=audio_path,
reply_to=reply_to,
metadata=metadata,
caption=caption,
outbound_message_type="audio",
)
"""Send audio to Feishu as a native voice message (opus) or file.
Feishu's voice channel only accepts Opus (msg_type='audio' with an
opus upload). Non-opus audio (mp3/wav/flac/...) is transcoded on the
fly via the shared ffmpeg engine so audio actually arrives as a
playable voice message; when ffmpeg is unavailable the original
file is sent as a file attachment (previous behavior).
"""
transcoded_path: Optional[str] = None
ext = Path(audio_path).suffix.lower()
if ext not in _FEISHU_OPUS_UPLOAD_EXTENSIONS:
from gateway.platforms.base import transcode_to_ogg_opus
transcoded_path = await asyncio.to_thread(transcode_to_ogg_opus, audio_path)
if transcoded_path:
audio_path = transcoded_path
try:
return await self._send_uploaded_file_message(
chat_id=chat_id,
file_path=audio_path,
reply_to=reply_to,
metadata=metadata,
caption=caption,
outbound_message_type="audio",
)
finally:
if transcoded_path:
try:
os.unlink(transcoded_path)
except OSError:
pass
async def send_document(
self,
plugins/platforms/telegram/adapter.py
+31 -1
View File
@@ -7728,10 +7728,34 @@ class TelegramAdapter(BasePlatformAdapter):
"""Send audio as a native Telegram voice message or audio file."""
if not self._bot:
return SendResult(success=False, error="Not connected")
_transcoded_voice_path: Optional[str] = None
try:
if not os.path.exists(audio_path):
return SendResult(success=False, error=self._missing_media_path_error("Audio", audio_path))
# Telegram sendVoice only accepts Ogg/Opus. When the caller
# explicitly asked for a voice bubble ([[audio_as_voice]] →
# is_voice=True in kwargs), transcode any other audio format
# (mp3/wav/flac/...) to Ogg/Opus on the fly via the shared
# ffmpeg engine — previously that intent dead-ended into
# document delivery. Without the explicit intent, extension
# behavior is unchanged (.mp3/.m4a → sendAudio; .ogg → here
# only when flagged; others → document fallback below).
_voice_ext = os.path.splitext(audio_path)[1].lower()
if kwargs.get("is_voice") and _voice_ext not in (".ogg", ".opus"):
from gateway.platforms.base import transcode_to_ogg_opus
_transcoded_voice_path = await asyncio.to_thread(
transcode_to_ogg_opus, audio_path
)
if _transcoded_voice_path:
audio_path = _transcoded_voice_path
else:
logger.warning(
"[%s] voice transcode unavailable for %s — sending "
"original format (install ffmpeg for voice bubbles)",
self.name, os.path.basename(audio_path),
)
# Compute duration locally — Telegram drops it for long clips
# (~5 min+), which then show 0:00 in the player.
@@ -7866,6 +7890,12 @@ class TelegramAdapter(BasePlatformAdapter):
exc_info=True,
)
return await super().send_voice(chat_id, audio_path, caption, reply_to, metadata=metadata)
finally:
if _transcoded_voice_path:
try:
os.unlink(_transcoded_voice_path)
except OSError:
pass
async def send_multiple_images(
self,
tests/gateway/test_tts_media_routing.py
+1
View File
@@ -82,6 +82,7 @@ async def test_base_adapter_routes_voice_tagged_telegram_ogg_media_tag_to_voice_
chat_id="chat-1",
audio_path=str(media_file),
metadata={"notify": True},
is_voice=True,
)
adapter.send_document.assert_not_awaited()
tests/gateway/test_voice_transcode.py
+129
View File
@@ -0,0 +1,129 @@
"""Voice transcode arc (#97873): [[audio_as_voice]] works for every audio format.
Found by a maintainer-directed all-platform hint verification: the telegram
hint claimed '.ogg sends as voice bubbles' but bare MEDIA:.ogg (no directive)
ships as a document, and mp3+directive DEAD-ENDED into document delivery.
Fix: shared transcode_to_ogg_opus in base.py; telegram/feishu send_voice
transcode non-opus input; should_send_media_as_audio honors explicit
is_voice for any audio ext on telegram; is_voice threaded through the three
dispatch call sites.
"""
import asyncio
import os
import sys
import tempfile
import unittest
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from gateway.platforms.base import should_send_media_as_audio, transcode_to_ogg_opus
class TestVoiceRouting(unittest.TestCase):
def test_telegram_explicit_voice_any_audio_ext(self):
"""[[audio_as_voice]] intent routes EVERY audio format to send_voice."""
for ext in (".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus"):
self.assertTrue(
should_send_media_as_audio("telegram", ext, is_voice=True),
f"{ext} with is_voice=True must route to the voice sender",
)
def test_telegram_without_intent_unchanged(self):
"""No directive: .mp3/.m4a -> sendAudio; .ogg NOT voice; .wav -> document."""
self.assertTrue(should_send_media_as_audio("telegram", ".mp3", is_voice=False))
self.assertTrue(should_send_media_as_audio("telegram", ".m4a", is_voice=False))
self.assertFalse(should_send_media_as_audio("telegram", ".ogg", is_voice=False))
self.assertFalse(should_send_media_as_audio("telegram", ".wav", is_voice=False))
def test_other_platforms_unchanged(self):
self.assertTrue(should_send_media_as_audio("feishu", ".mp3", is_voice=False))
self.assertFalse(should_send_media_as_audio("feishu", ".qzx7", is_voice=False))
class TestTranscodeEngine(unittest.TestCase):
def test_missing_ffmpeg_returns_none(self):
with patch("shutil.which", return_value=None):
self.assertIsNone(transcode_to_ogg_opus("/tmp/x.mp3"))
def test_transcode_failure_cleans_up(self):
"""A failing ffmpeg leaves no orphan temp file behind."""
import subprocess
fake = subprocess.CompletedProcess(args=[], returncode=1, stdout=b"", stderr=b"boom")
with patch("shutil.which", return_value="ffmpeg"), \
patch("subprocess.run", return_value=fake):
before = set(os.listdir(tempfile.gettempdir()))
self.assertIsNone(transcode_to_ogg_opus("/tmp/x.mp3"))
leaked = [f for f in os.listdir(tempfile.gettempdir())
if f.startswith("voice_transcode_") and f not in before]
self.assertEqual(leaked, [])
class TestTelegramSendVoiceTranscode(unittest.TestCase):
def test_mp3_with_intent_is_transcoded_and_sent_as_voice(self):
"""mp3 + is_voice=True: adapter transcodes, calls bot.send_voice, cleans up."""
from plugins.platforms.telegram.adapter import TelegramAdapter
adapter = TelegramAdapter.__new__(TelegramAdapter)
sent = {}
class _Msg:
message_id = 42
async def fake_retry(send_fn, kwargs_dict, *a, **kw):
sent["fn"] = getattr(send_fn, "__name__", str(send_fn))
sent["kwargs"] = kwargs_dict
return _Msg()
class _Bot:
async def send_voice(self, **kw): # identity only
raise AssertionError("should go through retry helper")
adapter._bot = _Bot()
adapter._reply_to_mode = None
# 'name' is a read-only property on the adapter; the send path only
# uses it for log strings, so bypass via the instance dict shim.
object.__setattr__(adapter, "_name", "telegram-test")
adapter._send_with_dm_topic_reply_anchor_retry = fake_retry
adapter._metadata_thread_id = lambda *_: None
adapter._reply_to_message_id_for_send = lambda *a, **k: None
adapter._thread_kwargs_for_send = lambda *a, **k: {}
adapter._notification_kwargs = lambda *_: {}
adapter._missing_media_path_error = lambda kind, p: f"missing {p}"
fd, mp3 = tempfile.mkstemp(suffix=".mp3")
os.write(fd, b"ID3fakebytes")
os.close(fd)
fd2, fake_ogg = tempfile.mkstemp(suffix=".ogg")
os.write(fd2, b"OggSfakeopus")
os.close(fd2)
transcode_calls = []
def fake_transcode(path, **kw):
transcode_calls.append(path)
return fake_ogg
try:
with patch("gateway.platforms.base.transcode_to_ogg_opus", side_effect=fake_transcode), \
patch("plugins.platforms.telegram.adapter._probe_voice_duration_seconds", return_value=1):
result = asyncio.run(
adapter.send_voice("123", mp3, is_voice=True)
)
self.assertTrue(result.success, result.error)
self.assertEqual(transcode_calls, [mp3])
self.assertIn("voice", sent["kwargs"]) # routed to send_voice branch
self.assertFalse(os.path.exists(fake_ogg), "transcoded temp must be cleaned up")
finally:
for p in (mp3, fake_ogg):
try:
os.unlink(p)
except OSError:
pass
if __name__ == "__main__":
unittest.main()