fix(voice): drop playback-phase barge transcripts that echo Hermes' own TTS
The full-duplex barge-in listener added in 5081551f0 stays active during
TTS playback with no acoustic echo cancellation. On some speaker/mic
combinations, TTS bleed alone crosses the barge threshold, gets
transcribed, and is queued as the next user turn -- whose reply is then
spoken, captured, and queued again, producing an unbounded TTS -> STT ->
TTS feedback loop (#75780).
Add a fail-closed transcript-level guard: when a barge trip happens during
the playback phase, compare the captured transcript against the TTS text
Hermes just spoke (tools/voice_mode.is_tts_echo, a language-agnostic
character-level similarity ratio). A close match is dropped instead of
queued, and the mic is handed back to the normal continuous-listening
loop. Generation-phase trips (no TTS playing, so no bleed is possible)
are unaffected.
This commit is contained in:
@@ -413,6 +413,79 @@ class TestVoiceBargeCaptureSubmit:
|
||||
assert not cli._voice_barge_capture.is_set()
|
||||
assert restarted.wait(2.0) # continuous mode resumes listening
|
||||
|
||||
def test_playback_phase_echo_of_own_tts_is_dropped(self, tmp_path, monkeypatch):
|
||||
"""#75780: a playback-phase capture that closely matches the TTS
|
||||
text Hermes just spoke is speaker bleed, not real user speech --
|
||||
it must be dropped instead of queued as the next turn, and the mic
|
||||
handed back so continuous mode keeps listening."""
|
||||
cli = _make_voice_cli(_voice_mode=True, _voice_continuous=True)
|
||||
cli._voice_barge_capture.set()
|
||||
cli._voice_barge_phase = "playback"
|
||||
cli._voice_last_tts_text = "네, 방금도 제 답변이 그대로 다시 입력됐어요."
|
||||
wav = tmp_path / "barge.wav"
|
||||
wav.write_bytes(b"RIFF")
|
||||
restarted = threading.Event()
|
||||
cli._voice_start_recording = lambda: restarted.set()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.voice_mode.transcribe_recording",
|
||||
lambda path, model=None: {
|
||||
"success": True,
|
||||
"transcript": "네 방금 네 방금도 제 답변이 그대로 다시 입력됐어요.",
|
||||
},
|
||||
)
|
||||
|
||||
cli._voice_submit_barge_utterance(str(wav))
|
||||
|
||||
assert cli._pending_input.empty() # not queued as a user turn
|
||||
assert not cli._voice_barge_capture.is_set()
|
||||
assert restarted.wait(2.0) # mic handed back instead of self-triggering another turn
|
||||
|
||||
def test_playback_phase_genuine_interjection_is_still_queued(self, tmp_path, monkeypatch):
|
||||
"""A real user interjection during playback -- unrelated to the TTS
|
||||
text -- must still reach the agent."""
|
||||
cli = _make_voice_cli()
|
||||
cli._voice_barge_capture.set()
|
||||
cli._voice_barge_phase = "playback"
|
||||
cli._voice_last_tts_text = "The weather today is sunny with a light breeze."
|
||||
wav = tmp_path / "barge.wav"
|
||||
wav.write_bytes(b"RIFF")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.voice_mode.transcribe_recording",
|
||||
lambda path, model=None: {
|
||||
"success": True,
|
||||
"transcript": "actually can you check my calendar for tomorrow",
|
||||
},
|
||||
)
|
||||
|
||||
cli._voice_submit_barge_utterance(str(wav))
|
||||
|
||||
queued = cli._pending_input.get_nowait()
|
||||
from cli import _VoiceInputMessage
|
||||
assert str(queued) == "actually can you check my calendar for tomorrow"
|
||||
|
||||
def test_generation_phase_transcript_not_echo_checked(self, tmp_path, monkeypatch):
|
||||
"""Generation-phase barges (no TTS playing) are never treated as
|
||||
echo, even if the transcript happens to match old TTS text."""
|
||||
cli = _make_voice_cli()
|
||||
cli._voice_barge_capture.set()
|
||||
cli._voice_barge_phase = "generation"
|
||||
cli._voice_last_tts_text = "stop, do it differently"
|
||||
wav = tmp_path / "barge.wav"
|
||||
wav.write_bytes(b"RIFF")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.voice_mode.transcribe_recording",
|
||||
lambda path, model=None: {"success": True, "transcript": "stop, do it differently"},
|
||||
)
|
||||
|
||||
cli._voice_submit_barge_utterance(str(wav))
|
||||
|
||||
queued = cli._pending_input.get_nowait()
|
||||
from cli import _VoiceInputMessage
|
||||
assert str(queued) == "stop, do it differently"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Full-duplex agent-turn listener — CLI phase behaviour
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Tests for the playback-phase TTS-echo guard (#75780).
|
||||
|
||||
Contract:
|
||||
- `is_tts_echo` flags a barge-in transcript as a likely self-capture of
|
||||
Hermes' own TTS output when it is a close character-level match for the
|
||||
text that was just spoken, regardless of language/tokenization.
|
||||
- A genuine, unrelated user interjection captured during playback must
|
||||
NOT be flagged, even though it happens to share some words.
|
||||
- `HermesCLI._voice_submit_barge_utterance` uses this guard ONLY for
|
||||
playback-phase barge captures (generation-phase speech can't be TTS
|
||||
bleed, since nothing is playing) and drops the echoed transcript
|
||||
instead of queuing it as the next user turn.
|
||||
"""
|
||||
|
||||
from tools.voice_mode import is_tts_echo
|
||||
|
||||
|
||||
class TestIsTtsEcho:
|
||||
def test_near_verbatim_repeat_is_echo(self):
|
||||
spoken = (
|
||||
"맞아요. 사용자가 마이크를 끄는 게 아니라 앱이 제 음성은 "
|
||||
"에코 제거로 걸러내고 실제 사용자 음성만 끼어들기로 받아야 해요."
|
||||
)
|
||||
transcript = spoken
|
||||
assert is_tts_echo(transcript, spoken) is True
|
||||
|
||||
def test_repeat_with_leading_stutter_is_echo(self):
|
||||
spoken = "네, 방금도 제 답변이 그대로 다시 입력됐어요."
|
||||
transcript = "네 방금 네 방금도 제 답변이 그대로 다시 입력됐어요."
|
||||
assert is_tts_echo(transcript, spoken) is True
|
||||
|
||||
def test_unrelated_interjection_is_not_echo(self):
|
||||
spoken = "The weather today is sunny with a light breeze from the west."
|
||||
transcript = "actually can you also check my calendar for tomorrow"
|
||||
assert is_tts_echo(transcript, spoken) is False
|
||||
|
||||
def test_short_unrelated_reply_is_not_echo(self):
|
||||
spoken = "I've finished summarizing the document you shared earlier."
|
||||
transcript = "stop"
|
||||
assert is_tts_echo(transcript, spoken) is False
|
||||
|
||||
def test_empty_inputs_are_not_echo(self):
|
||||
assert is_tts_echo("", "hello") is False
|
||||
assert is_tts_echo("hello", "") is False
|
||||
assert is_tts_echo("", "") is False
|
||||
|
||||
def test_case_and_whitespace_insensitive(self):
|
||||
spoken = "Sure, I can help with that right away."
|
||||
transcript = " SURE, I can help with that right away. "
|
||||
assert is_tts_echo(transcript, spoken) is True
|
||||
|
||||
def test_custom_threshold_is_honored(self):
|
||||
spoken = "This is a moderately similar sentence about testing."
|
||||
transcript = "This is a rather different sentence about coding."
|
||||
# Lenient threshold treats it as an echo, strict threshold does not.
|
||||
assert is_tts_echo(transcript, spoken, threshold=0.5) is True
|
||||
assert is_tts_echo(transcript, spoken, threshold=0.95) is False
|
||||
Reference in New Issue
Block a user