fix(memory/mem0): trim a synced message at the last sentence boundary, not the first separator kind
_truncate_for_sync documents "the last sentence boundary within max_len", but it looped over separator KINDS and returned on the first kind that qualified. An early "。" therefore outranked a "." 240 characters later, and in pure ASCII "." outranked a later "!" or "?" purely because it comes first in the tuple. With the 450-char default, "a"*200 + "。" + "b"*240 + "." + "c"*100 kept 201 of the 442 characters available: 241 characters the embedder would have accepted were discarded, so any fact in the second half of the turn never reached extraction. The add() call succeeds, so unlike #106235 nothing is logged — the turn is simply remembered from its first sentence. Raising sync_max_chars widens the gap rather than closing it. Take the max over every separator instead, from a named tuple so the set is not buried in the loop. ".\n" is dropped: its index can never exceed the bare "." it starts with, so under a max it is unreachable. The first-third guard and the hard-cut fallback for unsegmented input are unchanged. Fixes #108868
This commit is contained in:
committed by
Teknium
parent
e705dde6e4
commit
aca4dc86a3
@@ -42,6 +42,13 @@ _DEFAULT_USER_ID = "hermes-user"
|
||||
_SYNC_MSG_MAX_CHARS = 450
|
||||
|
||||
|
||||
# Sentence ends recognized when trimming a synced message. Deliberately unordered:
|
||||
# the LAST boundary of ANY kind wins, so one CJK stop early in a mixed-script turn
|
||||
# cannot outrank a Latin stop near the end of the window. ``".\n"`` is not listed —
|
||||
# its index can never exceed the bare ``"."`` it starts with.
|
||||
_SYNC_SENTENCE_ENDS = ("。", "!", "?", ".", "!", "?")
|
||||
|
||||
|
||||
def _truncate_for_sync(text: str, max_len: int = _SYNC_MSG_MAX_CHARS) -> str:
|
||||
"""Cap a synced message at its last sentence boundary within ``max_len``.
|
||||
|
||||
@@ -52,10 +59,10 @@ def _truncate_for_sync(text: str, max_len: int = _SYNC_MSG_MAX_CHARS) -> str:
|
||||
"""
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
for sep in ("。", "!", "?", ".\n", ".", "!", "?"):
|
||||
cut = text[:max_len].rfind(sep)
|
||||
if cut > max_len // 3:
|
||||
return text[:cut + 1]
|
||||
window = text[:max_len]
|
||||
cut = max(window.rfind(sep) for sep in _SYNC_SENTENCE_ENDS)
|
||||
if cut > max_len // 3:
|
||||
return text[:cut + 1]
|
||||
return text[:max_len]
|
||||
|
||||
|
||||
|
||||
@@ -181,6 +181,36 @@ class TestSyncTurnTruncation:
|
||||
assert len(sent[1]["content"]) <= mem0_plugin._SYNC_MSG_MAX_CHARS and sent[1]["content"].endswith(".")
|
||||
assert provider._consecutive_failures == 0
|
||||
|
||||
def test_the_boundary_kept_is_the_last_one_in_the_window_whatever_its_script(self):
|
||||
"""A mixed-script turn must not be cut back to an early CJK stop.
|
||||
|
||||
The trim exists to keep as much of the turn as the embedder can take; picking the
|
||||
first separator KIND that qualifies instead of the last boundary threw away most of
|
||||
the allowed window whenever two kinds appeared — an early ``。`` (or ``.``, which
|
||||
outranks ``!``/``?``) beat a boundary 240 characters later, so the facts stated in
|
||||
the rest of the message never reached extraction.
|
||||
"""
|
||||
cap = mem0_plugin._SYNC_MSG_MAX_CHARS
|
||||
early, late = cap // 2, cap - 9
|
||||
|
||||
for early_sep, late_sep in (("。", "."), (".", "!"), ("?", "?"), ("!", ".")):
|
||||
text = "a" * early + early_sep + "b" * (late - early - 1) + late_sep + "c" * cap
|
||||
assert text[late] == late_sep and len(text) > cap # both boundaries inside the window
|
||||
kept = mem0_plugin._truncate_for_sync(text)
|
||||
assert kept == text[:late + 1], f"{early_sep!r} before {late_sep!r} cut back to {len(kept)} chars"
|
||||
assert kept.endswith(late_sep)
|
||||
|
||||
def test_a_boundary_only_in_the_first_third_still_falls_back_to_a_hard_cut(self):
|
||||
"""Unsegmented input keeps the whole window rather than a sliver of a sentence."""
|
||||
cap = mem0_plugin._SYNC_MSG_MAX_CHARS
|
||||
text = "a" * 10 + "." + "b" * (cap * 2)
|
||||
assert mem0_plugin._truncate_for_sync(text) == text[:cap]
|
||||
|
||||
def test_a_message_inside_the_cap_is_never_touched(self):
|
||||
cap = mem0_plugin._SYNC_MSG_MAX_CHARS
|
||||
text = "Fine. Nothing to trim here! Really?"
|
||||
assert len(text) <= cap and mem0_plugin._truncate_for_sync(text) == text
|
||||
|
||||
def test_sync_max_chars_config_raises_cap(self, monkeypatch, tmp_path):
|
||||
"""8k-token embedders should not be stuck at the 512-token default (#106235)."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
Reference in New Issue
Block a user