fix(memory/mem0): trim a synced message at the last sentence boundary, not the first separator kind

_truncate_for_sync documents "the last sentence boundary within max_len", but it
looped over separator KINDS and returned on the first kind that qualified. An early
"。" therefore outranked a "." 240 characters later, and in pure ASCII "." outranked
a later "!" or "?" purely because it comes first in the tuple.

With the 450-char default, "a"*200 + "。" + "b"*240 + "." + "c"*100 kept 201 of the
442 characters available: 241 characters the embedder would have accepted were
discarded, so any fact in the second half of the turn never reached extraction. The
add() call succeeds, so unlike #106235 nothing is logged — the turn is simply
remembered from its first sentence. Raising sync_max_chars widens the gap rather
than closing it.

Take the max over every separator instead, from a named tuple so the set is not
buried in the loop. ".\n" is dropped: its index can never exceed the bare "." it
starts with, so under a max it is unreachable. The first-third guard and the hard-cut
fallback for unsegmented input are unchanged.

Fixes #108868
This commit is contained in:
John Paul Soliva
2026-09-12 15:16:32 +09:00
committed by Teknium
parent e705dde6e4
commit aca4dc86a3
2 changed files with 41 additions and 4 deletions
+11 -4
View File
@@ -42,6 +42,13 @@ _DEFAULT_USER_ID = "hermes-user"
_SYNC_MSG_MAX_CHARS = 450
# Sentence ends recognized when trimming a synced message. Deliberately unordered:
# the LAST boundary of ANY kind wins, so one CJK stop early in a mixed-script turn
# cannot outrank a Latin stop near the end of the window. ``".\n"`` is not listed —
# its index can never exceed the bare ``"."`` it starts with.
_SYNC_SENTENCE_ENDS = ("。", "!", "?", ".", "!", "?")
def _truncate_for_sync(text: str, max_len: int = _SYNC_MSG_MAX_CHARS) -> str:
"""Cap a synced message at its last sentence boundary within ``max_len``.
@@ -52,10 +59,10 @@ def _truncate_for_sync(text: str, max_len: int = _SYNC_MSG_MAX_CHARS) -> str:
"""
if len(text) <= max_len:
return text
for sep in ("。", "!", "?", ".\n", ".", "!", "?"):
cut = text[:max_len].rfind(sep)
if cut > max_len // 3:
return text[:cut + 1]
window = text[:max_len]
cut = max(window.rfind(sep) for sep in _SYNC_SENTENCE_ENDS)
if cut > max_len // 3:
return text[:cut + 1]
return text[:max_len]
+30
View File
@@ -181,6 +181,36 @@ class TestSyncTurnTruncation:
assert len(sent[1]["content"]) <= mem0_plugin._SYNC_MSG_MAX_CHARS and sent[1]["content"].endswith(".")
assert provider._consecutive_failures == 0
def test_the_boundary_kept_is_the_last_one_in_the_window_whatever_its_script(self):
"""A mixed-script turn must not be cut back to an early CJK stop.
The trim exists to keep as much of the turn as the embedder can take; picking the
first separator KIND that qualifies instead of the last boundary threw away most of
the allowed window whenever two kinds appeared — an early ``。`` (or ``.``, which
outranks ``!``/``?``) beat a boundary 240 characters later, so the facts stated in
the rest of the message never reached extraction.
"""
cap = mem0_plugin._SYNC_MSG_MAX_CHARS
early, late = cap // 2, cap - 9
for early_sep, late_sep in (("。", "."), (".", "!"), ("?", "?"), ("!", ".")):
text = "a" * early + early_sep + "b" * (late - early - 1) + late_sep + "c" * cap
assert text[late] == late_sep and len(text) > cap # both boundaries inside the window
kept = mem0_plugin._truncate_for_sync(text)
assert kept == text[:late + 1], f"{early_sep!r} before {late_sep!r} cut back to {len(kept)} chars"
assert kept.endswith(late_sep)
def test_a_boundary_only_in_the_first_third_still_falls_back_to_a_hard_cut(self):
"""Unsegmented input keeps the whole window rather than a sliver of a sentence."""
cap = mem0_plugin._SYNC_MSG_MAX_CHARS
text = "a" * 10 + "." + "b" * (cap * 2)
assert mem0_plugin._truncate_for_sync(text) == text[:cap]
def test_a_message_inside_the_cap_is_never_touched(self):
cap = mem0_plugin._SYNC_MSG_MAX_CHARS
text = "Fine. Nothing to trim here! Really?"
assert len(text) <= cap and mem0_plugin._truncate_for_sync(text) == text
def test_sync_max_chars_config_raises_cap(self, monkeypatch, tmp_path):
"""8k-token embedders should not be stuck at the 512-token default (#106235)."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))