From aca4dc86a3e943b20227d95f297a1bccb65c005f Mon Sep 17 00:00:00 2001 From: John Paul Soliva Date: Sat, 12 Sep 2026 15:16:32 +0900 Subject: [PATCH] fix(memory/mem0): trim a synced message at the last sentence boundary, not the first separator kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _truncate_for_sync documents "the last sentence boundary within max_len", but it looped over separator KINDS and returned on the first kind that qualified. An early "。" therefore outranked a "." 240 characters later, and in pure ASCII "." outranked a later "!" or "?" purely because it comes first in the tuple. With the 450-char default, "a"*200 + "。" + "b"*240 + "." + "c"*100 kept 201 of the 442 characters available: 241 characters the embedder would have accepted were discarded, so any fact in the second half of the turn never reached extraction. The add() call succeeds, so unlike #106235 nothing is logged — the turn is simply remembered from its first sentence. Raising sync_max_chars widens the gap rather than closing it. Take the max over every separator instead, from a named tuple so the set is not buried in the loop. ".\n" is dropped: its index can never exceed the bare "." it starts with, so under a max it is unreachable. The first-third guard and the hard-cut fallback for unsegmented input are unchanged. Fixes #108868 --- plugins/memory/mem0/__init__.py | 15 ++++++++++---- tests/plugins/memory/test_mem0_v3.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/plugins/memory/mem0/__init__.py b/plugins/memory/mem0/__init__.py index 8d41f71f40..d873167e0d 100644 --- a/plugins/memory/mem0/__init__.py +++ b/plugins/memory/mem0/__init__.py @@ -42,6 +42,13 @@ _DEFAULT_USER_ID = "hermes-user" _SYNC_MSG_MAX_CHARS = 450 +# Sentence ends recognized when trimming a synced message. Deliberately unordered: +# the LAST boundary of ANY kind wins, so one CJK stop early in a mixed-script turn +# cannot outrank a Latin stop near the end of the window. ``".\n"`` is not listed — +# its index can never exceed the bare ``"."`` it starts with. +_SYNC_SENTENCE_ENDS = ("。", "！", "？", ".", "!", "?") + + def _truncate_for_sync(text: str, max_len: int = _SYNC_MSG_MAX_CHARS) -> str: """Cap a synced message at its last sentence boundary within ``max_len``. @@ -52,10 +59,10 @@ def _truncate_for_sync(text: str, max_len: int = _SYNC_MSG_MAX_CHARS) -> str: """ if len(text) <= max_len: return text - for sep in ("。", "！", "？", ".\n", ".", "!", "?"): - cut = text[:max_len].rfind(sep) - if cut > max_len // 3: - return text[:cut + 1] + window = text[:max_len] + cut = max(window.rfind(sep) for sep in _SYNC_SENTENCE_ENDS) + if cut > max_len // 3: + return text[:cut + 1] return text[:max_len] diff --git a/tests/plugins/memory/test_mem0_v3.py b/tests/plugins/memory/test_mem0_v3.py index c1a1160143..481cb399a8 100644 --- a/tests/plugins/memory/test_mem0_v3.py +++ b/tests/plugins/memory/test_mem0_v3.py @@ -181,6 +181,36 @@ class TestSyncTurnTruncation: assert len(sent[1]["content"]) <= mem0_plugin._SYNC_MSG_MAX_CHARS and sent[1]["content"].endswith(".") assert provider._consecutive_failures == 0 + def test_the_boundary_kept_is_the_last_one_in_the_window_whatever_its_script(self): + """A mixed-script turn must not be cut back to an early CJK stop. + + The trim exists to keep as much of the turn as the embedder can take; picking the + first separator KIND that qualifies instead of the last boundary threw away most of + the allowed window whenever two kinds appeared — an early ``。`` (or ``.``, which + outranks ``!``/``?``) beat a boundary 240 characters later, so the facts stated in + the rest of the message never reached extraction. + """ + cap = mem0_plugin._SYNC_MSG_MAX_CHARS + early, late = cap // 2, cap - 9 + + for early_sep, late_sep in (("。", "."), (".", "!"), ("？", "?"), ("！", ".")): + text = "a" * early + early_sep + "b" * (late - early - 1) + late_sep + "c" * cap + assert text[late] == late_sep and len(text) > cap # both boundaries inside the window + kept = mem0_plugin._truncate_for_sync(text) + assert kept == text[:late + 1], f"{early_sep!r} before {late_sep!r} cut back to {len(kept)} chars" + assert kept.endswith(late_sep) + + def test_a_boundary_only_in_the_first_third_still_falls_back_to_a_hard_cut(self): + """Unsegmented input keeps the whole window rather than a sliver of a sentence.""" + cap = mem0_plugin._SYNC_MSG_MAX_CHARS + text = "a" * 10 + "." + "b" * (cap * 2) + assert mem0_plugin._truncate_for_sync(text) == text[:cap] + + def test_a_message_inside_the_cap_is_never_touched(self): + cap = mem0_plugin._SYNC_MSG_MAX_CHARS + text = "Fine. Nothing to trim here! Really?" + assert len(text) <= cap and mem0_plugin._truncate_for_sync(text) == text + def test_sync_max_chars_config_raises_cap(self, monkeypatch, tmp_path): """8k-token embedders should not be stuck at the 512-token default (#106235).""" monkeypatch.setenv("HERMES_HOME", str(tmp_path))