fix(compression): stamp _DB_PERSISTED_MARKER after in-place batch compaction commit (#98450)
compress() returns marker-swept copies (_strip_persistence_markers, #57491); the in-place branch committed them via archive_and_compact() but never stamped the persistence marker, so the next _persist_session -> _flush_messages_to_session_db_unlocked walk re-INSERTed the whole post-compaction transcript (live set regrew ~58K -> ~512K tokens). Centralize the post-commit contract in a shared helper, stamp_db_persisted_markers(), used by all three archive_and_compact callers: the in-place batch commit (previously missing), the micro-compaction sync, and the proactive tool-result prune. Co-authored-by: liuhao1024 <sunsky.lau@gmail.com>
This commit is contained in:
@@ -343,6 +343,33 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None:
|
||||
msg.pop(_DB_PERSISTED_MARKER, None)
|
||||
|
||||
|
||||
def stamp_db_persisted_markers(messages: List[Dict[str, Any]]) -> None:
|
||||
"""Fulfil the post-commit contract of ``SessionDB.archive_and_compact()``.
|
||||
|
||||
``archive_and_compact()`` atomically soft-archives the previous active
|
||||
rows and inserts *messages* as the new active set — after it returns,
|
||||
every dict in *messages* IS durably stored. Stamp ``_DB_PERSISTED_MARKER``
|
||||
on those exact dict instances so the append-only flush
|
||||
(``_persist_session`` → ``_flush_messages_to_session_db_unlocked``)
|
||||
skips them instead of re-INSERTing the whole compacted transcript.
|
||||
|
||||
This is the single stamp site for ALL ``archive_and_compact`` callers
|
||||
(in-place batch commit, micro-compaction sync, proactive prune). The
|
||||
marker must land on the dicts the caller actually keeps as the live
|
||||
message list: ``compress()`` output is marker-swept by design
|
||||
(``_strip_persistence_markers``, #57491 — the sweep protects the
|
||||
ROTATION flush to a child session), so a committed in-place set that
|
||||
is returned to the caller unstamped is re-written as "new" by the next
|
||||
persist walk and the live transcript doubles on every compaction
|
||||
(#98450: ~58K → ~512K tokens). Call this ONLY after the commit
|
||||
succeeded — an unstamped dict after a failed commit is correct
|
||||
(the flush then durably writes it).
|
||||
"""
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict):
|
||||
msg[_DB_PERSISTED_MARKER] = True
|
||||
|
||||
|
||||
def _prune_stale_reasoning_replay(messages: List[Dict[str, Any]]) -> int:
|
||||
"""Strip stale per-turn replay items (``codex_reasoning_items``) from
|
||||
assistant messages that belong to turns older than the active one.
|
||||
@@ -4269,9 +4296,9 @@ class ContextCompressor(ContextEngine):
|
||||
exc,
|
||||
)
|
||||
return messages, 0
|
||||
for msg in pruned_msgs:
|
||||
if isinstance(msg, dict):
|
||||
msg[_DB_PERSISTED_MARKER] = True
|
||||
# Shared post-commit contract with the in-place batch commit and
|
||||
# the micro-compaction sync (#98450) — one stamp site for the class.
|
||||
stamp_db_persisted_markers(pruned_msgs)
|
||||
self._proactive_prune_rearm_tokens = next_rearm_tokens
|
||||
return pruned_msgs, pruned_count
|
||||
|
||||
@@ -7260,9 +7287,9 @@ This compaction should PRIORITISE preserving all information related to the focu
|
||||
return
|
||||
try:
|
||||
session_db.archive_and_compact(session_id, compacted_messages)
|
||||
for msg in compacted_messages:
|
||||
if isinstance(msg, dict):
|
||||
msg[_DB_PERSISTED_MARKER] = True
|
||||
# Shared post-commit contract with the in-place batch commit and
|
||||
# the proactive prune (#98450) — one stamp site for the class.
|
||||
stamp_db_persisted_markers(compacted_messages)
|
||||
except Exception:
|
||||
logger.info(
|
||||
"Micro-compaction DB sync failed — resume will double-load "
|
||||
|
||||
@@ -4169,6 +4169,22 @@ def compress_context(
|
||||
lock_holder=_lock_holder,
|
||||
)
|
||||
split_status = "in_place_committed"
|
||||
# Post-commit contract (#98450, mirrors
|
||||
# _sync_micro_compact_to_db): archive_and_compact just
|
||||
# durably wrote every dict in `compressed` as the new
|
||||
# active set, but compress() returned marker-swept COPIES
|
||||
# (_strip_persistence_markers, #57491). These exact dict
|
||||
# instances become the live message list the caller keeps,
|
||||
# so without the stamp the next _persist_session →
|
||||
# _flush_messages_to_session_db_unlocked walk treats the
|
||||
# whole compacted transcript as unpersisted and re-INSERTs
|
||||
# it — the live set doubles on every compaction
|
||||
# (~58K → ~512K tokens in production).
|
||||
from agent.context_compressor import (
|
||||
stamp_db_persisted_markers,
|
||||
)
|
||||
|
||||
stamp_db_persisted_markers(compressed)
|
||||
# Reset the flush identity set so the next turn's appends are
|
||||
# diffed against the COMPACTED transcript: the compacted dicts
|
||||
# are passed as conversation_history next turn and skipped by
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Regression test for #98450: in-place batch compaction commit must stamp
|
||||
_DB_PERSISTED_MARKER on the committed dicts.
|
||||
|
||||
``compress()`` returns marker-swept COPIES (``_strip_persistence_markers``,
|
||||
#57491 — the sweep protects the ROTATION flush to a child session). The
|
||||
in-place branch commits those copies durably via
|
||||
``SessionDB.archive_and_compact()`` and hands the SAME dict instances back
|
||||
as the live message list. Without a post-commit stamp, the next
|
||||
``_persist_session`` → ``_flush_messages_to_session_db_unlocked`` walk sees
|
||||
every compacted row as unpersisted and re-INSERTs the whole post-compaction
|
||||
transcript — the live set doubles on every compaction (production:
|
||||
~58K → ~512K tokens in two hours).
|
||||
|
||||
The healthy sibling ``ContextCompressor._sync_micro_compact_to_db`` already
|
||||
stamps after its ``archive_and_compact`` call; both now share
|
||||
``stamp_db_persisted_markers``.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _make_agent(session_db, session_id):
|
||||
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
model="test/model",
|
||||
quiet_mode=True,
|
||||
session_db=session_db,
|
||||
session_id=session_id,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent.compression_in_place = True
|
||||
|
||||
def _fake_compress(messages, current_tokens=None, focus_topic=None, force=False):
|
||||
# Mirrors the real compress() contract: marker-swept fresh dicts.
|
||||
return [
|
||||
{"role": "user", "content": "[CONTEXT COMPACTION] summary of prior turns"},
|
||||
{"role": "assistant", "content": "recent reply 1"},
|
||||
{"role": "user", "content": "follow-up q"},
|
||||
{"role": "assistant", "content": "recent reply 2"},
|
||||
]
|
||||
|
||||
agent.context_compressor.compress = _fake_compress
|
||||
agent.context_compressor._last_compress_aborted = False
|
||||
agent.context_compressor._last_summary_error = None
|
||||
agent.context_compressor.compression_count = 1
|
||||
return agent
|
||||
|
||||
|
||||
def _seed(db, sid, n=8):
|
||||
db.create_session(sid, "cli", model="test/model")
|
||||
for i in range(n):
|
||||
db.append_message(
|
||||
session_id=sid,
|
||||
role="user" if i % 2 == 0 else "assistant",
|
||||
content=f"seed msg {i}",
|
||||
)
|
||||
|
||||
|
||||
def _row_counts(db, sid):
|
||||
total = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE session_id = ?", (sid,)
|
||||
).fetchone()[0]
|
||||
active = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE session_id = ? AND active = 1", (sid,)
|
||||
).fetchone()[0]
|
||||
return total, active
|
||||
|
||||
|
||||
class TestInPlaceCommitPersistMarker:
|
||||
def test_post_commit_persist_does_not_reinsert_compacted_rows(self):
|
||||
"""In-place commit → persist walk: row counts stay stable (#98450)."""
|
||||
from hermes_state import SessionDB
|
||||
from agent.conversation_compression import compress_context
|
||||
from agent.context_compressor import _DB_PERSISTED_MARKER
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = SessionDB(db_path=Path(tmp) / "t.db")
|
||||
sid = "20260830_120000_marker"
|
||||
_seed(db, sid, n=8)
|
||||
agent = _make_agent(db, sid)
|
||||
agent._session_db_created = True
|
||||
agent._last_flushed_db_idx = 8
|
||||
|
||||
messages = [{"role": "user", "content": f"m{i}"} for i in range(8)]
|
||||
compressed, _sp = compress_context(
|
||||
agent, messages, approx_tokens=100_000, system_message="sys"
|
||||
)
|
||||
|
||||
# ── Precondition: the compacted set genuinely went through
|
||||
# archive_and_compact — the 8 seeded rows were soft-archived
|
||||
# (kept on disk, active=0) and the 4 compacted dicts were
|
||||
# inserted as the new active set. Without this the "no
|
||||
# duplicates" assertion below would pass vacuously on a path
|
||||
# that never committed anything.
|
||||
total, active = _row_counts(db, sid)
|
||||
assert active == len(compressed) == 4
|
||||
assert total == 8 + len(compressed) # archived seeds + active set
|
||||
archived = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE session_id = ? AND active = 0",
|
||||
(sid,),
|
||||
).fetchone()[0]
|
||||
assert archived == 8
|
||||
|
||||
# ── The fix: every committed dict instance carries the marker.
|
||||
# These are the exact instances the caller keeps as the live
|
||||
# message list, so the stamp must be on THEM (compress() output
|
||||
# copies), not on some other collection.
|
||||
for msg in compressed:
|
||||
assert msg.get(_DB_PERSISTED_MARKER) is True, (
|
||||
f"committed dict missing persistence marker: {msg.get('content')!r}"
|
||||
)
|
||||
|
||||
# ── The symptom: run the post-compaction persist walk twice
|
||||
# (turn finalize + close safety-net in production). The flush
|
||||
# must skip the already-durable compacted rows: no re-INSERT,
|
||||
# counts stable, zero duplicate active contents.
|
||||
agent._persist_session(compressed, conversation_history=None)
|
||||
agent._persist_session(compressed, conversation_history=None)
|
||||
total2, active2 = _row_counts(db, sid)
|
||||
dup_groups = db._conn.execute(
|
||||
"SELECT content, COUNT(*) c FROM messages "
|
||||
"WHERE session_id = ? AND active = 1 "
|
||||
"GROUP BY content HAVING c > 1",
|
||||
(sid,),
|
||||
).fetchall()
|
||||
assert (total2, active2) == (total, active), (
|
||||
f"post-commit persist re-INSERTed compacted rows: "
|
||||
f"total {total}->{total2}, active {active}->{active2}"
|
||||
)
|
||||
assert dup_groups == [], f"duplicate active rows: {dup_groups}"
|
||||
|
||||
def test_stamp_helper_shared_by_micro_compact_sync(self):
|
||||
"""The micro-compaction sync fulfils the same post-commit contract
|
||||
via the shared helper (class-of-bug guard, not a change detector:
|
||||
asserts the behavioral outcome — dicts stamped after a successful
|
||||
archive_and_compact — for the sibling call path)."""
|
||||
from hermes_state import SessionDB
|
||||
from agent.context_compressor import (
|
||||
ContextCompressor,
|
||||
_DB_PERSISTED_MARKER,
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = SessionDB(db_path=Path(tmp) / "m.db")
|
||||
sid = "20260830_120001_micro0"
|
||||
_seed(db, sid, n=4)
|
||||
compressor = ContextCompressor.__new__(ContextCompressor)
|
||||
compressor._session_db = db
|
||||
compressor._session_id = sid
|
||||
compacted = [
|
||||
{"role": "user", "content": "u"},
|
||||
{"role": "assistant", "content": "micro summary"},
|
||||
]
|
||||
compressor._sync_micro_compact_to_db(compacted)
|
||||
# Precondition: the commit really happened.
|
||||
total, active = _row_counts(db, sid)
|
||||
assert active == 2 and total == 6
|
||||
for msg in compacted:
|
||||
assert msg.get(_DB_PERSISTED_MARKER) is True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user