fix(agent): preserve tool-output dedup state across context compaction (#84857)
This commit is contained in:
@@ -5574,9 +5574,13 @@ def compress_context(
|
||||
else:
|
||||
agent.context_compressor._verify_compaction_cleared_threshold = True
|
||||
|
||||
# Clear the file-read dedup cache. After compression the original
|
||||
# read content is summarised away — if the model re-reads the same
|
||||
# file it needs the full content, not a "file unchanged" stub.
|
||||
# Reset the file-read dedup stub-hit counters. The dedup mtime map
|
||||
# is preserved on purpose: a file that hasn't changed since the task
|
||||
# last read it still returns the lightweight "unchanged" stub after
|
||||
# compression, instead of re-sending the full content — which would
|
||||
# re-bloat the context we just reclaimed and defeat dedup exactly
|
||||
# when long sessions need it most (issue #84857). Only the stub-hit
|
||||
# counters reset so the 2-stub hard block restarts fresh.
|
||||
try:
|
||||
from tools.file_tools import reset_file_dedup
|
||||
reset_file_dedup(task_id)
|
||||
|
||||
@@ -591,10 +591,12 @@ class TestDedupStubLoopGuard(unittest.TestCase):
|
||||
|
||||
reset_file_dedup("loop")
|
||||
|
||||
# Fresh session — real read, no stub, no block
|
||||
# Post-compression: block counters cleared — the unchanged file
|
||||
# returns the lightweight dedup stub (dedup map survives), with
|
||||
# no error and no hard block.
|
||||
r4 = json.loads(read_file_tool(self._tmpfile, task_id="loop"))
|
||||
self.assertNotIn("error", r4)
|
||||
self.assertNotIn("dedup", r4)
|
||||
self.assertTrue(r4.get("dedup"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -602,8 +604,9 @@ class TestDedupStubLoopGuard(unittest.TestCase):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDedupResetOnCompression(unittest.TestCase):
|
||||
"""reset_file_dedup should clear the dedup cache so post-compression
|
||||
reads return full content."""
|
||||
"""reset_file_dedup should preserve the dedup mtime map so
|
||||
post-compression reads of unchanged files still return the lightweight
|
||||
stub (issue #84857)."""
|
||||
|
||||
def setUp(self):
|
||||
_read_tracker.clear()
|
||||
@@ -621,8 +624,8 @@ class TestDedupResetOnCompression(unittest.TestCase):
|
||||
pass
|
||||
|
||||
@patch("tools.file_tools._get_file_ops")
|
||||
def test_reset_clears_dedup(self, mock_ops):
|
||||
"""After reset_file_dedup, the same read returns full content."""
|
||||
def test_reset_preserves_dedup(self, mock_ops):
|
||||
"""After reset_file_dedup, the same read still returns the stub."""
|
||||
mock_ops.return_value = _make_fake_ops(
|
||||
content="original content\n", file_size=18,
|
||||
)
|
||||
@@ -636,10 +639,11 @@ class TestDedupResetOnCompression(unittest.TestCase):
|
||||
# Simulate compression
|
||||
reset_file_dedup("comp")
|
||||
|
||||
# Read again — should get full content
|
||||
# Read again — unchanged file still dedups: no full re-send, so
|
||||
# post-compaction re-reads don't re-inject the same content
|
||||
r_post = json.loads(read_file_tool(self._tmpfile, task_id="comp"))
|
||||
self.assertNotEqual(r_post.get("dedup"), True,
|
||||
"Post-compression read should return full content")
|
||||
self.assertEqual(r_post.get("dedup"), True,
|
||||
"Post-compression read of unchanged file should stub")
|
||||
|
||||
|
||||
@patch("tools.file_tools._get_file_ops")
|
||||
@@ -655,13 +659,12 @@ class TestDedupResetOnCompression(unittest.TestCase):
|
||||
|
||||
reset_file_dedup("loop")
|
||||
|
||||
# 3rd read — counter should still be at 2 from before reset
|
||||
# (dedup was hit for read 2, but consecutive counter was 1 for that)
|
||||
# After reset, this read goes through full path, incrementing to 2
|
||||
# 3rd read — still deduped (lightweight stub), not blocked
|
||||
r3 = json.loads(read_file_tool(self._tmpfile, task_id="loop"))
|
||||
# Should NOT be blocked or warned — counter restarted since dedup
|
||||
# intercepted reads before they reached the counter
|
||||
self.assertNotIn("error", r3)
|
||||
self.assertTrue(r3.get("dedup"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+16
-14
@@ -1132,9 +1132,11 @@ _file_ops_cache: dict = {}
|
||||
# "consecutive": how many times that exact call has been repeated in a row
|
||||
# "read_history": set of (path, offset, limit) tuples for get_read_files_summary
|
||||
# "dedup": dict mapping (resolved_path, offset, limit) → mtime float
|
||||
# Used to skip re-reads of unchanged files. Reset on
|
||||
# context compression (the original content is summarised
|
||||
# away so the model needs the full content again).
|
||||
# Used to skip re-reads of unchanged files. Survives
|
||||
# context compression: only the per-key stub-hit
|
||||
# counters are cleared on compression, so unchanged
|
||||
# files keep returning the lightweight stub instead of
|
||||
# re-sending full content (issue #84857).
|
||||
# "read_timestamps": dict mapping resolved_path → modification-time float
|
||||
# recorded when the file was last read (or written) by
|
||||
# this task. Used by write_file and patch to detect
|
||||
@@ -2015,28 +2017,28 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str =
|
||||
|
||||
|
||||
def reset_file_dedup(task_id: str = None):
|
||||
"""Clear the deduplication cache for file reads.
|
||||
"""Reset the read-dedup stub-hit counters after context compression.
|
||||
|
||||
Called after context compression — the original read content has been
|
||||
summarised away, so the model needs the full content if it reads the
|
||||
same file again. Without this, reads after compression would return
|
||||
a "file unchanged" stub pointing at content that no longer exists in
|
||||
context.
|
||||
Called after context compression. The per-key ``dedup`` mtime map is
|
||||
PRESERVED: a file that has not changed on disk since the task last read
|
||||
it still returns the lightweight "unchanged" stub instead of re-sending
|
||||
the full content — this is what stops long-session token usage from
|
||||
re-bloating the context we just reclaimed after every compaction
|
||||
(issue #84857). Only the per-key stub-hit counters (``dedup_hits``)
|
||||
are cleared, so the 2-stub hard block restarts fresh: a model that hits
|
||||
the stub right after compression is never blocked by hits accumulated
|
||||
before it.
|
||||
|
||||
Call with a task_id to clear just that task, or without to clear all.
|
||||
Call with a task_id to reset just that task, or without to reset all.
|
||||
"""
|
||||
with _read_tracker_lock:
|
||||
if task_id:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if task_data:
|
||||
if "dedup" in task_data:
|
||||
task_data["dedup"].clear()
|
||||
if "dedup_hits" in task_data:
|
||||
task_data["dedup_hits"].clear()
|
||||
else:
|
||||
for task_data in _read_tracker.values():
|
||||
if "dedup" in task_data:
|
||||
task_data["dedup"].clear()
|
||||
if "dedup_hits" in task_data:
|
||||
task_data["dedup_hits"].clear()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user