From d28938d3da56657a1d77412068d64c7fb520a3ca Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:11:49 -0700 Subject: [PATCH] fix(computer-use): screenshot dedup forgets its last frame at a compaction boundary The "screen unchanged" result points the model at its previous capture. After context compression that capture may be summarized away, so the note would refer to pixels no longer in context. Mirror read_file's reset_file_dedup: the compaction boundary (both the summary path and the codex app-server path) now clears the session's screenshot digest, and the first capture afterwards delivers the image again even when the screen is byte-identical. --- agent/conversation_compression.py | 11 ++++++++--- tests/tools/test_computer_use_screenshot_dedup.py | 10 ++++++++++ tools/computer_use/tool.py | 7 ++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 3bf4a23238..2cf2eaec0a 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -3040,14 +3040,19 @@ def _warn_summary_or_aux_fallback(agent: Any) -> None: ) -def _reset_read_dedup_caches(task_id: str, *, skills: bool = True) -> None: +def _reset_read_dedup_caches(task_id: str, *, session_id: str = "", skills: bool = True) -> None: """Advance the file-read (and skill_view) repeat-read dedup to a fresh generation after a boundary. The mtime map is kept: the first read of each unchanged key returns full content compaction may have omitted; later reads return stubs, and stub-hit counters restart at the same boundary (#84857). + The computer_use screenshot dedup is session-keyed and forgets its last frame for the same reason. """ with contextlib.suppress(Exception): from tools.file_tools_read_tracking import reset_file_dedup reset_file_dedup(task_id) + if session_id: + with contextlib.suppress(Exception): + from tools.computer_use.tool import reset_screenshot_dedup + reset_screenshot_dedup(session_id) if not skills: return with contextlib.suppress(Exception): @@ -3145,7 +3150,7 @@ def _finish_compaction_boundary( ) else: compressor._verify_compaction_cleared_threshold = True - _reset_read_dedup_caches(task_id) + _reset_read_dedup_caches(task_id, session_id=agent.session_id or "") return _compressed_est @@ -3832,7 +3837,7 @@ def _compress_context_via_codex_app_server( # armed until a later turn; minimal test engines may lack update_from_response. if hasattr(agent.context_compressor, "update_from_response"): _record_codex_app_server_usage(agent, result, messages=messages) - _reset_read_dedup_caches(task_id, skills=False) + _reset_read_dedup_caches(task_id, session_id=agent.session_id or "", skills=False) logger.info( "codex app-server compaction done: session=%s thread=%s turn=%s", _sid, getattr(result, "thread_id", None) or "", getattr(result, "turn_id", None) or "", diff --git a/tests/tools/test_computer_use_screenshot_dedup.py b/tests/tools/test_computer_use_screenshot_dedup.py index 6ba0653c32..5fd9dcb557 100644 --- a/tests/tools/test_computer_use_screenshot_dedup.py +++ b/tests/tools/test_computer_use_screenshot_dedup.py @@ -145,3 +145,13 @@ class TestScreenshotDedup: cu_tool._capture_response(_cap(seed=1), session_id="s1") cu_tool.release_computer_use_session("s1") assert _is_multimodal(cu_tool._capture_response(_cap(seed=1), session_id="s1")) + + def test_compaction_boundary_redelivers_pixels(self): + """The summary may have dropped the frame an "unchanged" note points at, so the first + capture after compaction must carry the image even when the screen is byte-identical.""" + from agent.conversation_compression import _reset_read_dedup_caches + + cu_tool._capture_response(_cap(seed=1), session_id="s1") + assert not _is_multimodal(cu_tool._capture_response(_cap(seed=1), session_id="s1")) + _reset_read_dedup_caches("task", session_id="s1") + assert _is_multimodal(cu_tool._capture_response(_cap(seed=1), session_id="s1")) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 58860cb996..f21239d3c3 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -116,13 +116,18 @@ def _screenshot_dedup_check(session_id: str, digest: str, target: Tuple[str, str return False def _reset_screenshot_dedup(session_id: Optional[str] = None) -> None: - """Forget dedup state (all sessions, or one).""" + """Forget dedup state (all sessions, or one scoped key).""" with _screenshot_dedup_lock: if session_id is None: _last_screenshot_state.clear() else: _last_screenshot_state.pop(session_id, None) +def reset_screenshot_dedup(session_id: str) -> None: + """Compaction boundary hook (mirrors ``reset_file_dedup``): the summary may have dropped the frame an + "unchanged" note would point at, so the next capture of this session must deliver pixels again.""" + _reset_screenshot_dedup(_scoped_sid(session_id)) + def _cua_permission_mode(session_id: str) -> str: """Map Hermes's approval bypass onto Cua's immutable mode; fails closed. Both identity namespaces are consulted (DB ``session_id`` and gateway ``session_key`` contextvar) or a gateway ``/yolo`` would be invisible here.