diff --git a/tests/tools/test_computer_use_fullscreen_capture.py b/tests/tools/test_computer_use_fullscreen_capture.py new file mode 100644 index 0000000000..d419d8b456 --- /dev/null +++ b/tests/tools/test_computer_use_fullscreen_capture.py @@ -0,0 +1,240 @@ +"""Full-screen vs desktop-shell capture routing (#60081, wallpaper-only bug). + +`capture(app='screen')` previously resolved to the OS shell/desktop window +(Progman/WorkerW on Windows) via list_windows — the wallpaper + icons layer — +so "screenshot my screen" always showed a bare desktop no matter what was +actually displayed. cua-driver's `get_desktop_state` does a real composited +full-screen grab; the `screen`/`fullscreen`/`all` sentinels now route there, +while `desktop` keeps the shell-window lane (with clickable elements). + +Salvaged from @2ndNatureAI's PR #60081 (enumeration-hang bypass) and extended +with the sentinel split + no-elements guidance note. +""" + +from __future__ import annotations + +import base64 +from typing import Any, Dict, List, Optional +from unittest.mock import MagicMock + +import pytest + +# 8×8 transparent PNG — decodes cleanly for dimension sniffing. +_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG" + "NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg==" +) + + +class _FakeSession: + """Records tool calls; serves get_config / set_config / get_desktop_state + and (for the desktop-shell path) list_windows / screenshot.""" + + def __init__( + self, + windows: Optional[List[Dict[str, Any]]] = None, + desktop_image: Optional[str] = _PNG_B64, + capture_scope: str = "window", + ): + self.calls: List[tuple] = [] + self._windows = windows or [] + self._desktop_image = desktop_image + self._scope = capture_scope + self.capabilities_discovered = True + + def _has_tool(self, name: str) -> bool: + return name in {"get_desktop_state", "screenshot", "list_windows"} + + def call_tool(self, name: str, args: Dict[str, Any], timeout: float = 30.0): + self.calls.append((name, dict(args or {}))) + if name == "get_config": + return { + "data": "", + "images": [], + "structuredContent": {"capture_scope": self._scope}, + "isError": False, + } + if name == "set_config": + self._scope = args["value"] + return {"data": "ok", "images": [], "structuredContent": None, + "isError": False} + if name == "get_desktop_state": + images = [self._desktop_image] if self._desktop_image else [] + return { + "data": "desktop state", + "images": images, + "image_mime_types": ["image/png"] if images else [], + "structuredContent": {"screen_width": 8, "screen_height": 8}, + "isError": False, + } + if name == "list_windows": + return { + "data": "", + "images": [], + "structuredContent": {"windows": self._windows}, + "isError": False, + } + if name == "screenshot": + return { + "data": "", + "images": [_PNG_B64], + "image_mime_types": ["image/png"], + "structuredContent": None, + "isError": False, + } + raise AssertionError(f"unexpected tool call: {name}") + + def called(self, name: str) -> List[Dict[str, Any]]: + return [a for (n, a) in self.calls if n == name] + + +def _make_backend(session: _FakeSession): + from tools.computer_use.cua_backend import CuaDriverBackend + + backend = CuaDriverBackend() + backend._session = session + backend._session_id = "test-session" + return backend + + +class TestFullScreenLane: + @pytest.mark.parametrize("sentinel", ["screen", "Screen", "fullscreen", + "full screen", "all"]) + def test_screen_sentinels_route_to_get_desktop_state(self, sentinel): + session = _FakeSession() + backend = _make_backend(session) + + cap = backend.capture(mode="som", app=sentinel) + + assert session.called("get_desktop_state"), ( + f"app={sentinel!r} must use the composited desktop lane" + ) + assert not session.called("list_windows"), ( + "full-screen capture must not enumerate windows (enumeration " + "can hang on Windows — trycua/cua#2110)" + ) + assert cap.png_b64 == _PNG_B64 + assert cap.elements == [] + assert cap.app == "screen" + + def test_full_screen_result_carries_interactive_lane_note(self): + session = _FakeSession() + backend = _make_backend(session) + + cap = backend.capture(mode="vision", app="screen") + + assert "no interactable elements" in cap.note + assert "capture(app='desktop')" in cap.note + assert "capture(app='')" in cap.note + + def test_capture_scope_switched_and_restored(self): + session = _FakeSession(capture_scope="window") + backend = _make_backend(session) + + backend.capture(mode="vision", app="screen") + + set_calls = session.called("set_config") + assert {"key": "capture_scope", "value": "desktop", + "session": "test-session"} in set_calls + assert {"key": "capture_scope", "value": "window", + "session": "test-session"} in set_calls + assert session._scope == "window", "prior scope must be restored" + + def test_scope_untouched_when_already_desktop(self): + session = _FakeSession(capture_scope="desktop") + backend = _make_backend(session) + + backend.capture(mode="vision", app="screen") + + assert not session.called("set_config") + + def test_imageless_desktop_state_fails_closed_with_guidance(self): + session = _FakeSession(desktop_image=None) + backend = _make_backend(session) + + cap = backend.capture(mode="vision", app="screen") + + assert cap.png_b64 is None + assert "get_desktop_state returned no image" in cap.window_title + + def test_dimensions_come_from_decoded_image(self): + session = _FakeSession() + backend = _make_backend(session) + + cap = backend.capture(mode="vision", app="screen") + + # The 8×8 PNG's real dimensions win over structuredContent. + assert (cap.width, cap.height) == (8, 8) + assert cap.png_bytes_len == len(base64.b64decode(_PNG_B64)) + + def test_exact_pid_window_target_bypasses_full_screen_lane(self): + session = _FakeSession() + backend = _make_backend(session) + + backend.capture(mode="vision", app="screen", pid=123, window_id=456) + + assert not session.called("get_desktop_state"), ( + "an exact pid/window target must win over the app sentinel" + ) + + +class TestDesktopShellLane: + _PROGMAN = { + "app_name": "Progman", + "title": "Program Manager", + "pid": 100, + "window_id": 1, + "off_screen": False, + "z_index": 0, + } + + def test_desktop_sentinel_keeps_shell_window_lane(self): + session = _FakeSession(windows=[self._PROGMAN]) + backend = _make_backend(session) + + cap = backend.capture(mode="vision", app="desktop") + + assert session.called("list_windows"), ( + "app='desktop' must still resolve the shell window so desktop " + "icons stay clickable" + ) + assert not session.called("get_desktop_state") + assert backend._active_pid == 100 + assert cap.note == "" + + def test_desktop_sentinel_without_shell_window_fails_with_guidance(self): + session = _FakeSession(windows=[{ + "app_name": "Notepad", "title": "Untitled", "pid": 7, + "window_id": 9, "off_screen": False, "z_index": 1, + }]) + backend = _make_backend(session) + + cap = backend.capture(mode="vision", app="desktop") + + assert cap.png_b64 is None + assert "no desktop/shell window found" in cap.window_title + + +class TestNoteInSummary: + def test_capture_response_appends_note_line(self, tmp_path, monkeypatch): + import hermes_constants + from tools.computer_use.backend import CaptureResult + from tools.computer_use import tool as cu_tool + + monkeypatch.setattr(hermes_constants, "get_hermes_dir", + lambda *a, **k: tmp_path) + monkeypatch.setattr(cu_tool, "_should_route_through_aux_vision", + lambda: False) + + cap = CaptureResult( + mode="vision", width=8, height=8, png_b64=_PNG_B64, + elements=[], app="screen", + window_title="Full screen (composited)", + png_bytes_len=len(base64.b64decode(_PNG_B64)), + note="full-screen capture has no interactable elements; " + "call capture(app='desktop') for the desktop shell", + ) + result = cu_tool._capture_response(cap) + + text = str(result) + assert "full-screen capture has no interactable elements" in text diff --git a/tools/computer_use/backend.py b/tools/computer_use/backend.py index 9e233459f7..df5d02ffc4 100644 --- a/tools/computer_use/backend.py +++ b/tools/computer_use/backend.py @@ -65,6 +65,10 @@ class CaptureResult: # When None, downstream consumers fall back to base64-prefix # sniffing for back-compat with older drivers. image_mime_type: Optional[str] = None + # Optional guidance appended to the human-readable summary — used by + # capture lanes that intentionally return no elements (e.g. full-screen + # composited grabs) to tell the model how to reach an interactive lane. + note: str = "" @dataclass diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index 1c0d322e41..35b9b5c86c 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -87,12 +87,12 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "Optional. Limit capture/action to a specific app " "(by name, e.g. 'Safari', or bundle ID, " "'com.apple.Safari'). If omitted, operates on the " - "frontmost app's window. Pass app='screen' (or " - "'desktop') to capture the OS desktop/shell surface — " - "e.g. to see the wallpaper or click the taskbar. Note: " - "capture is per-window; a single image cannot span " - "multiple monitors, so on a multi-screen setup capture " - "one window or display at a time." + "frontmost app's window. Pass app='screen' to capture " + "everything currently displayed (a composited " + "full-screen grab; image only, no clickable elements). " + "Pass app='desktop' to target the OS desktop/shell " + "surface itself (wallpaper, desktop icons, taskbar) " + "with its clickable elements." ), }, "pid": { diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index ab2824b8d0..221ae26262 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -1218,6 +1218,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME summary_lines.append( f" (shareable screenshot saved to {screenshot_path})" ) + if cap.note: + summary_lines.append(f" ({cap.note})") if elements_file: summary_lines.append( f" (full element tree with untruncated labels saved to " diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index 54fc0f8012..51b0552c84 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -301,6 +301,17 @@ you the saved file's path instead. Only the 20 most recent capture files are kept, and screenshots are never sent automatically — only when you ask for one. +### Whole screen vs. desktop surface + +"Screenshot my screen" captures **everything currently displayed** — a +composited grab of all visible windows, like pressing PrtScn. This image has +no clickable elements, so to *act* on something in it the agent re-captures +the specific app. + +Asking for the **desktop** instead targets the OS shell surface itself — +wallpaper, desktop icons, taskbar — with its clickable elements, so requests +like "open the Recycle Bin on my desktop" still work. + ## Provider compatibility | Provider | Vision? | Works? | Notes |