feat(computer_use): guide models from full-screen grabs to interactive lanes

Full-screen captures carry no element tree, so the CaptureResult now has a
'note' field surfaced in the tool summary telling the model to call
capture(app='<AppName>') or capture(app='desktop') when it needs to act on
what it sees. Schema description updated to distinguish app='screen'
(composited full-screen image) from app='desktop' (shell surface with
clickable elements); docs + regression tests (14, sabotage-verified) added.

tests/tools/test_computer_use_fullscreen_capture.py

@@ -0,0 +1,240 @@
+"""Full-screen vs desktop-shell capture routing (#60081, wallpaper-only bug).
+
+`capture(app='screen')` previously resolved to the OS shell/desktop window
+(Progman/WorkerW on Windows) via list_windows — the wallpaper + icons layer —
+so "screenshot my screen" always showed a bare desktop no matter what was
+actually displayed. cua-driver's `get_desktop_state` does a real composited
+full-screen grab; the `screen`/`fullscreen`/`all` sentinels now route there,
+while `desktop` keeps the shell-window lane (with clickable elements).
+
+Salvaged from @2ndNatureAI's PR #60081 (enumeration-hang bypass) and extended
+with the sentinel split + no-elements guidance note.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Any, Dict, List, Optional
+from unittest.mock import MagicMock
+
+import pytest
+
+# 8×8 transparent PNG — decodes cleanly for dimension sniffing.
+_PNG_B64 = (
+    "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
+    "NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
+)
+
+
+class _FakeSession:
+    """Records tool calls; serves get_config / set_config / get_desktop_state
+    and (for the desktop-shell path) list_windows / screenshot."""
+
+    def __init__(
+        self,
+        windows: Optional[List[Dict[str, Any]]] = None,
+        desktop_image: Optional[str] = _PNG_B64,
+        capture_scope: str = "window",
+    ):
+        self.calls: List[tuple] = []
+        self._windows = windows or []
+        self._desktop_image = desktop_image
+        self._scope = capture_scope
+        self.capabilities_discovered = True
+
+    def _has_tool(self, name: str) -> bool:
+        return name in {"get_desktop_state", "screenshot", "list_windows"}
+
+    def call_tool(self, name: str, args: Dict[str, Any], timeout: float = 30.0):
+        self.calls.append((name, dict(args or {})))
+        if name == "get_config":
+            return {
+                "data": "",
+                "images": [],
+                "structuredContent": {"capture_scope": self._scope},
+                "isError": False,
+            }
+        if name == "set_config":
+            self._scope = args["value"]
+            return {"data": "ok", "images": [], "structuredContent": None,
+                    "isError": False}
+        if name == "get_desktop_state":
+            images = [self._desktop_image] if self._desktop_image else []
+            return {
+                "data": "desktop state",
+                "images": images,
+                "image_mime_types": ["image/png"] if images else [],
+                "structuredContent": {"screen_width": 8, "screen_height": 8},
+                "isError": False,
+            }
+        if name == "list_windows":
+            return {
+                "data": "",
+                "images": [],
+                "structuredContent": {"windows": self._windows},
+                "isError": False,
+            }
+        if name == "screenshot":
+            return {
+                "data": "",
+                "images": [_PNG_B64],
+                "image_mime_types": ["image/png"],
+                "structuredContent": None,
+                "isError": False,
+            }
+        raise AssertionError(f"unexpected tool call: {name}")
+
+    def called(self, name: str) -> List[Dict[str, Any]]:
+        return [a for (n, a) in self.calls if n == name]
+
+
+def _make_backend(session: _FakeSession):
+    from tools.computer_use.cua_backend import CuaDriverBackend
+
+    backend = CuaDriverBackend()
+    backend._session = session
+    backend._session_id = "test-session"
+    return backend
+
+
+class TestFullScreenLane:
+    @pytest.mark.parametrize("sentinel", ["screen", "Screen", "fullscreen",
+                                          "full screen", "all"])
+    def test_screen_sentinels_route_to_get_desktop_state(self, sentinel):
+        session = _FakeSession()
+        backend = _make_backend(session)
+
+        cap = backend.capture(mode="som", app=sentinel)
+
+        assert session.called("get_desktop_state"), (
+            f"app={sentinel!r} must use the composited desktop lane"
+        )
+        assert not session.called("list_windows"), (
+            "full-screen capture must not enumerate windows (enumeration "
+            "can hang on Windows — trycua/cua#2110)"
+        )
+        assert cap.png_b64 == _PNG_B64
+        assert cap.elements == []
+        assert cap.app == "screen"
+
+    def test_full_screen_result_carries_interactive_lane_note(self):
+        session = _FakeSession()
+        backend = _make_backend(session)
+
+        cap = backend.capture(mode="vision", app="screen")
+
+        assert "no interactable elements" in cap.note
+        assert "capture(app='desktop')" in cap.note
+        assert "capture(app='<AppName>')" in cap.note
+
+    def test_capture_scope_switched_and_restored(self):
+        session = _FakeSession(capture_scope="window")
+        backend = _make_backend(session)
+
+        backend.capture(mode="vision", app="screen")
+
+        set_calls = session.called("set_config")
+        assert {"key": "capture_scope", "value": "desktop",
+                "session": "test-session"} in set_calls
+        assert {"key": "capture_scope", "value": "window",
+                "session": "test-session"} in set_calls
+        assert session._scope == "window", "prior scope must be restored"
+
+    def test_scope_untouched_when_already_desktop(self):
+        session = _FakeSession(capture_scope="desktop")
+        backend = _make_backend(session)
+
+        backend.capture(mode="vision", app="screen")
+
+        assert not session.called("set_config")
+
+    def test_imageless_desktop_state_fails_closed_with_guidance(self):
+        session = _FakeSession(desktop_image=None)
+        backend = _make_backend(session)
+
+        cap = backend.capture(mode="vision", app="screen")
+
+        assert cap.png_b64 is None
+        assert "get_desktop_state returned no image" in cap.window_title
+
+    def test_dimensions_come_from_decoded_image(self):
+        session = _FakeSession()
+        backend = _make_backend(session)
+
+        cap = backend.capture(mode="vision", app="screen")
+
+        # The 8×8 PNG's real dimensions win over structuredContent.
+        assert (cap.width, cap.height) == (8, 8)
+        assert cap.png_bytes_len == len(base64.b64decode(_PNG_B64))
+
+    def test_exact_pid_window_target_bypasses_full_screen_lane(self):
+        session = _FakeSession()
+        backend = _make_backend(session)
+
+        backend.capture(mode="vision", app="screen", pid=123, window_id=456)
+
+        assert not session.called("get_desktop_state"), (
+            "an exact pid/window target must win over the app sentinel"
+        )
+
+
+class TestDesktopShellLane:
+    _PROGMAN = {
+        "app_name": "Progman",
+        "title": "Program Manager",
+        "pid": 100,
+        "window_id": 1,
+        "off_screen": False,
+        "z_index": 0,
+    }
+
+    def test_desktop_sentinel_keeps_shell_window_lane(self):
+        session = _FakeSession(windows=[self._PROGMAN])
+        backend = _make_backend(session)
+
+        cap = backend.capture(mode="vision", app="desktop")
+
+        assert session.called("list_windows"), (
+            "app='desktop' must still resolve the shell window so desktop "
+            "icons stay clickable"
+        )
+        assert not session.called("get_desktop_state")
+        assert backend._active_pid == 100
+        assert cap.note == ""
+
+    def test_desktop_sentinel_without_shell_window_fails_with_guidance(self):
+        session = _FakeSession(windows=[{
+            "app_name": "Notepad", "title": "Untitled", "pid": 7,
+            "window_id": 9, "off_screen": False, "z_index": 1,
+        }])
+        backend = _make_backend(session)
+
+        cap = backend.capture(mode="vision", app="desktop")
+
+        assert cap.png_b64 is None
+        assert "no desktop/shell window found" in cap.window_title
+
+
+class TestNoteInSummary:
+    def test_capture_response_appends_note_line(self, tmp_path, monkeypatch):
+        import hermes_constants
+        from tools.computer_use.backend import CaptureResult
+        from tools.computer_use import tool as cu_tool
+
+        monkeypatch.setattr(hermes_constants, "get_hermes_dir",
+                            lambda *a, **k: tmp_path)
+        monkeypatch.setattr(cu_tool, "_should_route_through_aux_vision",
+                            lambda: False)
+
+        cap = CaptureResult(
+            mode="vision", width=8, height=8, png_b64=_PNG_B64,
+            elements=[], app="screen",
+            window_title="Full screen (composited)",
+            png_bytes_len=len(base64.b64decode(_PNG_B64)),
+            note="full-screen capture has no interactable elements; "
+                 "call capture(app='desktop') for the desktop shell",
+        )
+        result = cu_tool._capture_response(cap)
+
+        text = str(result)
+        assert "full-screen capture has no interactable elements" in text

tools/computer_use/backend.py

@@ -65,6 +65,10 @@ class CaptureResult:
     # When None, downstream consumers fall back to base64-prefix
     # sniffing for back-compat with older drivers.
     image_mime_type: Optional[str] = None
+    # Optional guidance appended to the human-readable summary — used by
+    # capture lanes that intentionally return no elements (e.g. full-screen
+    # composited grabs) to tell the model how to reach an interactive lane.
+    note: str = ""
 
 
 @dataclass

tools/computer_use/schema.py

@@ -87,12 +87,12 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = {
                     "Optional. Limit capture/action to a specific app "
                     "(by name, e.g. 'Safari', or bundle ID, "
                     "'com.apple.Safari'). If omitted, operates on the "
-                    "frontmost app's window. Pass app='screen' (or "
-                    "'desktop') to capture the OS desktop/shell surface — "
-                    "e.g. to see the wallpaper or click the taskbar. Note: "
-                    "capture is per-window; a single image cannot span "
-                    "multiple monitors, so on a multi-screen setup capture "
-                    "one window or display at a time."
+                    "frontmost app's window. Pass app='screen' to capture "
+                    "everything currently displayed (a composited "
+                    "full-screen grab; image only, no clickable elements). "
+                    "Pass app='desktop' to target the OS desktop/shell "
+                    "surface itself (wallpaper, desktop icons, taskbar) "
+                    "with its clickable elements."
                 ),
             },
             "pid": {

tools/computer_use/tool.py

@@ -1218,6 +1218,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
         summary_lines.append(
             f"  (shareable screenshot saved to {screenshot_path})"
         )
+    if cap.note:
+        summary_lines.append(f"  ({cap.note})")
     if elements_file:
         summary_lines.append(
             f"  (full element tree with untruncated labels saved to "

website/docs/user-guide/features/computer-use.md

@@ -301,6 +301,17 @@ you the saved file's path instead.
 Only the 20 most recent capture files are kept, and screenshots are never
 sent automatically — only when you ask for one.
 
+### Whole screen vs. desktop surface
+
+"Screenshot my screen" captures **everything currently displayed** — a
+composited grab of all visible windows, like pressing PrtScn. This image has
+no clickable elements, so to *act* on something in it the agent re-captures
+the specific app.
+
+Asking for the **desktop** instead targets the OS shell surface itself —
+wallpaper, desktop icons, taskbar — with its clickable elements, so requests
+like "open the Recycle Bin on my desktop" still work.
+
 ## Provider compatibility
 
 | Provider | Vision? | Works? | Notes |