"""Capture side of the cua-driver backend: window discovery, capture-target selection and the capture()/list_windows()/list_apps()/focus_app() methods (mixed into ``CuaDriverBackend``). Logger name is kept as ``tools.computer_use.cua_backend`` so log-based tests and operators see one backend logger. """ from __future__ import annotations import base64 import logging import os import re import subprocess import sys from typing import Any, Dict, List, Optional, Tuple from tools.computer_use.backend import ActionResult, CaptureResult, UIElement from tools.computer_use.cua_backend_input import _BTF_UNSUPPORTED_MSG from tools.computer_use.cua_backend_parse import ( _apps_from_windows, _image_dimensions_from_bytes, _image_from_tool_result, _ingest_windows, _is_placeholder_id, _is_real_app_window, _parse_elements_from_structured, _parse_elements_from_tree, _parse_xprop_net_active_window, _positive_int, _split_tree_text, _windows_from_tool_result, _z_index_uninformative, ) logger = logging.getLogger("tools.computer_use.cua_backend") # Whole-screen intents: app="screen"/... -> composited `get_desktop_state` # (pixels only); app="desktop" -> the OS shell window via list_windows, WITH # interactable elements (desktop icons, taskbar). _FULL_SCREEN_SENTINELS = {"screen", "fullscreen", "full screen", "all"} _DESKTOP_SHELL_SENTINELS = {"desktop"} # Shell window identifiers (substring of app_name + title, case-insensitive). # Windows: Progman/WorkerW = desktop, Shell_TrayWnd = taskbar; macOS: Finder/Dock. _DESKTOP_WINDOW_NAMES = ( "progman", "workerw", "program manager", "shell_traywnd", "taskbar", "finder", "desktop", "dock", ) # Backdrop subset preferred over the taskbar when both are present. _DESKTOP_BACKDROP_NAMES = ("progman", "workerw", "program manager", "finder", "desktop") _WINDOW_TITLE_RE = re.compile(r'AXWindow\s+"([^"]+)"') _LEGACY_APP_LINE_RE = re.compile(r'(.+?)\s+\(pid\s+(\d+)\)') _NO_DESKTOP_WINDOW_MSG = ( "" ) _NO_APP_MATCH_MSG = ( "" ) _NO_DESKTOP_IMAGE_MSG = ( "" ) _FULL_SCREEN_NOTE = ( "full-screen capture has no interactable elements; to act on what you see, " "call capture(app='') for that app's clickable element list, or " "capture(app='desktop') for the desktop shell (wallpaper icons / taskbar) " "with elements" ) def _linux_x11_active_window_id() -> Optional[int]: """Best-effort read of ``_NET_ACTIVE_WINDOW`` via xprop. Never raises.""" if sys.platform != "linux" or not os.environ.get("DISPLAY"): return None try: proc = subprocess.run(["xprop", "-root", "_NET_ACTIVE_WINDOW"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2, check=False, stdin=subprocess.DEVNULL) except Exception: return None return _parse_xprop_net_active_window(proc.stdout or "") if proc.returncode == 0 else None def _select_capture_target(windows: List[Dict[str, Any]], *, app_requested: bool, exact_target: bool = False) -> Dict[str, Any]: """Best window from z-sorted (frontmost-first) list_windows output. Unqualified default captures on Linux (no app filter, no exact target) skip desktop/shell helper windows first — targetable but capture as empty — and when every remaining candidate shares one ``z_index`` (the common X11 case) ``_NET_ACTIVE_WINDOW`` beats list order. Exact-target captures never pay for the ``xprop`` probe. """ pool = [w for w in windows if not w["off_screen"]] if not exact_target and not app_requested and sys.platform == "linux": pool = [w for w in pool if _is_real_app_window(w)] or pool if pool and _z_index_uninformative(pool): active_id = _linux_x11_active_window_id() if active_id is not None: for w in pool: if w.get("window_id") == active_id: return w return pool[0] if pool else windows[0] def _sorted_windows(out: Dict[str, Any]) -> List[Dict[str, Any]]: """Normalised windows from a list_windows result, ``z_index`` DESCENDING (frontmost at index 0 — the default target for capture()/focus_app()).""" windows = _ingest_windows(_windows_from_tool_result(out)) windows.sort(key=lambda w: w["z_index"], reverse=True) return windows def _tree_and_title(out: Dict[str, Any]) -> Tuple[str, str]: """``(tree_markdown, window_title)`` from a get_window_state result.""" data = out.get("data") _, tree = _split_tree_text(data if isinstance(data, str) else "") match = _WINDOW_TITLE_RE.search(tree) return tree, (match.group(1) if match else "") def _gws_is_empty(out: Dict[str, Any]) -> bool: """True when a get_window_state result carries neither a screenshot nor a parseable tree. Modern drivers put the payload in structuredContent with no markdown tree — that is NOT empty.""" if out.get("images"): return False sc_ = out.get("structuredContent") or {} if sc_.get("elements") or sc_.get("screenshot_png_b64"): return False tree, _ = _tree_and_title(out) return not tree.strip() def _png_metrics(png_b64: str, width: int, height: int) -> Tuple[int, int, int]: """Return ``(png_bytes_len, width, height)``, replacing the given size with the sniffed one when the bytes decode to a readable PNG/JPEG header.""" try: raw = base64.b64decode(png_b64, validate=False) png_bytes_len = len(raw) detected_width, detected_height = _image_dimensions_from_bytes(raw) if detected_width and detected_height: width, height = detected_width, detected_height except Exception: png_bytes_len = len(png_b64) * 3 // 4 return png_bytes_len, width, height def _is_desktop_window(w: Dict[str, Any], names: Tuple[str, ...] = _DESKTOP_WINDOW_NAMES) -> bool: haystack = f"{w.get('app_name', '')} {w.get('title', '')}".lower() return any(name in haystack for name in names) def _app_aliases(raw_app: Dict[str, Any]) -> set: return { value.strip().lower() for key in ("bundle_id", "bundleId", "name", "app_name", "display_name") if isinstance((value := raw_app.get(key)), str) and value.strip() } class _CaptureMixin: """capture()/list_windows()/list_apps()/focus_app() and their window-discovery helpers.""" # ── Failure plumbing ─────────────────────────────────────────── def _failed_capture(self, mode: str, message: str = "") -> CaptureResult: """Return an empty capture after disarming any prior target context.""" self._clear_active_target() return CaptureResult(mode=mode, width=0, height=0, png_b64=None, elements=[], app="", window_title=message, png_bytes_len=0) def _call_capture_tool(self, name: str, args: Dict[str, Any]) -> Dict[str, Any]: """Call a capture-stage tool and disarm state on transport or logical failure.""" try: out = self._session.call_tool(name, args) except Exception: self._clear_active_target() raise if out.get("isError") is True: message = out.get("data") self._clear_active_target() raise RuntimeError(f"cua-driver {name} failed" + (f": {message}" if isinstance(message, str) and message else "")) return out def _cli_refetch(self, name: str, args: Dict[str, Any], timeout: float, what: str) -> Optional[Dict[str, Any]]: """One-shot call over the CLI transport (different daemon socket) after MCP came back empty/imageless without raising. None on failure.""" try: cli_out = self._session._call_tool_via_cli(name, args, timeout) except Exception as cli_exc: logger.error("cua-driver CLI re-fetch for %s failed: %s", what, cli_exc) return None if cli_out.get("isError") is True: if name == "list_windows": logger.error("cua-driver CLI re-fetch for list_windows returned an error") self._clear_active_target() return None return cli_out # ── Window discovery ─────────────────────────────────────────── def _list_windows_args(self) -> Dict[str, Any]: return {"on_screen_only": True, "session": self._session_id} def _load_windows(self) -> List[Dict[str, Any]]: """Visible windows frontmost-first, re-fetching over the CLI transport when MCP returns nothing.""" windows = _sorted_windows(self._call_capture_tool("list_windows", self._list_windows_args())) if windows: return windows logger.warning("cua-driver list_windows returned no windows over MCP; re-fetching via CLI transport") cli_out = self._cli_refetch("list_windows", self._list_windows_args(), 20.0, "list_windows") return _sorted_windows(cli_out) if cli_out is not None else [] def _load_windows_or_disarm(self) -> List[Dict[str, Any]]: """``_load_windows`` that forgets the sticky target when discovery raises.""" try: return self._load_windows() except Exception: self._clear_active_target() raise def _match_windows_for_app(self, windows: List[Dict[str, Any]], app: str) -> List[Dict[str, Any]]: """Resolve ``app=``: exact window names, then exact list_apps aliases (Linux ``list_windows`` can omit the app name that ``list_apps`` keeps), then substrings — querying ``Code`` must not silently select ``Visual Studio Code`` because it is frontmost.""" app_lower = app.strip().lower() if not app_lower: return [] def _name(w: Dict[str, Any]) -> str: return str(w.get("app_name", "")).lower() direct_exact = [w for w in windows if app_lower == _name(w).strip()] if direct_exact: return direct_exact try: running_apps = self.list_apps() except Exception as exc: # A title can still be the only usable identity on X11 when app # enumeration is unavailable, so keep the title fallback below. logger.debug("computer_use list_apps fallback failed for %r: %s", app, exc) running_apps = [] exact_pids: set[int] = set() partial_pids: set[int] = set() for raw_app in running_apps: pid = _positive_int(raw_app.get("pid")) if isinstance(raw_app, dict) else None if pid is None or raw_app.get("running") is False: continue aliases = _app_aliases(raw_app) if app_lower in aliases: exact_pids.add(pid) elif any(app_lower in alias for alias in aliases): partial_pids.add(pid) for matched in ([w for w in windows if w.get("pid") in exact_pids], [w for w in windows if app_lower in _name(w)], [w for w in windows if w.get("pid") in partial_pids]): if matched: return matched # Some X11 backends expose a title but no app name. Restrict this final # fallback to nameless rows so a localized app name is not overridden # merely because its title happens to be in the caller's language. return [w for w in windows if not _name(w).strip() and app_lower in str(w.get("title", "")).lower()] def _resolve_capture_windows(self, mode: str, app: Optional[str], pid: Optional[int], window_id: Optional[int]) -> "List[Dict[str, Any]] | CaptureResult": """Candidate windows for capture(), or a failed CaptureResult.""" if pid is not None or window_id is not None: # An exact pid/window pair is both the stable capture_after target # and the escape hatch when discovery is unavailable on X11. if pid is None or window_id is None: return self._failed_capture(mode, "") target_pid, target_window_id = _positive_int(pid), _positive_int(window_id) if target_pid is None or target_window_id is None: return self._failed_capture( mode, "", ) return [{"app_name": app or "", "pid": target_pid, "window_id": target_window_id, "off_screen": False, "title": "", "z_index": 0}] windows = self._load_windows_or_disarm() if not windows: # Diagnose instead of a bare 0x0: the dominant real-world cause on # Linux is a locked desktop session. from tools.computer_use import cua_backend as _cb return self._failed_capture(mode, _cb._empty_discovery_reason()) if not app: return windows if app.strip().lower() in _DESKTOP_SHELL_SENTINELS: # Desktop-shell request: the OS shell window WITH its interactable # elements (desktop icons), so "click the taskbar" works. Prefer the # backdrop (Progman/WorkerW/Finder) over the taskbar so the capture # shows the full desktop rather than the task strip. desktop = [w for w in windows if _is_desktop_window(w)] if not desktop: return self._failed_capture(mode, _NO_DESKTOP_WINDOW_MSG.format(app=app)) return sorted(desktop, key=lambda w: 0 if _is_desktop_window(w, _DESKTOP_BACKDROP_NAMES) else 1) # When the filter matches nothing, say so instead of silently capturing # the frontmost window — on macOS list_windows returns the localized # app name (e.g. "計算機"), so `app="Calculator"` legitimately misses. return (self._match_windows_for_app(windows, app) or self._failed_capture(mode, _NO_APP_MATCH_MSG.format(app=app))) # ── Capture ──────────────────────────────────────────────────── def _gws_args(self) -> Dict[str, Any]: return {"pid": self._active_pid, "window_id": self._active_window_id, "session": self._session_id} def _capture_vision(self) -> Tuple[Optional[str], Optional[str], str]: """Pixels only, no elements: ``(png_b64, mime, window_title)``. Drivers advertising the cheaper standalone ``screenshot`` tool use it; current drivers folded PNG capture into ``get_window_state`` (tree DISCARDED here). Before discovery ran we still try ``screenshot`` first and fall back, so the path self-heals on any driver version. """ png_b64: Optional[str] = None image_mime_type: Optional[str] = None window_title = "" if self._session._has_tool("screenshot") or not self._session.capabilities_discovered: sc_out = self._call_capture_tool("screenshot", { "window_id": self._active_window_id, "format": "jpeg", "quality": 85, "session": self._session_id, }) png_b64, image_mime_type = _image_from_tool_result(sc_out) if not png_b64: # "Unknown tool: screenshot" or an empty image part -> get_window_state. gws_out = self._call_capture_tool("get_window_state", self._gws_args()) png_b64, image_mime_type = _image_from_tool_result(gws_out) # The title is cheap and useful; `elements` stays empty by contract. _, window_title = _tree_and_title(gws_out) if not png_b64: logger.warning("cua-driver vision capture returned no image over MCP (window_id=%s); " "re-fetching via CLI transport", self._active_window_id) cli_out = self._cli_refetch("get_window_state", self._gws_args(), 30.0, "vision screenshot") if cli_out is not None and cli_out.get("images"): png_b64, image_mime_type = cli_out["images"][0], "image/png" return png_b64, image_mime_type, window_title def _capture_window_state(self) -> Tuple[Optional[str], Optional[str], List[UIElement], str]: """AX tree + screenshot. Returns ``(png_b64, mime, elements, window_title)``.""" gws_out = self._call_capture_tool("get_window_state", self._gws_args()) # A flaky bridge can return a degenerate result (no screenshot AND no # parseable tree) WITHOUT raising — a silent 0x0 to the model. Distinct # from the EAGAIN path handled in call_tool: here MCP "succeeded". if _gws_is_empty(gws_out): logger.warning("cua-driver get_window_state returned an empty result over MCP " "(pid=%s window_id=%s); re-fetching via CLI transport", self._active_pid, self._active_window_id) cli_out = self._cli_refetch("get_window_state", self._gws_args(), 30.0, "get_window_state") if cli_out is not None and not _gws_is_empty(cli_out): gws_out = cli_out tree, window_title = _tree_and_title(gws_out) # Prefer the canonical structuredContent.elements (real frames); the # markdown regex fallback yields (0,0,0,0) bounds. sc_elements = (gws_out.get("structuredContent") or {}).get("elements") if isinstance(sc_elements, list) and sc_elements: elements = _parse_elements_from_structured(sc_elements) else: elements = _parse_elements_from_tree(tree) if tree else [] # Tokens are tied to this snapshot: overwrite the whole map (and clear # it when the new capture carries none). self._snapshot_tokens = {e.index: e.element_token for e in elements if e.element_token} png_b64, image_mime_type = _image_from_tool_result(gws_out) return png_b64, image_mime_type, elements, window_title def capture(self, mode: str = "som", app: Optional[str] = None, pid: Optional[int] = None, window_id: Optional[int] = None) -> CaptureResult: """Capture the frontmost on-screen window or an exact known target: `list_windows` + `get_window_state` (ax/som) or `screenshot` (vision). Only the structured ``structuredContent.windows`` shape is supported.""" # Schema-filler ids (models zero-fill optional properties) must not read # as a targeting request. pid = None if _is_placeholder_id(pid) else pid window_id = None if _is_placeholder_id(window_id) else window_id exact_target = pid is not None or window_id is not None # Full-screen lane bypasses enumeration entirely (also keeps # screenshots working when Windows UIA enumeration hangs). # app='desktop' deliberately does NOT take it: desktop icons stay clickable. if not exact_target and app and app.strip().lower() in _FULL_SCREEN_SENTINELS: return self._capture_full_screen(mode) windows = self._resolve_capture_windows(mode, app, pid, window_id) if isinstance(windows, CaptureResult): return windows target = _select_capture_target(windows, app_requested=bool(app), exact_target=exact_target) self._set_active_target(target) app_name = target["app_name"] # Record the resolved app so capture_after= follow-ups re-target the # same app rather than falling back to the frontmost window. if app or not self._last_app: self._last_app = app_name or app or "" elements: List[UIElement] = [] if mode == "vision": png_b64, image_mime_type, window_title = self._capture_vision() else: png_b64, image_mime_type, elements, window_title = self._capture_window_state() png_bytes_len, width, height = _png_metrics(png_b64, 0, 0) if png_b64 else (0, 0, 0) return CaptureResult(mode=mode, width=width, height=height, png_b64=png_b64, elements=elements, app=app_name, window_title=window_title, png_bytes_len=png_bytes_len, image_mime_type=image_mime_type) def _capture_full_screen(self, mode: str) -> CaptureResult: """Composited PrtScn-style grab via `get_desktop_state` (the shell window would only show wallpaper + icons). Never enumerates, so it also works when Windows UIA hangs. Pixels only — `elements` is empty and `note` points the model at the interactive lanes. ``capture_scope`` is switched to desktop for the call and restored afterwards.""" self._clear_active_target() previous_scope: Optional[str] = None try: cfg = self._session.call_tool("get_config", {"session": self._session_id}, timeout=10.0) sc = cfg.get("structuredContent") or {} if isinstance(sc, dict) and isinstance(sc.get("capture_scope"), str): previous_scope = sc["capture_scope"] except Exception as e: logger.debug("cua-driver get_config before full-screen capture failed: %s", e) def _set_scope(value: str) -> None: self._session.call_tool("set_config", {"key": "capture_scope", "value": value, "session": self._session_id}, timeout=10.0) try: if previous_scope != "desktop": _set_scope("desktop") out = self._call_capture_tool("get_desktop_state", {"session": self._session_id}) finally: if previous_scope and previous_scope != "desktop": try: _set_scope(previous_scope) except Exception as e: logger.debug("cua-driver restore capture_scope failed: %s", e) png_b64, image_mime_type = _image_from_tool_result(out) if not png_b64: return self._failed_capture(mode, _NO_DESKTOP_IMAGE_MSG) structured = out.get("structuredContent") or {} png_bytes_len, width, height = _png_metrics( png_b64, int(structured.get("screenshot_width") or structured.get("screen_width") or 0), int(structured.get("screenshot_height") or structured.get("screen_height") or 0), ) return CaptureResult( mode="vision", width=width, height=height, png_b64=png_b64, elements=[], app="screen", window_title="Full screen (composited)", png_bytes_len=png_bytes_len, image_mime_type=image_mime_type, note=_FULL_SCREEN_NOTE, ) # ── Introspection ────────────────────────────────────────────── def list_windows(self) -> List[Dict[str, Any]]: return self._load_windows() def list_apps(self) -> List[Dict[str, Any]]: out = self._session.call_tool("list_apps", {"session": self._session_id}) structured = out.get("structuredContent") data = out.get("data") # structuredContent is canonical; empty lists fall through so a # populated compatibility envelope (older drivers, CLI fallback) can # still recover. def _apps_in(container: Any) -> List[Any]: apps = container.get("apps") if isinstance(container, dict) else None return apps if isinstance(apps, list) else [] if _apps_in(structured): return _apps_in(structured) if isinstance(data, list) and data: return data for container in (data, out): if _apps_in(container): return _apps_in(container) derived = _apps_from_windows(_windows_from_tool_result(out)) if derived: return derived # Old text-only drivers retain a small, name/PID-only fallback. if isinstance(data, str): return [ {"name": m.group(1).strip(), "pid": int(m.group(2))} for m in map(_LEGACY_APP_LINE_RE.search, data.splitlines()) if m ] return [] def focus_app(self, app: str, raise_window: bool = False) -> ActionResult: """Pure window-selector (store pid/window_id so later input hits the right process) — background automation never needs to raise a window. ``raise_window=True`` is explicit, separately approved, and uses the standalone ``bring_to_front`` tool.""" matched = self._match_windows_for_app(self._load_windows_or_disarm(), app) # No silent fallback to the frontmost window: that hides the real # failure (often a localized macOS app-name mismatch). if not matched: self._clear_active_target() return ActionResult(ok=False, action="focus_app", message=f"No on-screen window found for app '{app}'.") target = matched[0] self._set_active_target(target) self._last_app = target["app_name"] or app # retained for back-compat diagnostics if raise_window: if not self._session._has_tool("bring_to_front"): return ActionResult(ok=False, action="focus_app", code="bring_to_front_unsupported", message=_BTF_UNSUPPORTED_MSG) focused = self.bring_to_front(pid=self._active_pid, window_id=self._active_window_id) if not focused.ok: return focused focused.action = "focus_app" focused.meta["target_selected"] = True return focused return ActionResult(ok=True, action="focus_app", message=f"Targeted {target['app_name']} (pid {self._active_pid}, " f"window {self._active_window_id}) without raising window.")