From a7c1ec6ecb6e4855cd6a8e4d6889a067d1ceed6e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:24:06 -0700 Subject: [PATCH] refactor(computer_use): capture mixin disarm-on-failure contextmanager + shared MCP-empty->CLI re-fetch helper; table-driven list_apps/_match_windows_for_app --- tools/computer_use/cua_backend_capture.py | 185 ++++++++++------------ 1 file changed, 83 insertions(+), 102 deletions(-) diff --git a/tools/computer_use/cua_backend_capture.py b/tools/computer_use/cua_backend_capture.py index 23c70e5f35..361260fe88 100644 --- a/tools/computer_use/cua_backend_capture.py +++ b/tools/computer_use/cua_backend_capture.py @@ -1,9 +1,7 @@ """Capture side of the cua-driver backend: window discovery, capture-target -selection and the capture()/list_windows()/list_apps()/focus_app() methods -(mixed into ``CuaDriverBackend``). - -Logger name is kept as ``tools.computer_use.cua_backend`` so log-based tests -and operators see one backend logger. +selection and capture()/list_windows()/list_apps()/focus_app() (mixed into +``CuaDriverBackend``). Logger name stays ``tools.computer_use.cua_backend`` so +log-based tests and operators see one backend logger. """ from __future__ import annotations @@ -14,7 +12,8 @@ import os import re import subprocess import sys -from typing import Any, Dict, List, Optional, Tuple +from contextlib import contextmanager +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple from tools.computer_use.backend import ActionResult, CaptureResult, UIElement from tools.computer_use.cua_backend_input import _BTF_UNSUPPORTED_MSG @@ -91,30 +90,25 @@ def _linux_x11_active_window_id() -> Optional[int]: def _select_capture_target(windows: List[Dict[str, Any]], *, app_requested: bool, exact_target: bool = False) -> Dict[str, Any]: """Best window from z-sorted (frontmost-first) list_windows output. - Unqualified default captures on Linux (no app filter, no exact target) skip desktop/shell helper windows first — targetable but capture as empty — and when every remaining candidate shares one ``z_index`` (the common X11 case) ``_NET_ACTIVE_WINDOW`` beats list order. Exact-target captures never pay - for the ``xprop`` probe. - """ + for the ``xprop`` probe.""" pool = [w for w in windows if not w["off_screen"]] if not exact_target and not app_requested and sys.platform == "linux": pool = [w for w in pool if _is_real_app_window(w)] or pool if pool and _z_index_uninformative(pool): active_id = _linux_x11_active_window_id() - if active_id is not None: - for w in pool: - if w.get("window_id") == active_id: - return w + for w in pool: + if active_id is not None and w.get("window_id") == active_id: + return w return pool[0] if pool else windows[0] def _sorted_windows(out: Dict[str, Any]) -> List[Dict[str, Any]]: """Normalised windows from a list_windows result, ``z_index`` DESCENDING (frontmost at index 0 — the default target for capture()/focus_app()).""" - windows = _ingest_windows(_windows_from_tool_result(out)) - windows.sort(key=lambda w: w["z_index"], reverse=True) - return windows + return sorted(_ingest_windows(_windows_from_tool_result(out)), key=lambda w: w["z_index"], reverse=True) def _tree_and_title(out: Dict[str, Any]) -> Tuple[str, str]: """``(tree_markdown, window_title)`` from a get_window_state result.""" @@ -132,12 +126,11 @@ def _gws_is_empty(out: Dict[str, Any]) -> bool: sc_ = out.get("structuredContent") or {} if sc_.get("elements") or sc_.get("screenshot_png_b64"): return False - tree, _ = _tree_and_title(out) - return not tree.strip() + return not _tree_and_title(out)[0].strip() def _png_metrics(png_b64: str, width: int, height: int) -> Tuple[int, int, int]: - """Return ``(png_bytes_len, width, height)``, replacing the given size with - the sniffed one when the bytes decode to a readable PNG/JPEG header.""" + """``(png_bytes_len, width, height)``, replacing the given size with the + sniffed one when the bytes decode to a readable PNG/JPEG header.""" try: raw = base64.b64decode(png_b64, validate=False) png_bytes_len = len(raw) @@ -164,6 +157,15 @@ class _CaptureMixin: """capture()/list_windows()/list_apps()/focus_app() and their window-discovery helpers.""" # ── Failure plumbing ─────────────────────────────────────────── + @contextmanager + def _disarming(self) -> Iterator[None]: + """Forget the sticky target when the wrapped capture-stage step raises.""" + try: + yield + except Exception: + self._clear_active_target() + raise + def _failed_capture(self, mode: str, message: str = "") -> CaptureResult: """Return an empty capture after disarming any prior target context.""" self._clear_active_target() @@ -172,22 +174,19 @@ class _CaptureMixin: def _call_capture_tool(self, name: str, args: Dict[str, Any]) -> Dict[str, Any]: """Call a capture-stage tool and disarm state on transport or logical failure.""" - try: + with self._disarming(): out = self._session.call_tool(name, args) - except Exception: - self._clear_active_target() - raise - if out.get("isError") is True: - message = out.get("data") - self._clear_active_target() - raise RuntimeError(f"cua-driver {name} failed" - + (f": {message}" if isinstance(message, str) and message else "")) + if out.get("isError") is True: + message = out.get("data") + raise RuntimeError(f"cua-driver {name} failed" + + (f": {message}" if isinstance(message, str) and message else "")) return out - def _cli_refetch(self, name: str, args: Dict[str, Any], timeout: float, - what: str) -> Optional[Dict[str, Any]]: - """One-shot call over the CLI transport (different daemon socket) after - MCP came back empty/imageless without raising. None on failure.""" + def _cli_refetch(self, name: str, args: Dict[str, Any], timeout: float, what: str, + warning: str, *warning_args: Any) -> Optional[Dict[str, Any]]: + """MCP came back empty/imageless without raising: log *warning*, then a + one-shot call over the CLI transport (different daemon socket). None on failure.""" + logger.warning(warning, *warning_args) try: cli_out = self._session._call_tool_via_cli(name, args, timeout) except Exception as cli_exc: @@ -200,27 +199,25 @@ class _CaptureMixin: return None return cli_out - # ── Window discovery ─────────────────────────────────────────── - def _list_windows_args(self) -> Dict[str, Any]: - return {"on_screen_only": True, "session": self._session_id} + def _fetch_or_refetch(self, name: str, args: Dict[str, Any], timeout: float, what: str, + empty: Callable[[Dict[str, Any]], bool], warning: str, *warning_args: Any) -> Dict[str, Any]: + """``_call_capture_tool`` whose result, when *empty*, is replaced by a + non-empty CLI re-fetch (the MCP result stands when the CLI fails too).""" + out = self._call_capture_tool(name, args) + if empty(out): + cli_out = self._cli_refetch(name, args, timeout, what, warning, *warning_args) + if cli_out is not None and not empty(cli_out): + out = cli_out + return out + # ── Window discovery ─────────────────────────────────────────── def _load_windows(self) -> List[Dict[str, Any]]: """Visible windows frontmost-first, re-fetching over the CLI transport when MCP returns nothing.""" - windows = _sorted_windows(self._call_capture_tool("list_windows", self._list_windows_args())) - if windows: - return windows - logger.warning("cua-driver list_windows returned no windows over MCP; re-fetching via CLI transport") - cli_out = self._cli_refetch("list_windows", self._list_windows_args(), 20.0, "list_windows") - return _sorted_windows(cli_out) if cli_out is not None else [] - - def _load_windows_or_disarm(self) -> List[Dict[str, Any]]: - """``_load_windows`` that forgets the sticky target when discovery raises.""" - try: - return self._load_windows() - except Exception: - self._clear_active_target() - raise + args = {"on_screen_only": True, "session": self._session_id} + return _sorted_windows(self._fetch_or_refetch( + "list_windows", args, 20.0, "list_windows", lambda out: not _sorted_windows(out), + "cua-driver list_windows returned no windows over MCP; re-fetching via CLI transport")) def _match_windows_for_app(self, windows: List[Dict[str, Any]], app: str) -> List[Dict[str, Any]]: """Resolve ``app=``: exact window names, then exact list_apps aliases @@ -255,16 +252,14 @@ class _CaptureMixin: exact_pids.add(pid) elif any(app_lower in alias for alias in aliases): partial_pids.add(pid) - for matched in ([w for w in windows if w.get("pid") in exact_pids], - [w for w in windows if app_lower in _name(w)], - [w for w in windows if w.get("pid") in partial_pids]): - if matched: - return matched - # Some X11 backends expose a title but no app name. Restrict this final + # Some X11 backends expose a title but no app name. Restrict the final # fallback to nameless rows so a localized app name is not overridden # merely because its title happens to be in the caller's language. - return [w for w in windows - if not _name(w).strip() and app_lower in str(w.get("title", "")).lower()] + tiers = ([w for w in windows if w.get("pid") in exact_pids], + [w for w in windows if app_lower in _name(w)], + [w for w in windows if w.get("pid") in partial_pids], + [w for w in windows if not _name(w).strip() and app_lower in str(w.get("title", "")).lower()]) + return next((matched for matched in tiers if matched), []) def _resolve_capture_windows(self, mode: str, app: Optional[str], pid: Optional[int], window_id: Optional[int]) -> "List[Dict[str, Any]] | CaptureResult": @@ -276,13 +271,12 @@ class _CaptureMixin: return self._failed_capture(mode, "") target_pid, target_window_id = _positive_int(pid), _positive_int(window_id) if target_pid is None or target_window_id is None: - return self._failed_capture( - mode, "", - ) + return self._failed_capture(mode, "") return [{"app_name": app or "", "pid": target_pid, "window_id": target_window_id, "off_screen": False, "title": "", "z_index": 0}] - windows = self._load_windows_or_disarm() + with self._disarming(): + windows = self._load_windows() if not windows: # Diagnose instead of a bare 0x0: the dominant real-world cause on # Linux is a locked desktop session. @@ -291,7 +285,6 @@ class _CaptureMixin: return self._failed_capture(mode, _cb._empty_discovery_reason()) if not app: return windows - if app.strip().lower() in _DESKTOP_SHELL_SENTINELS: # Desktop-shell request: the OS shell window WITH its interactable # elements (desktop icons), so "click the taskbar" works. Prefer the @@ -300,8 +293,7 @@ class _CaptureMixin: desktop = [w for w in windows if _is_desktop_window(w)] if not desktop: return self._failed_capture(mode, _NO_DESKTOP_WINDOW_MSG.format(app=app)) - return sorted(desktop, key=lambda w: 0 if _is_desktop_window(w, _DESKTOP_BACKDROP_NAMES) else 1) - + return sorted(desktop, key=lambda w: not _is_desktop_window(w, _DESKTOP_BACKDROP_NAMES)) # When the filter matches nothing, say so instead of silently capturing # the frontmost window — on macOS list_windows returns the localized # app name (e.g. "計算機"), so `app="Calculator"` legitimately misses. @@ -314,12 +306,10 @@ class _CaptureMixin: def _capture_vision(self) -> Tuple[Optional[str], Optional[str], str]: """Pixels only, no elements: ``(png_b64, mime, window_title)``. - Drivers advertising the cheaper standalone ``screenshot`` tool use it; current drivers folded PNG capture into ``get_window_state`` (tree DISCARDED here). Before discovery ran we still try ``screenshot`` first - and fall back, so the path self-heals on any driver version. - """ + and fall back, so the path self-heals on any driver version.""" png_b64: Optional[str] = None image_mime_type: Optional[str] = None window_title = "" @@ -336,27 +326,23 @@ class _CaptureMixin: # The title is cheap and useful; `elements` stays empty by contract. _, window_title = _tree_and_title(gws_out) if not png_b64: - logger.warning("cua-driver vision capture returned no image over MCP (window_id=%s); " - "re-fetching via CLI transport", self._active_window_id) - cli_out = self._cli_refetch("get_window_state", self._gws_args(), 30.0, "vision screenshot") + cli_out = self._cli_refetch( + "get_window_state", self._gws_args(), 30.0, "vision screenshot", + "cua-driver vision capture returned no image over MCP (window_id=%s); re-fetching via CLI transport", + self._active_window_id) if cli_out is not None and cli_out.get("images"): png_b64, image_mime_type = cli_out["images"][0], "image/png" return png_b64, image_mime_type, window_title def _capture_window_state(self) -> Tuple[Optional[str], Optional[str], List[UIElement], str]: """AX tree + screenshot. Returns ``(png_b64, mime, elements, window_title)``.""" - gws_out = self._call_capture_tool("get_window_state", self._gws_args()) # A flaky bridge can return a degenerate result (no screenshot AND no # parseable tree) WITHOUT raising — a silent 0x0 to the model. Distinct # from the EAGAIN path handled in call_tool: here MCP "succeeded". - if _gws_is_empty(gws_out): - logger.warning("cua-driver get_window_state returned an empty result over MCP " - "(pid=%s window_id=%s); re-fetching via CLI transport", - self._active_pid, self._active_window_id) - cli_out = self._cli_refetch("get_window_state", self._gws_args(), 30.0, "get_window_state") - if cli_out is not None and not _gws_is_empty(cli_out): - gws_out = cli_out - + gws_out = self._fetch_or_refetch( + "get_window_state", self._gws_args(), 30.0, "get_window_state", _gws_is_empty, + "cua-driver get_window_state returned an empty result over MCP (pid=%s window_id=%s); " + "re-fetching via CLI transport", self._active_pid, self._active_window_id) tree, window_title = _tree_and_title(gws_out) # Prefer the canonical structuredContent.elements (real frames); the # markdown regex fallback yields (0,0,0,0) bounds. @@ -390,7 +376,6 @@ class _CaptureMixin: windows = self._resolve_capture_windows(mode, app, pid, window_id) if isinstance(windows, CaptureResult): return windows - target = _select_capture_target(windows, app_requested=bool(app), exact_target=exact_target) self._set_active_target(target) app_name = target["app_name"] @@ -404,7 +389,6 @@ class _CaptureMixin: png_b64, image_mime_type, window_title = self._capture_vision() else: png_b64, image_mime_type, elements, window_title = self._capture_window_state() - png_bytes_len, width, height = _png_metrics(png_b64, 0, 0) if png_b64 else (0, 0, 0) return CaptureResult(mode=mode, width=width, height=height, png_b64=png_b64, elements=elements, app=app_name, window_title=window_title, @@ -419,8 +403,7 @@ class _CaptureMixin: self._clear_active_target() previous_scope: Optional[str] = None try: - cfg = self._session.call_tool("get_config", {"session": self._session_id}, timeout=10.0) - sc = cfg.get("structuredContent") or {} + sc = self._session.call_tool("get_config", {"session": self._session_id}, timeout=10.0).get("structuredContent") or {} if isinstance(sc, dict) and isinstance(sc.get("capture_scope"), str): previous_scope = sc["capture_scope"] except Exception as e: @@ -462,25 +445,21 @@ class _CaptureMixin: def list_apps(self) -> List[Dict[str, Any]]: out = self._session.call_tool("list_apps", {"session": self._session_id}) - structured = out.get("structuredContent") - data = out.get("data") + structured, data = out.get("structuredContent"), out.get("data") # structuredContent is canonical; empty lists fall through so a # populated compatibility envelope (older drivers, CLI fallback) can - # still recover. - def _apps_in(container: Any) -> List[Any]: - apps = container.get("apps") if isinstance(container, dict) else None - return apps if isinstance(apps, list) else [] - - if _apps_in(structured): - return _apps_in(structured) - if isinstance(data, list) and data: - return data - for container in (data, out): - if _apps_in(container): - return _apps_in(container) - derived = _apps_from_windows(_windows_from_tool_result(out)) - if derived: - return derived + # still recover, then apps derived from the windows payload. + candidates = ( + lambda: structured.get("apps") if isinstance(structured, dict) else None, + lambda: data, + lambda: data.get("apps") if isinstance(data, dict) else None, + lambda: out.get("apps"), + lambda: _apps_from_windows(_windows_from_tool_result(out)), + ) + for candidate in candidates: + apps = candidate() + if isinstance(apps, list) and apps: + return apps # Old text-only drivers retain a small, name/PID-only fallback. if isinstance(data, str): return [ @@ -494,7 +473,9 @@ class _CaptureMixin: right process) — background automation never needs to raise a window. ``raise_window=True`` is explicit, separately approved, and uses the standalone ``bring_to_front`` tool.""" - matched = self._match_windows_for_app(self._load_windows_or_disarm(), app) + with self._disarming(): + windows = self._load_windows() + matched = self._match_windows_for_app(windows, app) # No silent fallback to the frontmost window: that hides the real # failure (often a localized macOS app-name mismatch). if not matched: