diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index fe4ead2efd..3400daa9d3 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -88,15 +88,12 @@ def _computer_use_cfg() -> Dict[str, Any]: def _cua_no_overlay() -> bool: - """True when Hermes should pass ``--no-overlay`` to cua-driver. - - ``computer_use.no_overlay`` overrides when set. Auto-detect otherwise: - off on macOS (cursor-overlay redraw loop can peg a core after a session), - headless Linux / WSL2 / containers, and Linux X11 (the overlay is a - fullscreen always-on-top all-workspaces window with no compositor-owned - lifecycle, so an unclean session end can leave it wedged over every app); - on for Windows and Linux Wayland (compositor owns the surface). - """ + """Pass ``--no-overlay``? ``computer_use.no_overlay`` overrides; else off on + macOS (cursor-overlay redraw loop can peg a core after a session), headless + Linux / WSL2 / containers, and Linux X11 (the overlay is a fullscreen + always-on-top all-workspaces window with no compositor-owned lifecycle, so + an unclean session end can leave it wedged over every app); on for Windows + and Linux Wayland (compositor owns the surface).""" val = _computer_use_cfg().get("no_overlay") if val is not None: return bool(val) @@ -122,13 +119,10 @@ def _cua_telemetry_disabled() -> bool: def _cua_configured_permission_mode() -> str: - """``computer_use.permission_mode`` (default ``standard``). - - Only ``standard`` / ``bounded`` are honored: ``unrestricted`` is - deliberately NOT a config value — it stays tied to the per-session YOLO - toggle so a stale config line can never silently bypass approvals. - Unknown values fall closed to ``standard``. - """ + """``computer_use.permission_mode``: ``standard`` (default) or ``bounded``; + unknown values fall closed to ``standard``. ``unrestricted`` is deliberately + NOT a config value — it stays tied to the per-session YOLO toggle so a stale + config line can never silently bypass approvals.""" raw = str(_computer_use_cfg().get("permission_mode", "standard") or "").strip().lower() return raw if raw in {"standard", "bounded"} else "standard" @@ -141,14 +135,11 @@ def _cua_capability_manifest() -> Optional[str]: def _manifest_is_mode_independent(path: str) -> bool: - """True when this capability manifest may accompany any permission mode. - - v1/v2 manifests must declare ``mode: bounded`` and abort startup under an - unrestricted runtime; v3 must NOT declare a mode and is the mode- - independent ceiling the driver accepts alongside any mode. Unreadable / - unparseable -> False (forwarding one would turn a working session into a - hard startup failure; bounded forwards unconditionally anyway). - """ + """True when this manifest may accompany any permission mode: v1/v2 declare + ``mode: bounded`` and abort startup under an unrestricted runtime; v3 has no + mode and is the ceiling the driver accepts alongside any mode. Unreadable / + unparseable -> False (forwarding one would turn a working session into a hard + startup failure; bounded forwards unconditionally anyway).""" try: import yaml @@ -212,13 +203,11 @@ def _run_driver(driver_cmd: str, *args: str, timeout: float) -> subprocess.Compl # --------------------------------------------------------------------------- def _linux_session_locked() -> Optional[bool]: - """Best-effort: is the graphical session locked? (Linux only.) - - A locked KDE/GNOME session freezes renderers and half-disables the AX - tree, so window discovery legitimately returns nothing — which otherwise - reads as a driver bug. True/False when loginctl answers, None when - unavailable (non-Linux, no systemd-logind, probe failure). - """ + """Is the graphical session locked? (Linux; best-effort.) A locked KDE/GNOME + session freezes renderers and half-disables the AX tree, so discovery + legitimately returns nothing — which otherwise reads as a driver bug. + True/False when loginctl answers, None when unavailable (non-Linux, no + systemd-logind, probe failure).""" if sys.platform != "linux": return None @@ -355,21 +344,19 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend): ) self._bridge = _AsyncBridge() self._session = _CuaDriverSession(self._bridge, self._embedded_daemon) - # Sticky context — updated by capture()/focus_app(), used by actions. + # Sticky target — set by capture()/focus_app(), used by actions. self._active_pid: Optional[int] = None self._active_window_id: Optional[int] = None self._last_app: Optional[str] = None - # Exact identity for capture_after: app names may be generic on Linux + # Exact identity for capture_after: Linux app names may be generic # (several unrelated Qt windows can all say Qt6Application). self._last_target: Optional[Dict[str, Optional[int]]] = None - # Per-snapshot `element_index -> element_token` map from capture(). - # Actions attach the token so cua-driver detects "stale" explicitly - # instead of silently re-resolving to a different element. + # Per-snapshot `element_index -> element_token`; actions attach it so + # cua-driver reports "stale" instead of silently re-resolving. self._snapshot_tokens: Dict[int, str] = {} - # Public session label (one per backend = one per Hermes run) passed as - # `session` on every call: owns the agent cursor color and gives config - # / recording state a stable owner inside the transport-private - # lifecycle. Part of the 0.20 runtime contract checked at start(). + # Public session label (one per Hermes run) sent as `session` on every + # call: owns the cursor color and gives config/recording state a stable + # owner across transport restarts. Part of the 0.20 runtime contract. self._session_id: str = f"hermes-{uuid.uuid4().hex[:12]}" self._session.set_transport_reset_callback(self._handle_transport_reset) @@ -392,8 +379,8 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend): raise RuntimeError(f"cua-driver is not ready: {reason}. {repair}") _maybe_nudge_update() # `mcp` is an optional extra: lazy-install on first use (gated by - # `security.allow_lazy_installs`); on failure ensure() raises - # FeatureUnavailable with the exact `uv pip install` hint. + # `security.allow_lazy_installs`); failure raises FeatureUnavailable + # with the exact `uv pip install` hint. from tools.lazy_deps import ensure as _lazy_ensure _lazy_ensure("tool.computer_use", prompt=False) import importlib @@ -412,12 +399,10 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend): self._best_effort("start_session failed (continuing anonymous)", self._session.call_tool, "start_session", {"session": self._session_id}) - # Post-handshake tuning guards on `_started`: before the handshake - # flips it, call_tool would re-enter session.start() and tests that - # stub start() would recurse. + # Post-handshake tuning guards on `_started`: before the handshake flips + # it, call_tool would re-enter session.start() (stubbed start() recurses). if self._session._started: - # Cap screenshot size so every later capture pays less over the - # daemon socket and in the model turn. + # Smaller screenshots cost less over the daemon socket and per turn. max_dim = _computer_use_max_image_dimension() if max_dim: self._best_effort("set_config(max_image_dimension) failed", @@ -428,9 +413,9 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend): self.set_agent_cursor_enabled, False, cursor_id=self._session_id) def stop(self) -> None: - # Best-effort end_session first so the driver cleans per-session state - # (cursor overlay, recording ownership, config overrides); the - # connection drop below releases daemon-side state regardless. + # Best-effort end_session so the driver cleans per-session state (cursor + # overlay, recording ownership, config overrides); the connection drop + # below releases daemon-side state regardless. if self._session._started: self._best_effort("end_session failed (continuing teardown)", self._session.call_tool, "end_session", {"session": self._session_id}) @@ -531,10 +516,10 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend): # ── Internal ─────────────────────────────────────────────────── def _maybe_attach_element_token(self, tool: str, args: Dict[str, Any]) -> None: - """Attach the last snapshot's ``element_token`` for an ``element_index`` - call. The token takes precedence and yields an explicit 'stale' error - when the snapshot was superseded. Gated on the per-tool capability so - older drivers (``additionalProperties: false``) never see the field.""" + """Attach the snapshot's ``element_token`` to an ``element_index`` call so + a superseded snapshot yields an explicit 'stale' error. Gated on the + per-tool capability: older drivers (``additionalProperties: false``) + must never see the field.""" idx = args.get("element_index") token = self._snapshot_tokens.get(idx) if isinstance(idx, int) else None if token and self._session.supports_capability("accessibility.element_tokens", tool=tool): diff --git a/tools/computer_use/cua_backend_capture.py b/tools/computer_use/cua_backend_capture.py index 3ea89d049a..18a4a14e4f 100644 --- a/tools/computer_use/cua_backend_capture.py +++ b/tools/computer_use/cua_backend_capture.py @@ -96,14 +96,13 @@ def _select_capture_target( app_requested: bool, exact_target: bool = False, ) -> Dict[str, Any]: - """Select the best window for capture from z-sorted list_windows output. + """Best window from z-sorted (frontmost-first) list_windows output. - Windows arrive sorted by ``z_index`` descending (frontmost first). For - unqualified default captures on Linux (no app filter, no exact target), - desktop/shell helper windows are skipped first — they are targetable but - capture as empty — and when every remaining candidate shares the same - ``z_index`` (the common X11 case) ``_NET_ACTIVE_WINDOW`` beats list order. - Exact-target captures never pay for the ``xprop`` probe. + Unqualified default captures on Linux (no app filter, no exact target) skip + desktop/shell helper windows first — targetable but capture as empty — and + when every remaining candidate shares one ``z_index`` (the common X11 case) + ``_NET_ACTIVE_WINDOW`` beats list order. Exact-target captures never pay + for the ``xprop`` probe. """ pool = [w for w in windows if not w["off_screen"]] if not exact_target and not app_requested and sys.platform == "linux": @@ -243,13 +242,10 @@ class _CaptureMixin: def _match_windows_for_app( self, windows: List[Dict[str, Any]], app: str ) -> List[Dict[str, Any]]: - """Resolve ``app=`` through exact names before convenience substrings. - - Linux ``list_windows`` can omit an app name while ``list_apps`` keeps - name/bundle-ID metadata. Exact direct names and exact metadata aliases - win over substring matches: querying ``Code`` must not silently select - ``Visual Studio Code`` because it is frontmost. - """ + """Resolve ``app=``: exact window names, then exact list_apps aliases + (Linux ``list_windows`` can omit the app name that ``list_apps`` keeps), + then substrings — querying ``Code`` must not silently select + ``Visual Studio Code`` because it is frontmost.""" app_lower = app.strip().lower() if not app_lower: return [] @@ -354,13 +350,12 @@ class _CaptureMixin: return {"pid": self._active_pid, "window_id": self._active_window_id, "session": self._session_id} def _capture_vision(self) -> Tuple[Optional[str], Optional[str], str]: - """Pixels only, no elements. Returns ``(png_b64, mime, window_title)``. + """Pixels only, no elements: ``(png_b64, mime, window_title)``. - Drivers that advertise the (cheaper) standalone ``screenshot`` tool use - it; current drivers folded PNG capture into ``get_window_state``, whose - tree is DISCARDED here. When discovery hasn't run we still try - ``screenshot`` first and fall back, so the path self-heals on any - driver version. + Drivers advertising the cheaper standalone ``screenshot`` tool use it; + current drivers folded PNG capture into ``get_window_state`` (tree + DISCARDED here). Before discovery ran we still try ``screenshot`` first + and fall back, so the path self-heals on any driver version. """ png_b64: Optional[str] = None image_mime_type: Optional[str] = None @@ -425,14 +420,11 @@ class _CaptureMixin: pid: Optional[int] = None, window_id: Optional[int] = None, ) -> CaptureResult: - """Capture the frontmost on-screen window or an exact known target. - - Maps hermes `capture(mode, app)` -> cua-driver `list_windows` + - `get_window_state` (ax/som) or `screenshot` (vision). Only the - structured ``structuredContent.windows`` shape is supported. - """ - # Drop schema-filler ids (models that zero-fill every optional - # property) before they read as a targeting request. + """Capture the frontmost on-screen window or an exact known target: + `list_windows` + `get_window_state` (ax/som) or `screenshot` (vision). + Only the structured ``structuredContent.windows`` shape is supported.""" + # Schema-filler ids (models zero-fill optional properties) must not read + # as a targeting request. pid = None if _is_placeholder_id(pid) else pid window_id = None if _is_placeholder_id(window_id) else window_id exact_target = pid is not None or window_id is not None @@ -466,13 +458,11 @@ class _CaptureMixin: png_bytes_len=png_bytes_len, image_mime_type=image_mime_type) def _capture_full_screen(self, mode: str) -> CaptureResult: - """Composited grab of everything on screen via `get_desktop_state` - (like PrtScn) — the shell window would only show wallpaper + icons. - Never enumerates, so it also works when Windows UIA hangs. Pixels only: - `elements` is always empty; `note` tells the model how to reach the - interactive lanes. ``capture_scope`` is switched to desktop for the - call and restored afterwards. - """ + """Composited PrtScn-style grab via `get_desktop_state` (the shell window + would only show wallpaper + icons). Never enumerates, so it also works + when Windows UIA hangs. Pixels only — `elements` is empty and `note` + points the model at the interactive lanes. ``capture_scope`` is switched + to desktop for the call and restored afterwards.""" self._clear_active_target() previous_scope: Optional[str] = None try: @@ -550,11 +540,10 @@ class _CaptureMixin: return [] def focus_app(self, app: str, raise_window: bool = False) -> ActionResult: - """Target an app: a pure window-selector (store pid/window_id so later - input hits the right process) — background automation never needs to - raise a window. ``raise_window=True`` is explicit, separately approved, - and uses the standalone ``bring_to_front`` tool. - """ + """Pure window-selector (store pid/window_id so later input hits the + right process) — background automation never needs to raise a window. + ``raise_window=True`` is explicit, separately approved, and uses the + standalone ``bring_to_front`` tool.""" matched = self._match_windows_for_app(self._load_windows_or_disarm(), app) # No silent fallback to the frontmost window: that hides the real # failure (often a localized macOS app-name mismatch). diff --git a/tools/computer_use/cua_backend_driver.py b/tools/computer_use/cua_backend_driver.py index 840e9b03dc..9f312ad946 100644 --- a/tools/computer_use/cua_backend_driver.py +++ b/tools/computer_use/cua_backend_driver.py @@ -93,14 +93,11 @@ def _wsl_windows_path_to_posix(path: str) -> str: def _candidate_cua_driver_commands(override: Optional[str] = None) -> List[str]: - """Candidate cua-driver commands in resolution order. - - ``override`` / a non-empty ``HERMES_CUA_DRIVER_CMD`` is authoritative (if - it is wrong, report the driver missing rather than silently picking - another binary). Otherwise PATH, then canonical installer locations — - Desktop apps launched from Finder/Dock inherit a narrow PATH that omits - ``~/.local/bin``, and freshly installed Windows sessions inherit a stale one. - """ + """Candidate commands in resolution order. ``override`` / a non-empty + ``HERMES_CUA_DRIVER_CMD`` is authoritative (if wrong, report the driver + missing rather than silently picking another binary). Otherwise PATH, then + canonical installer locations — Finder/Dock-launched apps inherit a narrow + PATH without ``~/.local/bin``; fresh Windows sessions inherit a stale one.""" configured = (override if override is not None else os.environ.get(_CUA_DRIVER_CMD_ENV, "")).strip() if configured: return [configured] @@ -179,14 +176,11 @@ def _cua_driver_supports_no_overlay(driver_cmd: str) -> bool: def _resolve_mcp_invocation(driver_cmd: str, *, timeout: float = 6.0) -> Tuple[str, List[str]]: - """Return ``(command, args)`` that spawn cua-driver's stdio MCP server. - - Asks the driver itself via ``cua-driver manifest`` (``mcp_invocation`` - carries ``command`` + ``args``) so a future rename of the subcommand keeps - working. Falls back to ``(driver_cmd, ["mcp"])`` for older drivers or any - discovery failure — the wrapper must not refuse to start over a failed - discovery hop. ``--no-overlay`` is appended when policy + driver allow. - """ + """``(command, args)`` that spawn cua-driver's stdio MCP server, asked of + the driver itself via ``cua-driver manifest`` (``mcp_invocation``) so a + subcommand rename keeps working. Falls back to ``(driver_cmd, ["mcp"])`` on + older drivers or any discovery failure — the wrapper must not refuse to + start over a failed discovery hop. ``--no-overlay`` appended when allowed.""" manifest = _driver_json(driver_cmd, "manifest", timeout=timeout, require_ok=True) or {} invocation = manifest.get("mcp_invocation") invocation = invocation if isinstance(invocation, dict) else {} @@ -269,17 +263,13 @@ def cua_driver_runtime_contract_status(binary: Optional[str] = None) -> Dict[str def cua_driver_update_check(*, timeout: Optional[float] = None) -> Optional[Dict[str, Any]]: - """Run ``cua-driver check-update --json``; payload mirrors the - ``check_for_update`` MCP tool (``{current_version, latest_version, - update_available, ...}``). - - ``timeout`` defaults to 8s on POSIX and 25s on Windows (first spawn of the - exe routinely eats seconds in Defender scanning, and a false timeout is - expensive: callers treat ``None`` as indeterminate and the upgrade path - used to fall through to a full reinstall on it). Returns ``None`` when the - binary is missing, the driver predates the verb, the GitHub check failed - (``error`` set), or the output didn't parse. Never raises. - """ + """``cua-driver check-update --json`` payload (``{current_version, + latest_version, update_available, ...}``), or ``None`` when the binary is + missing, the driver predates the verb, the GitHub check failed (``error`` + set) or the output didn't parse. Never raises. ``timeout`` defaults to 8s + on POSIX / 25s on Windows: first spawn of the exe routinely eats seconds in + Defender scanning, and callers treat ``None`` as indeterminate (the upgrade + path used to fall through to a full reinstall on a false timeout).""" if timeout is None: timeout = 25.0 if sys.platform == "win32" else 8.0 driver_cmd = _cb().resolve_cua_driver_cmd()