refactor(computer_use): compact cua_backend docstrings/comments (WHYs preserved)

This commit is contained in:
Teknium
2026-09-02 16:18:47 -07:00
parent d2add66648
commit c3d789893f
3 changed files with 85 additions and 121 deletions
+39 -54
View File
@@ -88,15 +88,12 @@ def _computer_use_cfg() -> Dict[str, Any]:
def _cua_no_overlay() -> bool:
"""True when Hermes should pass ``--no-overlay`` to cua-driver.
``computer_use.no_overlay`` overrides when set. Auto-detect otherwise:
off on macOS (cursor-overlay redraw loop can peg a core after a session),
headless Linux / WSL2 / containers, and Linux X11 (the overlay is a
fullscreen always-on-top all-workspaces window with no compositor-owned
lifecycle, so an unclean session end can leave it wedged over every app);
on for Windows and Linux Wayland (compositor owns the surface).
"""
"""Pass ``--no-overlay``? ``computer_use.no_overlay`` overrides; else off on
macOS (cursor-overlay redraw loop can peg a core after a session), headless
Linux / WSL2 / containers, and Linux X11 (the overlay is a fullscreen
always-on-top all-workspaces window with no compositor-owned lifecycle, so
an unclean session end can leave it wedged over every app); on for Windows
and Linux Wayland (compositor owns the surface)."""
val = _computer_use_cfg().get("no_overlay")
if val is not None:
return bool(val)
@@ -122,13 +119,10 @@ def _cua_telemetry_disabled() -> bool:
def _cua_configured_permission_mode() -> str:
"""``computer_use.permission_mode`` (default ``standard``).
Only ``standard`` / ``bounded`` are honored: ``unrestricted`` is
deliberately NOT a config value — it stays tied to the per-session YOLO
toggle so a stale config line can never silently bypass approvals.
Unknown values fall closed to ``standard``.
"""
"""``computer_use.permission_mode``: ``standard`` (default) or ``bounded``;
unknown values fall closed to ``standard``. ``unrestricted`` is deliberately
NOT a config value — it stays tied to the per-session YOLO toggle so a stale
config line can never silently bypass approvals."""
raw = str(_computer_use_cfg().get("permission_mode", "standard") or "").strip().lower()
return raw if raw in {"standard", "bounded"} else "standard"
@@ -141,14 +135,11 @@ def _cua_capability_manifest() -> Optional[str]:
def _manifest_is_mode_independent(path: str) -> bool:
"""True when this capability manifest may accompany any permission mode.
v1/v2 manifests must declare ``mode: bounded`` and abort startup under an
unrestricted runtime; v3 must NOT declare a mode and is the mode-
independent ceiling the driver accepts alongside any mode. Unreadable /
unparseable -> False (forwarding one would turn a working session into a
hard startup failure; bounded forwards unconditionally anyway).
"""
"""True when this manifest may accompany any permission mode: v1/v2 declare
``mode: bounded`` and abort startup under an unrestricted runtime; v3 has no
mode and is the ceiling the driver accepts alongside any mode. Unreadable /
unparseable -> False (forwarding one would turn a working session into a hard
startup failure; bounded forwards unconditionally anyway)."""
try:
import yaml
@@ -212,13 +203,11 @@ def _run_driver(driver_cmd: str, *args: str, timeout: float) -> subprocess.Compl
# ---------------------------------------------------------------------------
def _linux_session_locked() -> Optional[bool]:
"""Best-effort: is the graphical session locked? (Linux only.)
A locked KDE/GNOME session freezes renderers and half-disables the AX
tree, so window discovery legitimately returns nothing — which otherwise
reads as a driver bug. True/False when loginctl answers, None when
unavailable (non-Linux, no systemd-logind, probe failure).
"""
"""Is the graphical session locked? (Linux; best-effort.) A locked KDE/GNOME
session freezes renderers and half-disables the AX tree, so discovery
legitimately returns nothing — which otherwise reads as a driver bug.
True/False when loginctl answers, None when unavailable (non-Linux, no
systemd-logind, probe failure)."""
if sys.platform != "linux":
return None
@@ -355,21 +344,19 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
)
self._bridge = _AsyncBridge()
self._session = _CuaDriverSession(self._bridge, self._embedded_daemon)
# Sticky context — updated by capture()/focus_app(), used by actions.
# Sticky target — set by capture()/focus_app(), used by actions.
self._active_pid: Optional[int] = None
self._active_window_id: Optional[int] = None
self._last_app: Optional[str] = None
# Exact identity for capture_after: app names may be generic on Linux
# Exact identity for capture_after: Linux app names may be generic
# (several unrelated Qt windows can all say Qt6Application).
self._last_target: Optional[Dict[str, Optional[int]]] = None
# Per-snapshot `element_index -> element_token` map from capture().
# Actions attach the token so cua-driver detects "stale" explicitly
# instead of silently re-resolving to a different element.
# Per-snapshot `element_index -> element_token`; actions attach it so
# cua-driver reports "stale" instead of silently re-resolving.
self._snapshot_tokens: Dict[int, str] = {}
# Public session label (one per backend = one per Hermes run) passed as
# `session` on every call: owns the agent cursor color and gives config
# / recording state a stable owner inside the transport-private
# lifecycle. Part of the 0.20 runtime contract checked at start().
# Public session label (one per Hermes run) sent as `session` on every
# call: owns the cursor color and gives config/recording state a stable
# owner across transport restarts. Part of the 0.20 runtime contract.
self._session_id: str = f"hermes-{uuid.uuid4().hex[:12]}"
self._session.set_transport_reset_callback(self._handle_transport_reset)
@@ -392,8 +379,8 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
raise RuntimeError(f"cua-driver is not ready: {reason}. {repair}")
_maybe_nudge_update()
# `mcp` is an optional extra: lazy-install on first use (gated by
# `security.allow_lazy_installs`); on failure ensure() raises
# FeatureUnavailable with the exact `uv pip install` hint.
# `security.allow_lazy_installs`); failure raises FeatureUnavailable
# with the exact `uv pip install` hint.
from tools.lazy_deps import ensure as _lazy_ensure
_lazy_ensure("tool.computer_use", prompt=False)
import importlib
@@ -412,12 +399,10 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
self._best_effort("start_session failed (continuing anonymous)",
self._session.call_tool, "start_session", {"session": self._session_id})
# Post-handshake tuning guards on `_started`: before the handshake
# flips it, call_tool would re-enter session.start() and tests that
# stub start() would recurse.
# Post-handshake tuning guards on `_started`: before the handshake flips
# it, call_tool would re-enter session.start() (stubbed start() recurses).
if self._session._started:
# Cap screenshot size so every later capture pays less over the
# daemon socket and in the model turn.
# Smaller screenshots cost less over the daemon socket and per turn.
max_dim = _computer_use_max_image_dimension()
if max_dim:
self._best_effort("set_config(max_image_dimension) failed",
@@ -428,9 +413,9 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
self.set_agent_cursor_enabled, False, cursor_id=self._session_id)
def stop(self) -> None:
# Best-effort end_session first so the driver cleans per-session state
# (cursor overlay, recording ownership, config overrides); the
# connection drop below releases daemon-side state regardless.
# Best-effort end_session so the driver cleans per-session state (cursor
# overlay, recording ownership, config overrides); the connection drop
# below releases daemon-side state regardless.
if self._session._started:
self._best_effort("end_session failed (continuing teardown)",
self._session.call_tool, "end_session", {"session": self._session_id})
@@ -531,10 +516,10 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
# ── Internal ───────────────────────────────────────────────────
def _maybe_attach_element_token(self, tool: str, args: Dict[str, Any]) -> None:
"""Attach the last snapshot's ``element_token`` for an ``element_index``
call. The token takes precedence and yields an explicit 'stale' error
when the snapshot was superseded. Gated on the per-tool capability so
older drivers (``additionalProperties: false``) never see the field."""
"""Attach the snapshot's ``element_token`` to an ``element_index`` call so
a superseded snapshot yields an explicit 'stale' error. Gated on the
per-tool capability: older drivers (``additionalProperties: false``)
must never see the field."""
idx = args.get("element_index")
token = self._snapshot_tokens.get(idx) if isinstance(idx, int) else None
if token and self._session.supports_capability("accessibility.element_tokens", tool=tool):
+29 -40
View File
@@ -96,14 +96,13 @@ def _select_capture_target(
app_requested: bool,
exact_target: bool = False,
) -> Dict[str, Any]:
"""Select the best window for capture from z-sorted list_windows output.
"""Best window from z-sorted (frontmost-first) list_windows output.
Windows arrive sorted by ``z_index`` descending (frontmost first). For
unqualified default captures on Linux (no app filter, no exact target),
desktop/shell helper windows are skipped first — they are targetable but
capture as empty — and when every remaining candidate shares the same
``z_index`` (the common X11 case) ``_NET_ACTIVE_WINDOW`` beats list order.
Exact-target captures never pay for the ``xprop`` probe.
Unqualified default captures on Linux (no app filter, no exact target) skip
desktop/shell helper windows first — targetable but capture as empty — and
when every remaining candidate shares one ``z_index`` (the common X11 case)
``_NET_ACTIVE_WINDOW`` beats list order. Exact-target captures never pay
for the ``xprop`` probe.
"""
pool = [w for w in windows if not w["off_screen"]]
if not exact_target and not app_requested and sys.platform == "linux":
@@ -243,13 +242,10 @@ class _CaptureMixin:
def _match_windows_for_app(
self, windows: List[Dict[str, Any]], app: str
) -> List[Dict[str, Any]]:
"""Resolve ``app=`` through exact names before convenience substrings.
Linux ``list_windows`` can omit an app name while ``list_apps`` keeps
name/bundle-ID metadata. Exact direct names and exact metadata aliases
win over substring matches: querying ``Code`` must not silently select
``Visual Studio Code`` because it is frontmost.
"""
"""Resolve ``app=``: exact window names, then exact list_apps aliases
(Linux ``list_windows`` can omit the app name that ``list_apps`` keeps),
then substrings — querying ``Code`` must not silently select
``Visual Studio Code`` because it is frontmost."""
app_lower = app.strip().lower()
if not app_lower:
return []
@@ -354,13 +350,12 @@ class _CaptureMixin:
return {"pid": self._active_pid, "window_id": self._active_window_id, "session": self._session_id}
def _capture_vision(self) -> Tuple[Optional[str], Optional[str], str]:
"""Pixels only, no elements. Returns ``(png_b64, mime, window_title)``.
"""Pixels only, no elements: ``(png_b64, mime, window_title)``.
Drivers that advertise the (cheaper) standalone ``screenshot`` tool use
it; current drivers folded PNG capture into ``get_window_state``, whose
tree is DISCARDED here. When discovery hasn't run we still try
``screenshot`` first and fall back, so the path self-heals on any
driver version.
Drivers advertising the cheaper standalone ``screenshot`` tool use it;
current drivers folded PNG capture into ``get_window_state`` (tree
DISCARDED here). Before discovery ran we still try ``screenshot`` first
and fall back, so the path self-heals on any driver version.
"""
png_b64: Optional[str] = None
image_mime_type: Optional[str] = None
@@ -425,14 +420,11 @@ class _CaptureMixin:
pid: Optional[int] = None,
window_id: Optional[int] = None,
) -> CaptureResult:
"""Capture the frontmost on-screen window or an exact known target.
Maps hermes `capture(mode, app)` -> cua-driver `list_windows` +
`get_window_state` (ax/som) or `screenshot` (vision). Only the
structured ``structuredContent.windows`` shape is supported.
"""
# Drop schema-filler ids (models that zero-fill every optional
# property) before they read as a targeting request.
"""Capture the frontmost on-screen window or an exact known target:
`list_windows` + `get_window_state` (ax/som) or `screenshot` (vision).
Only the structured ``structuredContent.windows`` shape is supported."""
# Schema-filler ids (models zero-fill optional properties) must not read
# as a targeting request.
pid = None if _is_placeholder_id(pid) else pid
window_id = None if _is_placeholder_id(window_id) else window_id
exact_target = pid is not None or window_id is not None
@@ -466,13 +458,11 @@ class _CaptureMixin:
png_bytes_len=png_bytes_len, image_mime_type=image_mime_type)
def _capture_full_screen(self, mode: str) -> CaptureResult:
"""Composited grab of everything on screen via `get_desktop_state`
(like PrtScn) — the shell window would only show wallpaper + icons.
Never enumerates, so it also works when Windows UIA hangs. Pixels only:
`elements` is always empty; `note` tells the model how to reach the
interactive lanes. ``capture_scope`` is switched to desktop for the
call and restored afterwards.
"""
"""Composited PrtScn-style grab via `get_desktop_state` (the shell window
would only show wallpaper + icons). Never enumerates, so it also works
when Windows UIA hangs. Pixels only — `elements` is empty and `note`
points the model at the interactive lanes. ``capture_scope`` is switched
to desktop for the call and restored afterwards."""
self._clear_active_target()
previous_scope: Optional[str] = None
try:
@@ -550,11 +540,10 @@ class _CaptureMixin:
return []
def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
"""Target an app: a pure window-selector (store pid/window_id so later
input hits the right process) — background automation never needs to
raise a window. ``raise_window=True`` is explicit, separately approved,
and uses the standalone ``bring_to_front`` tool.
"""
"""Pure window-selector (store pid/window_id so later input hits the
right process) — background automation never needs to raise a window.
``raise_window=True`` is explicit, separately approved, and uses the
standalone ``bring_to_front`` tool."""
matched = self._match_windows_for_app(self._load_windows_or_disarm(), app)
# No silent fallback to the frontmost window: that hides the real
# failure (often a localized macOS app-name mismatch).
+17 -27
View File
@@ -93,14 +93,11 @@ def _wsl_windows_path_to_posix(path: str) -> str:
def _candidate_cua_driver_commands(override: Optional[str] = None) -> List[str]:
"""Candidate cua-driver commands in resolution order.
``override`` / a non-empty ``HERMES_CUA_DRIVER_CMD`` is authoritative (if
it is wrong, report the driver missing rather than silently picking
another binary). Otherwise PATH, then canonical installer locations —
Desktop apps launched from Finder/Dock inherit a narrow PATH that omits
``~/.local/bin``, and freshly installed Windows sessions inherit a stale one.
"""
"""Candidate commands in resolution order. ``override`` / a non-empty
``HERMES_CUA_DRIVER_CMD`` is authoritative (if wrong, report the driver
missing rather than silently picking another binary). Otherwise PATH, then
canonical installer locations — Finder/Dock-launched apps inherit a narrow
PATH without ``~/.local/bin``; fresh Windows sessions inherit a stale one."""
configured = (override if override is not None else os.environ.get(_CUA_DRIVER_CMD_ENV, "")).strip()
if configured:
return [configured]
@@ -179,14 +176,11 @@ def _cua_driver_supports_no_overlay(driver_cmd: str) -> bool:
def _resolve_mcp_invocation(driver_cmd: str, *, timeout: float = 6.0) -> Tuple[str, List[str]]:
"""Return ``(command, args)`` that spawn cua-driver's stdio MCP server.
Asks the driver itself via ``cua-driver manifest`` (``mcp_invocation``
carries ``command`` + ``args``) so a future rename of the subcommand keeps
working. Falls back to ``(driver_cmd, ["mcp"])`` for older drivers or any
discovery failure — the wrapper must not refuse to start over a failed
discovery hop. ``--no-overlay`` is appended when policy + driver allow.
"""
"""``(command, args)`` that spawn cua-driver's stdio MCP server, asked of
the driver itself via ``cua-driver manifest`` (``mcp_invocation``) so a
subcommand rename keeps working. Falls back to ``(driver_cmd, ["mcp"])`` on
older drivers or any discovery failure — the wrapper must not refuse to
start over a failed discovery hop. ``--no-overlay`` appended when allowed."""
manifest = _driver_json(driver_cmd, "manifest", timeout=timeout, require_ok=True) or {}
invocation = manifest.get("mcp_invocation")
invocation = invocation if isinstance(invocation, dict) else {}
@@ -269,17 +263,13 @@ def cua_driver_runtime_contract_status(binary: Optional[str] = None) -> Dict[str
def cua_driver_update_check(*, timeout: Optional[float] = None) -> Optional[Dict[str, Any]]:
"""Run ``cua-driver check-update --json``; payload mirrors the
``check_for_update`` MCP tool (``{current_version, latest_version,
update_available, ...}``).
``timeout`` defaults to 8s on POSIX and 25s on Windows (first spawn of the
exe routinely eats seconds in Defender scanning, and a false timeout is
expensive: callers treat ``None`` as indeterminate and the upgrade path
used to fall through to a full reinstall on it). Returns ``None`` when the
binary is missing, the driver predates the verb, the GitHub check failed
(``error`` set), or the output didn't parse. Never raises.
"""
"""``cua-driver check-update --json`` payload (``{current_version,
latest_version, update_available, ...}``), or ``None`` when the binary is
missing, the driver predates the verb, the GitHub check failed (``error``
set) or the output didn't parse. Never raises. ``timeout`` defaults to 8s
on POSIX / 25s on Windows: first spawn of the exe routinely eats seconds in
Defender scanning, and callers treat ``None`` as indeterminate (the upgrade
path used to fall through to a full reinstall on a false timeout)."""
if timeout is None:
timeout = 25.0 if sys.platform == "win32" else 8.0
driver_cmd = _cb().resolve_cua_driver_cmd()