refactor(computer_use): compact cua_backend docstrings/comments (WHYs preserved)
This commit is contained in:
@@ -88,15 +88,12 @@ def _computer_use_cfg() -> Dict[str, Any]:
|
||||
|
||||
|
||||
def _cua_no_overlay() -> bool:
|
||||
"""True when Hermes should pass ``--no-overlay`` to cua-driver.
|
||||
|
||||
``computer_use.no_overlay`` overrides when set. Auto-detect otherwise:
|
||||
off on macOS (cursor-overlay redraw loop can peg a core after a session),
|
||||
headless Linux / WSL2 / containers, and Linux X11 (the overlay is a
|
||||
fullscreen always-on-top all-workspaces window with no compositor-owned
|
||||
lifecycle, so an unclean session end can leave it wedged over every app);
|
||||
on for Windows and Linux Wayland (compositor owns the surface).
|
||||
"""
|
||||
"""Pass ``--no-overlay``? ``computer_use.no_overlay`` overrides; else off on
|
||||
macOS (cursor-overlay redraw loop can peg a core after a session), headless
|
||||
Linux / WSL2 / containers, and Linux X11 (the overlay is a fullscreen
|
||||
always-on-top all-workspaces window with no compositor-owned lifecycle, so
|
||||
an unclean session end can leave it wedged over every app); on for Windows
|
||||
and Linux Wayland (compositor owns the surface)."""
|
||||
val = _computer_use_cfg().get("no_overlay")
|
||||
if val is not None:
|
||||
return bool(val)
|
||||
@@ -122,13 +119,10 @@ def _cua_telemetry_disabled() -> bool:
|
||||
|
||||
|
||||
def _cua_configured_permission_mode() -> str:
|
||||
"""``computer_use.permission_mode`` (default ``standard``).
|
||||
|
||||
Only ``standard`` / ``bounded`` are honored: ``unrestricted`` is
|
||||
deliberately NOT a config value — it stays tied to the per-session YOLO
|
||||
toggle so a stale config line can never silently bypass approvals.
|
||||
Unknown values fall closed to ``standard``.
|
||||
"""
|
||||
"""``computer_use.permission_mode``: ``standard`` (default) or ``bounded``;
|
||||
unknown values fall closed to ``standard``. ``unrestricted`` is deliberately
|
||||
NOT a config value — it stays tied to the per-session YOLO toggle so a stale
|
||||
config line can never silently bypass approvals."""
|
||||
raw = str(_computer_use_cfg().get("permission_mode", "standard") or "").strip().lower()
|
||||
return raw if raw in {"standard", "bounded"} else "standard"
|
||||
|
||||
@@ -141,14 +135,11 @@ def _cua_capability_manifest() -> Optional[str]:
|
||||
|
||||
|
||||
def _manifest_is_mode_independent(path: str) -> bool:
|
||||
"""True when this capability manifest may accompany any permission mode.
|
||||
|
||||
v1/v2 manifests must declare ``mode: bounded`` and abort startup under an
|
||||
unrestricted runtime; v3 must NOT declare a mode and is the mode-
|
||||
independent ceiling the driver accepts alongside any mode. Unreadable /
|
||||
unparseable -> False (forwarding one would turn a working session into a
|
||||
hard startup failure; bounded forwards unconditionally anyway).
|
||||
"""
|
||||
"""True when this manifest may accompany any permission mode: v1/v2 declare
|
||||
``mode: bounded`` and abort startup under an unrestricted runtime; v3 has no
|
||||
mode and is the ceiling the driver accepts alongside any mode. Unreadable /
|
||||
unparseable -> False (forwarding one would turn a working session into a hard
|
||||
startup failure; bounded forwards unconditionally anyway)."""
|
||||
try:
|
||||
import yaml
|
||||
|
||||
@@ -212,13 +203,11 @@ def _run_driver(driver_cmd: str, *args: str, timeout: float) -> subprocess.Compl
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _linux_session_locked() -> Optional[bool]:
|
||||
"""Best-effort: is the graphical session locked? (Linux only.)
|
||||
|
||||
A locked KDE/GNOME session freezes renderers and half-disables the AX
|
||||
tree, so window discovery legitimately returns nothing — which otherwise
|
||||
reads as a driver bug. True/False when loginctl answers, None when
|
||||
unavailable (non-Linux, no systemd-logind, probe failure).
|
||||
"""
|
||||
"""Is the graphical session locked? (Linux; best-effort.) A locked KDE/GNOME
|
||||
session freezes renderers and half-disables the AX tree, so discovery
|
||||
legitimately returns nothing — which otherwise reads as a driver bug.
|
||||
True/False when loginctl answers, None when unavailable (non-Linux, no
|
||||
systemd-logind, probe failure)."""
|
||||
if sys.platform != "linux":
|
||||
return None
|
||||
|
||||
@@ -355,21 +344,19 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
|
||||
)
|
||||
self._bridge = _AsyncBridge()
|
||||
self._session = _CuaDriverSession(self._bridge, self._embedded_daemon)
|
||||
# Sticky context — updated by capture()/focus_app(), used by actions.
|
||||
# Sticky target — set by capture()/focus_app(), used by actions.
|
||||
self._active_pid: Optional[int] = None
|
||||
self._active_window_id: Optional[int] = None
|
||||
self._last_app: Optional[str] = None
|
||||
# Exact identity for capture_after: app names may be generic on Linux
|
||||
# Exact identity for capture_after: Linux app names may be generic
|
||||
# (several unrelated Qt windows can all say Qt6Application).
|
||||
self._last_target: Optional[Dict[str, Optional[int]]] = None
|
||||
# Per-snapshot `element_index -> element_token` map from capture().
|
||||
# Actions attach the token so cua-driver detects "stale" explicitly
|
||||
# instead of silently re-resolving to a different element.
|
||||
# Per-snapshot `element_index -> element_token`; actions attach it so
|
||||
# cua-driver reports "stale" instead of silently re-resolving.
|
||||
self._snapshot_tokens: Dict[int, str] = {}
|
||||
# Public session label (one per backend = one per Hermes run) passed as
|
||||
# `session` on every call: owns the agent cursor color and gives config
|
||||
# / recording state a stable owner inside the transport-private
|
||||
# lifecycle. Part of the 0.20 runtime contract checked at start().
|
||||
# Public session label (one per Hermes run) sent as `session` on every
|
||||
# call: owns the cursor color and gives config/recording state a stable
|
||||
# owner across transport restarts. Part of the 0.20 runtime contract.
|
||||
self._session_id: str = f"hermes-{uuid.uuid4().hex[:12]}"
|
||||
self._session.set_transport_reset_callback(self._handle_transport_reset)
|
||||
|
||||
@@ -392,8 +379,8 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
|
||||
raise RuntimeError(f"cua-driver is not ready: {reason}. {repair}")
|
||||
_maybe_nudge_update()
|
||||
# `mcp` is an optional extra: lazy-install on first use (gated by
|
||||
# `security.allow_lazy_installs`); on failure ensure() raises
|
||||
# FeatureUnavailable with the exact `uv pip install` hint.
|
||||
# `security.allow_lazy_installs`); failure raises FeatureUnavailable
|
||||
# with the exact `uv pip install` hint.
|
||||
from tools.lazy_deps import ensure as _lazy_ensure
|
||||
_lazy_ensure("tool.computer_use", prompt=False)
|
||||
import importlib
|
||||
@@ -412,12 +399,10 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
|
||||
self._best_effort("start_session failed (continuing anonymous)",
|
||||
self._session.call_tool, "start_session", {"session": self._session_id})
|
||||
|
||||
# Post-handshake tuning guards on `_started`: before the handshake
|
||||
# flips it, call_tool would re-enter session.start() and tests that
|
||||
# stub start() would recurse.
|
||||
# Post-handshake tuning guards on `_started`: before the handshake flips
|
||||
# it, call_tool would re-enter session.start() (stubbed start() recurses).
|
||||
if self._session._started:
|
||||
# Cap screenshot size so every later capture pays less over the
|
||||
# daemon socket and in the model turn.
|
||||
# Smaller screenshots cost less over the daemon socket and per turn.
|
||||
max_dim = _computer_use_max_image_dimension()
|
||||
if max_dim:
|
||||
self._best_effort("set_config(max_image_dimension) failed",
|
||||
@@ -428,9 +413,9 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
|
||||
self.set_agent_cursor_enabled, False, cursor_id=self._session_id)
|
||||
|
||||
def stop(self) -> None:
|
||||
# Best-effort end_session first so the driver cleans per-session state
|
||||
# (cursor overlay, recording ownership, config overrides); the
|
||||
# connection drop below releases daemon-side state regardless.
|
||||
# Best-effort end_session so the driver cleans per-session state (cursor
|
||||
# overlay, recording ownership, config overrides); the connection drop
|
||||
# below releases daemon-side state regardless.
|
||||
if self._session._started:
|
||||
self._best_effort("end_session failed (continuing teardown)",
|
||||
self._session.call_tool, "end_session", {"session": self._session_id})
|
||||
@@ -531,10 +516,10 @@ class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
|
||||
|
||||
# ── Internal ───────────────────────────────────────────────────
|
||||
def _maybe_attach_element_token(self, tool: str, args: Dict[str, Any]) -> None:
|
||||
"""Attach the last snapshot's ``element_token`` for an ``element_index``
|
||||
call. The token takes precedence and yields an explicit 'stale' error
|
||||
when the snapshot was superseded. Gated on the per-tool capability so
|
||||
older drivers (``additionalProperties: false``) never see the field."""
|
||||
"""Attach the snapshot's ``element_token`` to an ``element_index`` call so
|
||||
a superseded snapshot yields an explicit 'stale' error. Gated on the
|
||||
per-tool capability: older drivers (``additionalProperties: false``)
|
||||
must never see the field."""
|
||||
idx = args.get("element_index")
|
||||
token = self._snapshot_tokens.get(idx) if isinstance(idx, int) else None
|
||||
if token and self._session.supports_capability("accessibility.element_tokens", tool=tool):
|
||||
|
||||
@@ -96,14 +96,13 @@ def _select_capture_target(
|
||||
app_requested: bool,
|
||||
exact_target: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Select the best window for capture from z-sorted list_windows output.
|
||||
"""Best window from z-sorted (frontmost-first) list_windows output.
|
||||
|
||||
Windows arrive sorted by ``z_index`` descending (frontmost first). For
|
||||
unqualified default captures on Linux (no app filter, no exact target),
|
||||
desktop/shell helper windows are skipped first — they are targetable but
|
||||
capture as empty — and when every remaining candidate shares the same
|
||||
``z_index`` (the common X11 case) ``_NET_ACTIVE_WINDOW`` beats list order.
|
||||
Exact-target captures never pay for the ``xprop`` probe.
|
||||
Unqualified default captures on Linux (no app filter, no exact target) skip
|
||||
desktop/shell helper windows first — targetable but capture as empty — and
|
||||
when every remaining candidate shares one ``z_index`` (the common X11 case)
|
||||
``_NET_ACTIVE_WINDOW`` beats list order. Exact-target captures never pay
|
||||
for the ``xprop`` probe.
|
||||
"""
|
||||
pool = [w for w in windows if not w["off_screen"]]
|
||||
if not exact_target and not app_requested and sys.platform == "linux":
|
||||
@@ -243,13 +242,10 @@ class _CaptureMixin:
|
||||
def _match_windows_for_app(
|
||||
self, windows: List[Dict[str, Any]], app: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Resolve ``app=`` through exact names before convenience substrings.
|
||||
|
||||
Linux ``list_windows`` can omit an app name while ``list_apps`` keeps
|
||||
name/bundle-ID metadata. Exact direct names and exact metadata aliases
|
||||
win over substring matches: querying ``Code`` must not silently select
|
||||
``Visual Studio Code`` because it is frontmost.
|
||||
"""
|
||||
"""Resolve ``app=``: exact window names, then exact list_apps aliases
|
||||
(Linux ``list_windows`` can omit the app name that ``list_apps`` keeps),
|
||||
then substrings — querying ``Code`` must not silently select
|
||||
``Visual Studio Code`` because it is frontmost."""
|
||||
app_lower = app.strip().lower()
|
||||
if not app_lower:
|
||||
return []
|
||||
@@ -354,13 +350,12 @@ class _CaptureMixin:
|
||||
return {"pid": self._active_pid, "window_id": self._active_window_id, "session": self._session_id}
|
||||
|
||||
def _capture_vision(self) -> Tuple[Optional[str], Optional[str], str]:
|
||||
"""Pixels only, no elements. Returns ``(png_b64, mime, window_title)``.
|
||||
"""Pixels only, no elements: ``(png_b64, mime, window_title)``.
|
||||
|
||||
Drivers that advertise the (cheaper) standalone ``screenshot`` tool use
|
||||
it; current drivers folded PNG capture into ``get_window_state``, whose
|
||||
tree is DISCARDED here. When discovery hasn't run we still try
|
||||
``screenshot`` first and fall back, so the path self-heals on any
|
||||
driver version.
|
||||
Drivers advertising the cheaper standalone ``screenshot`` tool use it;
|
||||
current drivers folded PNG capture into ``get_window_state`` (tree
|
||||
DISCARDED here). Before discovery ran we still try ``screenshot`` first
|
||||
and fall back, so the path self-heals on any driver version.
|
||||
"""
|
||||
png_b64: Optional[str] = None
|
||||
image_mime_type: Optional[str] = None
|
||||
@@ -425,14 +420,11 @@ class _CaptureMixin:
|
||||
pid: Optional[int] = None,
|
||||
window_id: Optional[int] = None,
|
||||
) -> CaptureResult:
|
||||
"""Capture the frontmost on-screen window or an exact known target.
|
||||
|
||||
Maps hermes `capture(mode, app)` -> cua-driver `list_windows` +
|
||||
`get_window_state` (ax/som) or `screenshot` (vision). Only the
|
||||
structured ``structuredContent.windows`` shape is supported.
|
||||
"""
|
||||
# Drop schema-filler ids (models that zero-fill every optional
|
||||
# property) before they read as a targeting request.
|
||||
"""Capture the frontmost on-screen window or an exact known target:
|
||||
`list_windows` + `get_window_state` (ax/som) or `screenshot` (vision).
|
||||
Only the structured ``structuredContent.windows`` shape is supported."""
|
||||
# Schema-filler ids (models zero-fill optional properties) must not read
|
||||
# as a targeting request.
|
||||
pid = None if _is_placeholder_id(pid) else pid
|
||||
window_id = None if _is_placeholder_id(window_id) else window_id
|
||||
exact_target = pid is not None or window_id is not None
|
||||
@@ -466,13 +458,11 @@ class _CaptureMixin:
|
||||
png_bytes_len=png_bytes_len, image_mime_type=image_mime_type)
|
||||
|
||||
def _capture_full_screen(self, mode: str) -> CaptureResult:
|
||||
"""Composited grab of everything on screen via `get_desktop_state`
|
||||
(like PrtScn) — the shell window would only show wallpaper + icons.
|
||||
Never enumerates, so it also works when Windows UIA hangs. Pixels only:
|
||||
`elements` is always empty; `note` tells the model how to reach the
|
||||
interactive lanes. ``capture_scope`` is switched to desktop for the
|
||||
call and restored afterwards.
|
||||
"""
|
||||
"""Composited PrtScn-style grab via `get_desktop_state` (the shell window
|
||||
would only show wallpaper + icons). Never enumerates, so it also works
|
||||
when Windows UIA hangs. Pixels only — `elements` is empty and `note`
|
||||
points the model at the interactive lanes. ``capture_scope`` is switched
|
||||
to desktop for the call and restored afterwards."""
|
||||
self._clear_active_target()
|
||||
previous_scope: Optional[str] = None
|
||||
try:
|
||||
@@ -550,11 +540,10 @@ class _CaptureMixin:
|
||||
return []
|
||||
|
||||
def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
|
||||
"""Target an app: a pure window-selector (store pid/window_id so later
|
||||
input hits the right process) — background automation never needs to
|
||||
raise a window. ``raise_window=True`` is explicit, separately approved,
|
||||
and uses the standalone ``bring_to_front`` tool.
|
||||
"""
|
||||
"""Pure window-selector (store pid/window_id so later input hits the
|
||||
right process) — background automation never needs to raise a window.
|
||||
``raise_window=True`` is explicit, separately approved, and uses the
|
||||
standalone ``bring_to_front`` tool."""
|
||||
matched = self._match_windows_for_app(self._load_windows_or_disarm(), app)
|
||||
# No silent fallback to the frontmost window: that hides the real
|
||||
# failure (often a localized macOS app-name mismatch).
|
||||
|
||||
@@ -93,14 +93,11 @@ def _wsl_windows_path_to_posix(path: str) -> str:
|
||||
|
||||
|
||||
def _candidate_cua_driver_commands(override: Optional[str] = None) -> List[str]:
|
||||
"""Candidate cua-driver commands in resolution order.
|
||||
|
||||
``override`` / a non-empty ``HERMES_CUA_DRIVER_CMD`` is authoritative (if
|
||||
it is wrong, report the driver missing rather than silently picking
|
||||
another binary). Otherwise PATH, then canonical installer locations —
|
||||
Desktop apps launched from Finder/Dock inherit a narrow PATH that omits
|
||||
``~/.local/bin``, and freshly installed Windows sessions inherit a stale one.
|
||||
"""
|
||||
"""Candidate commands in resolution order. ``override`` / a non-empty
|
||||
``HERMES_CUA_DRIVER_CMD`` is authoritative (if wrong, report the driver
|
||||
missing rather than silently picking another binary). Otherwise PATH, then
|
||||
canonical installer locations — Finder/Dock-launched apps inherit a narrow
|
||||
PATH without ``~/.local/bin``; fresh Windows sessions inherit a stale one."""
|
||||
configured = (override if override is not None else os.environ.get(_CUA_DRIVER_CMD_ENV, "")).strip()
|
||||
if configured:
|
||||
return [configured]
|
||||
@@ -179,14 +176,11 @@ def _cua_driver_supports_no_overlay(driver_cmd: str) -> bool:
|
||||
|
||||
|
||||
def _resolve_mcp_invocation(driver_cmd: str, *, timeout: float = 6.0) -> Tuple[str, List[str]]:
|
||||
"""Return ``(command, args)`` that spawn cua-driver's stdio MCP server.
|
||||
|
||||
Asks the driver itself via ``cua-driver manifest`` (``mcp_invocation``
|
||||
carries ``command`` + ``args``) so a future rename of the subcommand keeps
|
||||
working. Falls back to ``(driver_cmd, ["mcp"])`` for older drivers or any
|
||||
discovery failure — the wrapper must not refuse to start over a failed
|
||||
discovery hop. ``--no-overlay`` is appended when policy + driver allow.
|
||||
"""
|
||||
"""``(command, args)`` that spawn cua-driver's stdio MCP server, asked of
|
||||
the driver itself via ``cua-driver manifest`` (``mcp_invocation``) so a
|
||||
subcommand rename keeps working. Falls back to ``(driver_cmd, ["mcp"])`` on
|
||||
older drivers or any discovery failure — the wrapper must not refuse to
|
||||
start over a failed discovery hop. ``--no-overlay`` appended when allowed."""
|
||||
manifest = _driver_json(driver_cmd, "manifest", timeout=timeout, require_ok=True) or {}
|
||||
invocation = manifest.get("mcp_invocation")
|
||||
invocation = invocation if isinstance(invocation, dict) else {}
|
||||
@@ -269,17 +263,13 @@ def cua_driver_runtime_contract_status(binary: Optional[str] = None) -> Dict[str
|
||||
|
||||
|
||||
def cua_driver_update_check(*, timeout: Optional[float] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Run ``cua-driver check-update --json``; payload mirrors the
|
||||
``check_for_update`` MCP tool (``{current_version, latest_version,
|
||||
update_available, ...}``).
|
||||
|
||||
``timeout`` defaults to 8s on POSIX and 25s on Windows (first spawn of the
|
||||
exe routinely eats seconds in Defender scanning, and a false timeout is
|
||||
expensive: callers treat ``None`` as indeterminate and the upgrade path
|
||||
used to fall through to a full reinstall on it). Returns ``None`` when the
|
||||
binary is missing, the driver predates the verb, the GitHub check failed
|
||||
(``error`` set), or the output didn't parse. Never raises.
|
||||
"""
|
||||
"""``cua-driver check-update --json`` payload (``{current_version,
|
||||
latest_version, update_available, ...}``), or ``None`` when the binary is
|
||||
missing, the driver predates the verb, the GitHub check failed (``error``
|
||||
set) or the output didn't parse. Never raises. ``timeout`` defaults to 8s
|
||||
on POSIX / 25s on Windows: first spawn of the exe routinely eats seconds in
|
||||
Defender scanning, and callers treat ``None`` as indeterminate (the upgrade
|
||||
path used to fall through to a full reinstall on a false timeout)."""
|
||||
if timeout is None:
|
||||
timeout = 25.0 if sys.platform == "win32" else 8.0
|
||||
driver_cmd = _cb().resolve_cua_driver_cmd()
|
||||
|
||||
Reference in New Issue
Block a user