From 21372a16b23acb702ae5df7e77da2ffc83695d3c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:00:40 -0700 Subject: [PATCH] refactor(computer_use): compact response helpers, NoopBackend recorder, docstrings/comments (976->931 LOC) --- tools/computer_use/tool.py | 291 ++++++++++++++++--------------------- 1 file changed, 123 insertions(+), 168 deletions(-) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index d2b6490af2..73cfd4a94a 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -1,8 +1,7 @@ -"""Entry point for the `computer_use` tool: any-model desktop control (macOS/Windows/Linux) via -cua-driver, standard OpenAI function-calling schema. Return contract: text-only results are a JSON -string; captures / `capture_after=True` return ``{"_multimodal": True, "content": [text part, -image_url part], "text_summary": }`` which run_agent.py / the Anthropic adapter turn into -provider-specific image tool content.""" +"""`computer_use` tool entry point: any-model desktop control (macOS/Windows/Linux) via cua-driver. +Return contract: text-only results are a JSON string; captures / `capture_after=True` return +``{"_multimodal": True, "content": [text, image_url], "text_summary": }`` (run_agent.py / +the Anthropic adapter turn it into provider-specific image tool content).""" from __future__ import annotations @@ -61,8 +60,7 @@ def _canon_key_combo(keys: str) -> frozenset: def _reject_unsafe(action: str, args: Dict[str, Any]) -> Optional[str]: """JSON error for hard-blocked input, else None. Runs BEFORE the approval prompt.""" if action == "type": - text = args.get("text", "") - pat = next((p.pattern for p in _BLOCKED_TYPE_PATTERNS if p.search(text)), None) + pat = next((p.pattern for p in _BLOCKED_TYPE_PATTERNS if p.search(args.get("text", ""))), None) if pat: return json.dumps({"error": f"blocked pattern in type text: {pat!r}", "hint": "Dangerous shell patterns cannot be typed via computer_use."}) @@ -83,9 +81,7 @@ def _input_target_mismatch(backend, requested_app: str) -> Optional[str]: target -> None (fail open; the verify ladder catches wrong-window delivery).""" last_app = getattr(backend, "_last_app", None) current, wanted = (last_app or "").strip().lower(), requested_app.strip().lower() - if not current or not wanted or wanted in current or current in wanted: - return None - return last_app + return None if not current or not wanted or wanted in current or current in wanted else last_app # ── Backend selection — env-swappable for tests ───────────────────────────── @@ -103,13 +99,11 @@ _AUX_VISION_ROUTE_CACHE: Dict[Tuple[str, str], bool] = {} # process-scoped: (pr _approval_lock = threading.Lock() _session_auto_approve: Dict[str, bool] = {} # sid -> "always_approve everything" _always_allow: Dict[str, set] = {} # sid -> set of (action, delivery_mode) scope keys -# Sessions already warned that a bypass widened the driver mode (resolver runs per dispatch). -_escalation_warned: set = set() +_escalation_warned: set = set() # sids already warned that a bypass widened the driver mode def _warn_bypass_escalation(session_id: str) -> None: - """Warn once per session that ``-z``/``--yolo`` swapped the driver onto a private ``unrestricted`` - daemon, dropping the configured ceiling. Deliberate (``unrestricted`` is intentionally not a config - value), but easy to trigger by accident.""" + """Warn once per session that ``-z``/``--yolo`` swapped the driver onto a private ``unrestricted`` daemon, + dropping the configured ceiling. Deliberate (``unrestricted`` is intentionally not a config value), but easy to trigger.""" key = str(session_id or "") with _approval_lock: if key in _escalation_warned: @@ -132,16 +126,15 @@ def _configured_permission_mode() -> str: return "standard" def _cua_permission_mode(session_id: str) -> str: - """Map Hermes's approval bypass onto Cua's immutable mode. Both identity namespaces are consulted — - DB ``session_id`` and gateway ``session_key`` contextvar — or a gateway ``/yolo`` would be - invisible here. Fails closed.""" + """Map Hermes's approval bypass onto Cua's immutable mode. Both identity namespaces are consulted — DB + ``session_id`` and gateway ``session_key`` contextvar — or a gateway ``/yolo`` would be invisible here. Fails closed.""" try: from tools.approval import get_current_session_key, is_approval_bypass_active_for_session - if is_approval_bypass_active_for_session(session_id): - _warn_bypass_escalation(session_id) - return "unrestricted" - current_key = get_current_session_key(default="") - if current_key and is_approval_bypass_active_for_session(current_key): + bypassed = is_approval_bypass_active_for_session(session_id) + if not bypassed: + current_key = get_current_session_key(default="") + bypassed = bool(current_key) and is_approval_bypass_active_for_session(current_key) + if bypassed: _warn_bypass_escalation(session_id) return "unrestricted" except Exception: @@ -159,13 +152,11 @@ def _new_backend(permission_mode: str) -> ComputerUseBackend: def _install_backend(sid: str, backend: ComputerUseBackend, permission_mode: str) -> None: """Record a backend in the session caches. Caller holds ``_backend_lock``.""" - _backends[sid] = backend - _backend_call_locks[sid] = threading.RLock() - _backend_permission_modes[sid] = permission_mode + _backends[sid], _backend_call_locks[sid], _backend_permission_modes[sid] = backend, threading.RLock(), permission_mode def _detach_locked(sid: str) -> Tuple[Optional[ComputerUseBackend], Optional[threading.RLock]]: - """Remove one session's cache entries, and the ``_backend`` injection hook when it aliases the - empty session (older callers/tests may populate only the hook). Caller holds ``_backend_lock``.""" + """Remove one session's cache entries, and the ``_backend`` injection hook when it aliases the empty + session (older callers/tests may populate only the hook). Caller holds ``_backend_lock``.""" global _backend _backend_permission_modes.pop(sid, None) backend, call_lock = _backends.pop(sid, None), _backend_call_locks.pop(sid, None) @@ -186,34 +177,31 @@ def _get_backend(session_id: str = "") -> ComputerUseBackend: sid = str(session_id or "") while True: with _backend_lock: - # Resolve the mode under the cache lock; YOLO mutation never holds the approval lock - # while releasing this cache, so no lock cycle. + # Mode resolved under the cache lock; YOLO mutation never holds the approval lock while releasing + # this cache, so no lock cycle. permission_mode = _cua_permission_mode(sid) if sid == "" and _backend is not None and sid not in _backends: _install_backend(sid, _backend, permission_mode) # fold the injection hook into the cache cached = _backends.get(sid) if cached is None: backend = _new_backend(permission_mode) - # Starting under the cache lock preserves one-backend-per-session. A concurrent - # mode toggle releases this backend before returning. - backend.start() + backend.start() # under the cache lock: one backend per session; a concurrent toggle releases it _install_backend(sid, backend, permission_mode) if sid == "": _backend = backend return backend if _backend_permission_modes.get(sid, "standard") == permission_mode: return cached - # Cua's permission mode cannot change after daemon startup: a /yolo toggle replaces - # only this session's backend. + # Cua's permission mode cannot change after daemon startup: a /yolo toggle replaces only this + # session's backend. Stop it outside the cache lock; the loop re-reads the authoritative mode first. _, stale_lock = _detach_locked(sid) - # Stop outside the cache lock; the loop re-reads the authoritative mode before installing a replacement. with contextlib.suppress(Exception): _stop_backend(cached, stale_lock) def release_computer_use_session(session_id: str) -> bool: - """Release one session-owned backend (lifecycle seam for hosts/plugins). Cache entries are removed - BEFORE stopping so new lookups cannot retain the stale target/ref namespace; approval state is - cleared even without a backend. True when a backend was released, False if already absent; idempotent.""" + """Release one session-owned backend (lifecycle seam for hosts/plugins). Cache entries are removed BEFORE + stopping so new lookups cannot retain the stale target/ref namespace; approval state is cleared even without + a backend. True when a backend was released, False if already absent; idempotent.""" sid = str(session_id or "") with _backend_lock: backend, call_lock = _detach_locked(sid) @@ -229,11 +217,10 @@ def release_computer_use_session(session_id: str) -> bool: return True def _shutdown_backend_atexit() -> None: - """Stop all cached backends so cua-driver subprocesses don't outlive us. atexit only, no signal - handlers: a ``SystemExit`` from a prompt_toolkit key binding corrupts its coroutine state and makes - the process unkillable. Never raises.""" + """Stop all cached backends so cua-driver subprocesses don't outlive us. atexit only, no signal handlers: a + ``SystemExit`` from a prompt_toolkit key binding corrupts its coroutine state and makes the process + unkillable. Never raises. Drops the global lock before stop(): teardown budgets 5s and must not block spawns.""" global _backend - # Drop the global lock before stop() — teardown budgets 5s and shouldn't block an unrelated spawn. with _backend_lock: unique = {id(b): (b, _backend_call_locks.get(sid)) for sid, b in _backends.items()} if _backend is not None: @@ -268,26 +255,21 @@ class _NoopBackend(ComputerUseBackend): # pragma: no cover def stop(self) -> None: self._started = False def is_available(self) -> bool: return True - def _record(self, name: str, kw: Dict[str, Any]) -> ActionResult: + def _record(self, name: str, kw: Dict[str, Any], result: Any = None) -> Any: self.calls.append((name, kw)) - return ActionResult(ok=True, action=name) + return ActionResult(ok=True, action=name) if result is None else result - def _record_list(self, name: str) -> List[Dict[str, Any]]: - self.calls.append((name, {})) - return [] - - def capture(self, mode: str = "som", app: Optional[str] = None, - pid: Optional[int] = None, window_id: Optional[int] = None) -> CaptureResult: - self._record("capture", {"mode": mode, "app": app, "pid": pid, "window_id": window_id}) - return CaptureResult(mode=mode, width=1024, height=768, png_b64=None, elements=[], app=app or "", window_title="") + def capture(self, mode="som", app=None, pid=None, window_id=None) -> CaptureResult: + return self._record("capture", {"mode": mode, "app": app, "pid": pid, "window_id": window_id}, + CaptureResult(mode=mode, width=1024, height=768, png_b64=None, elements=[], app=app or "", window_title="")) def click(self, **kw) -> ActionResult: return self._record("click", kw) def drag(self, **kw) -> ActionResult: return self._record("drag", kw) def scroll(self, **kw) -> ActionResult: return self._record("scroll", kw) def type_text(self, text: str, **kw) -> ActionResult: return self._record("type", {"text": text, **kw}) def key(self, keys: str, **kw) -> ActionResult: return self._record("key", {"keys": keys, **kw}) - def list_apps(self) -> List[Dict[str, Any]]: return self._record_list("list_apps") - def list_windows(self) -> List[Dict[str, Any]]: return self._record_list("list_windows") + def list_apps(self) -> List[Dict[str, Any]]: return self._record("list_apps", {}, []) + def list_windows(self) -> List[Dict[str, Any]]: return self._record("list_windows", {}, []) def focus_app(self, app: str, raise_window: bool = False) -> ActionResult: return self._record("focus_app", {"app": app, "raise": raise_window}) def set_value(self, value: str, element: Optional[int] = None) -> ActionResult: @@ -297,8 +279,7 @@ class _NoopBackend(ComputerUseBackend): # pragma: no cover # ── Dispatch ──────────────────────────────────────────────────────────────── def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: - """Main entry point — dispatched by tools.registry. Returns a JSON string (text-only) or a dict - marked `_multimodal` (image + summary).""" + """Main entry point (tools.registry). Returns a JSON string (text-only) or a dict marked `_multimodal`.""" action = (args.get("action") or "").strip().lower() if not action: return json.dumps({"error": "missing `action`"}) @@ -306,8 +287,8 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: err = _reject_unsafe(action, args) if err is not None: return err - # Approval gate (destructive actions only). Persistent focus is a separate, visible side effect - # with its own scope even when the input rung is approved. + # Approval gate (destructive actions only). Persistent focus is a separate, visible side effect with its + # own scope even when the input rung is approved. scopes = [action] if action in _DESTRUCTIVE_ACTIONS else [] if args.get("bring_to_front") or (action == "focus_app" and args.get("raise_window")): scopes.append("bring_to_front") @@ -318,10 +299,9 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: try: backend = _get_backend(session_id=session_id) except Exception as e: - return json.dumps({ - "error": f"computer_use backend unavailable: {e}", - "hint": "If the cua-driver binary is missing, run `hermes computer-use install`. " - "If a Python dependency is missing, the error above shows the exact install command."}) + return json.dumps({"error": f"computer_use backend unavailable: {e}", + "hint": "If the cua-driver binary is missing, run `hermes computer-use install`. " + "If a Python dependency is missing, the error above shows the exact install command."}) try: with _backend_lock: call_lock = _backend_call_locks.setdefault(session_id, threading.RLock()) @@ -332,17 +312,15 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: return json.dumps({"error": f"{action} failed: {e}"}) def _request_approval(action: str, args: Dict[str, Any], session_id: str = "") -> Optional[str]: - """None if approved, else a JSON error string. Scoped by (action, delivery_mode) AND session_id: - foreground delivery is a visible focus change, so a background ``approve_session`` must NOT cover - it; the blanket ``always_approve`` does.""" + """None if approved, else a JSON error string. Scoped by (action, delivery_mode) AND session_id: foreground + delivery is a visible focus change, so a background ``approve_session`` must NOT cover it; the blanket + ``always_approve`` does.""" scope_key = (action, "foreground" if args.get("delivery_mode") == "foreground" else "background") with _approval_lock: if _session_auto_approve.get(session_id) or scope_key in _always_allow.get(session_id, set()): return None cb = _approval_callback - if cb is None: - # No CLI approval wired — default allow. Gateway approval is handled one layer out via the - # normal tool-approval infra. + if cb is None: # no CLI approval wired — default allow; gateway approval runs one layer out (tool-approval infra) return None try: verdict = cb(action, args, _summarize_action(action, args)) @@ -368,8 +346,7 @@ _CLICK_VARIANTS = {"click": (None, 1), "double_click": (None, 2), "right_click": def _summarize_click(action: str, args: Dict[str, Any], fg: str) -> str: if args.get("element") is not None: return f"{action} element #{args['element']}{fg}" - coord = args.get("coordinate") - return f"{action} at {tuple(coord)}{fg}" if coord else action + fg + return f"{action} at {tuple(args['coordinate'])}{fg}" if args.get("coordinate") else action + fg def _summarize_type(action: str, args: Dict[str, Any], fg: str) -> str: text = args.get("text", "") @@ -397,17 +374,15 @@ def _do_capture(backend: ComputerUseBackend, args: Dict[str, Any]) -> Any: mode = str(args.get("mode", "som")) if mode not in {"som", "vision", "ax"}: return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"}) - capture_kwargs: Dict[str, Any] = {"mode": mode, "app": args.get("app")} - # pid/window_id forwarded only when given so older backends keep their defaults. - if args.get("pid") is not None or args.get("window_id") is not None: - capture_kwargs.update(pid=args.get("pid"), window_id=args.get("window_id")) - return _capture_response(backend.capture(**capture_kwargs)) + kwargs: Dict[str, Any] = {"mode": mode, "app": args.get("app")} + if args.get("pid") is not None or args.get("window_id") is not None: # forwarded only when given (older backends) + kwargs.update(pid=args.get("pid"), window_id=args.get("window_id")) + return _capture_response(backend.capture(**kwargs)) def _do_focus_app(backend: ComputerUseBackend, args: Dict[str, Any]) -> Any: - app = args.get("app") - if not app: + if not args.get("app"): return json.dumps({"error": "focus_app requires `app`"}) - res = backend.focus_app(app, raise_window=bool(args.get("raise_window"))) + res = backend.focus_app(args["app"], raise_window=bool(args.get("raise_window"))) return _maybe_follow_capture(backend, res, bool(args.get("capture_after"))) def _listing(key: str, items: List[Dict[str, Any]]) -> str: @@ -450,10 +425,9 @@ def _do_scroll(backend, action, args, **delivery): element=args.get("element"), x=x, y=y, modifiers=args.get("modifiers"), **delivery) def _do_set_value(backend, action, args, **delivery): - value = args.get("value") - if value is None: + if args.get("value") is None: return json.dumps({"error": "set_value requires `value`"}) - return backend.set_value(value=str(value), element=args.get("element")) + return backend.set_value(value=str(args["value"]), element=args.get("element")) _INPUT_HANDLERS = { **dict.fromkeys(_CLICK_VARIANTS, _do_click), @@ -480,8 +454,8 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> handler = _INPUT_HANDLERS.get(action) if handler is None: hint = _ACTION_SUGGESTIONS.get(str(action)) - suffix = f" — did you mean {hint!r}? See the action enum in the tool schema." if hint else "" - return json.dumps({"error": f"unknown action {action!r}{suffix}"}) + return json.dumps({"error": f"unknown action {action!r}" + + (f" — did you mean {hint!r}? See the action enum in the tool schema." if hint else "")}) # app= guard: input goes to the sticky target from the last capture/focus_app and the backend drops # app= silently — refuse a clear mismatch rather than type into the wrong window while reporting ok:true. requested_app = args.get("app") @@ -510,9 +484,9 @@ def _classify_action_result(res: ActionResult) -> Dict[str, Any]: if res.effect == "confirmed" or res.verified is True: return {"decision": "done"} if res.effect == "unverifiable": - return {"decision": "verify_fresh_state", - "hint": ("Input was delivered but not confirmed. Re-capture and check the result BEFORE any " - "retry — do not repeat the input on an escalation recommendation alone.")} + return {"decision": "verify_fresh_state", "hint": ( + "Input was delivered but not confirmed. Re-capture and check the result BEFORE any " + "retry — do not repeat the input on an escalation recommendation alone.")} if res.effect == "suspected_noop" or not res.ok or res.code is not None: decision: Dict[str, Any] = {"decision": "escalate"} if isinstance(res.escalation, dict): @@ -544,15 +518,13 @@ def _action_payload(res: ActionResult) -> Dict[str, Any]: def _text_response(res: ActionResult) -> str: return json.dumps(_action_payload(res)) -# Cap for the AX `elements` array: dense UIs publish 500+ AX nodes, which would exhaust context after -# one capture. The full tree spills to `elements_file`. +# Cap for the AX `elements` array: dense UIs publish 500+ AX nodes, which would exhaust context after one +# capture. The full tree spills to `elements_file`. _DEFAULT_MAX_ELEMENTS = 100 -# Some providers reject images below 8x8 before the model sees the tool result; such captures fall -# back to the AX/SOM text payload. +# Some providers reject images below 8x8 before the model sees the result; such captures fall back to text. _MIN_PROVIDER_IMAGE_DIMENSION = 8 -# Some AX trees (Discord/Slack via UIA, Electron chat clients) expose ENTIRE message bodies as labels; -# uncapped they blew the tool-result budget and leaked private chat text. Labels identify a control; -# captures aren't text extraction. +# Some AX trees (Discord/Slack via UIA, Electron chat clients) expose ENTIRE message bodies as labels; uncapped +# they blew the tool-result budget and leaked private chat text. Labels identify a control, not text extraction. _MAX_ELEMENT_LABEL_CHARS = 120 # Bounded cache trails: every dense capture can spill, and CLI-only sessions never run the gateway's # periodic media-cache cleanup. @@ -567,8 +539,7 @@ def _image_dimensions_from_b64(image_b64: str) -> Optional[Tuple[int, int]]: return None def _capture_mime(cap: CaptureResult) -> str: - """Prefer cua-driver's explicit MIME type; sniff the base64 prefix for older builds (JPEG base64 - starts with /9j/, PNG with iVBOR).""" + """cua-driver's explicit MIME type, else sniff the base64 prefix (JPEG starts with /9j/, PNG with iVBOR).""" return cap.image_mime_type or ("image/jpeg" if (cap.png_b64 or "").startswith("/9j/") else "image/png") def _capture_image_ext(cap: CaptureResult) -> str: @@ -576,21 +547,19 @@ def _capture_image_ext(cap: CaptureResult) -> str: return ".jpg" if _capture_mime(cap).lower() == "image/jpeg" else ".png" def _bounds_unknown(bounds) -> bool: - """True when the AX tree reported no real geometry. KDE/Qt apps report ``[0, 0, 0, 0]`` for elements - clickable by index; serializing that as a rect invites ``coordinate=[0, 0]`` clicks.""" + """True when the AX tree reported no real geometry. KDE/Qt apps report ``[0, 0, 0, 0]`` for elements clickable + by index; serializing that as a rect invites ``coordinate=[0, 0]`` clicks.""" try: return all(int(v) == 0 for v in bounds) except (TypeError, ValueError): return False def _element_to_dict(e: UIElement) -> Dict[str, Any]: - # A zero rect is "geometry unknown", not a position — null it so no coordinate= is ever derived - # from it. The element index still works. - out: Dict[str, Any] = {"index": e.index, "role": e.role, "label": e.label[:_MAX_ELEMENT_LABEL_CHARS], - "bounds": None if _bounds_unknown(e.bounds) else list(e.bounds), "app": e.app} - if len(e.label) > _MAX_ELEMENT_LABEL_CHARS: - out["label_truncated"] = True - return out + # A zero rect is "geometry unknown", not a position — null it so no coordinate= is ever derived from it. + # The element index still works. + return {"index": e.index, "role": e.role, "label": e.label[:_MAX_ELEMENT_LABEL_CHARS], + "bounds": None if _bounds_unknown(e.bounds) else list(e.bounds), "app": e.app, + **({"label_truncated": True} if len(e.label) > _MAX_ELEMENT_LABEL_CHARS else {})} def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str]: out: List[str] = [] @@ -603,9 +572,8 @@ def _format_elements(elements: List[UIElement], max_lines: int = 40) -> List[str return out def _bounds_divergence(elements: List[UIElement], image_width: int, image_height: int) -> Optional[Tuple[int, int]]: - """(max right edge, max bottom edge) of element bounds when they exceed the screenshot, else None. - 5% slack: window chrome can hang a few px past the captured frame without implying a different - coordinate space.""" + """(max right edge, max bottom edge) of element bounds when they exceed the screenshot, else None. 5% slack: + window chrome can hang a few px past the captured frame without implying a different coordinate space.""" if not elements or image_width <= 0 or image_height <= 0: return None max_x = max_y = 0 @@ -621,10 +589,9 @@ def _bounds_divergence(elements: List[UIElement], image_width: int, image_height def _bounds_hints(elements: List[UIElement], image_width: int, image_height: int ) -> Tuple[Optional[float], Optional[str]]: - """(scale, note) when element bounds live in a different coordinate space than the screenshot, else - (None, None). On HiDPI displays AX bounds are native while the screenshot is downscaled, so coordinate= - clicks read off the screenshot miss by the scale factor. Scale is a heuristic: larger axis ratio wins - so real extent data drives it; 2 decimals.""" + """(scale, note) when element bounds live in a different coordinate space than the screenshot, else (None, + None). On HiDPI displays AX bounds are native while the screenshot is downscaled, so coordinate= clicks read + off the screenshot miss by the scale factor. Scale heuristic: larger axis ratio wins (real extent data drives it).""" extent = _bounds_divergence(elements, image_width, image_height) if extent is None: return None, None @@ -641,9 +608,8 @@ def _bounds_space_note(elements: List[UIElement], image_width: int, image_height @dataclass class _CaptureView: - """One capture's derived facts, computed once and shared by every response branch. ``width``/``height`` - are the decoded screenshot dims when an image is present, else the backend's; ``visible`` is the capped - element list every branch must use.""" + """One capture's derived facts, computed once and shared by every response branch. ``width``/``height`` are + the decoded screenshot dims when an image is present, else the backend's; ``visible`` is the capped list.""" cap: CaptureResult visible: List[UIElement] total: int @@ -673,8 +639,8 @@ def _capture_view(cap: CaptureResult, max_elements: int) -> _CaptureView: dims_omitted=dims if too_small else None, has_image=has_image) def _capture_summary_lines(v: _CaptureView) -> List[str]: - """Human-readable capture summary; line ORDER is contract. Indexes only what is surfaced in - `elements`, otherwise the summary names indices the model can't find.""" + """Human-readable capture summary; line ORDER is contract. Indexes only what is surfaced in `elements`, + otherwise the summary names indices the model can't find.""" cap, bounds_note = v.cap, v.bounds_note if bounds_note and v.bounds_scale: bounds_note += f"; estimated scale ~{v.bounds_scale}x (screenshot position x {v.bounds_scale} ≈ native coordinate)" @@ -711,8 +677,8 @@ def _multimodal_capture(v: _CaptureView, summary: str) -> Dict[str, Any]: } def _text_capture_payload(v: _CaptureView, summary: str, extra: Optional[Dict[str, Any]] = None) -> str: - """JSON text payload shared by the AX, vision-unavailable and aux-vision branches. Key order is - contract: fixed fields, ``extra`` branch markers, then set optionals.""" + """JSON text payload shared by the AX, vision-unavailable and aux-vision branches. Key order is contract: + fixed fields, ``extra`` branch markers, then set optionals.""" cap = v.cap payload: Dict[str, Any] = { "mode": cap.mode, "width": v.width, "height": v.height, "app": cap.app, "window_title": cap.window_title, @@ -729,9 +695,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME summary = "\n".join(lines) # multimodal/aux paths use this; text paths append notes and rebuild extra = None if v.has_image: - # Hand the screenshot to auxiliary.vision (text-only result) when the main model may not consume - # images natively; returning the multimodal envelope unconditionally tripped HTTP 404/400 at the - # provider boundary. + # Hand the screenshot to auxiliary.vision (text-only result) when the main model may not consume images + # natively; returning the multimodal envelope unconditionally tripped HTTP 404/400 at the provider. if not _should_route_through_aux_vision(): return _multimodal_capture(v, summary) routed = _route_capture_through_aux_vision( @@ -739,8 +704,8 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME elements_file=v.elements_file, screenshot_path=v.screenshot_path) if routed is not None: return routed - # Aux routing requested but failed (vision node down, empty analysis...). The multimodal envelope - # could now break with a provider error, so degrade to text. + # Aux routing requested but failed (vision node down, empty analysis...): the multimodal envelope could + # now break with a provider error, so degrade to text. lines.append(" (vision unavailable: the auxiliary vision model could not be reached; screenshot " "omitted. Element-index actions still work — drive via the element list above.)") extra = {"vision_unavailable": True} @@ -754,8 +719,8 @@ def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_cap if not do_capture or not res.ok: return _text_response(res) try: - # Recapture the exact window when known: on Linux several unrelated windows may share an app - # name, so app-only recapture can switch targets. + # Recapture the exact window when known: on Linux several unrelated windows may share an app name, so + # app-only recapture can switch targets. target = getattr(backend, "_last_target", None) or {} pid, window_id = target.get("pid"), target.get("window_id") mode = _capture_after_mode() @@ -769,8 +734,7 @@ def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_cap resp = _capture_response(cap) payload = _action_payload(res) if isinstance(resp, dict) and resp.get("_multimodal"): - # Keep the evidence/verdict contract visible alongside the image — it governs whether - # repeating input is allowed. + # Keep the evidence/verdict contract visible alongside the image — it governs whether repeating input is allowed. prefix = json.dumps(payload) + "\n\n" resp["content"][0]["text"] = prefix + resp["content"][0]["text"] resp["text_summary"] = prefix + resp["text_summary"] @@ -786,9 +750,9 @@ def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_cap # ── Cache files (screenshots, element spills, vision temps) ───────────────── def _cache_file(subdir: str, legacy: str, name: str, pattern: str = "", cap: int = 0): - """Path for a new file under ``$HERMES_HOME/`` (dir created). With ``pattern``/``cap``, first - unlinks the oldest matching files so at most ``cap - 1`` remain (best-effort). Imports lazily so - tests can patch ``hermes_constants.get_hermes_dir``.""" + """Path for a new file under ``$HERMES_HOME/`` (dir created). With ``pattern``/``cap``, first unlinks + the oldest matching files so at most ``cap - 1`` remain (best-effort). Lazy import so tests can patch + ``hermes_constants.get_hermes_dir``.""" from hermes_constants import get_hermes_dir cache_dir = get_hermes_dir(subdir, legacy) cache_dir.mkdir(parents=True, exist_ok=True) @@ -799,33 +763,25 @@ def _cache_file(subdir: str, legacy: str, name: str, pattern: str = "", cap: int stale.unlink(missing_ok=True) return cache_dir / name -def _best_effort_write(what: str, write: Callable[[], str]) -> Optional[str]: - """Run a cache write and return its path, or None on any failure: an unwritable cache must never - break desktop control.""" - try: - return write() - except Exception as exc: # pragma: no cover - defensive - logger.debug("computer_use: %s failed: %s", what, exc) - return None - def _persist_capture_image(cap: CaptureResult) -> Optional[str]: - """Save a bounded copy of the capture in Hermes' media cache so attachment surfaces can deliver it; - returns the path.""" + """Save a bounded copy of the capture in Hermes' media cache so attachment surfaces can deliver it; returns + the path. Best-effort: an unwritable cache must never break control.""" if not cap.png_b64: return None - - def write() -> str: + try: raw = base64.b64decode(cap.png_b64, validate=False) path = _cache_file("cache/images", "image_cache", f"computer_use_{uuid.uuid4().hex}{_capture_image_ext(cap)}", "computer_use_*.*", _MAX_CAPTURE_FILES) path.write_bytes(raw) return str(path) - return _best_effort_write("screenshot persistence", write) + except Exception as exc: # pragma: no cover - defensive + logger.debug("computer_use: screenshot persistence failed: %s", exc) + return None def _spill_elements_to_file(cap: CaptureResult) -> Optional[str]: - """Write the FULL element tree (untruncated labels) to a cache file — the read_file/search_files - escape hatch for capped text.""" - def write() -> str: + """Write the FULL element tree (untruncated labels) to a cache file — the read_file/search_files escape + hatch for capped text. Path, or None on any failure (a capture must never fail on an unwritable cache).""" + try: path = _cache_file("cache/computer_use", "computer_use_cache", f"elements_{uuid.uuid4().hex}.json", "elements_*.json", _MAX_SPILL_FILES) payload = {"app": cap.app, "window_title": cap.window_title, "total_elements": len(cap.elements), @@ -833,20 +789,21 @@ def _spill_elements_to_file(cap: CaptureResult) -> Optional[str]: "bounds": list(e.bounds), "app": e.app} for e in cap.elements]} path.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") return str(path) - return _best_effort_write("element spill", write) + except Exception as exc: # pragma: no cover - defensive + logger.debug("computer_use: element spill failed: %s", exc) + return None # ── auxiliary.vision routing for captured screenshots ─────────────────────── -# Longest image side handed to the aux vision model. Full-resolution desktop captures tokenize heavily -# and can overflow small local-model context windows; ~1456px keeps SOM badges legible while cutting -# per-capture vision latency. +# Longest image side handed to the aux vision model. Full-resolution desktop captures tokenize heavily and can +# overflow small local-model context windows; ~1456px keeps SOM badges legible while cutting vision latency. _MAX_VISION_DIM = 1456 def _shrink_capture_for_vision(raw: bytes, ext: str, max_dim: int = _MAX_VISION_DIM) -> tuple[bytes, Optional[str]]: - """Downscale encoded image bytes so the longest side is <= max_dim. Returns ``(bytes, scale_note)``; - note is None when unchanged (fits, or Pillow unavailable/failed), else it tells the vision model the - factor so reported coordinates map back to the real screen instead of being silently wrong.""" + """Downscale encoded image bytes so the longest side is <= max_dim. Returns ``(bytes, scale_note)``; note is + None when unchanged (fits, or Pillow unavailable/failed), else it tells the vision model the factor so + reported coordinates map back to the real screen instead of being silently wrong.""" try: from io import BytesIO from PIL import Image @@ -858,13 +815,11 @@ def _shrink_capture_for_vision(raw: bytes, ext: str, max_dim: int = _MAX_VISION_ new_w, new_h = img.size out = BytesIO() img.save(out, format="JPEG" if ext == ".jpg" else "PNG") - fx = orig_w / new_w if new_w else 1.0 - fy = orig_h / new_h if new_h else 1.0 - if f"{fx:.2f}" == f"{fy:.2f}": - factor_clause = f"multiply any coordinates you report by {fx:.2f} to map back to the real screen." - else: - factor_clause = (f"multiply any x coordinates you report by {fx:.2f} and " - f"any y coordinates by {fy:.2f} to map back to the real screen.") + fx, fy = (orig_w / new_w if new_w else 1.0), (orig_h / new_h if new_h else 1.0) + factor_clause = (f"multiply any coordinates you report by {fx:.2f} to map back to the real screen." + if f"{fx:.2f}" == f"{fy:.2f}" else + f"multiply any x coordinates you report by {fx:.2f} and " + f"any y coordinates by {fy:.2f} to map back to the real screen.") return out.getvalue(), f"Screenshot downscaled from {orig_w}x{orig_h} to {new_w}x{new_h} for vision; {factor_clause}" except Exception as exc: logger.debug("computer_use: vision downscale skipped: %s", exc) @@ -922,8 +877,8 @@ def _route_capture_through_aux_vision( cap: CaptureResult, summary: str, *, visible_elements: Optional[List[UIElement]] = None, truncated_elements: int = 0, elements_file: Optional[str] = None, screenshot_path: Optional[str] = None, ) -> Optional[str]: - """Pre-analyse the capture via ``vision_analyze_tool`` (temp file under ``$HERMES_HOME/cache/vision/``) - and merge the description with the AX/SOM summary into one text payload. JSON, or None on any failure.""" + """Pre-analyse the capture via ``vision_analyze_tool`` (temp file under ``$HERMES_HOME/cache/vision/``) and + merge the description with the AX/SOM summary into one text payload. JSON, or None on any failure.""" if not cap.png_b64: return None problem = "aux-vision import failed" @@ -954,8 +909,8 @@ def _route_capture_through_aux_vision( analysis_text = _vision_analysis_text(result_json) if not analysis_text: return None - # Same element cap as every other capture branch; dumping cap.elements in full would bypass - # max_elements exactly for non-vision main models. Dimensions are the backend's on this branch. + # Same element cap as every other capture branch; dumping cap.elements in full would bypass max_elements + # exactly for non-vision main models. Dimensions are the backend's on this branch. view = _CaptureView(cap, cap.elements if visible_elements is None else visible_elements, len(cap.elements), truncated_elements, cap.width, cap.height, elements_file=elements_file, screenshot_path=screenshot_path) return _text_capture_payload(view, summary, {"vision_analysis": analysis_text, "vision_analysis_routed_via": "auxiliary.vision"}) @@ -964,8 +919,8 @@ def _route_capture_through_aux_vision( # ── Availability check (used by the tool registry check_fn) ───────────────── def check_computer_use_requirements() -> bool: - """True iff computer_use can run here: macOS/Windows/Linux + cua-driver binary (or env override). - `hermes computer-use doctor` names blocked Linux checks.""" + """True iff computer_use can run here: macOS/Windows/Linux + cua-driver binary (or env override). `hermes + computer-use doctor` names blocked Linux checks.""" if sys.platform not in ("darwin", "win32", "linux"): return False from tools.computer_use.cua_backend import cua_driver_binary_available