diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index a42474bf7c..f065999b6f 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -642,8 +642,9 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str: "`computer_use.grant_existing_profile: true` (if unset, report the " "refusal and name that key — you can never grant it yourself); " "bounded mode authorizes via the user's reviewed capability manifest; " - "explicit Hermes YOLO uses a private unrestricted daemon after the " - "user's launch/session risk acceptance.\n\n" + "explicit Hermes YOLO uses an unrestricted runtime after the user's " + "launch/session risk acceptance. Permission mode and grants are fixed " + "when Hermes launches that runtime.\n\n" "## Background mode rules\n" "- Do NOT use `raise_window=true` on `focus_app` unless the user " "explicitly asked you to bring a window to front. Input routing to " @@ -653,9 +654,11 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str: "won't leak other windows the user has open.\n" + offscreen_line + "## The agent cursor you'll see on screen\n" - "Each computer-use run declares a session with cua-driver; that " - "session owns a tinted overlay cursor that glides to where you " - "act. It's a visual cue for the user — the REAL OS cursor never " + "Each computer-use run gives cua-driver a public session name. The " + "name labels its tinted overlay cursor and related state, while the " + "MCP transport owns a private lifecycle session inside the runtime. " + "The cursor glides " + "to where you act. It's a visual cue for the user; the REAL OS cursor never " "moves. Don't try to read it or click on it; it's UI feedback, " "not input.\n\n" "## Safety\n" diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 37be35baee..64c6a8958a 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -3312,21 +3312,23 @@ DEFAULT_CONFIG = { # True = always disable the overlay # False = always enable the overlay "no_overlay": None, - # cua-driver permission mode for this Hermes install. + # cua-driver permission mode for each Hermes computer-use runtime. # standard (default) — cua-driver's own approval boundary. Protected # operations (e.g. attaching to an existing signed-in browser # profile) fail closed unless grant_existing_profile is enabled # below. # bounded — repeatable automation under a user-reviewed session - # policy manifest (set capability_manifest below). No runtime + # capability manifest (set capability_manifest below). No runtime # prompts; anything outside the manifest fails closed inside # cua-driver. # `unrestricted` is intentionally NOT accepted here: it stays bound to # the explicit per-session YOLO toggle so a config line can never # silently bypass approvals. "permission_mode": "standard", - # Absolute or ~ path to the reviewed cua-driver session-policy - # manifest YAML used when permission_mode is "bounded". See + # Absolute or ~ path to the reviewed cua-driver capability + # manifest used when permission_mode is "bounded". Hermes passes the + # canonical --capability-manifest and --approve-capability-manifest + # flags when it launches the runtime. See # https://cua.ai/docs/reference/cua-driver/permission-modes "capability_manifest": "", # Pre-authorize existing-profile browser attachment in standard mode diff --git a/hermes_cli/main.py b/hermes_cli/main.py index bfd79dea4e..12f79b2208 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -12586,7 +12586,7 @@ def main(): build_tools_parser(subparsers, cmd_tools=cmd_tools) # ========================================================================= - # computer-use command — manage Computer Use (cua-driver) on macOS + # computer-use command — manage Computer Use (cua-driver) # ========================================================================= computer_use_parser = subparsers.add_parser( "computer-use", @@ -12689,39 +12689,6 @@ def main(): "grant", help="Request the grants (opens the dialog attributed to CuaDriver)", ) - computer_use_browser_approve = computer_use_sub.add_parser( - "browser-approve", - help="Mint a single-use token authorizing browser attachment for one exact window", - description=( - "Runs `cua-driver browser-approve` to mint a five-minute,\n" - "single-use token that authorizes ONE browser preparation for the\n" - "exact process (and window) you name. Give the printed token to\n" - "the agent; it passes it as approval_token on the\n" - "cua_browser_prepare action.\n\n" - "This is the explicit human boundary for attaching to a browser —\n" - "especially an existing signed-in profile, where the DevTools\n" - "protocol can see that profile's live pages, cookies, and storage.\n" - "Ordinary tool approval never substitutes for this grant, so a\n" - "model can never mint or guess the token itself.\n\n" - "Find the pid/window_id via the agent (list_windows) or ask it to\n" - "read them from a native capture." - ), - ) - computer_use_browser_approve.add_argument( - "--pid", type=int, required=True, - help="Exact browser process id to authorize", - ) - computer_use_browser_approve.add_argument( - "--window-id", type=int, default=None, - help="Exact native window id (required for existing-profile attachment)", - ) - computer_use_browser_approve.add_argument( - "--profile-mode", - choices=["isolated_new", "isolated_named", "existing_profile"], - default="isolated_new", - help="Preparation the token authorizes (default: isolated_new)", - ) - def cmd_computer_use(args): action = getattr(args, "computer_use_action", None) if action == "install": @@ -12777,36 +12744,6 @@ def main(): json_output=bool(getattr(args, "json", False)), ) sys.exit(code) - if action == "browser-approve": - import subprocess - from tools.computer_use.cua_backend import ( - cua_driver_child_env, - cua_driver_install_hint, - resolve_cua_driver_cmd, - ) - binary = resolve_cua_driver_cmd() - if not binary: - print(cua_driver_install_hint()) - sys.exit(2) - cmd = [binary, "browser-approve", "--pid", str(args.pid)] - window_id = getattr(args, "window_id", None) - if window_id is not None: - cmd += ["--window-id", str(window_id)] - cmd += ["--profile-mode", getattr(args, "profile_mode", "isolated_new")] - try: - # Interactive passthrough: cua-driver requires a TTY to mint - # the grant, prints the token itself, and owns the expiry. - proc = subprocess.run(cmd, env=cua_driver_child_env()) - except OSError as exc: - print(f"cua-driver browser-approve failed to launch: {exc}", file=sys.stderr) - sys.exit(2) - if proc.returncode == 0: - print( - "\nGive the token above to the agent — it passes it as " - "approval_token on cua_browser_prepare. Single use, " - "expires in ~5 minutes." - ) - sys.exit(proc.returncode) if action == "permissions": perms_action = getattr(args, "computer_use_perms_action", None) if perms_action == "grant": diff --git a/skills/autonomous-ai-agents/computer-use/SKILL.md b/skills/autonomous-ai-agents/computer-use/SKILL.md index 8e6b001650..ed280672cb 100644 --- a/skills/autonomous-ai-agents/computer-use/SKILL.md +++ b/skills/autonomous-ai-agents/computer-use/SKILL.md @@ -24,11 +24,12 @@ Everything here works with any tool-capable model — Claude, GPT, Gemini, or an open model on a local OpenAI-compatible endpoint. There is no Anthropic-native schema to learn. -Hermes drives [cua-driver](https://github.com/trycua/cua) under the hood -for the platform plumbing. The Hermes-side `computer_use` tool exposed -in this skill is a higher-level Hermes vocabulary; the raw cua-driver -MCP tools (which a different agent harness would see) are NOT what you -call — call the `computer_use` actions documented below. +Hermes drives [cua-driver](https://github.com/trycua/cua) under the hood. +This wrapper skill teaches the Hermes `computer_use` workflow and action +vocabulary. Call the actions documented below instead of raw cua-driver MCP +tools. For driver internals and platform-specific behavior, follow the Cua +skill installed by `cua-driver skills install`; that command detects Hermes +and links the skill pack automatically. ## The canonical workflow @@ -207,15 +208,17 @@ Authorization paths for `existing_profile`, in preference order: configured with a reviewed `capability_manifest`, prepares inside the manifest's scope succeed without prompts and everything else fails closed. 3. **Explicit Hermes YOLO** (`--yolo`, `/yolo`, or `approvals.mode: off`) - launches a private embedded cua-driver in `unrestricted` after that risk + launches a private cua-driver runtime in `unrestricted` after that risk acceptance, so there are no runtime Cua approval prompts. -A user may also paste a token from `hermes computer-use browser-approve` -(pass it as `approval_token`); current cua-driver builds treat that as a -disabled legacy path, so expect the config grant to be the working route. -Without any of these, `existing_profile` fails closed — report the refusal -and name the config key; do not retry, downgrade trust, or work around it. -Never invent, store, log, or reuse a grant token. +These settings belong to runtime launch. The agent cannot add or change them +after the runtime starts. Without one of these paths, `existing_profile` fails +closed. Report the refusal and name the config key; do not retry, downgrade +trust, or work around it. + +Every MCP transport owns a private lifecycle session inside the runtime. The +public session name only labels cursor identity and session-scoped state. It +does not select, share, or keep a runtime alive. Use the native capture/AX/pixel/foreground ladder for browser chrome, browser permission UI, OS prompts, native dialogs, extension surfaces, unsupported diff --git a/tests/hermes_cli/test_computer_use_cli.py b/tests/hermes_cli/test_computer_use_cli.py new file mode 100644 index 0000000000..452128ec73 --- /dev/null +++ b/tests/hermes_cli/test_computer_use_cli.py @@ -0,0 +1,31 @@ +"""CLI coverage for the public Computer Use command surface.""" + +from __future__ import annotations + +import subprocess +import sys + + +def _run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "hermes_cli.main", "computer-use", *args], + capture_output=True, + text=True, + timeout=30, + ) + + +def test_computer_use_help_omits_browser_approve() -> None: + result = _run("--help") + + assert result.returncode == 0 + assert "browser-approve" not in result.stdout + assert "doctor" in result.stdout + assert "permissions" in result.stdout + + +def test_computer_use_rejects_removed_browser_approve_command() -> None: + result = _run("browser-approve", "--pid", "123") + + assert result.returncode == 2 + assert "invalid choice: 'browser-approve'" in result.stderr diff --git a/tests/tools/test_computer_use.py b/tests/tools/test_computer_use.py index 7dc07239f9..6a61c1d25b 100644 --- a/tests/tools/test_computer_use.py +++ b/tests/tools/test_computer_use.py @@ -1089,6 +1089,77 @@ class TestCuaDriverSessionReconnect: assert bridge.calls[1][0] == ("call", "list_apps", {}) assert len(bridge.calls) == 2 + def test_mutation_is_not_replayed_after_closed_transport(self): + """A lost response cannot prove whether a click already happened.""" + from anyio import ClosedResourceError + + class FakeBridge: + def __init__(self): + self.calls = [] + + def run(self, value, timeout=None): + self.calls.append((value, timeout)) + raise ClosedResourceError() + + bridge = FakeBridge() + session = self._make_session(bridge) + + result = session.call_tool("click", {"x": 20, "y": 30}) + + assert result["isError"] is True + assert result["structuredContent"]["code"] == "transport_outcome_unknown" + assert result["structuredContent"]["next_step"] == "fresh_state" + assert session._reconnect_log == ["stop", "start"] + assert len(bridge.calls) == 1 + + def test_mutation_does_not_cross_to_cli_on_transient_proxy_error(self): + class FakeBridge: + def run(self, value, timeout=None): + raise RuntimeError("daemon proxy: Resource temporarily unavailable") + + session = self._make_session(FakeBridge()) + session._call_tool_via_cli = MagicMock() + reset = MagicMock() + session._transport_reset_callback = reset + + result = session.call_tool("type_text", {"text": "hello"}) + + assert result["structuredContent"]["code"] == "transport_outcome_unknown" + session._call_tool_via_cli.assert_not_called() + reset.assert_called_once_with() + + def test_reconnect_restores_public_label_before_replaying_read(self): + from anyio import ClosedResourceError + + class FakeBridge: + def __init__(self): + self.calls = [] + self.effects = [ + ClosedResourceError(), + {"isError": False}, + {"isError": False, "structuredContent": {"apps": []}}, + ] + + def run(self, value, timeout=None): + self.calls.append(value) + effect = self.effects.pop(0) + if isinstance(effect, Exception): + raise effect + return effect + + bridge = FakeBridge() + session = self._make_session(bridge) + session._declared_session_id = "hermes-label" + + result = session.call_tool("list_apps", {}) + + assert result["isError"] is False + assert bridge.calls == [ + ("call", "list_apps", {}), + ("call", "start_session", {"session": "hermes-label"}), + ("call", "list_apps", {}), + ] + def test_cli_fallback_reads_screenshot_from_file(self, tmp_path, monkeypatch): """_call_tool_via_cli must base64-read a screenshot written to disk diff --git a/tests/tools/test_computer_use_browser_authorization.py b/tests/tools/test_computer_use_browser_authorization.py index ff8176a066..1d8cc5fd18 100644 --- a/tests/tools/test_computer_use_browser_authorization.py +++ b/tests/tools/test_computer_use_browser_authorization.py @@ -1,11 +1,8 @@ """Authorization plumbing for the cua-driver typed browser route. -Covers the three rungs that let ``existing_profile`` attachment (and bounded -automation generally) actually work from Hermes: +Covers the authorization modes that let ``existing_profile`` attachment (and +bounded automation generally) work from Hermes: -* ``approval_token`` passthrough — the user-minted single-use token from - ``hermes computer-use browser-approve`` reaches ``browser_prepare`` and is - never fabricated by the wrapper. * ``bounded`` permission mode — a private embedded daemon launched with a user-reviewed capability manifest (``--capability-manifest`` + ``--approve-capability-manifest``), failing loudly when the manifest is @@ -47,16 +44,15 @@ def _route(driver: _PrepareDriver) -> CuaTypedBrowserRoute: ) -# ── approval_token passthrough ────────────────────────────────────────── +# ── existing-profile authorization ownership ─────────────────────────── -def test_existing_profile_prepare_forwards_user_minted_approval_token(): +def test_existing_profile_prepare_delegates_authorization_to_driver(): driver = _PrepareDriver() result = _route(driver).prepare( pid=101, window_id=202, profile_mode="existing_profile", - approval_token="tok-from-user-terminal", ) assert result["status"] == "ok" @@ -67,51 +63,13 @@ def test_existing_profile_prepare_forwards_user_minted_approval_token(): "pid": 101, "window_id": 202, "strategy": {"kind": "existing_profile"}, - "approval_token": "tok-from-user-terminal", "session": "hermes-a", }, ) ] -def test_existing_profile_prepare_without_token_sends_none(): - """No token → the field is absent; the driver's own gate decides.""" - driver = _PrepareDriver() - _route(driver).prepare(pid=101, window_id=202, profile_mode="existing_profile") - - (_, args), = driver.calls - assert "approval_token" not in args - - -@pytest.mark.parametrize("bogus", ["", None, 7, True]) -def test_non_string_or_empty_token_is_never_forwarded(bogus): - driver = _PrepareDriver() - _route(driver).prepare( - pid=101, - window_id=202, - profile_mode="existing_profile", - approval_token=bogus, - ) - - (_, args), = driver.calls - assert "approval_token" not in args - - -def test_isolated_prepare_ignores_approval_token(): - """The token authorizes existing-profile attachment only.""" - driver = _PrepareDriver() - _route(driver).prepare( - pid=101, - profile_mode="isolated_new", - allow_launch=True, - approval_token="tok", - ) - - (_, args), = driver.calls - assert "approval_token" not in args - - -def test_dispatch_forwards_approval_token_to_backend(): +def test_dispatch_does_not_forward_removed_approval_token(): from unittest.mock import Mock from tools.computer_use.tool import _dispatch @@ -126,22 +84,18 @@ def test_dispatch_forwards_approval_token_to_backend(): "pid": 101, "window_id": 202, "profile_mode": "existing_profile", - "approval_token": "tok-abc", }, ) kwargs = backend.typed_browser_prepare.call_args.kwargs - assert kwargs["approval_token"] == "tok-abc" + assert "approval_token" not in kwargs assert kwargs["profile_mode"] == "existing_profile" -def test_schema_documents_approval_token_as_user_minted(): +def test_schema_does_not_expose_approval_token(): from tools.computer_use.schema import COMPUTER_USE_SCHEMA - prop = COMPUTER_USE_SCHEMA["parameters"]["properties"]["approval_token"] - desc = prop["description"] - assert "browser-approve" in desc - assert "never invent" in desc.lower() + assert "approval_token" not in COMPUTER_USE_SCHEMA["parameters"]["properties"] # ── bounded embedded daemon ───────────────────────────────────────────── @@ -215,13 +169,12 @@ def test_bounded_daemon_serves_with_approved_manifest(tmp_path, monkeypatch): command = captured["command"] assert "--permission-mode" in command assert command[command.index("--permission-mode") + 1] == "bounded" - # Flag names live-verified against cua-driver 0.19.3. - assert "--session-policy" in command + assert "--capability-manifest" in command assert ( - command[command.index("--session-policy") + 1] + command[command.index("--capability-manifest") + 1] == str(manifest) ) - assert "--approve-session-policy" in command + assert "--approve-capability-manifest" in command assert "--dangerously-bypass-approvals" not in command @@ -255,7 +208,7 @@ def test_unrestricted_daemon_serve_command_unchanged(monkeypatch): command = captured["command"] assert "--dangerously-bypass-approvals" in command - assert "--session-policy" not in command + assert "--capability-manifest" not in command # ── standard-mode --grant existing-profile ────────────────────────────── diff --git a/tests/tools/test_computer_use_browser_contract_020.py b/tests/tools/test_computer_use_browser_contract_020.py new file mode 100644 index 0000000000..56e07b693f --- /dev/null +++ b/tests/tools/test_computer_use_browser_contract_020.py @@ -0,0 +1,133 @@ +"""Behavior coverage for the cua-driver 0.20 public browser contract.""" + +import json +from typing import Any, Dict +from unittest.mock import Mock + +from tools.computer_use.browser_route import CuaTypedBrowserRoute +from tools.computer_use.schema import COMPUTER_USE_SCHEMA +from tools.computer_use.tool import _dispatch + + +class _Driver: + def __init__(self, responses: list[Dict[str, Any]]) -> None: + self.responses = list(responses) + self.calls: list[tuple[str, Dict[str, Any]]] = [] + + def has_tool(self, _name: str) -> bool: + return True + + def call(self, name: str, args: Dict[str, Any]) -> Dict[str, Any]: + self.calls.append((name, dict(args))) + return self.responses.pop(0) + + +def _route(driver: _Driver) -> CuaTypedBrowserRoute: + return CuaTypedBrowserRoute( + session_id="hermes-browser-contract", + call_tool=driver.call, + has_tool=driver.has_tool, + ) + + +def test_public_schema_exposes_020_state_and_type_options(): + properties = COMPUTER_USE_SCHEMA["parameters"]["properties"] + + assert properties["include_screenshot"]["type"] == "boolean" + assert properties["replace"]["type"] == "boolean" + assert properties["browser_type_mode"]["enum"] == ["insert_text", "keystrokes"] + assert "approval_token" not in properties + + +def test_browser_state_forwards_screenshot_request_and_preserves_mcp_image(): + driver = _Driver([ + { + "structuredContent": { + "status": "ok", + "target_id": "target-a", + "binding_quality": "exact", + "mutation_allowed": True, + "tabs": [{"tab_id": "tab-a"}], + }, + "images": ["/9j/browser-shot"], + "image_mime_types": ["image/jpeg"], + } + ]) + + result = _route(driver).observe( + pid=101, + window_id=202, + include_screenshot=True, + ) + + assert driver.calls == [ + ( + "get_browser_state", + { + "pid": 101, + "window_id": 202, + "include_screenshot": True, + "session": "hermes-browser-contract", + }, + ) + ] + assert result["_mcp_images"] == [ + {"data": "/9j/browser-shot", "mime_type": "image/jpeg"} + ] + + +def test_browser_state_dispatch_returns_mcp_image_as_multimodal_content(): + backend = Mock() + backend.typed_browser_state.return_value = { + "status": "ok", + "url": "https://example.test/", + "_mcp_images": [ + {"data": "iVBORbrowser-shot", "mime_type": "image/png"} + ], + } + + result = _dispatch( + backend, + "cua_browser_state", + {"tab_id": "tab-a", "include_screenshot": True}, + ) + + backend.typed_browser_state.assert_called_once_with( + tab_id="tab-a", include_screenshot=True + ) + assert result["_multimodal"] is True + assert json.loads(result["content"][0]["text"])["url"] == "https://example.test/" + assert result["content"][1] == { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORbrowser-shot"}, + } + assert "iVBORbrowser-shot" not in result["text_summary"] + + +def test_browser_type_replace_reaches_typed_browser_backend(): + backend = Mock() + backend.typed_browser_action.return_value = {"status": "ok"} + + result = _dispatch( + backend, + "cua_browser_type", + { + "tab_id": "tab-a", + "ref": "field-a", + "text": "replacement", + "browser_type_mode": "keystrokes", + "replace": True, + }, + ) + + assert json.loads(result)["status"] == "ok" + backend.typed_browser_action.assert_called_once_with( + "browser_type", + tab_id="tab-a", + args={ + "ref": "field-a", + "text": "replacement", + "replace": True, + "mode": "keystrokes", + }, + ) diff --git a/tests/tools/test_computer_use_cua_0_10_permissions.py b/tests/tools/test_computer_use_cua_0_10_permissions.py index edcd388d75..6913487e25 100644 --- a/tests/tools/test_computer_use_cua_0_10_permissions.py +++ b/tests/tools/test_computer_use_cua_0_10_permissions.py @@ -205,3 +205,55 @@ def test_standard_backend_does_not_spawn_an_embedded_daemon(): assert standard._embedded_daemon is None assert unrestricted._embedded_daemon is not None + + +def test_standard_existing_profile_grant_owns_private_macos_runtime(): + from tools.computer_use.cua_backend import _standard_runtime_launch_args + + args, socket_path = _standard_runtime_launch_args( + ["mcp"], + grant_existing_profile=True, + platform="darwin", + socket_path="/tmp/hermes-cua-test.sock", + ) + + assert args == [ + "mcp", + "--grant", + "existing-profile", + "--socket", + "/tmp/hermes-cua-test.sock", + ] + assert socket_path == "/tmp/hermes-cua-test.sock" + + +def test_standard_existing_profile_grant_stays_in_process_off_macos(): + from tools.computer_use.cua_backend import _standard_runtime_launch_args + + args, socket_path = _standard_runtime_launch_args( + ["mcp"], grant_existing_profile=True, platform="linux" + ) + + assert args == ["mcp", "--grant", "existing-profile"] + assert socket_path is None + + +def test_transport_reset_invalidates_native_and_browser_capabilities(): + from tools.computer_use.cua_backend import CuaDriverBackend + + backend = CuaDriverBackend(permission_mode="standard") + backend._active_pid = 10 + backend._active_window_id = 20 + backend._snapshot_tokens = {1: "old-token"} + backend._typed_browser.state.pid = 10 + backend._typed_browser.state.window_id = 20 + backend._typed_browser.state.target_id = "old-target" + backend._typed_browser.state.refs = {"old-ref": {"click"}} + + backend._handle_transport_reset() + + assert backend._active_pid is None + assert backend._active_window_id is None + assert backend._snapshot_tokens == {} + assert backend._typed_browser.state.target_id is None + assert backend._typed_browser.state.refs == {} diff --git a/tests/tools/test_computer_use_cua_0_9.py b/tests/tools/test_computer_use_cua_0_9.py index 940a2c7e23..7e8191480b 100644 --- a/tests/tools/test_computer_use_cua_0_9.py +++ b/tests/tools/test_computer_use_cua_0_9.py @@ -788,7 +788,6 @@ def test_namespaced_state_and_prepare_actions_use_typed_backend_wrappers(): profile_mode="isolated_new", profile_name=None, allow_launch=True, - approval_token=None, ) diff --git a/tools/computer_use/browser_route.py b/tools/computer_use/browser_route.py index c6bbd77b07..7d72409556 100644 --- a/tools/computer_use/browser_route.py +++ b/tools/computer_use/browser_route.py @@ -39,7 +39,7 @@ def _positive_int(value: Any) -> Optional[int]: def _tool_payload(out: Dict[str, Any]) -> Dict[str, Any]: - """Return the structured driver payload without discarding refusals.""" + """Return structured data without discarding refusals or MCP images.""" structured = out.get("structuredContent") data = out.get("data") payload: Dict[str, Any] = {} @@ -49,6 +49,23 @@ def _tool_payload(out: Dict[str, Any]) -> Dict[str, Any]: payload["message"] = data if isinstance(structured, dict): payload.update(structured) + images = out.get("images") + mime_types = out.get("image_mime_types") + if isinstance(images, list): + preserved_images = [] + for index, image in enumerate(images): + if not isinstance(image, str) or not image: + continue + mime_type = "" + if ( + isinstance(mime_types, list) + and index < len(mime_types) + and isinstance(mime_types[index], str) + ): + mime_type = mime_types[index] + preserved_images.append({"data": image, "mime_type": mime_type}) + if preserved_images: + payload["_mcp_images"] = preserved_images if out.get("isError") is True: payload.setdefault("isError", True) return payload @@ -225,6 +242,7 @@ class CuaTypedBrowserRoute: query: Optional[str] = None, scope_ref: Optional[str] = None, continuation: Optional[str] = None, + include_screenshot: bool = False, ) -> Dict[str, Any]: """Bind an exact native window or snapshot a bound tab.""" missing = self._require_tool("get_browser_state") @@ -242,10 +260,13 @@ class CuaTypedBrowserRoute: "Typed browser binding requires an exact positive pid and window_id pair.", native_fallback=True, ) - payload = self._call( - "get_browser_state", - {"pid": exact_pid, "window_id": exact_window}, - ) + bind_args: Dict[str, Any] = { + "pid": exact_pid, + "window_id": exact_window, + } + if include_screenshot: + bind_args["include_screenshot"] = True + payload = self._call("get_browser_state", bind_args) if payload.get("status") != "ok": code = _refusal_code(payload) payload.setdefault("ok", False) @@ -318,6 +339,8 @@ class CuaTypedBrowserRoute: args["scope_ref"] = scope_ref if continuation: args["continuation"] = continuation + if include_screenshot: + args["include_screenshot"] = True continuing = continuation is not None if not continuing: @@ -351,7 +374,6 @@ class CuaTypedBrowserRoute: profile_mode: str, profile_name: Optional[str] = None, allow_launch: bool = False, - approval_token: Optional[str] = None, ) -> Dict[str, Any]: """Run explicit setup through the driver's authoritative mode gate.""" missing = self._require_tool("browser_prepare") @@ -370,21 +392,15 @@ class CuaTypedBrowserRoute: "Existing-profile attachment requires an exact positive pid and window_id pair.", ) # The driver owns the immutable standard/bounded/unrestricted - # decision. Standard fails closed without a certified host, - # a user-minted single-use approval token, or an approved - # bounded manifest; explicit Hermes YOLO owns a private - # unrestricted daemon. + # decision. Standard fails closed without a certified host; + # bounded mode uses its approved capability manifest and explicit + # Hermes YOLO owns a private unrestricted daemon. self.state.clear() args: Dict[str, Any] = { "pid": exact_pid, "window_id": exact_window, "strategy": {"kind": "existing_profile"}, } - if isinstance(approval_token, str) and approval_token: - # Minted out-of-band by `hermes computer-use browser-approve` - # (cua-driver browser-approve): five-minute, single-use. The - # user, never the model, is the source of this value. - args["approval_token"] = approval_token return self._call("browser_prepare", args) if profile_mode not in {"isolated_new", "isolated_named"}: return _refusal( diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 2067684cfa..c0a899c3c2 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -270,19 +270,46 @@ def _cua_grant_existing_profile() -> bool: """True when the user pre-authorized existing-profile browser attachment. Reads ``computer_use.grant_existing_profile`` (default False). This is - cua-driver's trusted-launcher grant: Hermes appends - ``--grant existing-profile`` when spawning the standard-mode runtime, so - ``browser_prepare`` with ``strategy: existing_profile`` succeeds without - a per-use token (live-verified against cua-driver 0.19.3, where the - interactive ``browser-approve`` token is a legacy compatibility path - that is disabled by default). The user flips this in config.yaml — a - deliberate, durable statement that agents on this machine may attach to - their signed-in browser. It never applies to bounded (the manifest owns - that decision) or unrestricted (already bypassed) daemons. + cua-driver's trusted-launcher grant. Hermes passes + ``--grant existing-profile`` when it launches the standard-mode runtime. + On macOS it also selects a private socket so the newly configured + CuaDriver.app runtime cannot collide with an already-running default + daemon. The setting never applies to bounded mode, where the manifest owns + authorization, or unrestricted mode, which already bypasses approvals. """ return bool(_computer_use_cfg().get("grant_existing_profile", False)) +def _standard_runtime_launch_args( + args: List[str], + *, + grant_existing_profile: bool, + platform: str, + socket_path: Optional[str] = None, +) -> Tuple[List[str], Optional[str]]: + """Return MCP args and any private runtime socket owned by this transport. + + Windows and Linux run the standard runtime in the MCP process, so the + launch grant can be passed directly. macOS proxies through CuaDriver.app; + a grant must therefore launch a fresh app daemon on a private socket + instead of trying to reconfigure the default daemon. + + ``platform`` is explicit so this policy can be tested as a pure function + on every CI host. + """ + result = list(args) + if not grant_existing_profile: + return result, None + result.extend(["--grant", "existing-profile"]) + if platform != "darwin": + return result, None + private_socket = socket_path or os.path.join( + tempfile.gettempdir(), f"hermes-cua-standard-{uuid.uuid4().hex[:12]}.sock" + ) + result.extend(["--socket", private_socket]) + return result, private_socket + + def _computer_use_max_image_dimension() -> Optional[int]: """Longest-edge cap for cua-driver screenshots, or None to leave unset. @@ -586,14 +613,11 @@ class _EmbeddedCuaDaemon: if self.permission_mode == "unrestricted": command.append("--dangerously-bypass-approvals") else: # bounded — manifest validated in __init__ - # Live-verified against cua-driver 0.19.3: the serve flags are - # --session-policy/--approve-session-policy (the docs' older - # --capability-manifest names are not accepted). command.extend( [ - "--session-policy", + "--capability-manifest", str(self.capability_manifest), - "--approve-session-policy", + "--approve-capability-manifest", ] ) self._process = subprocess.Popen( @@ -1271,6 +1295,12 @@ class _CuaDriverSession: # Used to revive a logical ended-session rejection without # recursive call_tool re-entry or backend-owned state (#71166). self._declared_session_id: Optional[str] = None + # A macOS standard-mode launch grant belongs to the app daemon that + # receives it. Select and own a private endpoint so an existing + # default daemon cannot reject or silently miss the requested grant. + self._owned_standard_runtime_socket: Optional[str] = None + self._transport_generation = 0 + self._transport_reset_callback: Optional[Any] = None def _require_started(self) -> None: if not self._started: @@ -1312,15 +1342,13 @@ class _CuaDriverSession: child_env = self._embedded_daemon.child_env() else: command, args = _resolve_mcp_invocation(driver_cmd) - # Standard-mode trusted-launcher grant: the user opted in via - # config.yaml (computer_use.grant_existing_profile), so the - # runtime is launched pre-authorized for existing-profile - # browser attachment (`cua-driver mcp --grant - # existing-profile`, live-verified on 0.19.3). Never applied - # to embedded daemons: bounded's manifest and unrestricted's - # bypass own that decision. - if _cua_grant_existing_profile(): - args = [*args, "--grant", "existing-profile"] + args, owned_socket = _standard_runtime_launch_args( + args, + grant_existing_profile=_cua_grant_existing_profile(), + platform=sys.platform, + socket_path=self._owned_standard_runtime_socket, + ) + self._owned_standard_runtime_socket = owned_socket child_env = cua_driver_child_env() _t_manifest = _time.monotonic() params = StdioServerParameters( @@ -1429,8 +1457,17 @@ class _CuaDriverSession: with self._lock: if self._started: return + # A previous transport may have died without taking down its + # private app daemon. Stop that exact endpoint before relaunching + # with --grant; grants cannot modify an already-running runtime. + if self._owned_standard_runtime_socket is not None: + self._stop_owned_standard_runtime_locked() self._bridge.start() - self._start_lifecycle_locked() + try: + self._start_lifecycle_locked() + except Exception: + self._stop_owned_standard_runtime_locked() + raise self._started = True def _start_lifecycle_locked(self) -> None: @@ -1469,13 +1506,59 @@ class _CuaDriverSession: raise RuntimeError( f"cua-driver session setup failed: {self._setup_error}" ) from self._setup_error + self._transport_generation += 1 + if self._transport_generation > 1: + self._notify_transport_reset() def stop(self) -> None: with self._lock: if not self._started: + self._stop_owned_standard_runtime_locked() return self._started = False self._stop_lifecycle_locked() + self._stop_owned_standard_runtime_locked() + + def set_transport_reset_callback(self, callback: Any) -> None: + """Register a synchronous cache invalidation hook for transport swaps.""" + self._transport_reset_callback = callback + + def _notify_transport_reset(self) -> None: + callback = getattr(self, "_transport_reset_callback", None) + if callback is None: + return + try: + callback() + except Exception as exc: + logger.debug("cua-driver transport reset callback failed: %s", exc) + + def _stop_owned_standard_runtime_locked(self) -> None: + """Stop the exact private macOS app daemon launched for a grant.""" + socket_path = getattr(self, "_owned_standard_runtime_socket", None) + if not socket_path: + return + self._owned_standard_runtime_socket = None + driver_command = resolve_cua_driver_cmd() + if driver_command: + from tools.environments.local import _sanitize_subprocess_env + + try: + subprocess.run( + [driver_command, "stop", "--socket", socket_path], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=3.0, + creationflags=windows_hide_flags(), + env=_sanitize_subprocess_env(cua_driver_child_env()), + ) + except (OSError, subprocess.SubprocessError): + pass + if os.path.exists(socket_path): + try: + os.remove(socket_path) + except OSError: + pass def _stop_lifecycle_locked(self) -> None: """Signal shutdown + wait for the lifecycle coroutine to unwind. @@ -1628,6 +1711,22 @@ class _CuaDriverSession: timeout=timeout, ) + def _restore_declared_session_after_transport_reset(self, timeout: float) -> None: + """Re-attach the public label inside a replacement private lifecycle.""" + session_id = getattr(self, "_declared_session_id", None) + if not session_id: + return + result = self._bridge.run( + self._call_tool_async("start_session", {"session": session_id}), + timeout=timeout, + ) + if result.get("isError") is True: + logger.warning( + "cua-driver public session label %s could not be restored: %s", + session_id, + self._logical_error_text(result), + ) + @staticmethod def _is_closed_session_error(exc: Exception) -> bool: """Return True for MCP/stdio failures that are recoverable by reconnecting.""" @@ -1673,8 +1772,10 @@ class _CuaDriverSession: except Exception as e: logger.debug("cua-driver session cleanup before reconnect failed: %s", e) self._started = False + self._stop_owned_standard_runtime_locked() # Clear stale capability state; the next start populates from scratch. self._capabilities = {} + self._tool_schemas = {} self._capability_version = "" self._start_lifecycle_locked() self._started = True @@ -1831,6 +1932,44 @@ class _CuaDriverSession: # into start() when the session-start hasn't flipped _started yet. _LIFECYCLE_CALLS = frozenset({"start_session", "end_session"}) + # Retrying these calls after a broken transport is safe. The first call + # either had no side effect or is explicitly idempotent. Mutations stay + # out of this set because a lost response does not prove they failed. + _TRANSPORT_REPLAY_SAFE_TOOLS = frozenset({ + "get_cursor_position", + "get_displays", + "get_screen_size", + "get_window_state", + "list_apps", + "list_windows", + }) + + @classmethod + def _transport_replay_is_safe(cls, name: str) -> bool: + return name in cls._TRANSPORT_REPLAY_SAFE_TOOLS + + @staticmethod + def _unknown_transport_outcome(name: str, exc: Exception) -> Dict[str, Any]: + message = ( + f"cua-driver transport failed during {name}; the action outcome is " + "unknown, so Hermes did not replay it. Take fresh state before " + "deciding whether to act again." + ) + return { + "data": message, + "images": [], + "image_mime_types": [], + "structuredContent": { + "ok": False, + "code": "transport_outcome_unknown", + "message": message, + "operation": name, + "next_step": "fresh_state", + "detail": str(exc), + }, + "isError": True, + } + def call_tool(self, name: str, args: Dict[str, Any], timeout: float = 30.0) -> Dict[str, Any]: # A prior session may have died (MCP drop / driver crash): its # lifecycle coro reset _started to False in its finally (#55048). @@ -1839,6 +1978,7 @@ class _CuaDriverSession: "cua-driver session not active on %s; (re)starting before call", name ) self.start() + self._restore_declared_session_after_transport_reset(timeout) self._require_started() try: @@ -1848,6 +1988,9 @@ class _CuaDriverSession: ) except Exception as e: if self._is_transient_daemon_error(e): + if not self._transport_replay_is_safe(name): + self._notify_transport_reset() + return self._unknown_transport_outcome(name, e) logger.warning( "cua-driver MCP transport failed on %s (%s); " "falling back to CLI transport", name, e, @@ -1858,6 +2001,9 @@ class _CuaDriverSession: logger.warning("cua-driver MCP session closed during %s; reconnecting once", name) with self._lock: self._restart_session_locked() + self._restore_declared_session_after_transport_reset(timeout) + if not self._transport_replay_is_safe(name): + return self._unknown_transport_outcome(name, e) result = self._bridge.run( self._call_tool_async(name, args), timeout=timeout, @@ -2137,19 +2283,18 @@ class CuaDriverBackend(ComputerUseBackend): # element. Cleared whenever a fresh capture overwrites the # snapshot context. self._snapshot_tokens: Dict[int, str] = {} - # Per-instance cua-driver session id. cua-driver's MCP server - # instructions ask every consumer to declare a stable session - # at the start of a run (start_session) and tear it down at - # the end (end_session). Doing so: + # Per-instance public cua-driver session label. The MCP transport owns + # the private lifecycle and releases it when the connection closes. + # start_session/end_session attach this stable label to cursor, + # recording, and config state within that lifecycle. Doing so: # - Gets a distinct agent-cursor color per Hermes run, with # overlay rendering visualising where actions land # (without moving the real OS cursor). - # - Isolates per-session config + recording ownership so - # concurrent Hermes runs / subagents don't step on each - # other. + # - Gives config and recording state a stable owner label inside the + # transport-private lifecycle. # We mint a UUID4-based id once per CuaDriverBackend instance — - # one Hermes run = one backend = one session — and pass it as - # `session` on every cua-driver tool call. Sessions are an + # one Hermes run = one backend = one label — and pass it as + # `session` on every cua-driver tool call. Labels are an # additive feature on the cua-driver side: when our id is # unknown to the driver (older builds), the tool calls # degrade to the anonymous / unsynced path documented in the @@ -2160,6 +2305,14 @@ class CuaDriverBackend(ComputerUseBackend): call_tool=self._session.call_tool, has_tool=self._session._has_tool, ) + self._session.set_transport_reset_callback(self._handle_transport_reset) + + def _handle_transport_reset(self) -> None: + """Invalidate every capability minted by the replaced transport.""" + self._clear_active_target() + route = getattr(self, "_typed_browser", None) + if route is not None: + route.state.clear() def _browser_route(self) -> CuaTypedBrowserRoute: """Return the per-backend typed route, including test-constructed instances.""" diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index 049940316c..72651f6bc3 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -283,6 +283,13 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "enum": ["semantic_v2", "dom_refs_v1"], "description": "Typed-browser snapshot format; semantic_v2 is the default.", }, + "include_screenshot": { + "type": "boolean", + "description": ( + "For cua_browser_state, include the current browser screenshot " + "as image content in the tool result. Defaults to false." + ), + }, "query": {"type": "string", "description": "Optional browser-state query."}, "scope_ref": {"type": "string", "description": "Optional current ref to scope a snapshot."}, "continuation": {"type": "string", "description": "Continuation minted by the current snapshot."}, @@ -301,16 +308,6 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { ), }, "profile_name": {"type": "string", "description": "Name for isolated_named setup."}, - "approval_token": { - "type": "string", - "description": ( - "Optional single-use setup token the USER minted with " - "`hermes computer-use browser-approve` and pasted to you; " - "never invent one. Legacy path on current cua-driver " - "builds — the supported existing-profile route is the " - "computer_use.grant_existing_profile config opt-in." - ), - }, "allow_launch": { "type": "boolean", "description": "Explicitly allow launch of a driver-owned isolated browser.", @@ -330,6 +327,14 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "enum": ["insert_text", "keystrokes"], "description": "Delivery form for cua_browser_type; defaults to insert_text.", }, + "replace": { + "type": "boolean", + "description": ( + "For cua_browser_type, select the target's complete value " + "before typing so the supplied text replaces it. Defaults " + "to false; true with empty text clears the field." + ), + }, "dialog_id": {"type": "string", "description": "Opaque page-dialog capability."}, "prompt_text": {"type": "string", "description": "Optional text for a page prompt dialog."}, "files": { diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 93dc89dfaf..c6f89c66b3 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -673,10 +673,11 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> ("query", "query"), ("scope_ref", "scope_ref"), ("continuation", "continuation"), + ("include_screenshot", "include_screenshot"), ): if args.get(public) is not None: state_args[internal] = args[public] - return json.dumps(backend.typed_browser_state(**state_args)) + return _browser_state_response(backend.typed_browser_state(**state_args)) if action == "cua_browser_prepare": return json.dumps(backend.typed_browser_prepare( @@ -685,7 +686,6 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> profile_mode=args.get("profile_mode", "isolated_new"), profile_name=args.get("profile_name"), allow_launch=bool(args.get("allow_launch")), - approval_token=args.get("approval_token"), )) browser_tools = { @@ -703,7 +703,7 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> allowed_fields = { "browser_navigate": ("url",), "browser_click": ("ref", "input_route", "x", "y"), - "browser_type": ("ref", "text"), + "browser_type": ("ref", "text", "replace"), "browser_pointer": ( "ref", "destination_ref", "input_route", "x", "y", "to_x", "to_y", "delta_x", "delta_y", @@ -872,6 +872,41 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> # Response shaping # --------------------------------------------------------------------------- +def _browser_state_response(payload: Dict[str, Any]) -> Any: + """Return browser state as JSON, preserving requested MCP image parts.""" + state = dict(payload) + raw_images = state.pop("_mcp_images", None) + if not isinstance(raw_images, list) or not raw_images: + return json.dumps(state) + + text_summary = json.dumps(state) + content: List[Dict[str, Any]] = [ + {"type": "text", "text": text_summary}, + ] + image_count = 0 + for image in raw_images: + if not isinstance(image, dict): + continue + data = image.get("data") + if not isinstance(data, str) or not data: + continue + mime_type = image.get("mime_type") + if not isinstance(mime_type, str) or not mime_type.startswith("image/"): + mime_type = "image/jpeg" if data.startswith("/9j/") else "image/png" + content.append({ + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{data}"}, + }) + image_count += 1 + if image_count == 0: + return text_summary + return { + "_multimodal": True, + "content": content, + "text_summary": text_summary, + "meta": {"action": "cua_browser_state", "images": image_count}, + } + def _classify_action_result(res: ActionResult) -> Dict[str, Any]: """Choose the next ladder step from semantic evidence, in precedence order. diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index 18acbc934a..5a388fa62f 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -1422,6 +1422,9 @@ Subcommands: | `install` | Run the upstream cua-driver installer (macOS, Windows, and Linux). | | `install --upgrade` | Re-run the installer even if cua-driver is already on PATH. The upstream script always pulls the latest release, so this performs an in-place upgrade. | | `status` | Print whether `cua-driver` is on `$PATH` and which version is installed. | +| `doctor [--include CHECK] [--skip CHECK] [--json]` | Run cua-driver's health report and show its platform checks. | +| `permissions status [--json]` | Report macOS Accessibility and Screen Recording grants. | +| `permissions grant` | Ask macOS to grant Accessibility and Screen Recording to Cua Driver. | `hermes computer-use install` is the stable entry point for installing the [cua-driver](https://github.com/trycua/cua) binary used by the @@ -1430,6 +1433,17 @@ Subcommands: to use for re-running the install if the toolset toggle didn't trigger it (for example, on returning-user setups). +The built-in `computer_use` toolset is the recommended Hermes integration. +Registering raw Cua MCP tools is an alternative when you need Cua's low-level +tool vocabulary. `cua-driver skills install` detects Hermes and links Cua's +skill pack into the Hermes skills directory automatically. + +Permission mode, capability-manifest approval, and the existing-profile grant +belong to runtime launch. In bounded mode Hermes passes Cua's canonical +`--capability-manifest` and `--approve-capability-manifest` flags. Every MCP +transport owns a private lifecycle session inside its runtime. Public session +names label cursor and session state; they do not own or share the runtime. + `hermes update` automatically re-runs the upstream installer at the end of the update if cua-driver is on PATH, so most users will not need to call `--upgrade` manually. Use it when upstream ships a fix you want diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index f14fc93bfc..11bacd6fc2 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -18,7 +18,8 @@ about. ## How it works -The `computer_use` toolset speaks MCP over stdio to +The built-in `computer_use` toolset is the recommended Hermes integration. It +speaks MCP over stdio to [`cua-driver`](https://github.com/trycua/cua), an open-source background computer-use driver. Each platform uses the appropriate accessibility + input stack under the hood: @@ -61,12 +62,18 @@ This fetches and runs the upstream cua-driver installer — `install.sh` on macOS/Linux, `install.ps1` on Windows. Use `hermes computer-use status` to verify the install. +If you install Cua Driver first, `cua-driver skills install` detects Hermes +and links Cua's skill pack into the Hermes skills directory automatically. +You can also register raw Cua MCP tools as a custom MCP server, but that is an +alternative for users who need the low-level interface. The built-in toolset +provides Hermes actions, configuration, approvals, and diagnostics. + After installing, regardless of which path you took, grant the platform-appropriate prereqs: | Platform | Prereqs | |---|---| -| **macOS** | System Settings → Privacy & Security → **Accessibility** + **Screen Recording** → allow your terminal (or Hermes app). `hermes computer-use doctor` will tell you which permission is missing. | +| **macOS** | System Settings → Privacy & Security → **Accessibility** + **Screen Recording**. Grant the identity named by `hermes computer-use doctor`. Standard mode uses CuaDriver.app; bounded and unrestricted modes use the Hermes host identity. | | **Windows** | None at install time. If you're driving over SSH (not RDP / console), you need the autostart pattern — see [cua.ai/docs/how-to-guides/driver/windows-ssh](https://cua.ai/docs/how-to-guides/driver/windows-ssh) for the Session 0 ↔ Session 1+ proxy. | | **Linux** | A reachable display server: `DISPLAY` set for X11, or `XDG_SESSION_TYPE=wayland`. Wayland sessions need an XWayland bridge for capture. AT-SPI must be on (default on GNOME/KDE/Xfce). | @@ -80,8 +87,9 @@ or add `computer_use` to your enabled toolsets in `~/.hermes/config.yaml`. ## Permission modes and logged-in browser profiles -Hermes maps its existing approval UX onto cua-driver's immutable daemon -modes. There is no second permission toggle to keep in sync: +Hermes maps its existing approval UX onto cua-driver's immutable runtime +modes. Permission mode, capability manifest approval, and the existing-profile +grant are launch settings. They cannot change after the runtime starts: | Hermes session | cua-driver mode | Human intervention | `existing_profile` | |---|---|---|---| @@ -103,23 +111,16 @@ computer_use: ``` Hermes then launches the cua-driver runtime with the trusted-launcher grant -(`--grant existing-profile`, live-verified against cua-driver 0.19.3), and +(`--grant existing-profile`), and `cua_browser_prepare` with an existing profile succeeds against the exact `(pid, window_id)` the agent proves. Leave it `false` (the default) and existing-profile attachment fails closed; driver-owned isolated profiles work either way and are what the agent prefers. -Older cua-driver builds also shipped an interactive `browser-approve` token -verb; current drivers treat that token as a disabled legacy compatibility -path. Hermes still exposes `hermes computer-use browser-approve` as a -passthrough and forwards a pasted token as `approval_token`, but the config -grant above is the supported route. - ### Bounded mode for repeatable automation For recurring browser automation (cron jobs, scheduled research against an -authenticated app), per-run tokens are impractical. `bounded` mode replaces -prompts with a capability manifest you review once: +authenticated app), `bounded` mode uses a capability manifest you review once: ```yaml # config.yaml @@ -131,21 +132,25 @@ computer_use: The manifest names the apps, browser profile kinds, allowed origins, and typed tools the session may use (see the [cua-driver permission modes reference](https://cua.ai/docs/reference/cua-driver/permission-modes) -for the format). Hermes launches a private per-session daemon with +for the format). Hermes launches a private runtime with `--capability-manifest ... --approve-capability-manifest`; anything outside the manifest fails closed inside cua-driver. A missing or unreadable manifest fails loudly at session start rather than silently downgrading. Session YOLO still overrides bounded for that one session. -The bounded and unrestricted daemons are private to that Hermes session. -Turning `/yolo` off, resetting/closing the session, cancellation cleanup, or -process exit ends the Cua session and stops that daemon. It never changes the -machine-wide daemon's mode or grants another Hermes conversation the same -authority. +Each MCP transport owns a private lifecycle session inside its runtime. A +public session name is only a label for cursor identity and session-scoped +state. It does not select, share, or keep a runtime alive. Turning `/yolo` off, +resetting or closing the Hermes session, cancellation cleanup, or process exit +closes that transport session. Hermes also stops private runtimes that it +launched for bounded, unrestricted, or existing-profile access. One Hermes +conversation cannot change another runtime's mode or grants. On macOS, a +standard runtime with an existing-profile grant uses a fresh CuaDriver.app +daemon on a private socket. Bounded and unrestricted modes use a private +embedded service under the Hermes host identity. -`smart` approval remains `standard`: an LLM classification is not protected -human consent, and it cannot mint a `browser-approve` token or stand in for a -reviewed manifest. +`smart` approval remains `standard`: an LLM classification cannot stand in for +a reviewed manifest or a launch-time grant.