From 191cc8d30aa28a7cd4684f54956a3637a6724c6e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:23:21 -0700 Subject: [PATCH] refactor(plugins/web): BaseWebSearchProvider + _common helpers across all search providers; keyless_mcp failover cleanup --- plugins/web/_common.py | 254 +++++++++++ plugins/web/brave_free/__init__.py | 7 +- plugins/web/brave_free/provider.py | 120 ++--- plugins/web/ddgs/__init__.py | 8 +- plugins/web/ddgs/_search_worker.py | 80 ++-- plugins/web/ddgs/provider.py | 305 +++++-------- plugins/web/exa/__init__.py | 8 +- plugins/web/exa/provider.py | 280 +++--------- plugins/web/firecrawl/__init__.py | 21 +- plugins/web/firecrawl/provider.py | 684 ++++++++--------------------- plugins/web/keenable/__init__.py | 6 +- plugins/web/keenable/provider.py | 220 +++------- plugins/web/keyless_mcp.py | 644 ++++++++------------------- plugins/web/parallel/__init__.py | 9 +- plugins/web/parallel/provider.py | 367 +++------------- plugins/web/searxng/__init__.py | 8 +- plugins/web/searxng/provider.py | 130 +----- plugins/web/tavily/__init__.py | 1 - plugins/web/tavily/provider.py | 301 +++---------- plugins/web/xai/__init__.py | 7 +- plugins/web/xai/provider.py | 474 +++++++------------- 21 files changed, 1209 insertions(+), 2725 deletions(-) create mode 100644 plugins/web/_common.py diff --git a/plugins/web/_common.py b/plugins/web/_common.py new file mode 100644 index 0000000000..f5fba3b161 --- /dev/null +++ b/plugins/web/_common.py @@ -0,0 +1,254 @@ +"""Shared plumbing for the bundled web-search provider plugins. + +Every helper resolves patched-in-tests collaborators (``get_provider_env``, +``tools.interrupt``, ``plugins.web.keyless_mcp``, ``tools.web_tools`` client +slots) lazily at call time so monkeypatching the source module keeps working. +""" + +from __future__ import annotations + +import logging +from typing import Any, Awaitable, Callable, Dict, List, Optional + +import httpx + +from agent.web_search_provider import WebSearchProvider + +SEARCH_LIMIT_CAP = 20 # every vendor here caps max_results at 20 server-side + + +def provider_env(name: str) -> str: + """Config-aware env lookup (os.environ, then ~/.hermes/.env).""" + from agent.web_search_provider import get_provider_env + + return get_provider_env(name) + + +def use_keyless(name: str, api_key: str) -> bool: + from plugins.web.keyless_mcp import use_keyless as _use_keyless + + return _use_keyless(name, api_key) + + +def _interrupted() -> bool: + from tools.interrupt import is_interrupted + + return is_interrupted() + + +# --- Result shapes (key order is part of the contract — it reaches the model as JSON) --- + + +def search_ok(web_results: List[Dict[str, Any]]) -> Dict[str, Any]: + return {"success": True, "data": {"web": web_results}} + + +def search_fail(error: str) -> Dict[str, Any]: + return {"success": False, "error": error} + + +def web_hit(url: str, title: str, description: str, position: int) -> Dict[str, Any]: + return {"url": url, "title": title, "description": description, "position": position} + + +def document(url: str, title: str, content: str, *, source_url: Optional[str] = None) -> Dict[str, Any]: + """Successful extract entry; ``raw_content`` mirrors ``content`` for the legacy pipeline.""" + return { + "url": url, + "title": title, + "content": content, + "raw_content": content, + "metadata": {"sourceURL": url if source_url is None else source_url, "title": title}, + } + + +def extract_fail(urls: List[str], error: str) -> List[Dict[str, Any]]: + return [{"url": u, "title": "", "content": "", "error": error} for u in urls] + + +# --- Guarded execution: interrupt check + uniform failure classification --- + + +def _failure_message( + vendor: str, kind: str, exc: Exception, logger: logging.Logger, *, sdk: bool, verbatim_value_error: bool +) -> str: + """Map an exception to the user-facing error string. + + ``verbatim_value_error``: ValueError carries a pre-formatted message (missing + key, HTTP body) and is returned as-is. ``sdk``: ImportError means the lazily + installed vendor SDK is missing. Anything else is logged and wrapped. + """ + if verbatim_value_error and isinstance(exc, ValueError): + return str(exc) + if sdk and isinstance(exc, ImportError): + return f"{vendor} SDK not installed: {exc}" + logger.warning("%s %s error: %s", vendor, kind, exc) + return f"{vendor} {kind} failed: {exc}" + + +def run_search( + vendor: str, logger: logging.Logger, body: Callable[[], Dict[str, Any]], *, sdk: bool = False, verbatim_value_error: bool = True +) -> Dict[str, Any]: + try: + if _interrupted(): + return search_fail("Interrupted") + return body() + except Exception as exc: # noqa: BLE001 — surface as failure dict + return search_fail(_failure_message(vendor, "search", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error)) + + +def _extract_interrupted(urls: List[str]) -> List[Dict[str, Any]]: + return [{"url": u, "error": "Interrupted", "title": ""} for u in urls] + + +def run_extract( + vendor: str, logger: logging.Logger, urls: List[str], body: Callable[[], List[Dict[str, Any]]], + *, sdk: bool = False, verbatim_value_error: bool = True, +) -> List[Dict[str, Any]]: + """Per-URL failures are returned as entries with ``error`` — never raised.""" + try: + if _interrupted(): + return _extract_interrupted(urls) + return body() + except Exception as exc: # noqa: BLE001 + return extract_fail(urls, _failure_message(vendor, "extract", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error)) + + +async def run_extract_async( + vendor: str, logger: logging.Logger, urls: List[str], body: Callable[[], Awaitable[List[Dict[str, Any]]]], + *, sdk: bool = False, verbatim_value_error: bool = True, +) -> List[Dict[str, Any]]: + try: + if _interrupted(): + return _extract_interrupted(urls) + return await body() + except Exception as exc: # noqa: BLE001 + return extract_fail(urls, _failure_message(vendor, "extract", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error)) + + +# --- HTTP + SDK client helpers --- + + +def http_status_detail(response: Any) -> str: + """Response body text for a >=400 reply, or ``HTTP `` when the body is empty.""" + return (response.text or "").strip() or f"HTTP {response.status_code}" + + +def http_get_json( + label: str, url: str, *, params: Dict[str, Any], headers: Dict[str, str], timeout: int, + logger: logging.Logger, reach_target: Optional[str] = None, +) -> tuple[Any, Optional[Dict[str, Any]]]: + """GET ``url`` and parse JSON. Returns ``(data, None)`` or ``(None, failure_dict)``. + + ``label`` names the vendor in error strings; ``reach_target`` overrides the + "Could not reach ..." subject (SearXNG includes its instance URL). + """ + try: + resp = httpx.get(url, params=params, headers=headers, timeout=timeout) + resp.raise_for_status() + except httpx.HTTPStatusError as exc: + logger.warning("%s HTTP error: %s", label, exc) + return None, search_fail(f"{label} returned HTTP {exc.response.status_code}") + except httpx.RequestError as exc: + logger.warning("%s request error: %s", label, exc) + return None, search_fail(f"Could not reach {reach_target or label}: {exc}") + try: + return resp.json(), None + except Exception as exc: # noqa: BLE001 + logger.warning("%s response parse error: %s", label, exc) + return None, search_fail(f"Could not parse {label} response as JSON") + + +def cached_sdk_client(slot: str, env_var: str, missing_key_error: str, feature: str, factory: Callable[[str], Any]) -> Any: + """Lazy-build + cache a vendor SDK client on ``tools.web_tools.``. + + The cache slot lives on ``tools.web_tools`` so tests that reset + ``tools.web_tools.__client = None`` between cases see fresh state. + Raises ValueError when the key is unset; lazy_deps install hints are + re-raised as ImportError (its own ImportError is benign and swallowed). + """ + import tools.web_tools as _wt + + cached = getattr(_wt, slot, None) + if cached is not None: + return cached + + api_key = provider_env(env_var) + if not api_key: + raise ValueError(missing_key_error) + + try: + from tools.lazy_deps import ensure as _lazy_ensure + + _lazy_ensure(feature, prompt=False) + except ImportError: + pass + except Exception as exc: # noqa: BLE001 + raise ImportError(str(exc)) + + client = factory(api_key) + setattr(_wt, slot, client) + return client + + +# --- Provider base --- + + +class BaseWebSearchProvider(WebSearchProvider): + """Common surface for the bundled providers. + + Subclasses set ``NAME`` / ``DISPLAY_NAME`` / ``KEY_ENV`` and flip + ``EXTRACT`` / ``KEYLESS``. ``is_available`` deliberately ignores the + keyless tier: otherwise the legacy preference walk would route users + holding a key for a lower-priority backend onto this vendor's free tier. + ``is_keyless_available`` is True for ring members and opt-in keyless + vendors unless the user pinned ``web.provider_tier.: paid``. + """ + + NAME: str = "" + DISPLAY_NAME: str = "" + KEY_ENV: str = "" + EXTRACT: bool = False + KEYLESS: bool = False + + @property + def name(self) -> str: + return self.NAME + + @property + def display_name(self) -> str: + return self.DISPLAY_NAME + + def is_available(self) -> bool: + return bool(provider_env(self.KEY_ENV)) + + def is_keyless_available(self) -> bool: + from plugins.web.keyless_mcp import keyless_enabled, provider_tier + + return self.KEYLESS and keyless_enabled() and provider_tier(self.NAME) != "paid" + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return self.EXTRACT + + +def keyless_variant_schema(display: str, key_env: str, key_url: str, *, free_tag: str, paid_tag: str) -> Dict[str, Any]: + """``hermes tools`` picker entry for a keyless-ring vendor with a paid variant.""" + return { + "name": f"{display} · Free (keyless)", + "badge": "free · no key", + "tag": free_tag, + "env_vars": [], + "web_tier": "free", + "variants": [ + { + "name": f"{display} · Paid (API key)", + "badge": "paid", + "tag": paid_tag, + "env_vars": [{"key": key_env, "prompt": f"{display} API key", "url": key_url}], + "web_tier": "paid", + }, + ], + } diff --git a/plugins/web/brave_free/__init__.py b/plugins/web/brave_free/__init__.py index 6499d54672..abd05807b9 100644 --- a/plugins/web/brave_free/__init__.py +++ b/plugins/web/brave_free/__init__.py @@ -1,8 +1,4 @@ -"""Brave Search (free tier) plugin — bundled, auto-loaded. - -Mirrors the ``plugins/image_gen/openai/`` layout: ``provider.py`` holds the -provider class, ``__init__.py::register(ctx)`` registers an instance. -""" +"""Brave Search (free tier) plugin — bundled, auto-loaded.""" from __future__ import annotations @@ -10,5 +6,4 @@ from plugins.web.brave_free.provider import BraveFreeWebSearchProvider def register(ctx) -> None: - """Register the Brave-free provider with the plugin context.""" ctx.register_web_search_provider(BraveFreeWebSearchProvider()) diff --git a/plugins/web/brave_free/provider.py b/plugins/web/brave_free/provider.py index 0da8d11c99..1ffd6a6c15 100644 --- a/plugins/web/brave_free/provider.py +++ b/plugins/web/brave_free/provider.py @@ -1,111 +1,46 @@ -"""Brave Search (free tier) — plugin form. +"""Brave Search (free-tier Data-for-Search API) — search only, 2,000 queries/month. -Subclasses :class:`agent.web_search_provider.WebSearchProvider` (the -plugin-facing ABC). The legacy in-tree module -``tools.web_providers.brave_free`` was removed in the same commit that -moved this code under ``plugins/``; this file is now the canonical -implementation. - -Config keys this provider responds to:: - - web: - search_backend: "brave-free" # explicit per-capability - backend: "brave-free" # shared fallback - -Auth env var:: - - BRAVE_SEARCH_API_KEY=... # https://brave.com/search/api/ (free tier) +Config: ``web.search_backend`` / ``web.backend: "brave-free"`` (hyphen form kept for +existing user configs). Env: ``BRAVE_SEARCH_API_KEY``. Pair with Firecrawl/Tavily/Exa +for ``web_extract``. """ from __future__ import annotations import logging -import os from typing import Any, Dict -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import BaseWebSearchProvider, http_get_json, provider_env, search_fail, search_ok logger = logging.getLogger(__name__) _BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search" -class BraveFreeWebSearchProvider(WebSearchProvider): - """Search-only Brave provider using the free-tier Data-for-Search API. +class BraveFreeWebSearchProvider(BaseWebSearchProvider): + """Search-only Brave provider using the free-tier Data-for-Search API.""" - Free tier is 2,000 queries/month (1 qps). No content-extraction capability — - users pair this with Firecrawl/Tavily/Exa for ``web_extract``. - """ - - @property - def name(self) -> str: - # Hyphen form preserved for backward compat with the existing - # ``web.search_backend: "brave-free"`` config keys users have set. - return "brave-free" - - @property - def display_name(self) -> str: - return "Brave Search (Free)" - - def is_available(self) -> bool: - """Return True when ``BRAVE_SEARCH_API_KEY`` is set to a non-empty value.""" - from agent.web_search_provider import get_provider_env - - return bool(get_provider_env("BRAVE_SEARCH_API_KEY")) - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return False + NAME = "brave-free" + DISPLAY_NAME = "Brave Search (Free)" + KEY_ENV = "BRAVE_SEARCH_API_KEY" def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a search against the Brave Search API. - - Returns ``{"success": True, "data": {"web": [{"title", "url", "description", "position"}]}}`` - on success, or ``{"success": False, "error": str}`` on failure. - """ - import httpx - - from agent.web_search_provider import get_provider_env - - api_key = get_provider_env("BRAVE_SEARCH_API_KEY") + api_key = provider_env("BRAVE_SEARCH_API_KEY") if not api_key: - return {"success": False, "error": "BRAVE_SEARCH_API_KEY is not set"} + return search_fail("BRAVE_SEARCH_API_KEY is not set") - # Brave's `count` is capped at 20. - count = max(1, min(int(limit), 20)) - - try: - resp = httpx.get( - _BRAVE_ENDPOINT, - params={"q": query, "count": count}, - headers={ - "X-Subscription-Token": api_key, - "Accept": "application/json", - }, - timeout=15, - ) - resp.raise_for_status() - except httpx.HTTPStatusError as exc: - logger.warning("Brave Search HTTP error: %s", exc) - return { - "success": False, - "error": f"Brave Search returned HTTP {exc.response.status_code}", - } - except httpx.RequestError as exc: - logger.warning("Brave Search request error: %s", exc) - return {"success": False, "error": f"Could not reach Brave Search: {exc}"} - - try: - data = resp.json() - except Exception as exc: # noqa: BLE001 - logger.warning("Brave Search response parse error: %s", exc) - return {"success": False, "error": "Could not parse Brave Search response as JSON"} + data, failure = http_get_json( + "Brave Search", + _BRAVE_ENDPOINT, + params={"q": query, "count": max(1, min(int(limit), 20))}, # Brave caps count at 20 + headers={"X-Subscription-Token": api_key, "Accept": "application/json"}, + timeout=15, + logger=logger, + ) + if failure is not None: + return failure raw_results = (data.get("web") or {}).get("results", []) or [] - truncated = raw_results[:limit] - web_results = [ { "title": str(r.get("title", "")), @@ -113,18 +48,13 @@ class BraveFreeWebSearchProvider(WebSearchProvider): "description": str(r.get("description", "")), "position": i + 1, } - for i, r in enumerate(truncated) + for i, r in enumerate(raw_results[:limit]) ] - logger.info( "Brave Search '%s': %d results (from %d raw, limit %d)", - query, - len(web_results), - len(raw_results), - limit, + query, len(web_results), len(raw_results), limit, ) - - return {"success": True, "data": {"web": web_results}} + return search_ok(web_results) def get_setup_schema(self) -> Dict[str, Any]: return { diff --git a/plugins/web/ddgs/__init__.py b/plugins/web/ddgs/__init__.py index 26eb6407ef..ee14d558fc 100644 --- a/plugins/web/ddgs/__init__.py +++ b/plugins/web/ddgs/__init__.py @@ -1,9 +1,4 @@ -"""DuckDuckGo search plugin — bundled, auto-loaded. - -Backed by the community ``ddgs`` Python package which scrapes DDG's HTML -results page. No API key required, but the package itself must be installed -(it's an optional dep — gated via :meth:`is_available`). -""" +"""DuckDuckGo search plugin (``ddgs`` package, optional dep) — bundled, auto-loaded.""" from __future__ import annotations @@ -11,5 +6,4 @@ from plugins.web.ddgs.provider import DDGSWebSearchProvider def register(ctx) -> None: - """Register the DDGS provider with the plugin context.""" ctx.register_web_search_provider(DDGSWebSearchProvider()) diff --git a/plugins/web/ddgs/_search_worker.py b/plugins/web/ddgs/_search_worker.py index 0521284c53..9ac3795160 100644 --- a/plugins/web/ddgs/_search_worker.py +++ b/plugins/web/ddgs/_search_worker.py @@ -1,18 +1,9 @@ -"""DDGS search child-process entrypoint (#68096). +"""DDGS search child-process entrypoint, run as ``python plugins/web/ddgs/_search_worker.py``. -Invoked as ``python plugins/web/ddgs/_search_worker.py`` (script path from the -parent provider). Reads one JSON request from stdin, writes one JSON envelope -to stdout, then exits. - -Request:: - {"query": str, "safe_limit": int} - -Envelope:: - {"ok": true, "results": [...]} - {"ok": false, "error": str} - -Optional test hooks (only when ``HERMES_DDGS_ALLOW_TEST_HOOKS=1``):: - {"query": ..., "safe_limit": ..., "test_hook": "sleep"|"gil"|"success"|"error"|"empty"} +Reads one JSON request ``{"query": str, "safe_limit": int}`` from stdin, writes one +envelope ``{"ok": true, "results": [...]}`` / ``{"ok": false, "error": str}`` to +stdout, exits. Test hooks (``"test_hook": "sleep"|"gil"|"success"|"error"|"empty"``) +are honored only when ``HERMES_DDGS_ALLOW_TEST_HOOKS=1``. """ from __future__ import annotations @@ -24,11 +15,8 @@ import time def _hold_gil(secs: int) -> None: - """Block in a foreign call that keeps the GIL (ctypes.PyDLL). - - Mirrors native ``primp`` holding the interpreter lock. ``PyDLL`` (unlike - ``CDLL``/``WinDLL``) does not release the GIL around the call. - """ + """Block in a foreign call that keeps the GIL — mirrors native ``primp``. + ``ctypes.PyDLL`` (unlike ``CDLL``/``WinDLL``) does not release the GIL.""" import ctypes if sys.platform == "win32": @@ -46,30 +34,26 @@ def _hold_gil(secs: int) -> None: sleep(int(secs)) -def _run_test_hook(hook: str) -> dict: - if hook == "sleep": - time.sleep(30) - return {"ok": False, "error": "sleep hook returned unexpectedly"} - if hook == "gil": - _hold_gil(30) - return {"ok": False, "error": "gil hook returned unexpectedly"} - if hook == "success": - return { - "ok": True, - "results": [ - { - "title": "Hit", - "url": "https://example.com", - "description": "body", - "position": 1, - } - ], - } - if hook == "empty": - return {"ok": True, "results": []} - if hook == "error": - return {"ok": False, "error": "RuntimeError: boom"} - return {"ok": False, "error": f"unknown test_hook: {hook!r}"} +def _hook_sleep() -> dict: + time.sleep(30) + return {"ok": False, "error": "sleep hook returned unexpectedly"} + + +def _hook_gil() -> dict: + _hold_gil(30) + return {"ok": False, "error": "gil hook returned unexpectedly"} + + +_TEST_HOOKS = { + "sleep": _hook_sleep, + "gil": _hook_gil, + "success": lambda: { + "ok": True, + "results": [{"title": "Hit", "url": "https://example.com", "description": "body", "position": 1}], + }, + "empty": lambda: {"ok": True, "results": []}, + "error": lambda: {"ok": False, "error": "RuntimeError: boom"}, +} def _write_envelope(envelope: dict) -> None: @@ -87,19 +71,17 @@ def main() -> int: hook = request.get("test_hook") if hook: if os.environ.get("HERMES_DDGS_ALLOW_TEST_HOOKS") != "1": - _write_envelope( - {"ok": False, "error": "test_hook refused (hooks not enabled)"} - ) + _write_envelope({"ok": False, "error": "test_hook refused (hooks not enabled)"}) return 3 - envelope = _run_test_hook(str(hook)) + fn = _TEST_HOOKS.get(str(hook)) + envelope = fn() if fn else {"ok": False, "error": f"unknown test_hook: {hook!r}"} _write_envelope(envelope) return 0 if envelope.get("ok") else 1 query = str(request.get("query") or "") safe_limit = max(1, int(request.get("safe_limit") or 1)) try: - # Import inside main so script startup stays light / patchable. - from plugins.web.ddgs.provider import _run_ddgs_search + from plugins.web.ddgs.provider import _run_ddgs_search # lazy: light startup, patchable results = _run_ddgs_search(query, safe_limit) _write_envelope({"ok": True, "results": results}) diff --git a/plugins/web/ddgs/provider.py b/plugins/web/ddgs/provider.py index 824f3edba6..cca68b18ca 100644 --- a/plugins/web/ddgs/provider.py +++ b/plugins/web/ddgs/provider.py @@ -1,19 +1,12 @@ -"""DuckDuckGo search — plugin form (via the ``ddgs`` package). +"""DuckDuckGo search via the optional ``ddgs`` package (search only, no key). -Subclasses the plugin-facing :class:`agent.web_search_provider.WebSearchProvider`. -The legacy in-tree module ``tools.web_providers.ddgs`` was removed in the -same commit that moved this code under ``plugins/``; this file is now the -canonical implementation. +``is_available()`` reflects package importability; the plugin registers either way +so ``hermes tools`` can offer to install it. -The ``ddgs`` package is an optional dependency. ``is_available()`` reflects -whether the package is importable; the plugin still registers either way so -``hermes tools`` can prompt the user to install it. - -Isolation note (#68096): ``ddgs``/``primp`` can block inside native code while -holding the Python GIL. A ``ThreadPoolExecutor`` + ``future.result(timeout=…)`` -cap (see #52118) cannot fire in that state — the waiter never reacquires the -GIL — so the whole Hermes process freezes through Ctrl+C/SIGTERM. Each search -therefore runs in a disposable child process the parent can terminate/kill. +Isolation: ``ddgs``/``primp`` can block inside native code while holding the GIL, so +a thread-pool ``future.result(timeout=…)`` cap can never fire and Ctrl+C/SIGTERM +freeze the whole process. Each search therefore runs in a disposable child process +the parent can terminate/kill. """ from __future__ import annotations @@ -27,22 +20,22 @@ import sys import time from typing import Any, Dict, Optional -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import BaseWebSearchProvider, search_fail, search_ok logger = logging.getLogger(__name__) -# Overall wall-clock cap for a single ddgs search. The DDGS constructor's -# ``timeout`` only bounds individual HTTP requests; ddgs's multi-engine retry -# loop has no overall cap, so a slow/rate-limited DuckDuckGo response can hang -# the (single, shared) agent loop indefinitely (#36776). Enforce a hard cap -# here by killing a disposable worker process (#68096). +# Hard wall-clock cap per search. ``DDGS(timeout=…)`` only bounds individual HTTP +# requests; ddgs's multi-engine retry loop has no overall cap, so a rate-limited +# response could otherwise hang the shared agent loop indefinitely. _SEARCH_TIMEOUT_SECS = 30 +_POLL_INTERVAL_SECS = 0.1 # parent stdout / interrupt-flag poll cadence +_TERMINATE_GRACE_SECS = 1.0 # wait after terminate() before escalating to kill() -# How often the parent polls stdout / interrupt flag while waiting. -_POLL_INTERVAL_SECS = 0.1 +# Test-only hook name forwarded to the child (see _search_worker.py); never set in production. +_test_hook: Optional[str] = None -# After terminate(), wait this long before escalating to kill(). -_TERMINATE_GRACE_SECS = 1.0 +# Last worker Popen started by ``_run_ddgs_search_bounded`` (test reap checks). +_last_worker_proc: Optional[subprocess.Popen] = None class _SearchInterrupted(Exception): @@ -50,13 +43,8 @@ class _SearchInterrupted(Exception): def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]: - """Run the blocking ddgs query and return normalized hits. - - Module-level (not a closure) so the child worker can import it and so - tests can patch it for in-process unit tests. ``DDGS(timeout=…)`` bounds - each individual HTTP request; the overall wall-clock cap is enforced by - the parent via process timeout (#68096). - """ + """Blocking ddgs query → normalized hits. Module-level so the child worker can + import it and tests can patch it for in-process runs.""" from ddgs import DDGS # type: ignore results: list[dict[str, Any]] = [] @@ -64,11 +52,10 @@ def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]: for i, hit in enumerate(client.text(query, max_results=safe_limit)): if i >= safe_limit: break - url = str(hit.get("href") or hit.get("url") or "") results.append( { "title": str(hit.get("title", "")), - "url": url, + "url": str(hit.get("href") or hit.get("url") or ""), "description": str(hit.get("body", "")), "position": i + 1, } @@ -76,20 +63,9 @@ def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]: return results -# Optional test-only hook name forwarded to the child (see _search_worker.py). -# Production search() never sets this. -_test_hook: Optional[str] = None - -# Last worker Popen started by ``_run_ddgs_search_bounded`` (test reap checks). -_last_worker_proc: Optional[subprocess.Popen] = None - - def _plugins_path_entry() -> str: - """Return the ``sys.path`` entry that makes ``import plugins`` work. - - Prefer the live ``plugins`` package location over counting ``dirname``s from - this file — that stays correct for source checkouts and site-packages. - """ + """``sys.path`` entry that makes ``import plugins`` work in the child (prefers + the live package location; correct for source checkouts and site-packages).""" try: import plugins as plugins_pkg @@ -98,32 +74,24 @@ def _plugins_path_entry() -> str: return os.path.dirname(os.path.dirname(os.path.abspath(pkg_file))) except Exception: # noqa: BLE001 — fall through to path-walk fallback pass - return os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - ) + here = os.path.abspath(__file__) + for _ in range(4): + here = os.path.dirname(here) + return here -def _terminate_and_reap( - proc: Optional[subprocess.Popen], - *, - grace: float = _TERMINATE_GRACE_SECS, -) -> None: +def _terminate_and_reap(proc: Optional[subprocess.Popen], *, grace: float = _TERMINATE_GRACE_SECS) -> None: """Terminate a worker, escalate to kill, and wait so no orphan remains. - Does not close the parent's pipe ends — the caller must finish any - ``communicate()``/reader first. Closing stdout while another thread is - blocked in ``read()`` deadlocks on some platforms. + Does not close the parent's pipe ends — closing stdout while another thread is + blocked in ``read()`` deadlocks on some platforms; the caller drains first. """ if proc is None: return def _wait_until_dead(seconds: float) -> bool: deadline = time.monotonic() + seconds - while time.monotonic() < deadline: - if proc.poll() is not None: - return True + while proc.poll() is None and time.monotonic() < deadline: time.sleep(0.05) return proc.poll() is not None @@ -139,57 +107,28 @@ def _terminate_and_reap( logger.debug("DDGS worker reap error: %s", exc) -def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]]: - """Run ``_run_ddgs_search`` in a disposable process with a hard deadline. - - The parent never joins the child while it may be inside native code holding - *its* GIL — it only polls a communicator thread and, on timeout/interrupt, - terminates the child OS process. Raises ``TimeoutError``, - ``_SearchInterrupted``, or ``RuntimeError``. - """ - # Imported lazily so plugin import stays light for ``hermes tools`` probes. - from tools.interrupt import is_interrupted - - global _last_worker_proc - - request: dict[str, Any] = {"query": query, "safe_limit": safe_limit} - if _test_hook: - request["test_hook"] = _test_hook - - from tools.environments.local import _sanitize_subprocess_env - - env = _sanitize_subprocess_env(dict(os.environ)) - if _test_hook: - env["HERMES_DDGS_ALLOW_TEST_HOOKS"] = "1" - +def _spawn_worker(env: dict[str, str]) -> subprocess.Popen: + """Start ``_search_worker.py`` as a script with ``plugins`` importable.""" # Running the worker as a script puts ``plugins/web/ddgs/`` on ``sys.path[0]``, - # which breaks ``import plugins...``. Prepend the path entry that makes the - # live ``plugins`` package importable (source tree or site-packages). + # which breaks ``import plugins...``; prepend the real package location. child_pythonpath = env.get("PYTHONPATH", "") path_entry = _plugins_path_entry() if path_entry and path_entry not in child_pythonpath.split(os.pathsep): - env["PYTHONPATH"] = ( - path_entry + os.pathsep + child_pythonpath if child_pythonpath else path_entry - ) + env["PYTHONPATH"] = path_entry + os.pathsep + child_pythonpath if child_pythonpath else path_entry worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_search_worker.py") - # Platform-only spawn knobs — stdin/stdout/stderr must stay as explicit - # keyword args on the Popen call so scripts/check_subprocess_stdin.py can - # see them (TUI gateway inherits stdin; #14036). - extra_kwargs: dict[str, Any] = {} - if sys.platform == "win32": - # New process group so terminate/kill reach the worker cleanly on Windows. - extra_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP - else: - # Own session so a hung primp/libcurl grandchild can be reaped with the worker. - extra_kwargs["start_new_session"] = True - - proc = subprocess.Popen( + # stdin/stdout/stderr must stay explicit keyword args on the Popen call so + # scripts/check_subprocess_stdin.py can see them (TUI gateway inherits stdin). + extra_kwargs: dict[str, Any] = ( + {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} # so terminate/kill reach the worker + if sys.platform == "win32" + else {"start_new_session": True} # own session so a hung primp grandchild is reaped too + ) + return subprocess.Popen( [sys.executable, worker_path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - # DEVNULL avoids the classic deadlock where a chatty child fills the - # stderr pipe buffer while the parent only drains stdout. + # DEVNULL: a chatty child filling the stderr pipe while we only drain stdout would deadlock. stderr=subprocess.DEVNULL, env=env, text=True, @@ -197,14 +136,53 @@ def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any] errors="replace", **extra_kwargs, ) + + +def _parse_envelope(raw: str, proc: subprocess.Popen) -> list[dict[str, Any]]: + """Decode the worker's stdout envelope; raise ``RuntimeError`` on any malformed shape.""" + raw = raw.strip() + if not raw: + raise RuntimeError(f"DDGS worker exited without a result (code={proc.poll()})") + try: + envelope = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError(f"DDGS worker returned invalid JSON: {raw[:200]!r}") from exc + if not isinstance(envelope, dict): + raise RuntimeError(f"DDGS worker returned an invalid envelope: {envelope!r}") + if envelope.get("ok"): + results = envelope.get("results") or [] + if not isinstance(results, list): + raise RuntimeError("DDGS worker returned non-list results") + return results + raise RuntimeError(str(envelope.get("error") or "DDGS worker failed")) + + +def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]]: + """Run ``_run_ddgs_search`` in a disposable process with a hard deadline. + + The parent never joins a child that may be in native code holding *its* GIL — + it polls a communicator thread and, on timeout/interrupt, kills the OS process. + Raises ``TimeoutError``, ``_SearchInterrupted``, or ``RuntimeError``. + """ + from tools.interrupt import is_interrupted # lazy: keep plugin import light + from tools.environments.local import _sanitize_subprocess_env + + global _last_worker_proc + + request: dict[str, Any] = {"query": query, "safe_limit": safe_limit} + env = _sanitize_subprocess_env(dict(os.environ)) + if _test_hook: + request["test_hook"] = _test_hook + env["HERMES_DDGS_ALLOW_TEST_HOOKS"] = "1" + + proc = _spawn_worker(env) _last_worker_proc = proc # ``communicate`` runs in a side thread so the parent can poll interrupt / - # deadline without blocking. Killing the child unblocks communicate. + # deadline without blocking; killing the child unblocks it. pool = cf.ThreadPoolExecutor(max_workers=1) fut = pool.submit(proc.communicate, json.dumps(request)) - timed_out = False - interrupted = False + timed_out = interrupted = False raw = "" try: deadline = time.monotonic() + _SEARCH_TIMEOUT_SECS @@ -237,56 +215,20 @@ def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any] if interrupted: raise _SearchInterrupted("DuckDuckGo search interrupted") if timed_out: - raise TimeoutError( - f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s" - ) - - raw = raw.strip() - if not raw: - raise RuntimeError( - f"DDGS worker exited without a result (code={proc.poll()})" - ) - - try: - envelope = json.loads(raw) - except json.JSONDecodeError as exc: - raise RuntimeError( - f"DDGS worker returned invalid JSON: {raw[:200]!r}" - ) from exc - - if not isinstance(envelope, dict): - raise RuntimeError(f"DDGS worker returned an invalid envelope: {envelope!r}") - if envelope.get("ok"): - results = envelope.get("results") or [] - if not isinstance(results, list): - raise RuntimeError("DDGS worker returned non-list results") - return results - raise RuntimeError(str(envelope.get("error") or "DDGS worker failed")) + raise TimeoutError(f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s") + return _parse_envelope(raw, proc) -class DDGSWebSearchProvider(WebSearchProvider): - """DuckDuckGo HTML-scrape search provider. +class DDGSWebSearchProvider(BaseWebSearchProvider): + """DuckDuckGo HTML-scrape search provider (no API key; DDG rate-limits server-side). + ddgs errors surface as ``{"success": False, "error": ...}`` rather than raising.""" - No API key needed. Rate limits are enforced server-side by DuckDuckGo; - the provider surfaces ``DuckDuckGoSearchException`` and other ddgs errors - as ``{"success": False, "error": ...}`` rather than raising. - """ - - @property - def name(self) -> str: - return "ddgs" - - @property - def display_name(self) -> str: - return "DuckDuckGo (ddgs)" + NAME = "ddgs" + DISPLAY_NAME = "DuckDuckGo (ddgs)" def is_available(self) -> bool: - """Return True when the ``ddgs`` package is importable. - - Probes the import once; cheap because Python caches the import. Must - NOT perform network I/O — runs at tool-registration time and on every - ``hermes tools`` paint. - """ + """True when ``ddgs`` is importable. Must NOT do network I/O — runs at + tool-registration time and on every ``hermes tools`` paint.""" try: import ddgs # noqa: F401 @@ -294,61 +236,33 @@ class DDGSWebSearchProvider(WebSearchProvider): except ImportError: return False - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return False - def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a DuckDuckGo search and return normalized results. + """Run the search in a disposable child with a hard wall-clock timeout so a + hung native ``primp`` call cannot freeze the Hermes process.""" + if not self.is_available(): + return search_fail("ddgs package is not installed — run `pip install ddgs`") - The synchronous ``ddgs`` call runs in a disposable child process with - a hard wall-clock timeout (``_SEARCH_TIMEOUT_SECS``) so a hung native - ``primp`` call cannot freeze the Hermes process (#36776, #68096). - """ - try: - import ddgs # type: ignore # noqa: F401 — availability probe - except ImportError: - return { - "success": False, - "error": "ddgs package is not installed — run `pip install ddgs`", - } - - # DDGS().text yields at most `max_results` items; we cap defensively - # in case the package ignores the hint. + # Defensive cap in case the package ignores its max_results hint. safe_limit = max(1, int(limit)) try: web_results = _run_ddgs_search_bounded(query, safe_limit) except TimeoutError: - logger.warning( - "DDGS search timed out after %ds for query: %r", - _SEARCH_TIMEOUT_SECS, - query, + logger.warning("DDGS search timed out after %ds for query: %r", _SEARCH_TIMEOUT_SECS, query) + return search_fail( + f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — " + "DuckDuckGo may be rate-limiting or slow. Try again later " + "or switch to a different search provider." ) - return { - "success": False, - "error": ( - f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — " - "DuckDuckGo may be rate-limiting or slow. Try again later " - "or switch to a different search provider." - ), - } except _SearchInterrupted: logger.info("DDGS search interrupted for query: %r", query) - return { - "success": False, - "error": "DuckDuckGo search interrupted", - } + return search_fail("DuckDuckGo search interrupted") except Exception as exc: # noqa: BLE001 — ddgs raises its own exceptions logger.warning("DDGS search error: %s", exc) - return {"success": False, "error": f"DuckDuckGo search failed: {exc}"} + return search_fail(f"DuckDuckGo search failed: {exc}") - logger.info( - "DDGS search '%s': %d results (limit %d)", query, len(web_results), limit - ) - return {"success": True, "data": {"web": web_results}} + logger.info("DDGS search '%s': %d results (limit %d)", query, len(web_results), limit) + return search_ok(web_results) def get_setup_schema(self) -> Dict[str, Any]: return { @@ -356,7 +270,6 @@ class DDGSWebSearchProvider(WebSearchProvider): "badge": "free · no key · search only", "tag": "Search via the ddgs Python package — no API key (pair with any extract provider)", "env_vars": [], - # Trigger `_run_post_setup("ddgs")` after the user picks this row - # so the ddgs Python package gets pip-installed on first selection. + # Triggers `_run_post_setup("ddgs")` so the package gets pip-installed on first pick. "post_setup": "ddgs", } diff --git a/plugins/web/exa/__init__.py b/plugins/web/exa/__init__.py index d2ef3f16cf..47a989fb59 100644 --- a/plugins/web/exa/__init__.py +++ b/plugins/web/exa/__init__.py @@ -1,9 +1,4 @@ -"""Exa web search + extract plugin — bundled, auto-loaded. - -Backed by the official Exa SDK (``exa-py``). Both search and extract are -sync; the dispatcher in :mod:`tools.web_tools` handles the wrap when the -caller is async. -""" +"""Exa web search + extract plugin — bundled, auto-loaded (sync SDK; dispatcher wraps async callers).""" from __future__ import annotations @@ -11,5 +6,4 @@ from plugins.web.exa.provider import ExaWebSearchProvider def register(ctx) -> None: - """Register the Exa provider with the plugin context.""" ctx.register_web_search_provider(ExaWebSearchProvider()) diff --git a/plugins/web/exa/provider.py b/plugins/web/exa/provider.py index a388e3ad0a..562e117230 100644 --- a/plugins/web/exa/provider.py +++ b/plugins/web/exa/provider.py @@ -1,268 +1,86 @@ -"""Exa web search + content extraction — plugin form. +"""Exa web search + content extraction via the ``exa-py`` SDK (lazy-installed). -Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses the -official Exa SDK (``exa-py``) which is lazy-loaded via -:func:`tools.lazy_deps.ensure` so that cold-start CLI users don't pay the -SDK import cost when Exa isn't configured. - -Config keys this provider responds to:: - - web: - search_backend: "exa" # explicit per-capability - extract_backend: "exa" # explicit per-capability - backend: "exa" # shared fallback for both - -Env var:: - - EXA_API_KEY=... # https://exa.ai (paid tier; free trial available) - -The previous in-tree implementation lived at -``tools.web_tools._exa_search`` / ``_exa_extract``; this file is the -canonical replacement. Behavior is bit-for-bit identical aside from the -ABC method-name change. +Env: ``EXA_API_KEY`` (https://exa.ai). Both methods are sync — Exa's SDK is +sync-only; the dispatcher threads extract when the caller is async. """ from __future__ import annotations import logging -import os from typing import Any, Dict, List -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import ( + BaseWebSearchProvider, + cached_sdk_client, + document, + keyless_variant_schema, + provider_env, + run_extract, + run_search, + search_ok, + use_keyless, + web_hit, +) logger = logging.getLogger(__name__) -# Module-level note: the canonical ``_exa_client`` cache slot lives on -# :mod:`tools.web_tools` so tests that do ``tools.web_tools._exa_client = -# None`` between cases see fresh state. The plugin reads/writes through -# that public module (see :func:`_get_exa_client`). +_MISSING_KEY = "EXA_API_KEY environment variable not set. Get your API key at https://exa.ai" def _get_exa_client() -> Any: - """Lazy-import and cache an Exa SDK client. + def _factory(api_key: str) -> Any: + from exa_py import Exa # deliberately lazy - Cache lives on :mod:`tools.web_tools` (as ``_exa_client``) so unit - tests that reset that name between cases keep working. Raises - ``ValueError`` when ``EXA_API_KEY`` is unset. - """ - import tools.web_tools as _wt + client = Exa(api_key=api_key) + client.headers["x-exa-integration"] = "hermes-agent" + return client - cached = getattr(_wt, "_exa_client", None) - if cached is not None: - return cached - - from agent.web_search_provider import get_provider_env - - api_key = get_provider_env("EXA_API_KEY") - if not api_key: - raise ValueError( - "EXA_API_KEY environment variable not set. " - "Get your API key at https://exa.ai" - ) - - try: - from tools.lazy_deps import ensure as _lazy_ensure - - _lazy_ensure("search.exa", prompt=False) - except ImportError: - pass - except Exception as exc: # noqa: BLE001 — lazy_deps surfaces install hints - raise ImportError(str(exc)) - - from exa_py import Exa # noqa: WPS433 — deliberately lazy - - client = Exa(api_key=api_key) - client.headers["x-exa-integration"] = "hermes-agent" - _wt._exa_client = client - return client + return cached_sdk_client("_exa_client", "EXA_API_KEY", _MISSING_KEY, "search.exa", _factory) -def _reset_client_for_tests() -> None: - """Drop the cached Exa client so tests can re-instantiate cleanly.""" - import tools.web_tools as _wt +class ExaWebSearchProvider(BaseWebSearchProvider): + """Exa search + extract provider.""" - _wt._exa_client = None - - -class ExaWebSearchProvider(WebSearchProvider): - """Exa search + extract provider. - - Both methods are sync — Exa's SDK is sync-only. The web_extract_tool - dispatcher wraps sync extracts via ``asyncio.to_thread`` when it - needs to keep the event loop responsive. - """ - - @property - def name(self) -> str: - return "exa" - - @property - def display_name(self) -> str: - return "Exa" - - def is_available(self) -> bool: - """Return True when ``EXA_API_KEY`` is set to a non-empty value. - - Deliberately does NOT consider the keyless free tier — that would - let the legacy preference walk route keyed users of lower-priority - backends onto Exa's anonymous tier. Keyless availability is a - separate, last-resort signal (:meth:`is_keyless_available`). - """ - from agent.web_search_provider import get_provider_env - - return bool(get_provider_env("EXA_API_KEY")) - - def is_keyless_available(self) -> bool: - """Exa serves anonymous free-tier calls via its public MCP endpoint. - - False when the user forced ``web.provider_tier.exa: paid`` — an - explicit paid selection must never silently resolve keyless. - """ - from plugins.web.keyless_mcp import keyless_enabled, provider_tier - - return keyless_enabled() and provider_tier("exa") != "paid" - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return True + NAME = "exa" + DISPLAY_NAME = "Exa" + KEY_ENV = "EXA_API_KEY" + EXTRACT = True + KEYLESS = True def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute an Exa search. + def _body() -> Dict[str, Any]: + from plugins.web.keyless_mcp import search_with_failover - Returns ``{"success": True, "data": {"web": [{...}, ...]}}`` on - success, ``{"success": False, "error": str}`` on failure (incl. - missing API key and SDK install errors). - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return {"success": False, "error": "Interrupted"} - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import search_with_failover, use_keyless - - if use_keyless("exa", get_provider_env("EXA_API_KEY")): - # Keyless free tier — public MCP endpoint, no SDK needed. - logger.info( - "Exa keyless search: '%s' (limit=%d)", query, limit - ) + if use_keyless("exa", provider_env("EXA_API_KEY")): + logger.info("Exa keyless search: '%s' (limit=%d)", query, limit) return search_with_failover("exa", query, limit) logger.info("Exa search: '%s' (limit=%d)", query, limit) - response = _get_exa_client().search( - query, - num_results=limit, - contents={"highlights": True}, - ) + response = _get_exa_client().search(query, num_results=limit, contents={"highlights": True}) + return search_ok([ + web_hit(r.url or "", r.title or "", " ".join(r.highlights or []), i + 1) + for i, r in enumerate(response.results or []) + ]) - web_results = [] - for i, result in enumerate(response.results or []): - highlights = result.highlights or [] - web_results.append( - { - "url": result.url or "", - "title": result.title or "", - "description": " ".join(highlights) if highlights else "", - "position": i + 1, - } - ) - - return {"success": True, "data": {"web": web_results}} - except ValueError as exc: - # Raised by _get_exa_client when EXA_API_KEY missing - return {"success": False, "error": str(exc)} - except ImportError as exc: - return {"success": False, "error": f"Exa SDK not installed: {exc}"} - except Exception as exc: # noqa: BLE001 — surface as failure - logger.warning("Exa search error: %s", exc) - return {"success": False, "error": f"Exa search failed: {exc}"} + return run_search("Exa", logger, _body, sdk=True) def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: - """Extract content from one or more URLs via Exa. + def _body() -> List[Dict[str, Any]]: + from plugins.web.keyless_mcp import extract_with_failover - Returns a list of result dicts shaped for the legacy LLM - post-processing pipeline. On per-URL or whole-batch failure, - results carry an ``error`` field rather than raising. - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return [ - {"url": u, "error": "Interrupted", "title": ""} for u in urls - ] - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import extract_with_failover, use_keyless - - if use_keyless("exa", get_provider_env("EXA_API_KEY")): - # Keyless free tier — public MCP endpoint, no SDK needed. + if use_keyless("exa", provider_env("EXA_API_KEY")): logger.info("Exa keyless extract: %d URL(s)", len(urls)) return extract_with_failover("exa", list(urls)) logger.info("Exa extract: %d URL(s)", len(urls)) response = _get_exa_client().get_contents(urls, text=True) + return [document(r.url or "", r.title or "", r.text or "") for r in response.results or []] - results: List[Dict[str, Any]] = [] - for result in response.results or []: - content = result.text or "" - url = result.url or "" - title = result.title or "" - results.append( - { - "url": url, - "title": title, - "content": content, - "raw_content": content, - "metadata": {"sourceURL": url, "title": title}, - } - ) - return results - except ValueError as exc: - return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls] - except ImportError as exc: - return [ - {"url": u, "title": "", "content": "", "error": f"Exa SDK not installed: {exc}"} - for u in urls - ] - except Exception as exc: # noqa: BLE001 - logger.warning("Exa extract error: %s", exc) - return [ - {"url": u, "title": "", "content": "", "error": f"Exa extract failed: {exc}"} - for u in urls - ] + return run_extract("Exa", logger, urls, _body, sdk=True) def get_setup_schema(self) -> Dict[str, Any]: - return { - "name": "Exa · Free (keyless)", - "badge": "free · no key", - "tag": ( - "Semantic + neural web search with content extraction on " - "Exa's anonymous free tier. Rate-limited under burst load." - ), - "env_vars": [], - "web_tier": "free", - "variants": [ - { - "name": "Exa · Paid (API key)", - "badge": "paid", - "tag": ( - "Semantic + neural web search with content extraction " - "via the Exa SDK. Unthrottled, guaranteed service." - ), - "env_vars": [ - { - "key": "EXA_API_KEY", - "prompt": "Exa API key", - "url": "https://exa.ai", - }, - ], - "web_tier": "paid", - }, - ], - } + return keyless_variant_schema( + "Exa", "EXA_API_KEY", "https://exa.ai", + free_tag="Semantic + neural web search with content extraction on Exa's anonymous free tier. Rate-limited under burst load.", + paid_tag="Semantic + neural web search with content extraction via the Exa SDK. Unthrottled, guaranteed service.", + ) diff --git a/plugins/web/firecrawl/__init__.py b/plugins/web/firecrawl/__init__.py index 4cb9dd63d0..d52314d909 100644 --- a/plugins/web/firecrawl/__init__.py +++ b/plugins/web/firecrawl/__init__.py @@ -1,22 +1,4 @@ -"""Firecrawl web search + extract plugin — bundled, auto-loaded. - -Largest single plugin in this PR. Captures everything the previous -inline implementation in tools/web_tools.py did: - - - Lazy import of the firecrawl SDK (~200ms cold-start cost) via a - callable proxy that defers the actual import to first use. - - Dual client paths: direct (FIRECRAWL_API_KEY / FIRECRAWL_API_URL) - OR Nous-hosted tool-gateway routing for subscribers, with - web.use_gateway as the tie-breaker. - - Per-URL scrape loop with 60s timeout, SSRF re-check after redirect, - website-policy gating, and format-aware content selection. - - Robust response shape normalization across SDK / direct API / - gateway variants (search returns differ by transport). - -The plugin re-exports ``Firecrawl`` (the lazy proxy) and -``check_firecrawl_api_key`` for backward-compatibility with tests and -external code that imports those names from ``tools.web_tools``. -""" +"""Firecrawl web search + extract plugin — bundled, auto-loaded.""" from __future__ import annotations @@ -24,5 +6,4 @@ from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider def register(ctx) -> None: - """Register the Firecrawl provider with the plugin context.""" ctx.register_web_search_provider(FirecrawlWebSearchProvider()) diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.py index 34784a034a..f35c50e3a1 100644 --- a/plugins/web/firecrawl/provider.py +++ b/plugins/web/firecrawl/provider.py @@ -1,85 +1,37 @@ -"""Firecrawl web search + extract — plugin form. +"""Firecrawl web search + extract provider (direct SDK, keyless cloud, or Nous tool-gateway). -Subclasses :class:`agent.web_search_provider.WebSearchProvider`. This is -the largest provider migrated in this PR; it captures the full inline -firecrawl implementation that previously lived in tools/web_tools.py: - - - :data:`Firecrawl` lazy proxy that defers the ~200ms SDK import to - first use (re-exported by tools.web_tools for backward compat with - existing tests that mock that name). - - :func:`_get_firecrawl_client` with direct + managed-gateway dual - mode, controlled by ``web.use_gateway`` config when both are - configured. - - :func:`check_firecrawl_api_key` re-exported (tests + tools_config - setup hint depend on this name living in tools.web_tools). - - :func:`_extract_web_search_results` / :func:`_extract_scrape_payload` - response-shape normalizers that handle SDK / direct API / gateway - response variants. - - Per-URL extract loop with 60s timeout, redirect-aware SSRF re-check, - website-policy gating, and format-aware content selection. - -Async note: the underlying SDK is sync. ``extract()`` is declared -``async def`` because it performs per-URL I/O that benefits from -running in an executor; the implementation wraps each scrape in -:func:`asyncio.to_thread` with :func:`asyncio.wait_for(timeout=60)` to -guard against hung fetches. - -Config keys this provider responds to:: - - web: - search_backend: "firecrawl" # explicit per-capability - extract_backend: "firecrawl" # explicit per-capability - backend: "firecrawl" # shared fallback (default) - use_gateway: false # prefer managed gateway when both - # direct + gateway credentials exist - -Env vars:: - - FIRECRAWL_API_KEY=... # direct cloud auth - FIRECRAWL_API_URL=... # self-hosted Firecrawl - FIRECRAWL_GATEWAY_URL=... # Nous tool-gateway (subscribers) - TOOL_GATEWAY_DOMAIN=... # alternate gateway env - TOOL_GATEWAY_SCHEME=... - TOOL_GATEWAY_USER_TOKEN=... +Config: ``web.backend`` / ``web.search_backend`` / ``web.extract_backend: firecrawl``. +Env: FIRECRAWL_API_KEY, FIRECRAWL_API_URL (self-hosted), FIRECRAWL_GATEWAY_URL / TOOL_GATEWAY_*. """ from __future__ import annotations import asyncio import logging -import os -from typing import Any, Dict, List, NoReturn, Optional, TYPE_CHECKING +from typing import Any, Dict, List, Optional import httpx -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import BaseWebSearchProvider, search_fail, search_ok from tools.url_safety import is_safe_url +# Module-level (cheap import) so tests can monkeypatch the policy gate on this module. from tools.website_policy import check_website_access logger = logging.getLogger(__name__) _FIRECRAWL_CLOUD_API_URL = "https://api.firecrawl.dev" +_SELECTION_KEYS = ("backend", "search_backend", "extract_backend") -# --------------------------------------------------------------------------- -# Lazy Firecrawl SDK proxy -# --------------------------------------------------------------------------- -# The firecrawl SDK pulls ~200ms of imports (httpcore, firecrawl.v1/v2 type -# trees) on a cold CLI. We only need it when the backend is actually -# "firecrawl", so defer the import to first use via a callable proxy. -# -# Tests that do ``patch("tools.web_tools.Firecrawl", ...)`` continue to -# work because tools/web_tools.py re-exports ``Firecrawl`` from this -# module — so the patched name still references the same proxy instance. - -if TYPE_CHECKING: - from firecrawl import Firecrawl as FirecrawlSDK # noqa: F401 — type hints only +# --- Lazy Firecrawl SDK proxy ------------------------------------------------- +# The SDK costs ~200ms of imports on a cold CLI; defer to first use. tools.web_tools +# re-exports ``Firecrawl`` so ``patch("tools.web_tools.Firecrawl")`` keeps working. _FIRECRAWL_CLS_CACHE: Optional[type] = None def _load_firecrawl_cls() -> type: - """Import and cache ``firecrawl.Firecrawl``.""" + """Import and cache ``firecrawl.Firecrawl`` (lazy_deps install hint → ImportError).""" global _FIRECRAWL_CLS_CACHE if _FIRECRAWL_CLS_CACHE is None: try: @@ -90,7 +42,7 @@ def _load_firecrawl_cls() -> type: pass except Exception as exc: # noqa: BLE001 — surface install hint raise ImportError(str(exc)) - from firecrawl import Firecrawl as _cls # noqa: WPS433 — deliberately lazy + from firecrawl import Firecrawl as _cls _FIRECRAWL_CLS_CACHE = _cls return _FIRECRAWL_CLS_CACHE @@ -114,72 +66,53 @@ class _FirecrawlProxy: Firecrawl = _FirecrawlProxy() -# --------------------------------------------------------------------------- -# Client construction (direct vs managed-gateway) -# --------------------------------------------------------------------------- -# -# The canonical cache slots live on :mod:`tools.web_tools` so tests that do -# ``tools.web_tools._firecrawl_client = None`` between cases see fresh -# state. The plugin reads/writes through that public module — see -# :func:`_get_firecrawl_client` below. +# --- Client construction (direct vs managed-gateway) --------------------------- +# Client cache slots and gateway/token helpers are read through tools.web_tools so +# tests that reset ``tools.web_tools._firecrawl_client`` or patch +# ``tools.web_tools._peek_nous_access_token`` see their changes. + + +def _env(name: str) -> str: + from hermes_cli.config import get_env_value + + return (get_env_value(name) or "").strip() def _get_direct_firecrawl_config() -> Optional[tuple]: - """Return direct Firecrawl (mode, kwargs, cache key), or None when unavailable. + """Return direct Firecrawl ``(mode, kwargs, cache_key)`` or None. - ``mode`` is ``"sdk"`` (keyed / self-hosted via the Firecrawl SDK) or - ``"keyless"`` (explicit Firecrawl selection with no credentials — served - by :class:`_KeylessFirecrawlClient` against the public cloud API, which - accepts anonymous rate-limited requests). Keyless requires the explicit - selection so an unconfigured install never silently routes to it. + ``mode`` is ``"sdk"`` (keyed / self-hosted) or ``"keyless"`` (explicit + Firecrawl selection with no credentials — public cloud API, anonymous + rate-limited). Keyless requires the explicit selection so an unconfigured + install never silently routes to it. """ - from hermes_cli.config import get_env_value - - api_key = (get_env_value("FIRECRAWL_API_KEY") or "").strip() - api_url = (get_env_value("FIRECRAWL_API_URL") or "").strip().rstrip("/") + api_key = _env("FIRECRAWL_API_KEY") + api_url = _env("FIRECRAWL_API_URL").rstrip("/") if not api_key and not api_url: if _is_explicit_firecrawl_selection(): - return ( - "keyless", - {"api_url": _FIRECRAWL_CLOUD_API_URL}, - ("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None), - ) + return "keyless", {"api_url": _FIRECRAWL_CLOUD_API_URL}, ("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None) return None - kwargs: Dict[str, str] = {} - if api_key: - kwargs["api_key"] = api_key - if api_url: - kwargs["api_url"] = api_url - + kwargs = {k: v for k, v in (("api_key", api_key), ("api_url", api_url)) if v} return "sdk", kwargs, ("direct", api_url or None, api_key or None) def _is_explicit_firecrawl_selection() -> bool: - """Return True when config explicitly selects Firecrawl for web tools.""" + """True when config explicitly selects Firecrawl for web tools.""" import tools.web_tools as _wt cfg = _wt._load_web_config() - return any( - (cfg.get(key) or "").lower().strip() == "firecrawl" - for key in ("backend", "search_backend", "extract_backend") - ) + return any((cfg.get(key) or "").lower().strip() == "firecrawl" for key in _SELECTION_KEYS) def _use_keyless_ring() -> bool: """True when Firecrawl calls should route via the keyless ring. - Ring dispatch applies when there are no direct credentials, the - managed Nous gateway isn't the selected path, and the keyless tier - isn't disabled or pinned paid. Keyed/self-hosted/gateway setups never - reach the ring. + Only when there are no direct credentials, the managed Nous gateway isn't + the selected path, and the keyless tier isn't disabled or pinned paid. """ - from hermes_cli.config import get_env_value - - if (get_env_value("FIRECRAWL_API_KEY") or "").strip(): - return False - if (get_env_value("FIRECRAWL_API_URL") or "").strip(): + if _env("FIRECRAWL_API_KEY") or _env("FIRECRAWL_API_URL"): return False import tools.web_tools as _wt from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection @@ -202,9 +135,7 @@ def _use_keyless_ring() -> bool: class _KeylessFirecrawlClient: """Minimal REST client for Firecrawl's keyless cloud mode. - Duck-types the two SDK methods the provider calls (``search`` / - ``scrape``) so the rest of the pipeline (result normalizers, caching) - is unchanged. No Authorization header is ever sent. + Duck-types the SDK's ``search`` / ``scrape``; never sends an Authorization header. """ def __init__(self, api_url: str = _FIRECRAWL_CLOUD_API_URL): @@ -235,44 +166,24 @@ def _get_firecrawl_gateway_url() -> str: def _is_tool_gateway_ready() -> bool: - """Return True when gateway URL + Nous Subscriber token are available. - - Reads ``peek_nous_access_token`` and ``resolve_managed_tool_gateway`` - via :mod:`tools.web_tools` rather than direct imports, so unit tests - that ``patch("tools.web_tools._peek_nous_access_token", ...)`` see - their patches honored. The names are re-exported on - :mod:`tools.web_tools` for exactly this reason. - """ + """True when gateway URL + Nous Subscriber token are available.""" import tools.web_tools as _wt - return _wt.resolve_managed_tool_gateway( - "firecrawl", token_reader=_wt._peek_nous_access_token - ) is not None - - -def _has_direct_firecrawl_config() -> bool: - """Return True when direct Firecrawl config is explicitly configured.""" - return _get_direct_firecrawl_config() is not None + return _wt.resolve_managed_tool_gateway("firecrawl", token_reader=_wt._peek_nous_access_token) is not None def check_firecrawl_api_key() -> bool: - """Return True when the Firecrawl backend selected via `hermes tools` - (or, on a never-configured install, either route) is usable. - - Re-exported by :mod:`tools.web_tools` for backward compatibility with - existing tests and the ``hermes tools`` setup flow. - """ - from tools.tool_backend_helpers import ( - NOUS_MANAGED_PROVIDER, - read_selection, - ) + """True when the Firecrawl route selected via ``hermes tools`` (or, on a + never-configured install, either route) is usable. Re-exported by tools.web_tools.""" + from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection selected = read_selection("web") if selected == NOUS_MANAGED_PROVIDER: return _is_tool_gateway_ready() + has_direct = _get_direct_firecrawl_config() is not None if selected is not None: - return _has_direct_firecrawl_config() - return _has_direct_firecrawl_config() or _is_tool_gateway_ready() + return has_direct + return has_direct or _is_tool_gateway_ready() def _firecrawl_backend_help_suffix() -> str: @@ -281,55 +192,16 @@ def _firecrawl_backend_help_suffix() -> str: if not _wt.managed_nous_tools_enabled(): return "" - return ( - ", or use the Nous Tool Gateway via your subscription " - "(FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)" - ) - - -def _raise_web_backend_configuration_error() -> "NoReturn": - """Raise a clear error for unsupported web backend configuration.""" - import tools.web_tools as _wt - - message = ( - "Web tools are not configured. " - "Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL " - "for a self-hosted Firecrawl instance." - ) - if _wt.managed_nous_tools_enabled(): - message += ( - " With your Nous subscription you can also use the Tool Gateway. " - "run `hermes tools` and select Nous Subscription as the web provider." - ) - else: - message += " " + _wt.nous_tool_gateway_unavailable_message( - "managed Firecrawl web tools", - ) - raise ValueError(message) + return ", or use the Nous Tool Gateway via your subscription (FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)" def _get_firecrawl_client() -> Any: """Get or create the cached Firecrawl client. - Strict selection semantics (switch on the stored ``web`` selection): - - ``"nous"`` (or legacy ``use_gateway: true``) → managed Tool Gateway - ONLY; unavailable is a selection-naming error (a present - FIRECRAWL_API_KEY does not reroute). - - any other stored web backend → direct Firecrawl ONLY; missing config - is a selection-naming error — never a silent managed fallback billed - to Nous. - - never-configured web section → legacy behavior: direct config when - present, else the managed gateway. - - Raises ValueError when the resolved path is unusable. - - The cached client is stored on :mod:`tools.web_tools` (as - ``_firecrawl_client`` and ``_firecrawl_client_config``) rather than on - this plugin module so that unit tests that reset the cache via - ``tools.web_tools._firecrawl_client = None`` keep working. Helper - functions (``resolve_managed_tool_gateway``, ``_read_nous_access_token``, - ``Firecrawl``) are also looked up via :mod:`tools.web_tools` for the same - reason — see :func:`_is_tool_gateway_ready`. + Strict selection semantics on the stored ``web`` selection: ``"nous"`` → + managed Tool Gateway ONLY; any other stored backend → direct Firecrawl ONLY + (never a silent managed fallback billed to Nous); never-configured → direct + when present, else managed. Raises ValueError when the resolved path is unusable. """ import tools.web_tools as _wt from tools.tool_backend_helpers import ( @@ -340,25 +212,16 @@ def _get_firecrawl_client() -> Any: ) selected = read_selection("web") - direct_config = _get_direct_firecrawl_config() def _managed_kwargs(): - managed_gateway = _wt.resolve_managed_tool_gateway( - "firecrawl", token_reader=_wt._read_nous_access_token - ) - if managed_gateway is None: + gw = _wt.resolve_managed_tool_gateway("firecrawl", token_reader=_wt._read_nous_access_token) + if gw is None: return None - kwargs = { - "api_key": managed_gateway.nous_user_token, - "api_url": managed_gateway.gateway_origin, - } - return kwargs, ( - "tool-gateway", - kwargs["api_url"], - managed_gateway.nous_user_token, - ) + kwargs = {"api_key": gw.nous_user_token, "api_url": gw.gateway_origin} + return kwargs, ("tool-gateway", kwargs["api_url"], gw.nous_user_token) + client_mode = "sdk" if selected == NOUS_MANAGED_PROVIDER: managed = _managed_kwargs() if managed is None: @@ -368,16 +231,12 @@ def _get_firecrawl_client() -> Any: "is unavailable." ) raise ValueError(selection_error( - "web", - NOUS_MANAGED_PROVIDER, - "the Nous Tool Gateway is not available (not entitled or " - "unreachable)", + "web", NOUS_MANAGED_PROVIDER, + "the Nous Tool Gateway is not available (not entitled or unreachable)", )) kwargs, client_config = managed - client_mode = "sdk" elif selected is not None or selection_exists("web"): - # Stored vendor selection (or per-capability web keys routing to - # firecrawl): direct Firecrawl only. With no credentials, the + # Stored vendor selection: direct Firecrawl only. With no credentials the # explicit selection unlocks keyless cloud mode instead of erroring. if direct_config is None: logger.error( @@ -385,9 +244,7 @@ def _get_firecrawl_client() -> Any: "selected but FIRECRAWL_API_KEY/FIRECRAWL_API_URL is not set." ) raise ValueError(selection_error( - "web", - selected or "firecrawl", - "neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set", + "web", selected or "firecrawl", "neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set", )) client_mode, kwargs, client_config = direct_config elif direct_config is not None: @@ -396,21 +253,26 @@ def _get_firecrawl_client() -> Any: # Never-configured web section: legacy managed fallback. managed = _managed_kwargs() if managed is None: - logger.error( - "Firecrawl client initialization failed: " - "missing direct config and tool-gateway auth." + logger.error("Firecrawl client initialization failed: missing direct config and tool-gateway auth.") + message = ( + "Web tools are not configured. " + "Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL " + "for a self-hosted Firecrawl instance." ) - _raise_web_backend_configuration_error() + if _wt.managed_nous_tools_enabled(): + message += ( + " With your Nous subscription you can also use the Tool Gateway. " + "run `hermes tools` and select Nous Subscription as the web provider." + ) + else: + message += " " + _wt.nous_tool_gateway_unavailable_message("managed Firecrawl web tools") + raise ValueError(message) kwargs, client_config = managed - client_mode = "sdk" cached = getattr(_wt, "_firecrawl_client", None) - cached_config = getattr(_wt, "_firecrawl_client_config", None) - if cached is not None and cached_config == client_config: + if cached is not None and getattr(_wt, "_firecrawl_client_config", None) == client_config: return cached - # Construct via the re-exported Firecrawl proxy on tools.web_tools so - # unit tests patching ``tools.web_tools.Firecrawl`` see their mock. if client_mode == "keyless": _wt._firecrawl_client = _KeylessFirecrawlClient(api_url=kwargs["api_url"]) else: @@ -419,43 +281,23 @@ def _get_firecrawl_client() -> Any: return _wt._firecrawl_client -def _reset_client_for_tests() -> None: - """Drop the cached Firecrawl client so tests can re-instantiate cleanly. - - Clears the canonical slots on :mod:`tools.web_tools` (where - :func:`_get_firecrawl_client` reads/writes them). - """ - import tools.web_tools as _wt - - _wt._firecrawl_client = None - _wt._firecrawl_client_config = None - - -# --------------------------------------------------------------------------- -# Response shape normalization (SDK / direct / gateway differ) -# --------------------------------------------------------------------------- +# --- Response shape normalization (SDK / direct / gateway differ) -------------- def _to_plain_object(value: Any) -> Any: - """Convert SDK objects to plain python data structures when possible.""" - if value is None: - return None - - if isinstance(value, (dict, list, str, int, float, bool)): + """Convert SDK objects (pydantic ``model_dump`` / ``__dict__``) to plain data when possible.""" + if value is None or isinstance(value, (dict, list, str, int, float, bool)): return value - if hasattr(value, "model_dump"): try: return value.model_dump() except Exception: # noqa: BLE001 pass - if hasattr(value, "__dict__"): try: return {k: v for k, v in value.__dict__.items() if not k.startswith("_")} except Exception: # noqa: BLE001 pass - return value @@ -463,13 +305,8 @@ def _normalize_result_list(values: Any) -> List[Dict[str, Any]]: """Normalize mixed SDK/list payloads into a list of dicts.""" if not isinstance(values, list): return [] - - normalized: List[Dict[str, Any]] = [] - for item in values: - plain = _to_plain_object(item) - if isinstance(plain, dict): - normalized.append(plain) - return normalized + plain = (_to_plain_object(item) for item in values) + return [p for p in plain if isinstance(p, dict)] def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]: @@ -480,26 +317,17 @@ def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]: data = response_plain.get("data") if isinstance(data, list): return _normalize_result_list(data) - + candidates = [] if isinstance(data, dict): - data_web = _normalize_result_list(data.get("web")) - if data_web: - return data_web - data_results = _normalize_result_list(data.get("results")) - if data_results: - return data_results - - top_web = _normalize_result_list(response_plain.get("web")) - if top_web: - return top_web - - top_results = _normalize_result_list(response_plain.get("results")) - if top_results: - return top_results + candidates += [data.get("web"), data.get("results")] + candidates += [response_plain.get("web"), response_plain.get("results")] + for candidate in candidates: + normalized = _normalize_result_list(candidate) + if normalized: + return normalized if hasattr(response, "web"): return _normalize_result_list(getattr(response, "web", [])) - return [] @@ -508,292 +336,158 @@ def _extract_scrape_payload(scrape_result: Any) -> Dict[str, Any]: result_plain = _to_plain_object(scrape_result) if not isinstance(result_plain, dict): return {} - nested = result_plain.get("data") - if isinstance(nested, dict): - return nested - - return result_plain + return nested if isinstance(nested, dict) else result_plain -# --------------------------------------------------------------------------- -# Provider class -# --------------------------------------------------------------------------- +def _error_entry( + url: str, + error: str, + *, + title: str = "", + raw: bool = False, + blocked: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Per-URL extract failure item. ``raw`` adds the ``raw_content`` key (post-scrape + failures carry it, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``.""" + entry: Dict[str, Any] = {"url": url, "title": title, "content": ""} + if raw: + entry["raw_content"] = "" + entry["error"] = error + if blocked: + entry["blocked_by_policy"] = {k: blocked[k] for k in ("host", "rule", "source")} + return entry -class FirecrawlWebSearchProvider(WebSearchProvider): +_SCRAPE_TIMEOUT_MSG = ( + "Scrape timed out after 60s — page may be too large " + "or unresponsive. Try browser_navigate instead." +) +_UNSAFE_REDIRECT_MSG = "Blocked: URL targets a private or internal network address" + + +async def _scrape_one(url: str, formats: List[str], format: Optional[str]) -> Dict[str, Any]: + """Scrape one URL (60s timeout) and re-check SSRF + website policy against the + post-redirect URL. Never raises for scrape errors; returns an error entry instead.""" + blocked = check_website_access(url) + if blocked: + logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"]) + return _error_entry(url, blocked["message"], blocked=blocked) + + try: + logger.info("Firecrawl scraping: %s", url) + try: + scrape_result = await asyncio.wait_for( + asyncio.to_thread(_get_firecrawl_client().scrape, url=url, formats=formats), + timeout=60, + ) + except asyncio.TimeoutError: + logger.warning("Firecrawl scrape timed out for %s", url) + return _error_entry(url, _SCRAPE_TIMEOUT_MSG) + + scrape_payload = _extract_scrape_payload(scrape_result) + metadata = scrape_payload.get("metadata", {}) + content_markdown = scrape_payload.get("markdown") + content_html = scrape_payload.get("html") + + # SDK may return a typed object for metadata (raw __dict__ here, unlike _to_plain_object). + if not isinstance(metadata, dict): + if hasattr(metadata, "model_dump"): + metadata = metadata.model_dump() + elif hasattr(metadata, "__dict__"): + metadata = metadata.__dict__ + else: + metadata = {} + + title = metadata.get("title", "") + final_url = metadata.get("sourceURL", url) + + if not is_safe_url(final_url): + logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url) + return _error_entry(final_url, _UNSAFE_REDIRECT_MSG, title=title, raw=True) + + final_blocked = check_website_access(final_url) + if final_blocked: + logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"]) + return _error_entry(final_url, final_blocked["message"], title=title, raw=True, blocked=final_blocked) + + if format == "markdown" or (format is None and content_markdown): + chosen_content = content_markdown + else: + chosen_content = content_html or content_markdown or "" + return {"url": final_url, "title": title, "content": chosen_content, "raw_content": chosen_content, "metadata": metadata} + except Exception as scrape_err: # noqa: BLE001 + logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err) + return _error_entry(url, str(scrape_err), raw=True) + + +# --- Provider class ------------------------------------------------------------ + + +class FirecrawlWebSearchProvider(BaseWebSearchProvider): """Firecrawl search + extract provider with dual auth paths.""" - @property - def name(self) -> str: - return "firecrawl" - - @property - def display_name(self) -> str: - return "Firecrawl" + NAME = "firecrawl" + DISPLAY_NAME = "Firecrawl" + EXTRACT = True + KEYLESS = True # default-on ring member unless pinned ``paid`` def is_available(self) -> bool: - """Return True when direct Firecrawl OR managed-gateway path is configured.""" return check_firecrawl_api_key() - def is_keyless_available(self) -> bool: - """Firecrawl serves keyless cloud requests (public API, no auth). - - Default-on ring member of the keyless free tier: fresh installs - rotate across Exa/Parallel/Firecrawl/Keenable. False when - the user pinned ``web.provider_tier.firecrawl: paid``. - """ - from plugins.web.keyless_mcp import keyless_enabled, provider_tier - - return keyless_enabled() and provider_tier("firecrawl") != "paid" - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return True - def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a Firecrawl search. - - Sync; matches the legacy ``_get_firecrawl_client().search(...)`` - call directly. Normalizes the response across SDK/direct/gateway - shapes via :func:`_extract_web_search_results`. - - Pre-flight errors (``ValueError`` from configuration check, - ``ImportError`` from missing SDK) propagate to the dispatcher's - top-level handler, which wraps them as ``tool_error(...)`` — - matching the legacy ``{"error": "Error searching web: ..."}`` - envelope. Only in-flight errors are caught and surfaced as - ``{"success": False, "error": ...}``. - """ + """Sync search. Pre-flight errors (ValueError / ImportError) propagate so the + dispatcher emits the legacy ``tool_error`` envelope; in-flight errors are + returned as ``{"success": False, "error": ...}``.""" from tools.interrupt import is_interrupted if is_interrupted(): - return {"success": False, "error": "Interrupted"} + return search_fail("Interrupted") if _use_keyless_ring(): - # No credentials and no managed gateway: ring dispatch with - # next-in-line failover on rate limits (default-on free tier). from plugins.web.keyless_mcp import search_with_failover - logger.info( - "Firecrawl keyless search: '%s' (limit=%d)", query, limit - ) + logger.info("Firecrawl keyless search: '%s' (limit=%d)", query, limit) return search_with_failover("firecrawl", query, limit) logger.info("Firecrawl search: '%s' (limit=%d)", query, limit) - # _get_firecrawl_client() raises ValueError on unconfigured systems — - # let it propagate so the dispatcher emits the legacy envelope shape. client = _get_firecrawl_client() try: response = client.search(query=query, limit=limit) web_results = _extract_web_search_results(response) logger.info("Firecrawl: found %d search results", len(web_results)) - return {"success": True, "data": {"web": web_results}} + return search_ok(web_results) except Exception as exc: # noqa: BLE001 logger.warning("Firecrawl search error: %s", exc) - return {"success": False, "error": f"Firecrawl search failed: {exc}"} + return search_fail(f"Firecrawl search failed: {exc}") async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: - """Extract content from one or more URLs via Firecrawl. - - Async; each URL is scraped in a background thread with a 60s - timeout. After scraping, the final URL (post-redirect) is - re-checked against website-access policy. - - Accepted kwargs (others ignored for forward compat): - - ``format``: ``"markdown"`` or ``"html"``; default is both - (request both, return markdown when available). - - Returns the legacy per-URL list-of-results shape. Per-URL failures - (timeout, SSRF block, scrape error, policy block) become items - with an ``error`` field rather than raising. - """ + """Per-URL scrape via :func:`_scrape_one`; failures become items with an + ``error`` field. ``format``: "markdown" | "html" | both (markdown preferred).""" from tools.interrupt import is_interrupted as _is_interrupted if _is_interrupted(): return [{"url": u, "error": "Interrupted", "title": ""} for u in urls] if _use_keyless_ring(): - # No credentials and no managed gateway: ring dispatch with - # next-in-line failover on rate limits (default-on free tier). - import asyncio as _asyncio - from plugins.web.keyless_mcp import extract_with_failover logger.info("Firecrawl keyless extract: %d URL(s)", len(urls)) - return await _asyncio.to_thread( - extract_with_failover, "firecrawl", list(urls) - ) + return await asyncio.to_thread(extract_with_failover, "firecrawl", list(urls)) format = kwargs.get("format") - formats: List[str] = [] - if format == "markdown": - formats = ["markdown"] - elif format == "html": - formats = ["html"] - else: - formats = ["markdown", "html"] + formats = [format] if format in ("markdown", "html") else ["markdown", "html"] - # check_website_access is the legacy policy gate; imported at - # module level (lazy-friendly because the website_policy import is - # cheap) so monkeypatching it in tests works as expected. - - results: List[Dict[str, Any]] = [] - - for url in urls: - if _is_interrupted(): - results.append({"url": url, "error": "Interrupted", "title": ""}) - continue - - # Pre-scrape website policy gate - blocked = check_website_access(url) - if blocked: - logger.info( - "Blocked web_extract for %s by rule %s", - blocked["host"], - blocked["rule"], - ) - results.append( - { - "url": url, - "title": "", - "content": "", - "error": blocked["message"], - "blocked_by_policy": { - "host": blocked["host"], - "rule": blocked["rule"], - "source": blocked["source"], - }, - } - ) - continue - - try: - logger.info("Firecrawl scraping: %s", url) - try: - scrape_result = await asyncio.wait_for( - asyncio.to_thread( - _get_firecrawl_client().scrape, - url=url, - formats=formats, - ), - timeout=60, - ) - except asyncio.TimeoutError: - logger.warning("Firecrawl scrape timed out for %s", url) - results.append( - { - "url": url, - "title": "", - "content": "", - "error": ( - "Scrape timed out after 60s — page may be too large " - "or unresponsive. Try browser_navigate instead." - ), - } - ) - continue - - scrape_payload = _extract_scrape_payload(scrape_result) - metadata = scrape_payload.get("metadata", {}) - content_markdown = scrape_payload.get("markdown") - content_html = scrape_payload.get("html") - - # Ensure metadata is a dict (SDK may return a typed object) - if not isinstance(metadata, dict): - if hasattr(metadata, "model_dump"): - metadata = metadata.model_dump() - elif hasattr(metadata, "__dict__"): - metadata = metadata.__dict__ - else: - metadata = {} - - title = metadata.get("title", "") - final_url = metadata.get("sourceURL", url) - - # Re-check SSRF safety after any redirect reported by Firecrawl. - if not is_safe_url(final_url): - logger.info( - "Blocked redirected web_extract for unsafe final URL: %s", - final_url, - ) - results.append( - { - "url": final_url, - "title": title, - "content": "", - "raw_content": "", - "error": ( - "Blocked: URL targets a private or internal " - "network address" - ), - } - ) - continue - - # Re-check website-access policy after any redirect - final_blocked = check_website_access(final_url) - if final_blocked: - logger.info( - "Blocked redirected web_extract for %s by rule %s", - final_blocked["host"], - final_blocked["rule"], - ) - results.append( - { - "url": final_url, - "title": title, - "content": "", - "raw_content": "", - "error": final_blocked["message"], - "blocked_by_policy": { - "host": final_blocked["host"], - "rule": final_blocked["rule"], - "source": final_blocked["source"], - }, - } - ) - continue - - # Choose markdown vs html according to the requested format - if format == "markdown" or (format is None and content_markdown): - chosen_content = content_markdown - else: - chosen_content = content_html or content_markdown or "" - - results.append( - { - "url": final_url, - "title": title, - "content": chosen_content, - "raw_content": chosen_content, - "metadata": metadata, - } - ) - except Exception as scrape_err: # noqa: BLE001 - logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err) - results.append( - { - "url": url, - "title": "", - "content": "", - "raw_content": "", - "error": str(scrape_err), - } - ) - - return results + return [ + {"url": url, "error": "Interrupted", "title": ""} if _is_interrupted() else await _scrape_one(url, formats, format) + for url in urls + ] def get_setup_schema(self) -> Dict[str, Any]: return { "name": "Firecrawl", "badge": "keyless/paid · optional gateway", - "tag": ( - "Full search + extract; supports keyless cloud, direct API, " - "and Nous tool-gateway routing." - ), + "tag": "Full search + extract; supports keyless cloud, direct API, and Nous tool-gateway routing.", "env_vars": [ { "key": "FIRECRAWL_API_KEY", diff --git a/plugins/web/keenable/__init__.py b/plugins/web/keenable/__init__.py index 1897ca0926..ffb4affe2e 100644 --- a/plugins/web/keenable/__init__.py +++ b/plugins/web/keenable/__init__.py @@ -1,7 +1,4 @@ -"""Keenable web search + extract plugin — bundled, auto-loaded. - -Keyless-ring member (keyed via KEENABLE_API_KEY for higher limits). -""" +"""Keenable web search + extract plugin — bundled, auto-loaded; keyless-ring member.""" from __future__ import annotations @@ -9,5 +6,4 @@ from plugins.web.keenable.provider import KeenableWebSearchProvider def register(ctx) -> None: - """Register the Keenable provider with the plugin context.""" ctx.register_web_search_provider(KeenableWebSearchProvider()) diff --git a/plugins/web/keenable/provider.py b/plugins/web/keenable/provider.py index 2795189107..c61c885d41 100644 --- a/plugins/web/keenable/provider.py +++ b/plugins/web/keenable/provider.py @@ -1,27 +1,7 @@ -"""Keenable web search + content extraction — bundled plugin. +"""Keenable (https://keenable.ai) web search + fetch — keyless-ring member. -Keenable (https://keenable.ai) operates an independent web index for AI -apps with public keyless endpoints (rate-limited free tier; keyed access -via KEENABLE_API_KEY for higher limits). Integrated as a keyless-ring -member following the Exa/Parallel/Firecrawl pattern: fresh installs with -zero web credentials rotate across the ring vendors' free tiers. - -Credit: Keenable integration originally proposed by Ilya Gusev (Keenable) -in PR #49758; the native provider form follows the salvage of that work -plus the keyless-ring design. - -Config keys this provider responds to:: - - web: - search_backend: "keenable" # explicit per-capability - extract_backend: "keenable" # explicit per-capability - backend: "keenable" # shared fallback - provider_tier: - keenable: free|paid # pin the tier (unset = auto) - -Env var:: - - KEENABLE_API_KEY=... # optional — keyless free tier works without it +Env: ``KEENABLE_API_KEY`` (optional; keyless free tier works without it). +Config: ``web.provider_tier.keenable: free|paid`` pins the tier (unset = auto). """ from __future__ import annotations @@ -29,7 +9,20 @@ from __future__ import annotations import logging from typing import Any, Dict, List -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import ( + SEARCH_LIMIT_CAP, + BaseWebSearchProvider, + document, + http_status_detail, + keyless_variant_schema, + provider_env, + run_extract, + run_search, + search_fail, + search_ok, + use_keyless, + web_hit, +) logger = logging.getLogger(__name__) @@ -37,67 +30,29 @@ _KEENABLE_API_URL = "https://api.keenable.ai" def _keenable_headers(api_key: str) -> Dict[str, str]: - """Build Keenable request headers for keyed or keyless access. - - Their keyless tier structurally requires an app-identifier header - (X-Keenable-Title); no user identifiers are sent. - """ + # The keyless tier structurally requires an app-identifier header; no user identifiers are sent. headers = {"X-Keenable-Title": "hermes-agent"} if api_key: headers["Authorization"] = f"Bearer {api_key}" return headers -class KeenableWebSearchProvider(WebSearchProvider): +class KeenableWebSearchProvider(BaseWebSearchProvider): """Keenable search + extract provider (keyed or keyless).""" - @property - def name(self) -> str: - return "keenable" - - @property - def display_name(self) -> str: - return "Keenable" - - def is_available(self) -> bool: - """Return True when ``KEENABLE_API_KEY`` is set to a non-empty value.""" - from agent.web_search_provider import get_provider_env - - return bool(get_provider_env("KEENABLE_API_KEY")) - - def is_keyless_available(self) -> bool: - """Keenable serves anonymous free-tier calls via its public endpoints. - - Default-on ring member of the keyless free tier. False when the - user pinned ``web.provider_tier.keenable: paid``. - """ - from plugins.web.keyless_mcp import keyless_enabled, provider_tier - - return keyless_enabled() and provider_tier("keenable") != "paid" - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return True + NAME = "keenable" + DISPLAY_NAME = "Keenable" + KEY_ENV = "KEENABLE_API_KEY" + EXTRACT = True + KEYLESS = True def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a Keenable search (keyed path or keyless ring).""" - try: - from tools.interrupt import is_interrupted + def _body() -> Dict[str, Any]: + from plugins.web.keyless_mcp import search_with_failover - if is_interrupted(): - return {"success": False, "error": "Interrupted"} - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import search_with_failover, use_keyless - - api_key = get_provider_env("KEENABLE_API_KEY") + api_key = provider_env("KEENABLE_API_KEY") if use_keyless("keenable", api_key): - logger.info( - "Keenable keyless search: '%s' (limit=%d)", query, limit - ) + logger.info("Keenable keyless search: '%s' (limit=%d)", query, limit) return search_with_failover("keenable", query, limit) import requests @@ -105,52 +60,24 @@ class KeenableWebSearchProvider(WebSearchProvider): logger.info("Keenable search: '%s' (limit=%d)", query, limit) response = requests.post( f"{_KEENABLE_API_URL}/v1/search", - json={"query": query, "max_results": min(max(1, int(limit)), 20)}, + json={"query": query, "max_results": min(max(1, int(limit)), SEARCH_LIMIT_CAP)}, headers=_keenable_headers(api_key), timeout=30, ) if response.status_code >= 400: - detail = (response.text or "").strip() or f"HTTP {response.status_code}" - return {"success": False, "error": f"Keenable search failed: {detail}"} - data = response.json() + return search_fail(f"Keenable search failed: {http_status_detail(response)}") + return search_ok([ + web_hit(r.get("url") or "", r.get("title") or "", r.get("snippet") or r.get("description") or "", i + 1) + for i, r in enumerate(response.json().get("results") or []) + ]) - web_results = [] - for i, result in enumerate(data.get("results") or []): - web_results.append( - { - "url": result.get("url") or "", - "title": result.get("title") or "", - "description": result.get("snippet") - or result.get("description") - or "", - "position": i + 1, - } - ) - return {"success": True, "data": {"web": web_results}} - except Exception as exc: # noqa: BLE001 — surface as failure - logger.warning("Keenable search error: %s", exc) - return {"success": False, "error": f"Keenable search failed: {exc}"} + return run_search("Keenable", logger, _body, verbatim_value_error=False) def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: - """Extract content via Keenable's fetch endpoint (per-URL). + def _body() -> List[Dict[str, Any]]: + from plugins.web.keyless_mcp import extract_with_failover - Sync — the dispatcher wraps in a thread when the caller is async. - Returns the legacy list-of-results shape; per-URL failures become - items with an ``error`` field. - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return [ - {"url": u, "error": "Interrupted", "title": ""} for u in urls - ] - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import extract_with_failover, use_keyless - - api_key = get_provider_env("KEENABLE_API_KEY") + api_key = provider_env("KEENABLE_API_KEY") if use_keyless("keenable", api_key): logger.info("Keenable keyless extract: %d URL(s)", len(urls)) return extract_with_failover("keenable", list(urls)) @@ -162,72 +89,23 @@ class KeenableWebSearchProvider(WebSearchProvider): for url in urls: try: response = requests.get( - f"{_KEENABLE_API_URL}/v1/fetch", - params={"url": url}, - headers=_keenable_headers(api_key), - timeout=30, + f"{_KEENABLE_API_URL}/v1/fetch", params={"url": url}, headers=_keenable_headers(api_key), timeout=30 ) if response.status_code >= 400: - raise ValueError( - (response.text or "").strip() - or f"HTTP {response.status_code}" - ) + raise ValueError(http_status_detail(response)) data = response.json() - content = data.get("content") or "" - title = data.get("title") or "" results.append( - { - "url": data.get("url") or url, - "title": title, - "content": content, - "raw_content": content, - "metadata": {"sourceURL": url, "title": title}, - } + document(data.get("url") or url, data.get("title") or "", data.get("content") or "", source_url=url) ) except Exception as exc: # noqa: BLE001 — per-URL error entry - results.append( - { - "url": url, - "title": "", - "content": "", - "error": f"Keenable extract failed: {exc}", - } - ) + results.append({"url": url, "title": "", "content": "", "error": f"Keenable extract failed: {exc}"}) return results - except Exception as exc: # noqa: BLE001 - logger.warning("Keenable extract error: %s", exc) - return [ - {"url": u, "title": "", "content": "", - "error": f"Keenable extract failed: {exc}"} - for u in urls - ] + + return run_extract("Keenable", logger, urls, _body, verbatim_value_error=False) def get_setup_schema(self) -> Dict[str, Any]: - return { - "name": "Keenable · Free (keyless)", - "badge": "free · no key", - "tag": ( - "Independent web index for AI apps — fast search + page " - "fetch on Keenable's anonymous free tier." - ), - "env_vars": [], - "web_tier": "free", - "variants": [ - { - "name": "Keenable · Paid (API key)", - "badge": "paid", - "tag": ( - "Independent web index for AI apps. Keyed access " - "with higher limits and guaranteed service." - ), - "env_vars": [ - { - "key": "KEENABLE_API_KEY", - "prompt": "Keenable API key", - "url": "https://keenable.ai", - }, - ], - "web_tier": "paid", - }, - ], - } + return keyless_variant_schema( + "Keenable", "KEENABLE_API_KEY", "https://keenable.ai", + free_tag="Independent web index for AI apps — fast search + page fetch on Keenable's anonymous free tier.", + paid_tag="Independent web index for AI apps. Keyed access with higher limits and guaranteed service.", + ) diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.py index a2f39a7c05..98af33012e 100644 --- a/plugins/web/keyless_mcp.py +++ b/plugins/web/keyless_mcp.py @@ -1,42 +1,30 @@ -"""Keyless web search/extract via public MCP endpoints. +"""Keyless web search/extract via public free-tier endpoints (Exa, Parallel, Firecrawl, Keenable). -Exa and Parallel both operate public, anonymous MCP endpoints with a free -tier (the same endpoints the opencode CLI ships as its default search -path): - -- Exa: https://mcp.exa.ai/mcp (tools: web_search_exa, web_fetch_exa) -- Parallel: https://search.parallel.ai/mcp (tools: web_search, web_fetch) - -This module implements a minimal JSON-RPC ``tools/call`` client for those -two endpoints so a fresh Hermes install with **zero web credentials** still -gets working ``web_search`` / ``web_extract`` tools. The keyless tier is -resolved strictly LAST — after every keyed backend, the managed tool -gateway, ddgs, and custom plugin providers — so it never pre-empts a -deliberate setup (see ``tools.web_tools._get_backend`` and the registry's -``_KEYLESS_PREFERENCE`` walk). - -Privacy: requests carry no user identifiers. Parallel's free tier asks for -a ``session_id`` used for rate limiting; we send a random per-process UUID -(rotates every restart, never persisted). Their optional ``model_name`` -analytics field is deliberately omitted. - -Disable the whole tier with ``web.keyless_fallback: false`` in config.yaml. +Resolved strictly LAST — after every keyed backend, the managed gateway, ddgs and +custom plugin providers — so it never pre-empts a deliberate setup. Privacy: no user +identifiers are sent; Parallel gets a random per-process ``session_id`` (rate +limiting only) and its optional ``model_name`` analytics field is deliberately +omitted. Disable the tier with ``web.keyless_fallback: false``. """ from __future__ import annotations import json import logging +import threading import uuid -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional + +from plugins.web._common import document as _page, search_fail, search_ok, web_hit as _row logger = logging.getLogger(__name__) EXA_MCP_URL = "https://mcp.exa.ai/mcp" PARALLEL_MCP_URL = "https://search.parallel.ai/mcp" +KEENABLE_API_URL = "https://api.keenable.ai" +_KEENABLE_TITLE = "hermes-agent" -# Free-tier rate-limit correlation id for Parallel — random per process, -# never persisted, not derived from any user/machine identifier. +# Parallel free-tier rate-limit correlation id — random per process, never persisted. _SESSION_ID = uuid.uuid4().hex _TIMEOUT_SECONDS = 30 @@ -46,30 +34,38 @@ class KeylessMCPError(RuntimeError): """A keyless MCP call failed (transport, rate limit, or tool error).""" -_RATE_LIMIT_MARKERS = ( - "rate limit", - "rate-limit", - "ratelimit", - "too many requests", - "429", - "quota exceeded", - "slow down", -) +_RATE_LIMIT_MARKERS = ("rate limit", "rate-limit", "ratelimit", "too many requests", "429", "quota exceeded", "slow down") + +# vendor -> (display label, env key, signup URL) for the standard failure hint. +_VENDOR_HINTS = { + "exa": ("Exa", "EXA_API_KEY", "https://exa.ai"), + "parallel": ("Parallel", "PARALLEL_API_KEY", "https://parallel.ai"), + "firecrawl": ("Firecrawl", "FIRECRAWL_API_KEY", "https://firecrawl.dev"), + "keenable": ("Keenable", "KEENABLE_API_KEY", "https://keenable.ai"), +} def _is_rate_limitish(message: str) -> bool: """Heuristic: does an error message look like free-tier throttling?""" - lowered = (message or "").lower() - return any(marker in lowered for marker in _RATE_LIMIT_MARKERS) + return any(marker in (message or "").lower() for marker in _RATE_LIMIT_MARKERS) + + +def _fail_msg(vendor: str, kind: str, exc: Any, *, other_backends: bool = True) -> str: + label, env_key, site = _VENDOR_HINTS[vendor] + alt = " or another web backend via `hermes tools`" if other_backends else "" + return f"Keyless {label} {kind} failed: {exc}. Set {env_key} ({site}){alt} for reliable service." + + +def _page_error(url: str, message: str) -> Dict[str, Any]: + return {"url": url, "title": "", "content": "", "error": message} + + +# --- Tier / config ------------------------------------------------------------ def keyless_enabled() -> bool: - """Return True when the keyless fallback tier is enabled. - - Delegates to :func:`agent.web_search_registry._keyless_tier_enabled` so - the config chokepoint (``web.keyless_fallback``, default on) lives in - one place alongside the rest of backend resolution. - """ + """True when the keyless tier is enabled. Delegates to the registry so the + ``web.keyless_fallback`` (default on) chokepoint lives with backend resolution.""" try: from agent.web_search_registry import _keyless_tier_enabled @@ -80,16 +76,8 @@ def keyless_enabled() -> bool: def provider_tier(name: str) -> str: - """Return the user-selected tier for *name*: ``free``, ``paid``, or ``auto``. - - Reads ``web.provider_tier.`` from config.yaml (set by the - ``hermes tools`` picker's Free/Paid rows). ``free`` forces the keyless - public endpoint even when the vendor API key is present; ``paid`` - forces the keyed SDK path (missing key surfaces the standard - "X_API_KEY not set" error instead of silently downgrading to the free - tier). Anything else — including unset — is ``auto``: key present → - keyed, otherwise keyless when the tier is enabled. - """ + """Return ``web.provider_tier.`` (set by the ``hermes tools`` Free/Paid rows): + ``free``, ``paid``, or ``auto`` for anything else including unset.""" try: from hermes_cli.config import load_config @@ -103,17 +91,10 @@ def provider_tier(name: str) -> str: def use_keyless(name: str, api_key: str) -> bool: - """Decide whether provider *name* should route via the keyless endpoint. - - Single chokepoint shared by the Exa/Parallel search + extract paths so - tier semantics can't drift between capabilities: - - - tier ``free`` → keyless, even when *api_key* is set - - tier ``paid`` → keyed, even when *api_key* is missing (the keyed - path then raises its usual missing-key error) - - tier ``auto`` → keyed when *api_key* is set; otherwise keyless when - ``web.keyless_fallback`` is enabled - """ + """Single chokepoint for search + extract so tier semantics can't drift: + ``free`` → keyless even with a key; ``paid`` → keyed even without one (the keyed + path raises its usual missing-key error); ``auto`` → keyless only when no key + and the tier is enabled.""" tier = provider_tier(name) if tier == "free": return True @@ -122,13 +103,15 @@ def use_keyless(name: str, api_key: str) -> bool: return not api_key and keyless_enabled() +# --- MCP transport ------------------------------------------------------------ + + def _parse_mcp_body(body: str) -> str: """Extract the first text content item from an MCP tools/call response. - Handles both plain-JSON bodies and SSE (``data: {...}`` lines) — the - Exa endpoint answers as an event stream, Parallel as direct JSON. - Raises :class:`KeylessMCPError` for JSON-RPC errors and ``isError`` - tool results (e.g. Exa's free-tier rate-limit message). + Handles plain-JSON bodies (Parallel) and SSE ``data: {...}`` lines (Exa). Raises + :class:`KeylessMCPError` for JSON-RPC errors and ``isError`` tool results (e.g. + Exa's free-tier rate-limit message). """ def _from_payload(payload: str) -> Optional[str]: @@ -143,9 +126,7 @@ def _parse_mcp_body(body: str) -> str: content = result.get("content") or [] if result.get("isError"): texts = [c.get("text", "") for c in content if isinstance(c, dict)] - raise KeylessMCPError( - " ".join(t for t in texts if t) or "MCP tool call failed" - ) + raise KeylessMCPError(" ".join(t for t in texts if t) or "MCP tool call failed") for item in content: if isinstance(item, dict) and item.get("text"): return str(item["text"]) @@ -173,25 +154,15 @@ def _parse_mcp_body(body: str) -> str: raise KeylessMCPError("Unrecognized MCP response shape") -def mcp_call( - url: str, - tool: str, - arguments: Dict[str, Any], - timeout: int = _TIMEOUT_SECONDS, -) -> str: +def mcp_call(url: str, tool: str, arguments: Dict[str, Any], timeout: int = _TIMEOUT_SECONDS) -> str: """POST a JSON-RPC ``tools/call`` to *url* and return the text payload. - Raises :class:`KeylessMCPError` on transport failures, non-2xx - statuses, JSON-RPC errors, and error-shaped tool results. + Raises :class:`KeylessMCPError` on transport failures, non-2xx statuses, + JSON-RPC errors, and error-shaped tool results. """ import requests - payload = { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": tool, "arguments": arguments}, - } + payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": arguments}} headers = { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", @@ -202,146 +173,74 @@ def mcp_call( except requests.RequestException as exc: raise KeylessMCPError(f"request failed: {exc}") from exc if response.status_code >= 400: - raise KeylessMCPError( - f"HTTP {response.status_code}: {response.text[:300]}" - ) + raise KeylessMCPError(f"HTTP {response.status_code}: {response.text[:300]}") return _parse_mcp_body(response.text) -# --------------------------------------------------------------------------- -# Parallel (search.parallel.ai) — JSON text payloads -# --------------------------------------------------------------------------- +# --- Parallel (search.parallel.ai) — JSON text payloads ----------------------- def parallel_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: """Keyless Parallel web search → legacy search response shape.""" try: text = mcp_call( - PARALLEL_MCP_URL, - "web_search", - { - "objective": query, - "search_queries": [query], - "session_id": _SESSION_ID, - }, + PARALLEL_MCP_URL, "web_search", + {"objective": query, "search_queries": [query], "session_id": _SESSION_ID}, ) data = json.loads(text) web_results = [] for i, result in enumerate(data.get("results") or []): if limit and i >= limit: break - excerpts = result.get("excerpts") or [] - web_results.append( - { - "url": result.get("url") or "", - "title": result.get("title") or "", - "description": " ".join(excerpts) if excerpts else "", - "position": i + 1, - } - ) - return {"success": True, "data": {"web": web_results}} + web_results.append(_row( + result.get("url") or "", result.get("title") or "", + " ".join(result.get("excerpts") or []), i + 1, + )) + return search_ok(web_results) except KeylessMCPError as exc: - return { - "success": False, - "error": ( - f"Keyless Parallel search failed: {exc}. " - "Set PARALLEL_API_KEY (https://parallel.ai) or another web " - "backend via `hermes tools` for reliable service." - ), - } + return search_fail(_fail_msg("parallel", "search", exc)) except (json.JSONDecodeError, TypeError, KeyError) as exc: - return {"success": False, "error": f"Keyless Parallel search returned an unexpected payload: {exc}"} + return search_fail(f"Keyless Parallel search returned an unexpected payload: {exc}") def parallel_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: """Keyless Parallel web fetch → legacy extract result list.""" try: text = mcp_call( - PARALLEL_MCP_URL, - "web_fetch", - { - "urls": list(urls), - "objective": "Full page content", - "session_id": _SESSION_ID, - }, + PARALLEL_MCP_URL, "web_fetch", + {"urls": list(urls), "objective": "Full page content", "session_id": _SESSION_ID}, ) data = json.loads(text) except (KeylessMCPError, json.JSONDecodeError, TypeError) as exc: - message = ( - f"Keyless Parallel extract failed: {exc}. " - "Set PARALLEL_API_KEY (https://parallel.ai) or another web " - "backend via `hermes tools` for reliable service." - ) - return [ - {"url": u, "title": "", "content": "", "error": message} - for u in urls - ] + message = _fail_msg("parallel", "extract", exc) + return [_page_error(u, message) for u in urls] results: List[Dict[str, Any]] = [] seen = set() for result in data.get("results") or []: url = result.get("url") or "" - title = result.get("title") or "" - content = ( - result.get("full_content") - or result.get("content") - or "\n\n".join(result.get("excerpts") or []) - ) + content = result.get("full_content") or result.get("content") or "\n\n".join(result.get("excerpts") or []) seen.add(url) - results.append( - { - "url": url, - "title": title, - "content": content, - "raw_content": content, - "metadata": {"sourceURL": url, "title": title}, - } - ) + results.append(_page(url, result.get("title") or "", content)) for error in data.get("errors") or []: url = error.get("url") or "" seen.add(url) - results.append( - { - "url": url, - "title": "", - "content": "", - "error": str( - error.get("content") or error.get("error_type") or "extraction failed" - ), - "metadata": {"sourceURL": url}, - } - ) - # Any URL the endpoint silently dropped still gets an error entry so the - # caller's per-URL contract holds. - for u in urls: - if u not in seen: - results.append( - {"url": u, "title": "", "content": "", "error": "no content returned"} - ) + entry = _page_error(url, str(error.get("content") or error.get("error_type") or "extraction failed")) + entry["metadata"] = {"sourceURL": url} + results.append(entry) + # URLs the endpoint silently dropped still get an error entry (per-URL contract). + results.extend(_page_error(u, "no content returned") for u in urls if u not in seen) return results -# --------------------------------------------------------------------------- -# Exa (mcp.exa.ai) — formatted plain-text payloads -# --------------------------------------------------------------------------- +# --- Exa (mcp.exa.ai) — formatted plain-text payloads ------------------------- def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]: - """Parse Exa's formatted search text into result dicts. - - The payload is blocks separated by ``---`` lines, each shaped like:: - - Title: - URL: <url> - Published: ... - Author: ... - Highlights: - <free text> - """ + """Parse Exa's ``---``-separated ``Title:/URL:/Published:/Author:/Highlights:`` blocks.""" results: List[Dict[str, Any]] = [] for block in text.split("\n---\n"): - title = "" - url = "" + title = url = "" highlight_lines: List[str] = [] in_highlights = False for line in block.splitlines(): @@ -359,14 +258,7 @@ def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]: elif in_highlights and stripped: highlight_lines.append(stripped) if url: - results.append( - { - "url": url, - "title": title, - "description": " ".join(highlight_lines), - "position": len(results) + 1, - } - ) + results.append(_row(url, title, " ".join(highlight_lines), len(results) + 1)) if limit and len(results) >= limit: break return results @@ -375,46 +267,21 @@ def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]: def exa_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: """Keyless Exa web search → legacy search response shape.""" try: - text = mcp_call( - EXA_MCP_URL, - "web_search_exa", - {"query": query, "numResults": max(1, int(limit))}, - ) + text = mcp_call(EXA_MCP_URL, "web_search_exa", {"query": query, "numResults": max(1, int(limit))}) except KeylessMCPError as exc: - return { - "success": False, - "error": ( - f"Keyless Exa search failed: {exc}. " - "Set EXA_API_KEY (https://exa.ai) or another web backend " - "via `hermes tools` for reliable service." - ), - } - return {"success": True, "data": {"web": _parse_exa_search_text(text, limit)}} + return search_fail(_fail_msg("exa", "search", exc)) + return search_ok(_parse_exa_search_text(text, limit)) def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: - """Keyless Exa web fetch → legacy extract result list. - - ``web_fetch_exa`` takes a ``urls`` array but returns one combined text - payload; we call it per-URL so each result maps cleanly. - """ + """Keyless Exa web fetch → legacy extract result list (called per-URL; the + tool returns one combined text payload).""" results: List[Dict[str, Any]] = [] for url in urls: try: text = mcp_call(EXA_MCP_URL, "web_fetch_exa", {"urls": [url]}) except KeylessMCPError as exc: - results.append( - { - "url": url, - "title": "", - "content": "", - "error": ( - f"Keyless Exa extract failed: {exc}. " - "Set EXA_API_KEY (https://exa.ai) or another web " - "backend via `hermes tools` for reliable service." - ), - } - ) + results.append(_page_error(url, _fail_msg("exa", "extract", exc))) continue title = "" for line in text.splitlines(): @@ -425,234 +292,107 @@ def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: if stripped.startswith("Title:"): title = stripped[len("Title:"):].strip() break - results.append( - { - "url": url, - "title": title, - "content": text, - "raw_content": text, - "metadata": {"sourceURL": url, "title": title}, - } - ) + results.append(_page(url, title, text)) return results - -# --------------------------------------------------------------------------- -# Firecrawl keyless (public cloud API, no auth header) -# --------------------------------------------------------------------------- +# --- Firecrawl keyless (public cloud API, no auth header) --------------------- def firecrawl_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: """Keyless Firecrawl cloud search → legacy search response shape.""" - from plugins.web.firecrawl.provider import ( - _KeylessFirecrawlClient, - _extract_web_search_results, - ) + from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_web_search_results try: response = _KeylessFirecrawlClient().search(query=query, limit=limit) - return {"success": True, "data": {"web": _extract_web_search_results(response)}} + return search_ok(_extract_web_search_results(response)) except Exception as exc: # noqa: BLE001 — normalized below - return { - "success": False, - "error": ( - f"Keyless Firecrawl search failed: {exc}. " - "Set FIRECRAWL_API_KEY (https://firecrawl.dev) or another web " - "backend via `hermes tools` for reliable service." - ), - } + return search_fail(_fail_msg("firecrawl", "search", exc)) def firecrawl_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: """Keyless Firecrawl cloud scrape → legacy extract result list.""" - from plugins.web.firecrawl.provider import ( - _KeylessFirecrawlClient, - _extract_scrape_payload, - ) + from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload client = _KeylessFirecrawlClient() results: List[Dict[str, Any]] = [] for url in urls: try: - response = client.scrape(url=url, formats=["markdown"]) - payload = _extract_scrape_payload(response) or {} + payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {} metadata = payload.get("metadata") or {} if not isinstance(metadata, dict): metadata = {} content = payload.get("markdown") or payload.get("html") or "" - title = metadata.get("title") or "" - results.append( - { - "url": url, - "title": title, - "content": content, - "raw_content": content, - "metadata": {"sourceURL": url, "title": title}, - } - ) + results.append(_page(url, metadata.get("title") or "", content)) except Exception as exc: # noqa: BLE001 — per-URL error entry - results.append( - { - "url": url, - "title": "", - "content": "", - "error": ( - f"Keyless Firecrawl extract failed: {exc}. " - "Set FIRECRAWL_API_KEY (https://firecrawl.dev) for " - "reliable service." - ), - } - ) + results.append(_page_error(url, _fail_msg("firecrawl", "extract", exc, other_backends=False))) return results -# --------------------------------------------------------------------------- -# Keenable keyless (api.keenable.ai public endpoints) -# --------------------------------------------------------------------------- +# --- Keenable keyless (api.keenable.ai public endpoints) ---------------------- -KEENABLE_API_URL = "https://api.keenable.ai" -_KEENABLE_TITLE = "hermes-agent" +def _keenable_request(method: str, path: str, **kwargs: Any) -> Dict[str, Any]: + """Call a Keenable public endpoint with the mandatory X-Keenable-Title app id.""" + import requests + + headers = {"X-Keenable-Title": _KEENABLE_TITLE} + if method == "post": + headers["Content-Type"] = "application/json" + response = getattr(requests, method)(f"{KEENABLE_API_URL}{path}", headers=headers, timeout=_TIMEOUT_SECONDS, **kwargs) + if response.status_code >= 400: + raise KeylessMCPError((response.text or "").strip() or f"HTTP {response.status_code}") + return response.json() def keenable_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: - """Keyless Keenable search → legacy search response shape. - - POST /v1/search/public with the mandatory X-Keenable-Title app - identifier (their keyless tier requires an app name; no user - identifiers are sent). Response: {results: [{title, url, snippet}]}. - """ - import requests - + """Keyless Keenable search (POST /v1/search/public) → legacy search response shape.""" try: - response = requests.post( - f"{KEENABLE_API_URL}/v1/search/public", - json={"query": query, "max_results": max(1, int(limit))}, - headers={ - "Content-Type": "application/json", - "X-Keenable-Title": _KEENABLE_TITLE, - }, - timeout=_TIMEOUT_SECONDS, - ) - if response.status_code >= 400: - raise KeylessMCPError( - (response.text or "").strip() or f"HTTP {response.status_code}" - ) - data = response.json() + data = _keenable_request("post", "/v1/search/public", json={"query": query, "max_results": max(1, int(limit))}) except KeylessMCPError as exc: - return { - "success": False, - "error": ( - f"Keyless Keenable search failed: {exc}. " - "Set KEENABLE_API_KEY (https://keenable.ai) or another web " - "backend via `hermes tools` for reliable service." - ), - } + return search_fail(_fail_msg("keenable", "search", exc)) except Exception as exc: # noqa: BLE001 — transport/JSON errors - return { - "success": False, - "error": f"Keyless Keenable search failed: {exc}.", - } - web_results = [] - for i, result in enumerate(data.get("results") or []): - web_results.append( - { - "url": result.get("url") or "", - "title": result.get("title") or "", - "description": result.get("snippet") - or result.get("description") - or "", - "position": i + 1, - } - ) - return {"success": True, "data": {"web": web_results}} + return search_fail(f"Keyless Keenable search failed: {exc}.") + return search_ok([ + _row(r.get("url") or "", r.get("title") or "", r.get("snippet") or r.get("description") or "", i + 1) + for i, r in enumerate(data.get("results") or []) + ]) def keenable_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: - """Keyless Keenable page fetch → legacy extract result list. - - GET /v1/fetch/public?url=... returns {url, title, content} (markdown). - Called per-URL; failures become per-URL error entries. - """ - import requests - + """Keyless Keenable page fetch (GET /v1/fetch/public, per-URL) → legacy extract list.""" results: List[Dict[str, Any]] = [] for url in urls: try: - response = requests.get( - f"{KEENABLE_API_URL}/v1/fetch/public", - params={"url": url}, - headers={"X-Keenable-Title": _KEENABLE_TITLE}, - timeout=_TIMEOUT_SECONDS, - ) - if response.status_code >= 400: - raise KeylessMCPError( - (response.text or "").strip() or f"HTTP {response.status_code}" - ) - data = response.json() - content = data.get("content") or "" - title = data.get("title") or "" - results.append( - { - "url": data.get("url") or url, - "title": title, - "content": content, - "raw_content": content, - "metadata": {"sourceURL": url, "title": title}, - } - ) + data = _keenable_request("get", "/v1/fetch/public", params={"url": url}) + results.append(_page(data.get("url") or url, data.get("title") or "", data.get("content") or "", source_url=url)) except Exception as exc: # noqa: BLE001 — per-URL error entry - results.append( - { - "url": url, - "title": "", - "content": "", - "error": ( - f"Keyless Keenable extract failed: {exc}. " - "Set KEENABLE_API_KEY (https://keenable.ai) for " - "reliable service." - ), - } - ) + results.append(_page_error(url, _fail_msg("keenable", "extract", exc, other_backends=False))) return results -# --------------------------------------------------------------------------- -# Round-robin ring + next-in-line failover (rate-limited free tiers) -# --------------------------------------------------------------------------- +# --- Round-robin ring + next-in-line failover (rate-limited free tiers) ------- _KEYLESS_RING = ("exa", "parallel", "firecrawl", "keenable") -_KEYLESS_SEARCHERS = { - "exa": lambda query, limit: exa_search_keyless(query, limit), - "parallel": lambda query, limit: parallel_search_keyless(query, limit), - "firecrawl": lambda query, limit: firecrawl_search_keyless(query, limit), - "keenable": lambda query, limit: keenable_search_keyless(query, limit), +# Late-bound lookups (not bare references) so ``patch.object(keyless_mcp, "<vendor>_search_keyless")`` +# is honored at call time. Tests also ``setitem`` these dicts directly. +_KEYLESS_SEARCHERS: Dict[str, Callable[[str, int], Dict[str, Any]]] = { + v: (lambda query, limit, _v=v: globals()[f"{_v}_search_keyless"](query, limit)) for v in _KEYLESS_RING +} +_KEYLESS_EXTRACTORS: Dict[str, Callable[[List[str]], List[Dict[str, Any]]]] = { + v: (lambda urls, _v=v: globals()[f"{_v}_extract_keyless"](urls)) for v in _KEYLESS_RING } -_KEYLESS_EXTRACTORS = { - "exa": lambda urls: exa_extract_keyless(urls), - "parallel": lambda urls: parallel_extract_keyless(urls), - "firecrawl": lambda urls: firecrawl_extract_keyless(urls), - "keenable": lambda urls: keenable_extract_keyless(urls), -} - -# Per-process round-robin cursor, seeded by the random session id so the -# fleet spreads evenly across all five free tiers; advances once per -# unpinned keyless request so a single process also rotates. -_ring_lock = __import__("threading").Lock() +# Per-process round-robin cursor, seeded by the random session id so the fleet +# spreads across vendors; advances once per unpinned keyless request. +_ring_lock = threading.Lock() _ring_cursor = int(_SESSION_ID, 16) % len(_KEYLESS_RING) def _vendor_pinned(name: str) -> bool: - """True when config explicitly routes web traffic to *name*. - - A pinned vendor starts every keyless request (rotation off); the ring - is only walked past it on throttle. Pin signals: web.backend / - web.search_backend / web.extract_backend naming the vendor, or a - free-tier pin in web.provider_tier. - """ + """True when config explicitly routes web traffic to *name* (backend keys or a + ``free`` tier pin). A pinned vendor starts every keyless request.""" if provider_tier(name) == "free": return True try: @@ -669,14 +409,9 @@ def _vendor_pinned(name: str) -> bool: def _ring_order(name: str) -> List[str]: - """Return the vendor walk order for a request entering via *name*. - - Pinned vendor → start at it (its position in the ring determines the - failover succession). Unpinned → true round-robin: start at the next - cursor position, advancing the cursor per request. Vendors whose tier - is pinned ``paid`` are excluded entirely (an explicit paid selection - opts that vendor's free endpoint out). - """ + """Vendor walk order: pinned → start at *name* (its ring position fixes the + failover succession); else round-robin from the cursor, advancing it per request. + Vendors pinned ``paid`` are excluded (explicit paid opts their free endpoint out).""" global _ring_cursor if _vendor_pinned(name): start = _KEYLESS_RING.index(name) if name in _KEYLESS_RING else 0 @@ -684,77 +419,60 @@ def _ring_order(name: str) -> List[str]: with _ring_lock: start = _ring_cursor _ring_cursor = (_ring_cursor + 1) % len(_KEYLESS_RING) - ordered = [ - _KEYLESS_RING[(start + i) % len(_KEYLESS_RING)] - for i in range(len(_KEYLESS_RING)) - ] + ordered = _KEYLESS_RING[start:] + _KEYLESS_RING[:start] return [v for v in ordered if provider_tier(v) != "paid"] -def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]: - """Keyless search across the vendor ring with next-in-line failover. +_ALL_PAID_MSG = "All keyless web providers are pinned to paid tiers." - Starts at *name* when the user pinned it, otherwise at the round-robin - cursor. Rate-limit-shaped errors advance to the next ring vendor; - non-throttle errors stop the walk (a malformed query fails everywhere). - The result notes the serving vendor via ``data.served_by`` whenever it - differs from *name*. - """ + +def _walk_ring(name: str, kind: str, call, throttled) -> tuple: + """Shared ring walk: call each vendor from :func:`_ring_order` until a result is + not ``throttled``. Returns ``(order, vendor, result, exhausted)``; ``order`` is + empty (and result None) when every vendor is pinned paid.""" order = _ring_order(name) - if not order: - return { - "success": False, - "error": "All keyless web providers are pinned to paid tiers.", - } - last: Dict[str, Any] = {} + vendor = None + result: Any = None for i, vendor in enumerate(order): - result = _KEYLESS_SEARCHERS[vendor](query, limit) - if result.get("success"): - if vendor != name: - result.setdefault("data", {})["served_by"] = vendor - return result - last = result - if not _is_rate_limitish(result.get("error", "")): - return result - nxt = order[i + 1] if i + 1 < len(order) else None - if nxt: - logger.info( - "keyless %s search throttled; failing over to %s", vendor, nxt - ) - last["error"] = ( - f"{last.get('error', '')} (all keyless vendors throttled: " - f"{', '.join(order)})" + result = call(vendor) + if not throttled(result): + return order, vendor, result, False + if i + 1 < len(order): + logger.info("keyless %s %s throttled; failing over to %s", vendor, kind, order[i + 1]) + return order, vendor, result, True + + +def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]: + """Keyless search across the ring; rate-limit-shaped errors advance to the + next vendor, other errors stop the walk (a malformed query fails everywhere). + ``data.served_by`` is set when the serving vendor differs from *name*.""" + + def _throttled(result: Dict[str, Any]) -> bool: + return not result.get("success") and _is_rate_limitish(result.get("error", "")) + + order, vendor, result, exhausted = _walk_ring( + name, "search", lambda v: _KEYLESS_SEARCHERS[v](query, limit), _throttled ) - return last + if not order: + return search_fail(_ALL_PAID_MSG) + if exhausted: + result["error"] = f"{result.get('error', '')} (all keyless vendors throttled: {', '.join(order)})" + elif result.get("success") and vendor != name: + result.setdefault("data", {})["served_by"] = vendor + return result def extract_with_failover(name: str, urls: List[str]) -> List[Dict[str, Any]]: - """Keyless extract across the vendor ring, failing over per-batch. + """Keyless extract across the ring; fails over only when EVERY url in a batch + is rate-limit-shaped (partial failures are page problems, returned as-is).""" - Advances to the next ring vendor only when EVERY url in a batch comes - back with a rate-limit-shaped error — partial failures are page - problems, not throttling, and return as-is. - """ - order = _ring_order(name) - if not order: - return [ - {"url": u, "title": "", "content": "", - "error": "All keyless web providers are pinned to paid tiers."} - for u in urls - ] - last: List[Dict[str, Any]] = [] - for i, vendor in enumerate(order): - results = _KEYLESS_EXTRACTORS[vendor](list(urls)) + def _all_throttled(results: List[Dict[str, Any]]) -> bool: errors = [r.get("error", "") for r in results] - all_throttled = bool(results) and all( - e and _is_rate_limitish(e) for e in errors - ) - if not all_throttled: - return results - last = results - nxt = order[i + 1] if i + 1 < len(order) else None - if nxt: - logger.info( - "keyless %s extract throttled; failing over to %s", vendor, nxt - ) - return last + return bool(results) and all(e and _is_rate_limitish(e) for e in errors) + + order, _vendor, results, _exhausted = _walk_ring( + name, "extract", lambda v: _KEYLESS_EXTRACTORS[v](list(urls)), _all_throttled + ) + if not order: + return [_page_error(u, _ALL_PAID_MSG) for u in urls] + return results diff --git a/plugins/web/parallel/__init__.py b/plugins/web/parallel/__init__.py index 2a109894dc..8e2d1c91bf 100644 --- a/plugins/web/parallel/__init__.py +++ b/plugins/web/parallel/__init__.py @@ -1,10 +1,4 @@ -"""Parallel.ai web search + extract plugin — bundled, auto-loaded. - -First plugin in this repo to expose an async :meth:`extract` — Parallel's -SDK is async-native (``AsyncParallel.beta.extract``). The web_extract_tool -dispatcher detects coroutines via :func:`inspect.iscoroutinefunction` and -awaits. -""" +"""Parallel.ai web search + extract plugin — bundled, auto-loaded; async-native ``extract``.""" from __future__ import annotations @@ -12,5 +6,4 @@ from plugins.web.parallel.provider import ParallelWebSearchProvider def register(ctx) -> None: - """Register the Parallel provider with the plugin context.""" ctx.register_web_search_provider(ParallelWebSearchProvider()) diff --git a/plugins/web/parallel/provider.py b/plugins/web/parallel/provider.py index 5a9390e92f..05be452496 100644 --- a/plugins/web/parallel/provider.py +++ b/plugins/web/parallel/provider.py @@ -1,353 +1,120 @@ -"""Parallel.ai web search + content extraction — plugin form. +"""Parallel.ai web search (sync ``Parallel`` SDK) + async extract (``AsyncParallel``). -Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses two -distinct Parallel SDK clients: - -- ``Parallel`` (sync) — for :meth:`search` -- ``AsyncParallel`` (async) — for :meth:`extract` - -This is the first plugin to exercise the **async-extract** code path in -the ABC: :meth:`extract` is declared ``async def``, and the dispatcher -in :func:`tools.web_tools.web_extract_tool` detects coroutines via -:func:`inspect.iscoroutinefunction` and awaits. - -Config keys this provider responds to:: - - web: - search_backend: "parallel" # explicit per-capability - extract_backend: "parallel" # explicit per-capability - backend: "parallel" # shared fallback - # Optional: search mode (default "agentic"; also "fast" or "one-shot") - # via the PARALLEL_SEARCH_MODE env var. - -Env vars:: - - PARALLEL_API_KEY=... # https://parallel.ai (required) - PARALLEL_SEARCH_MODE=agentic # optional: agentic|fast|one-shot +Env: ``PARALLEL_API_KEY`` (https://parallel.ai), optional +``PARALLEL_SEARCH_MODE`` = agentic (default) | fast | one-shot. """ from __future__ import annotations +import asyncio import logging import os from typing import Any, Dict, List -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import ( + SEARCH_LIMIT_CAP, + BaseWebSearchProvider, + cached_sdk_client, + document, + keyless_variant_schema, + provider_env, + run_extract_async, + run_search, + search_ok, + use_keyless, + web_hit, +) logger = logging.getLogger(__name__) -# Module-level note: the canonical cache slots ``_parallel_client`` and -# ``_async_parallel_client`` live on :mod:`tools.web_tools` so tests that do -# ``tools.web_tools._parallel_client = None`` between cases see fresh state. -# The plugin reads/writes through that public module (see -# :func:`_get_sync_client` / :func:`_get_async_client`). +_MISSING_KEY = "PARALLEL_API_KEY environment variable not set. Get your API key at https://parallel.ai" -def _ensure_parallel_sdk_installed() -> None: - """Trigger lazy install of the parallel SDK if it isn't present. +def _client(slot: str, cls_name: str) -> Any: + def _factory(api_key: str) -> Any: + import parallel # deliberately lazy - Mirrors the lazy-deps pattern used by the legacy implementation. - Swallows benign ImportError from the lazy_deps helper itself; if the - SDK is genuinely missing the subsequent ``from parallel import ...`` - raises ImportError that the caller can handle. - """ - try: - from tools.lazy_deps import ensure as _lazy_ensure + return getattr(parallel, cls_name)(api_key=api_key) - _lazy_ensure("search.parallel", prompt=False) - except ImportError: - pass - except Exception as exc: # noqa: BLE001 — surface install hint as ImportError - raise ImportError(str(exc)) + return cached_sdk_client(slot, "PARALLEL_API_KEY", _MISSING_KEY, "search.parallel", _factory) def _get_sync_client() -> Any: - """Lazy-load + cache the sync Parallel client. - - Cache lives on :mod:`tools.web_tools` (as ``_parallel_client``) so unit - tests that reset that name between cases keep working. - """ - import tools.web_tools as _wt - - cached = getattr(_wt, "_parallel_client", None) - if cached is not None: - return cached - - from agent.web_search_provider import get_provider_env - - api_key = get_provider_env("PARALLEL_API_KEY") - if not api_key: - raise ValueError( - "PARALLEL_API_KEY environment variable not set. " - "Get your API key at https://parallel.ai" - ) - - _ensure_parallel_sdk_installed() - from parallel import Parallel # noqa: WPS433 — deliberately lazy - - client = Parallel(api_key=api_key) - _wt._parallel_client = client - return client + return _client("_parallel_client", "Parallel") def _get_async_client() -> Any: - """Lazy-load + cache the async Parallel client. - - Cache lives on :mod:`tools.web_tools` (as ``_async_parallel_client``). - """ - import tools.web_tools as _wt - - cached = getattr(_wt, "_async_parallel_client", None) - if cached is not None: - return cached - - from agent.web_search_provider import get_provider_env - - api_key = get_provider_env("PARALLEL_API_KEY") - if not api_key: - raise ValueError( - "PARALLEL_API_KEY environment variable not set. " - "Get your API key at https://parallel.ai" - ) - - _ensure_parallel_sdk_installed() - from parallel import AsyncParallel # noqa: WPS433 — deliberately lazy - - client = AsyncParallel(api_key=api_key) - _wt._async_parallel_client = client - return client + return _client("_async_parallel_client", "AsyncParallel") -def _reset_clients_for_tests() -> None: - """Drop both cached clients so tests can re-instantiate cleanly. - - Clears the canonical slots on :mod:`tools.web_tools` (where - :func:`_get_sync_client` / :func:`_get_async_client` read/write them). - """ - import tools.web_tools as _wt - - _wt._parallel_client = None - _wt._async_parallel_client = None - - -# Backward-compatible aliases for the names that lived in tools.web_tools -# before the migration (matches existing tests + external callers). +# Names re-exported by tools.web_tools for existing tests/callers. _get_parallel_client = _get_sync_client _get_async_parallel_client = _get_async_client def _resolve_search_mode() -> str: - """Return the validated PARALLEL_SEARCH_MODE value (default "agentic").""" mode = os.getenv("PARALLEL_SEARCH_MODE", "agentic").lower().strip() - if mode not in {"fast", "one-shot", "agentic"}: - mode = "agentic" - return mode + return mode if mode in {"fast", "one-shot", "agentic"} else "agentic" -class ParallelWebSearchProvider(WebSearchProvider): +class ParallelWebSearchProvider(BaseWebSearchProvider): """Parallel.ai search + async extract provider.""" - @property - def name(self) -> str: - return "parallel" - - @property - def display_name(self) -> str: - return "Parallel" - - def is_available(self) -> bool: - """Return True when ``PARALLEL_API_KEY`` is set to a non-empty value. - - Deliberately does NOT consider the keyless free tier — that would - let the legacy preference walk route keyed users of lower-priority - backends onto Parallel's anonymous tier. Keyless availability is a - separate, last-resort signal (:meth:`is_keyless_available`). - """ - from agent.web_search_provider import get_provider_env - - return bool(get_provider_env("PARALLEL_API_KEY")) - - def is_keyless_available(self) -> bool: - """Parallel serves anonymous free-tier calls via its public MCP endpoint. - - False when the user forced ``web.provider_tier.parallel: paid`` — - an explicit paid selection must never silently resolve keyless. - """ - from plugins.web.keyless_mcp import keyless_enabled, provider_tier - - return keyless_enabled() and provider_tier("parallel") != "paid" - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return True + NAME = "parallel" + DISPLAY_NAME = "Parallel" + KEY_ENV = "PARALLEL_API_KEY" + EXTRACT = True + KEYLESS = True def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a Parallel search (sync). + def _body() -> Dict[str, Any]: + from plugins.web.keyless_mcp import search_with_failover - Uses the ``beta.search`` endpoint with the configured mode - (``PARALLEL_SEARCH_MODE`` env var, default "agentic"). Limit is - capped at 20 server-side. - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return {"success": False, "error": "Interrupted"} - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import search_with_failover, use_keyless - - if use_keyless("parallel", get_provider_env("PARALLEL_API_KEY")): - # Keyless free tier — public MCP endpoint, no SDK needed. - logger.info( - "Parallel keyless search: '%s' (limit=%d)", query, limit - ) + if use_keyless("parallel", provider_env("PARALLEL_API_KEY")): + logger.info("Parallel keyless search: '%s' (limit=%d)", query, limit) return search_with_failover("parallel", query, limit) mode = _resolve_search_mode() - logger.info( - "Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit - ) + logger.info("Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit) response = _get_sync_client().beta.search( - search_queries=[query], - objective=query, - mode=mode, - max_results=min(limit, 20), + search_queries=[query], objective=query, mode=mode, max_results=min(limit, SEARCH_LIMIT_CAP) ) + return search_ok([ + web_hit(r.url or "", r.title or "", " ".join(r.excerpts or []), i + 1) + for i, r in enumerate(response.results or []) + ]) - web_results = [] - for i, result in enumerate(response.results or []): - excerpts = result.excerpts or [] - web_results.append( - { - "url": result.url or "", - "title": result.title or "", - "description": " ".join(excerpts) if excerpts else "", - "position": i + 1, - } - ) + return run_search("Parallel", logger, _body, sdk=True) - return {"success": True, "data": {"web": web_results}} - except ValueError as exc: - return {"success": False, "error": str(exc)} - except ImportError as exc: - return { - "success": False, - "error": f"Parallel SDK not installed: {exc}", - } - except Exception as exc: # noqa: BLE001 - logger.warning("Parallel search error: %s", exc) - return {"success": False, "error": f"Parallel search failed: {exc}"} - - async def extract( - self, urls: List[str], **kwargs: Any - ) -> List[Dict[str, Any]]: - """Extract content from one or more URLs via the async SDK. - - Returns the legacy list-of-results shape that - :func:`tools.web_tools.web_extract_tool` expects: one entry per - successful URL plus one entry per failed URL with an ``error`` - field. Errors are not raised — they're returned as per-URL items. - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return [ - {"url": u, "error": "Interrupted", "title": ""} for u in urls - ] - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import extract_with_failover, use_keyless - - if use_keyless("parallel", get_provider_env("PARALLEL_API_KEY")): - # Keyless free tier — blocking HTTP, so hop off the loop. - import asyncio + async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: + async def _body() -> List[Dict[str, Any]]: + from plugins.web.keyless_mcp import extract_with_failover + if use_keyless("parallel", provider_env("PARALLEL_API_KEY")): + # Keyless ring is blocking HTTP — hop off the event loop. logger.info("Parallel keyless extract: %d URL(s)", len(urls)) - return await asyncio.to_thread( - extract_with_failover, "parallel", list(urls) - ) + return await asyncio.to_thread(extract_with_failover, "parallel", list(urls)) logger.info("Parallel extract: %d URL(s)", len(urls)) - response = await _get_async_client().beta.extract( - urls=urls, - full_content=True, - ) - - results: List[Dict[str, Any]] = [] - for result in response.results or []: - content = result.full_content or "" - if not content: - content = "\n\n".join(result.excerpts or []) - url = result.url or "" - title = result.title or "" - results.append( - { - "url": url, - "title": title, - "content": content, - "raw_content": content, - "metadata": {"sourceURL": url, "title": title}, - } - ) + response = await _get_async_client().beta.extract(urls=urls, full_content=True) + results = [ + document(r.url or "", r.title or "", r.full_content or "\n\n".join(r.excerpts or [])) + for r in response.results or [] + ] for error in response.errors or []: - results.append( - { - "url": error.url or "", - "title": "", - "content": "", - "error": error.content or error.error_type or "extraction failed", - "metadata": {"sourceURL": error.url or ""}, - } - ) - + results.append({ + "url": error.url or "", "title": "", "content": "", + "error": error.content or error.error_type or "extraction failed", + "metadata": {"sourceURL": error.url or ""}, + }) return results - except ValueError as exc: - return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls] - except ImportError as exc: - return [ - {"url": u, "title": "", "content": "", "error": f"Parallel SDK not installed: {exc}"} - for u in urls - ] - except Exception as exc: # noqa: BLE001 - logger.warning("Parallel extract error: %s", exc) - return [ - {"url": u, "title": "", "content": "", "error": f"Parallel extract failed: {exc}"} - for u in urls - ] + + return await run_extract_async("Parallel", logger, urls, _body, sdk=True) def get_setup_schema(self) -> Dict[str, Any]: - return { - "name": "Parallel · Free (keyless)", - "badge": "free · no key", - "tag": ( - "Objective-tuned search + page extraction on Parallel's " - "anonymous free tier. Rate-limited under burst load." - ), - "env_vars": [], - "web_tier": "free", - "variants": [ - { - "name": "Parallel · Paid (API key)", - "badge": "paid", - "tag": ( - "Objective-tuned search + parallel page extraction " - "via the Parallel SDK. Unthrottled, guaranteed service." - ), - "env_vars": [ - { - "key": "PARALLEL_API_KEY", - "prompt": "Parallel API key", - "url": "https://parallel.ai", - }, - ], - "web_tier": "paid", - }, - ], - } + return keyless_variant_schema( + "Parallel", "PARALLEL_API_KEY", "https://parallel.ai", + free_tag="Objective-tuned search + page extraction on Parallel's anonymous free tier. Rate-limited under burst load.", + paid_tag="Objective-tuned search + parallel page extraction via the Parallel SDK. Unthrottled, guaranteed service.", + ) diff --git a/plugins/web/searxng/__init__.py b/plugins/web/searxng/__init__.py index cea8eabb18..5ed1d8c73d 100644 --- a/plugins/web/searxng/__init__.py +++ b/plugins/web/searxng/__init__.py @@ -1,9 +1,4 @@ -"""SearXNG search plugin — bundled, auto-loaded. - -Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``). -Search-only — pair with an extract provider (firecrawl/tavily/exa) for -``web_extract`` calls. -""" +"""SearXNG search-only plugin — bundled, auto-loaded (``SEARXNG_URL``).""" from __future__ import annotations @@ -11,5 +6,4 @@ from plugins.web.searxng.provider import SearXNGWebSearchProvider def register(ctx) -> None: - """Register the SearXNG provider with the plugin context.""" ctx.register_web_search_provider(SearXNGWebSearchProvider()) diff --git a/plugins/web/searxng/provider.py b/plugins/web/searxng/provider.py index 6f747fc3f9..35e6e22ff1 100644 --- a/plugins/web/searxng/provider.py +++ b/plugins/web/searxng/provider.py @@ -1,123 +1,46 @@ -"""SearXNG search — plugin form. +"""SearXNG search via a user-hosted instance (``/search?format=json``). -Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Same JSON -API call (``/search?format=json``), same result normalization. The legacy -in-tree module ``tools.web_providers.searxng`` was removed in the same -commit that moved this code under ``plugins/``; this file is now the -canonical implementation. - -Search-only — SearXNG aggregates results from upstream engines but does not -fetch/extract arbitrary URLs. ``supports_extract()`` returns False. - -Config keys this provider responds to:: - - web: - search_backend: "searxng" # explicit per-capability - backend: "searxng" # shared fallback - -Env var:: - - SEARXNG_URL=http://localhost:8080 +Search-only — SearXNG aggregates upstream engines but does not fetch URLs. +Env: ``SEARXNG_URL=http://localhost:8080``. """ from __future__ import annotations import logging -import os from typing import Any, Dict -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import BaseWebSearchProvider, http_get_json, provider_env, search_fail, search_ok logger = logging.getLogger(__name__) -def _searxng_url() -> str: - """Return SEARXNG_URL from Hermes config-aware env, falling back to process env.""" - try: - from hermes_cli.config import get_env_value - - val = get_env_value("SEARXNG_URL") - except Exception: - val = None - if val is None: - val = os.getenv("SEARXNG_URL", "") - return (val or "").strip() - - -class SearXNGWebSearchProvider(WebSearchProvider): +class SearXNGWebSearchProvider(BaseWebSearchProvider): """Search via a user-hosted SearXNG instance.""" - @property - def name(self) -> str: - return "searxng" - - @property - def display_name(self) -> str: - return "SearXNG" - - def is_available(self) -> bool: - """Return True when ``SEARXNG_URL`` is set.""" - return bool(_searxng_url()) - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return False + NAME = "searxng" + DISPLAY_NAME = "SearXNG" + KEY_ENV = "SEARXNG_URL" def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a search against the configured SearXNG instance.""" - import httpx - - base_url = _searxng_url().rstrip("/") + base_url = provider_env("SEARXNG_URL").rstrip("/") if not base_url: - return {"success": False, "error": "SEARXNG_URL is not set"} + return search_fail("SEARXNG_URL is not set") - params: Dict[str, Any] = { - "q": query, - "format": "json", - "pageno": 1, - } - - try: - resp = httpx.get( - f"{base_url}/search", - params=params, - timeout=15, - headers={"Accept": "application/json"}, - ) - resp.raise_for_status() - except httpx.HTTPStatusError as exc: - logger.warning("SearXNG HTTP error: %s", exc) - return { - "success": False, - "error": f"SearXNG returned HTTP {exc.response.status_code}", - } - except httpx.RequestError as exc: - logger.warning("SearXNG request error: %s", exc) - return { - "success": False, - "error": f"Could not reach SearXNG at {base_url}: {exc}", - } - - try: - data = resp.json() - except Exception as exc: # noqa: BLE001 - logger.warning("SearXNG response parse error: %s", exc) - return { - "success": False, - "error": "Could not parse SearXNG response as JSON", - } + data, failure = http_get_json( + "SearXNG", + f"{base_url}/search", + params={"q": query, "format": "json", "pageno": 1}, + headers={"Accept": "application/json"}, + timeout=15, + logger=logger, + reach_target=f"SearXNG at {base_url}", + ) + if failure is not None: + return failure raw_results = data.get("results", []) - # SearXNG may return a score field; sort descending and cap to limit. - sorted_results = sorted( - raw_results, - key=lambda r: float(r.get("score", 0)), - reverse=True, - )[:limit] - + sorted_results = sorted(raw_results, key=lambda r: float(r.get("score", 0)), reverse=True)[:limit] web_results = [ { "title": str(r.get("title", "")), @@ -127,16 +50,11 @@ class SearXNGWebSearchProvider(WebSearchProvider): } for i, r in enumerate(sorted_results) ] - logger.info( "SearXNG search '%s': %d results (from %d raw, limit %d)", - query, - len(web_results), - len(raw_results), - limit, + query, len(web_results), len(raw_results), limit, ) - - return {"success": True, "data": {"web": web_results}} + return search_ok(web_results) def get_setup_schema(self) -> Dict[str, Any]: return { diff --git a/plugins/web/tavily/__init__.py b/plugins/web/tavily/__init__.py index 1e0ced61d1..71da624b6b 100644 --- a/plugins/web/tavily/__init__.py +++ b/plugins/web/tavily/__init__.py @@ -6,5 +6,4 @@ from plugins.web.tavily.provider import TavilyWebSearchProvider def register(ctx) -> None: - """Register the Tavily provider with the plugin context.""" ctx.register_web_search_provider(TavilyWebSearchProvider()) diff --git a/plugins/web/tavily/provider.py b/plugins/web/tavily/provider.py index 621aa3ea03..e29f16de0b 100644 --- a/plugins/web/tavily/provider.py +++ b/plugins/web/tavily/provider.py @@ -1,34 +1,9 @@ -"""Tavily web search + content extraction — plugin form. +"""Tavily web search + content extraction (``/search``, ``/extract``; sync httpx). -Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two -capabilities advertised: - -- ``supports_search()`` -> True (Tavily ``/search``) -- ``supports_extract()`` -> True (Tavily ``/extract``) - -Both are sync — the underlying call is ``httpx.post(...)``. - -Config keys this provider responds to:: - - web: - search_backend: "tavily" # explicit per-capability - extract_backend: "tavily" # explicit per-capability - backend: "tavily" # shared fallback for both - -Env vars:: - - TAVILY_API_KEY=... # https://app.tavily.com/home (optional) - TAVILY_BASE_URL=... # optional override of https://api.tavily.com - -Auth is header-based. A key uses ``Authorization: Bearer``; without a -key the request is keyless (``X-Tavily-Access-Mode: keyless``). Both -paths send ``X-Client-Name: hermes-agent``. - -Tavily is **not** a member of the zero-config keyless ring -(``plugins.web.keyless_mcp._KEYLESS_RING``). Keyless access is opt-in: -select Tavily in ``hermes tools`` (or set ``web.backend: tavily``). -Fresh installs with no web credentials rotate across Exa / Parallel / -Firecrawl / Keenable instead. +Env: ``TAVILY_API_KEY`` (https://app.tavily.com/home, optional), ``TAVILY_BASE_URL``. +Keyed requests use ``Authorization: Bearer``; without a key the request is +keyless (``X-Tavily-Access-Mode: keyless``). Tavily is NOT in the zero-config +keyless ring — keyless access is opt-in by selecting Tavily in ``hermes tools``. """ from __future__ import annotations @@ -38,20 +13,28 @@ from typing import Any, Dict, List, Optional import httpx -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import ( + SEARCH_LIMIT_CAP, + BaseWebSearchProvider, + document, + extract_fail, + http_status_detail, + provider_env, + run_extract, + run_search, + search_fail, + search_ok, + use_keyless, +) logger = logging.getLogger(__name__) _CLIENT_NAME = "hermes-agent" -_SEARCH_PAYLOAD = { - "include_raw_content": False, - "include_images": False, -} +_SEARCH_PAYLOAD = {"include_raw_content": False, "include_images": False} def _tavily_headers(api_key: str) -> Dict[str, str]: - """Build Tavily request headers for keyed or keyless access.""" headers = {"X-Client-Name": _CLIENT_NAME} if api_key: headers["Authorization"] = f"Bearer {api_key}" @@ -60,249 +43,101 @@ def _tavily_headers(api_key: str) -> Dict[str, str]: return headers -def _tavily_request( - endpoint: str, - payload: Dict[str, Any], - *, - api_key: Optional[str] = None, -) -> Dict[str, Any]: - """POST to the Tavily API and return the parsed JSON response. +def _tavily_request(endpoint: str, payload: Dict[str, Any], *, api_key: Optional[str] = None) -> Dict[str, Any]: + """POST to Tavily and return parsed JSON. - Keyed when *api_key* (or ``TAVILY_API_KEY``) is set (Bearer auth); - otherwise keyless. Pass ``api_key=""`` to force the keyless header even - when a key is present (``web.provider_tier.tavily: free``). Non-2xx - responses raise ``ValueError`` with the response body so Tavily's - keyless rate-limit / upgrade text reaches the model. + ``api_key=None`` reads ``TAVILY_API_KEY``; pass ``""`` to force the keyless + header even when a key exists (``web.provider_tier.tavily: free``). Non-2xx + raises ValueError with the body so Tavily's rate-limit/upgrade text reaches the model. """ - from agent.web_search_provider import get_provider_env - if api_key is None: - api_key = get_provider_env("TAVILY_API_KEY") - base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com" + api_key = provider_env("TAVILY_API_KEY") + base_url = provider_env("TAVILY_BASE_URL") or "https://api.tavily.com" url = f"{base_url}/{endpoint.lstrip('/')}" logger.info("Tavily %s request to %s", endpoint, url) - response = httpx.post( - url, - json=payload, - timeout=60, - headers=_tavily_headers(api_key), - ) + response = httpx.post(url, json=payload, timeout=60, headers=_tavily_headers(api_key)) if response.status_code >= 400: - body = (response.text or "").strip() - detail = body or f"HTTP {response.status_code}" - raise ValueError(detail) + raise ValueError(http_status_detail(response)) return response.json() def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]: - """Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``.""" - web_results = [] - for i, result in enumerate(response.get("results", [])): - web_results.append( - { - "title": result.get("title", ""), - "url": result.get("url", ""), - "description": result.get("content", ""), - "position": i + 1, - } - ) - return {"success": True, "data": {"web": web_results}} + # title-first key order is Tavily's historical wire shape (differs from web_hit). + return search_ok([ + {"title": r.get("title", ""), "url": r.get("url", ""), "description": r.get("content", ""), "position": i + 1} + for i, r in enumerate(response.get("results", [])) + ]) -def _normalize_tavily_documents( - response: Dict[str, Any], fallback_url: str = "" -) -> List[Dict[str, Any]]: - """Map Tavily ``/extract`` response to standard documents. - - Documents follow the legacy LLM post-processing shape:: - - {"url", "title", "content", "raw_content", "metadata"} - - Failures (``failed_results``, ``failed_urls``) become result entries - with an ``error`` field rather than raising. - """ +def _normalize_tavily_documents(response: Dict[str, Any], fallback_url: str = "") -> List[Dict[str, Any]]: + """Map ``/extract`` to documents; ``failed_results`` / ``failed_urls`` become ``error`` entries.""" documents: List[Dict[str, Any]] = [] for result in response.get("results", []): url = result.get("url", fallback_url) raw = result.get("raw_content", "") or result.get("content", "") - documents.append( - { - "url": url, - "title": result.get("title", ""), - "content": raw, - "raw_content": raw, - "metadata": {"sourceURL": url, "title": result.get("title", "")}, - } - ) + documents.append(document(url, result.get("title", ""), raw)) for fail in response.get("failed_results", []): - documents.append( - { - "url": fail.get("url", fallback_url), - "title": "", - "content": "", - "raw_content": "", - "error": fail.get("error", "extraction failed"), - "metadata": {"sourceURL": fail.get("url", fallback_url)}, - } - ) + url = fail.get("url", fallback_url) + documents.append(_failed_document(url, fail.get("error", "extraction failed"))) for fail_url in response.get("failed_urls", []): - url_str = fail_url if isinstance(fail_url, str) else str(fail_url) - documents.append( - { - "url": url_str, - "title": "", - "content": "", - "raw_content": "", - "error": "extraction failed", - "metadata": {"sourceURL": url_str}, - } - ) + documents.append(_failed_document(str(fail_url), "extraction failed")) return documents +def _failed_document(url: str, error: str) -> Dict[str, Any]: + return {"url": url, "title": "", "content": "", "raw_content": "", "error": error, "metadata": {"sourceURL": url}} + + def _missing_key_error(action: str) -> str: return ( - f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home " + "TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home " f"or select Tavily in `hermes tools` for opt-in keyless {action}." ) -class TavilyWebSearchProvider(WebSearchProvider): +class TavilyWebSearchProvider(BaseWebSearchProvider): """Tavily search + extract provider (keyed, or opt-in keyless).""" - @property - def name(self) -> str: - return "tavily" - - @property - def display_name(self) -> str: - return "Tavily" - - def is_available(self) -> bool: - """Return True when ``TAVILY_API_KEY`` is set to a non-empty value.""" - from agent.web_search_provider import get_provider_env - - return bool(get_provider_env("TAVILY_API_KEY")) - - def is_keyless_available(self) -> bool: - """Tavily serves anonymous keyless requests (X-Tavily-Access-Mode). - - Opt-in only — Tavily is not a member of the zero-config keyless - ring. ``is_keyless_available`` is True so an explicit - ``web.backend: tavily`` (or ``hermes tools`` pick) works without a - key. False when the user pinned ``web.provider_tier.tavily: paid``. - """ - from plugins.web.keyless_mcp import keyless_enabled, provider_tier - - return keyless_enabled() and provider_tier("tavily") != "paid" - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return True + NAME = "tavily" + DISPLAY_NAME = "Tavily" + KEY_ENV = "TAVILY_API_KEY" + EXTRACT = True + KEYLESS = True def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a Tavily search (keyed path or opt-in keyless).""" - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return {"success": False, "error": "Interrupted"} - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import use_keyless - - api_key = get_provider_env("TAVILY_API_KEY") + def _body() -> Dict[str, Any]: + api_key = provider_env("TAVILY_API_KEY") force_keyless = use_keyless("tavily", api_key) if not force_keyless and not api_key: - return {"success": False, "error": _missing_key_error("search")} + return search_fail(_missing_key_error("search")) - logger.info( - "Tavily %ssearch: '%s' (limit=%d)", - "keyless " if force_keyless else "", - query, - limit, + logger.info("Tavily %ssearch: '%s' (limit=%d)", "keyless " if force_keyless else "", query, limit) + payload = {"query": query, "max_results": min(limit, SEARCH_LIMIT_CAP), **_SEARCH_PAYLOAD} + return _normalize_tavily_search_results( + _tavily_request("search", payload, api_key="" if force_keyless else api_key) ) - raw = _tavily_request( - "search", - { - "query": query, - "max_results": min(limit, 20), - **_SEARCH_PAYLOAD, - }, - api_key="" if force_keyless else api_key, - ) - return _normalize_tavily_search_results(raw) - except ValueError as exc: - return {"success": False, "error": str(exc)} - except Exception as exc: # noqa: BLE001 — including httpx errors - logger.warning("Tavily search error: %s", exc) - return {"success": False, "error": f"Tavily search failed: {exc}"} + + return run_search("Tavily", logger, _body) def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: - """Extract content from one or more URLs via Tavily. - - Sync — the underlying call is httpx.post(...). Returns the legacy - list-of-results shape; per-URL failures become items with ``error``. - Keyless uses Tavily's own endpoint, not the keyless ring. - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return [ - {"url": u, "error": "Interrupted", "title": ""} for u in urls - ] - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import use_keyless - - api_key = get_provider_env("TAVILY_API_KEY") + def _body() -> List[Dict[str, Any]]: + api_key = provider_env("TAVILY_API_KEY") force_keyless = use_keyless("tavily", api_key) if not force_keyless and not api_key: - err = _missing_key_error("extract") - return [ - {"url": u, "title": "", "content": "", "error": err} - for u in urls - ] + return extract_fail(urls, _missing_key_error("extract")) - logger.info( - "Tavily %sextract: %d URL(s)", - "keyless " if force_keyless else "", - len(urls), - ) - raw = _tavily_request( - "extract", - { - "urls": urls, - "include_images": False, - }, - api_key="" if force_keyless else api_key, - ) - return _normalize_tavily_documents( - raw, fallback_url=urls[0] if urls else "" - ) - except ValueError as exc: - return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls] - except Exception as exc: # noqa: BLE001 - logger.warning("Tavily extract error: %s", exc) - return [ - {"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"} - for u in urls - ] + logger.info("Tavily %sextract: %d URL(s)", "keyless " if force_keyless else "", len(urls)) + raw = _tavily_request("extract", {"urls": urls, "include_images": False}, api_key="" if force_keyless else api_key) + return _normalize_tavily_documents(raw, fallback_url=urls[0] if urls else "") + + return run_extract("Tavily", logger, urls, _body) def get_setup_schema(self) -> Dict[str, Any]: return { "name": "Tavily", "badge": "free · key optional", - "tag": ( - "Search + extract. Opt-in keyless; " - "set TAVILY_API_KEY for higher limits." - ), + "tag": "Search + extract. Opt-in keyless; set TAVILY_API_KEY for higher limits.", "env_vars": [ { "key": "TAVILY_API_KEY", diff --git a/plugins/web/xai/__init__.py b/plugins/web/xai/__init__.py index 9ec4a58899..a3aed4577d 100644 --- a/plugins/web/xai/__init__.py +++ b/plugins/web/xai/__init__.py @@ -1,8 +1,4 @@ -"""xAI web search plugin — bundled, auto-loaded. - -Mirrors the ``plugins/web/brave_free/`` layout: ``provider.py`` holds the -provider class, ``__init__.py::register(ctx)`` registers an instance. -""" +"""xAI web search plugin — bundled, auto-loaded.""" from __future__ import annotations @@ -10,5 +6,4 @@ from plugins.web.xai.provider import XAIWebSearchProvider def register(ctx) -> None: - """Register the xAI Web Search provider with the plugin context.""" ctx.register_web_search_provider(XAIWebSearchProvider()) diff --git a/plugins/web/xai/provider.py b/plugins/web/xai/provider.py index 77d80a4398..8620aa2345 100644 --- a/plugins/web/xai/provider.py +++ b/plugins/web/xai/provider.py @@ -1,33 +1,13 @@ -"""xAI Web Search — plugin form. +"""xAI Web Search — search-only provider backed by Grok's server-side ``web_search`` +tool on the Responses API. Grok searches/browses server-side; we ask for structured +JSON so results match the ``{title, url, description, position}`` rows every other +Hermes web provider produces. Reference: https://docs.x.ai/developers/tools/web-search -Routes ``web_search`` tool calls through xAI's agentic Web Search tool -(server-side ``web_search`` on the Responses API). Grok runs the actual -searching and page-browsing server-side; we ask it to return the top -results as structured JSON so we can hand back the same -``{title, url, description, position}`` rows every other Hermes web -provider produces. - -Reference: https://docs.x.ai/developers/tools/web-search - -Config keys this provider responds to:: - - web: - search_backend: "xai" # explicit per-capability - backend: "xai" # shared fallback - -Optional knobs (under ``web.xai`` in ``config.yaml``):: - - web: - xai: - model: "grok-build-0.1" # reasoning model required by web_search - allowed_domains: ["x.ai"] # max 5 — mutually exclusive with excluded_domains - excluded_domains: ["bad.com"] # max 5 — mutually exclusive with allowed_domains - timeout: 90 # seconds (default 90) - -Auth: reuses :func:`tools.xai_http.resolve_xai_http_credentials`, which -prefers Hermes-managed xAI Grok OAuth (via ``hermes auth``) and falls back -to ``XAI_API_KEY`` (resolved through ``~/.hermes/.env``, then -``os.environ``). +Config: ``web.search_backend`` / ``web.backend: "xai"``. Optional ``web.xai``: +``model`` (reasoning model, default grok-build-0.1), ``allowed_domains`` / +``excluded_domains`` (max 5, mutually exclusive), ``timeout`` (seconds, default 90). +Auth: :func:`tools.xai_http.resolve_xai_http_credentials` (Grok OAuth via +``hermes auth``, else ``XAI_API_KEY``). """ from __future__ import annotations @@ -37,7 +17,7 @@ import logging import re from typing import Any, Dict, List, Optional -from agent.web_search_provider import WebSearchProvider +from plugins.web._common import BaseWebSearchProvider, search_fail as _fail, search_ok from tools.xai_http import ( has_xai_credentials, hermes_xai_user_agent, @@ -50,17 +30,11 @@ DEFAULT_MODEL = "grok-build-0.1" DEFAULT_TIMEOUT = 90 _MAX_DOMAIN_FILTERS = 5 # xAI hard cap on allowed_domains / excluded_domains -# Match the JSON object Grok is asked to emit. Tolerates leading/trailing -# prose since reasoning models occasionally narrate before the JSON block -# even when explicitly asked not to. +# The JSON object Grok is asked to emit; tolerates leading/trailing prose since +# reasoning models occasionally narrate before the JSON block. _JSON_BLOCK_RE = re.compile(r"\{[\s\S]*\}", re.MULTILINE) -# --------------------------------------------------------------------------- -# Config -# --------------------------------------------------------------------------- - - def _load_xai_web_config() -> Dict[str, Any]: """Read ``web.xai`` from config.yaml (returns {} on miss).""" try: @@ -79,83 +53,53 @@ def _coerce_domain_list(value: Any) -> List[str]: """Coerce a config value to a clean list of <=5 domain strings.""" if not isinstance(value, list): return [] - cleaned: List[str] = [] - for item in value: - if isinstance(item, str) and item.strip(): - cleaned.append(item.strip()) - if len(cleaned) >= _MAX_DOMAIN_FILTERS: - break - return cleaned + cleaned = [item.strip() for item in value if isinstance(item, str) and item.strip()] + return cleaned[:_MAX_DOMAIN_FILTERS] -# --------------------------------------------------------------------------- -# Provider -# --------------------------------------------------------------------------- +def _coerce(cast, value: Any, default: Any) -> Any: + try: + return cast(value) + except (TypeError, ValueError): + return default -class XAIWebSearchProvider(WebSearchProvider): +def _row(title: str, url: str, description: str, position: int) -> Dict[str, Any]: + # Key order (title first) is this provider's wire shape; keep it distinct from web_hit. + return {"title": title, "url": url, "description": description, "position": position} + + +class XAIWebSearchProvider(BaseWebSearchProvider): """Search-only provider backed by xAI's agentic Web Search tool. - Sends a structured prompt to Grok with ``tools=[{"type": "web_search"}]`` - enabled and asks it to return the top *limit* results as JSON. Falls - back to the Responses API ``citations`` list if Grok ignores the JSON - schema instruction (rare for grok-4.3 but cheap insurance). + Sends a structured prompt with ``tools=[{"type": "web_search"}]`` and parses the + JSON Grok returns; falls back to message annotations, then the ``citations`` + list, if Grok ignores the schema. No extract capability. - No extract capability — pair with Firecrawl / Tavily / Exa for - ``web_extract`` if you need page content. - - Trust model - ----------- - Unlike index-backed providers (Brave / Tavily / Exa) which return - verbatim search-engine results, this backend is an LLM in a trench - coat: Grok decides which URLs to surface, generates the titles and - descriptions itself, and is influenced by the *content of the query*. - A maliciously crafted query (e.g. injected via untrusted upstream - input the agent picked up) can in principle steer Grok into emitting - attacker-chosen URLs. Callers that pipe untrusted text directly into - ``web_search`` should treat returned URLs the same way they would - treat any model-generated link — validate before fetching. + Trust model: unlike index-backed providers, Grok *generates* the URLs, titles + and descriptions and is steerable by the query text itself — treat returned + URLs like any model-generated link and validate before fetching. """ - @property - def name(self) -> str: - return "xai" - - @property - def display_name(self) -> str: - return "xAI Web Search (Grok)" + NAME = "xai" + DISPLAY_NAME = "xAI Web Search (Grok)" def is_available(self) -> bool: - """Cheap availability probe — env var OR auth-store has OAuth tokens. - - Delegates to :func:`tools.xai_http.has_xai_credentials`, which is - deliberately *not* the same as :func:`resolve_xai_http_credentials`: - it never triggers OAuth token refresh or acquires the auth-store - lock. The ABC contract requires this method to be safe to call on - every ``hermes tools`` repaint and at tool-registration time. - Token freshness / refresh is handled inside :meth:`search`. - """ + """Cheap probe (env var OR auth-store tokens). Deliberately NOT + ``resolve_xai_http_credentials``: must never refresh tokens or take the + auth-store lock, since this runs on every ``hermes tools`` repaint.""" return has_xai_credentials() - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return False - # -- Search ----------------------------------------------------------- def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a Grok-backed web search. - - Returns ``{"success": True, "data": {"web": [{title, url, description, position}, ...]}}`` - on success, ``{"success": False, "error": str}`` on failure. - """ + """Grok-backed web search → ``{"success": True, "data": {"web": [...]}}`` + or ``{"success": False, "error": str}``.""" try: from tools.interrupt import is_interrupted if is_interrupted(): - return {"success": False, "error": "Interrupted"} + return _fail("Interrupted") except Exception: # noqa: BLE001 — interrupt module is best-effort pass @@ -163,103 +107,105 @@ class XAIWebSearchProvider(WebSearchProvider): api_key = str(creds.get("api_key") or "").strip() base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/") if not api_key: - return { - "success": False, - "error": ( - "No xAI credentials found. Run `hermes auth` to sign in with " - "xAI Grok OAuth, or set XAI_API_KEY." - ), - } + return _fail( + "No xAI credentials found. Run `hermes auth` to sign in with " + "xAI Grok OAuth, or set XAI_API_KEY." + ) - # Clamp limit to the same range the caller (web_search_tool) accepts, - # so we don't silently downgrade explicit limits. Grok happily - # produces longer lists; cost scales linearly with the requested - # count via reasoning tokens, but that's the caller's call to make. - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 5 - limit = max(1, min(limit, 100)) + # Same clamp range as web_search_tool so explicit limits aren't downgraded; + # cost scales with the count via reasoning tokens, but that's the caller's call. + limit = max(1, min(_coerce(int, limit, 5), 100)) cfg = _load_xai_web_config() model = cfg.get("model") if isinstance(cfg.get("model"), str) else DEFAULT_MODEL model = model.strip() or DEFAULT_MODEL + timeout = _coerce(float, cfg.get("timeout", DEFAULT_TIMEOUT), DEFAULT_TIMEOUT) - try: - timeout = float(cfg.get("timeout", DEFAULT_TIMEOUT)) - except (TypeError, ValueError): - timeout = DEFAULT_TIMEOUT - - allowed = _coerce_domain_list(cfg.get("allowed_domains")) - excluded = _coerce_domain_list(cfg.get("excluded_domains")) - if allowed and excluded: - # xAI explicitly rejects this combo — surface a clear error - # rather than a 400 from the API. - return { - "success": False, - "error": ( - "web.xai.allowed_domains and web.xai.excluded_domains " - "cannot both be set (xAI restriction)." - ), - } - - web_search_tool: Dict[str, Any] = {"type": "web_search"} - if allowed: - web_search_tool["filters"] = {"allowed_domains": allowed} - elif excluded: - web_search_tool["filters"] = {"excluded_domains": excluded} - - prompt = self._build_prompt(query, limit) + web_search_tool = self._web_search_tool(cfg) + if web_search_tool is None: + # xAI rejects this combo — surface a clear error rather than an API 400. + return _fail( + "web.xai.allowed_domains and web.xai.excluded_domains " + "cannot both be set (xAI restriction)." + ) payload: Dict[str, Any] = { "model": model, - "input": [{"role": "user", "content": prompt}], + "input": [{"role": "user", "content": self._build_prompt(query, limit)}], "tools": [web_search_tool], - # Drop inline citation markdown — we want the JSON block clean, - # and we read URLs from annotations / citations separately. + # Keep the JSON block clean; URLs are read from annotations/citations. "include": ["no_inline_citations"], } + try: + import httpx # noqa: F401 — availability probe + except ImportError: + return _fail("httpx is not installed (required for xAI web search)") + + logger.info("xAI web search via %s: '%s' (limit=%d, model=%s)", base_url, query, limit, model) + + data, error = self._post_responses( + base_url, payload, api_key, timeout, + is_oauth_path=(creds.get("provider") == "xai-oauth"), + ) + if error: + return error + + # xAI sometimes returns HTTP 200 with an error envelope (overloaded, refusal); + # without this check we'd report success-with-no-rows and mask a real failure. + api_error = data.get("error") if isinstance(data, dict) else None + if isinstance(api_error, dict): + err_msg = api_error.get("message") or api_error.get("code") or "unknown error" + logger.warning("xAI web search returned error envelope: %s", err_msg) + return _fail(f"xAI returned an error: {err_msg}") + + # Empty list on 0 hits is a success (matches brave-free / exa) so the model + # can decide whether to retry. + return search_ok(self._extract_results(data, limit=limit)) + + @staticmethod + def _web_search_tool(cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Build the ``web_search`` tool spec with optional domain filters; None when + both allowed and excluded are set (xAI rejects the combination).""" + allowed = _coerce_domain_list(cfg.get("allowed_domains")) + excluded = _coerce_domain_list(cfg.get("excluded_domains")) + if allowed and excluded: + return None + tool: Dict[str, Any] = {"type": "web_search"} + if allowed: + tool["filters"] = {"allowed_domains": allowed} + elif excluded: + tool["filters"] = {"excluded_domains": excluded} + return tool + + @staticmethod + def _post_responses( + base_url: str, + payload: Dict[str, Any], + api_key: str, + timeout: float, + *, + is_oauth_path: bool, + ) -> tuple[Any, Optional[Dict[str, Any]]]: + """POST to ``/responses`` → ``(parsed_json, None)`` or ``(None, failure_envelope)`` + on transport/HTTP/JSON failure. + + Two attempts: on a first-call 401 with OAuth creds, force-refresh once and + retry. Covers opaque (non-JWT) tokens the resolver can't pre-check and + mid-window revocation/rotation. XAI_API_KEY creds can't be refreshed, so + they skip the retry rather than burn quota. + """ + import httpx + headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "User-Agent": hermes_xai_user_agent(), } - - try: - import httpx - except ImportError: - return { - "success": False, - "error": "httpx is not installed (required for xAI web search)", - } - - logger.info( - "xAI web search via %s: '%s' (limit=%d, model=%s)", - base_url, query, limit, model, - ) - - # Two-attempt loop: if the first call returns 401 and our creds came - # from the OAuth path, force-refresh the token once and retry. This - # closes two gaps the proactive resolver check doesn't cover: - # (1) opaque (non-JWT) access tokens — `_xai_access_token_is_expiring` - # can't decode them and returns False, so refresh never fires - # until the server hands us a 401. - # (2) mid-window revocation — admin revoke, refresh-token rotation, - # or clock skew can produce 401s on a token whose JWT `exp` claim - # is still in the future. - # Env-var (`XAI_API_KEY`) credentials skip the retry entirely — we - # can't refresh those and an immediate retry would just burn quota. - is_oauth_path = (creds.get("provider") == "xai-oauth") resp = None for attempt in range(2): try: - resp = httpx.post( - f"{base_url}/responses", - headers=headers, - json=payload, - timeout=timeout, - ) + resp = httpx.post(f"{base_url}/responses", headers=headers, json=payload, timeout=timeout) resp.raise_for_status() break except httpx.HTTPStatusError as exc: @@ -270,84 +216,40 @@ class XAIWebSearchProvider(WebSearchProvider): "refresh and retrying once.", ) try: - refreshed = resolve_xai_http_credentials( - force_refresh=True, - api_key_hint=api_key, - ) + refreshed = resolve_xai_http_credentials(force_refresh=True, api_key_hint=api_key) refreshed_key = str(refreshed.get("api_key") or "").strip() if refreshed_key and refreshed_key != api_key: api_key = refreshed_key headers["Authorization"] = f"Bearer {api_key}" continue - # Refresh returned the same (or empty) token — no point - # in retrying. Fall through to the error return below. + # Same/empty token back — retrying is pointless; fall through. except Exception as refresh_exc: # noqa: BLE001 - logger.warning( - "xAI web search OAuth refresh after 401 failed: %s", - refresh_exc, - ) - body = "" + logger.warning("xAI web search OAuth refresh after 401 failed: %s", refresh_exc) try: body = exc.response.text[:300] if exc.response is not None else "" except Exception: body = "" logger.warning("xAI web search HTTP %d: %s", status, body) - return { - "success": False, - "error": f"xAI web search returned HTTP {status}: {body}".rstrip(), - } + return None, _fail(f"xAI web search returned HTTP {status}: {body}".rstrip()) except httpx.RequestError as exc: logger.warning("xAI web search request error: %s", exc) - return {"success": False, "error": f"Could not reach xAI: {exc}"} + return None, _fail(f"Could not reach xAI: {exc}") if resp is None: - # Defensive — both attempts somehow exited the loop without resp. - return {"success": False, "error": "xAI web search produced no response"} + return None, _fail("xAI web search produced no response") try: - data = resp.json() + return resp.json(), None except Exception as exc: # noqa: BLE001 logger.warning("xAI web search bad JSON: %s", exc) - return { - "success": False, - "error": "Could not parse xAI Responses API reply as JSON", - } - - # xAI's Responses surface sometimes returns HTTP 200 with an error - # envelope (model overloaded, content-policy refusal, etc.). Without - # this check, ``_extract_results`` would silently produce an empty - # list and we'd report success-with-no-rows — masking a real failure - # the agent should see and decide whether to retry. - api_error = data.get("error") if isinstance(data, dict) else None - if isinstance(api_error, dict): - err_msg = ( - api_error.get("message") - or api_error.get("code") - or "unknown error" - ) - logger.warning("xAI web search returned error envelope: %s", err_msg) - return {"success": False, "error": f"xAI returned an error: {err_msg}"} - - web_results = self._extract_results(data, limit=limit) - if not web_results: - # Successful call, just no usable rows — return success with an - # empty list so the model can decide whether to retry. Matches - # what brave-free / exa do when the upstream API returns 0 hits. - return {"success": True, "data": {"web": []}} - - return {"success": True, "data": {"web": web_results}} + return None, _fail("Could not parse xAI Responses API reply as JSON") # -- Prompt + parsing ------------------------------------------------- @staticmethod def _build_prompt(query: str, limit: int) -> str: - """Compose the prompt that asks Grok to act as a search engine. - - We deliberately ask for a JSON object (not bare array) so we can - match it cheaply with ``_JSON_BLOCK_RE``; we explicitly forbid - prose, markdown fences, and inline-citation links to keep the - payload parseable. - """ + """Ask Grok for a JSON *object* (cheap to match with ``_JSON_BLOCK_RE``) and + forbid prose/fences/inline citations to keep the payload parseable.""" return ( "Use the web_search tool to find current information for the query below, " "then respond with ONLY a single JSON object — no prose, no markdown " @@ -361,56 +263,28 @@ class XAIWebSearchProvider(WebSearchProvider): ) @classmethod - def _extract_results( - cls, - response_data: Dict[str, Any], - *, - limit: int, - ) -> List[Dict[str, Any]]: - """Pull a ``[{title, url, description, position}, ...]`` list out of a - Responses-API reply. - - Strategy: - - 1. Walk ``output[*].content[*].text`` for ``output_text`` blocks and - try to parse the first JSON object that has a ``results`` list. - 2. If the JSON path fails, fall back to the message annotations - (``url_citation`` entries) — every annotation carries a URL and - a ``title`` (citation number); we pair those URLs with surrounding - text from the message body as a best-effort description. - """ + def _extract_results(cls, response_data: Dict[str, Any], *, limit: int) -> List[Dict[str, Any]]: + """Result rows from a Responses-API reply, in order of preference: + (1) the JSON object in ``output_text`` blocks, (2) ``url_citation`` + annotations paired with surrounding text, (3) the raw ``citations`` list. + Only short-circuit on (2) when it yields rows, so future annotation types + don't mask real data in ``citations``.""" text_blocks, annotations = cls._collect_output_text(response_data) - # Primary path: parse the JSON object Grok was asked for. for block in text_blocks: parsed = cls._try_parse_json_results(block, limit=limit) if parsed: return parsed - # Secondary path: derive results from message annotations + raw text. - # Only short-circuit when annotations actually yielded usable rows; - # otherwise fall through to the citations list. (xAI currently only - # emits ``url_citation`` annotations, but future annotation types - # would silently produce an empty result set if we returned here - # unconditionally — masking real data in ``citations``.) if annotations: - joined_text = "\n".join(text_blocks) - annotation_results = cls._results_from_annotations( - annotations, joined_text, limit=limit, - ) + annotation_results = cls._results_from_annotations(annotations, "\n".join(text_blocks), limit=limit) if annotation_results: return annotation_results - # Last-ditch: raw citations list (no titles or descriptions). citations = response_data.get("citations") or [] if isinstance(citations, list): return [ - { - "title": "", - "url": str(u), - "description": "", - "position": i + 1, - } + _row("", str(u), "", i + 1) for i, u in enumerate(citations[:limit]) if isinstance(u, str) and u.strip() ] @@ -418,10 +292,8 @@ class XAIWebSearchProvider(WebSearchProvider): return [] @staticmethod - def _collect_output_text( - response_data: Dict[str, Any], - ) -> tuple[List[str], List[Dict[str, Any]]]: - """Return (text_blocks, annotations) extracted from ``response.output``.""" + def _collect_output_text(response_data: Dict[str, Any]) -> tuple[List[str], List[Dict[str, Any]]]: + """Return (text_blocks, annotations) from ``response.output`` message chunks.""" text_blocks: List[str] = [] annotations: List[Dict[str, Any]] = [] output = response_data.get("output") @@ -442,25 +314,14 @@ class XAIWebSearchProvider(WebSearchProvider): text_blocks.append(text) chunk_annotations = chunk.get("annotations") if isinstance(chunk_annotations, list): - for ann in chunk_annotations: - if isinstance(ann, dict): - annotations.append(ann) + annotations.extend(a for a in chunk_annotations if isinstance(a, dict)) return text_blocks, annotations @staticmethod - def _try_parse_json_results( - text: str, - *, - limit: int, - ) -> Optional[List[Dict[str, Any]]]: - """Parse a JSON object with a ``results`` array out of ``text``. - - Returns the normalized result list on success, ``None`` when the - block has no valid JSON object or no ``results`` key. Tolerates - leading/trailing prose because reasoning models sometimes prefix a - short narration even when told not to. - """ - # Try the whole string first — cheapest path when Grok obeys. + def _try_parse_json_results(text: str, *, limit: int) -> Optional[List[Dict[str, Any]]]: + """Parse a JSON object with a ``results`` array out of ``text``; None when + absent. Tries the whole string first, then the regex-matched block, since + reasoning models sometimes prefix narration.""" candidates = [text] match = _JSON_BLOCK_RE.search(text) if match and match.group(0) != text: @@ -483,34 +344,21 @@ class XAIWebSearchProvider(WebSearchProvider): url = str(row.get("url", "")).strip() if not url: continue - normalized.append( - { - "title": str(row.get("title", "")).strip(), - "url": url, - "description": str(row.get("description", "")).strip(), - # Renumber from the kept results, not the raw input - # index, so a dropped malformed row doesn't leave a - # gap in the positions handed back to the agent. - "position": len(normalized) + 1, - } - ) + # Renumber from kept rows so a dropped malformed row leaves no gap. + normalized.append(_row( + str(row.get("title", "")).strip(), url, + str(row.get("description", "")).strip(), len(normalized) + 1, + )) if normalized: return normalized return None @staticmethod def _results_from_annotations( - annotations: List[Dict[str, Any]], - joined_text: str, - *, - limit: int, + annotations: List[Dict[str, Any]], joined_text: str, *, limit: int ) -> List[Dict[str, Any]]: - """Best-effort fallback when JSON parsing fails. - - Uses each ``url_citation`` annotation's ``url`` (the citation - title is just the integer label, so we don't surface it) and - slices ~200 characters of surrounding text as the description. - """ + """Fallback rows from ``url_citation`` annotations: URL plus ~200 chars of + preceding text as the description (the annotation title is just a number).""" seen: set[str] = set() results: List[Dict[str, Any]] = [] for ann in annotations: @@ -525,19 +373,11 @@ class XAIWebSearchProvider(WebSearchProvider): start = ann.get("start_index") end = ann.get("end_index") if isinstance(start, int) and isinstance(end, int) and 0 <= start < end <= len(joined_text): - window_start = max(0, start - 200) - description = joined_text[window_start:start].strip() + description = joined_text[max(0, start - 200):start].strip() if len(description) > 200: description = description[-200:].strip() - results.append( - { - "title": "", - "url": url, - "description": description, - "position": len(results) + 1, - } - ) + results.append(_row("", url, description, len(results) + 1)) if len(results) >= limit: break return results @@ -545,16 +385,12 @@ class XAIWebSearchProvider(WebSearchProvider): # -- Setup picker ----------------------------------------------------- def get_setup_schema(self) -> Dict[str, Any]: - # Auth resolution is delegated to the shared ``xai_grok`` post_setup - # hook (same one image_gen.xai and tts.xai use) so users see the - # familiar OAuth-or-API-key prompt for every xAI service. + # Auth resolution is delegated to the shared ``xai_grok`` post_setup hook + # (same one image_gen.xai / tts.xai use) for a consistent OAuth-or-key prompt. return { "name": "xAI Web Search (Grok)", "badge": "paid", - "tag": ( - "Agentic web search via Grok's web_search tool — uses xAI " - "Grok OAuth or XAI_API_KEY." - ), + "tag": "Agentic web search via Grok's web_search tool — uses xAI Grok OAuth or XAI_API_KEY.", "env_vars": [], "post_setup": "xai_grok", }