ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

refactor(plugins/web): BaseWebSearchProvider + _common helpers across all search providers; keyless_mcp failover cleanup

Browse Source
This commit is contained in:
Teknium
2026-09-02 11:23:21 -07:00
parent e43f381b4e
commit 191cc8d30a
21 changed files with 1209 additions and 2725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/web/_common.py
+254
View File
@@ -0,0 +1,254 @@
"""Shared plumbing for the bundled web-search provider plugins.
Every helper resolves patched-in-tests collaborators (``get_provider_env``,
``tools.interrupt``, ``plugins.web.keyless_mcp``, ``tools.web_tools`` client
slots) lazily at call time so monkeypatching the source module keeps working.
"""
from __future__ import annotations
import logging
from typing import Any, Awaitable, Callable, Dict, List, Optional
import httpx
from agent.web_search_provider import WebSearchProvider
SEARCH_LIMIT_CAP = 20 # every vendor here caps max_results at 20 server-side
def provider_env(name: str) -> str:
"""Config-aware env lookup (os.environ, then ~/.hermes/.env)."""
from agent.web_search_provider import get_provider_env
return get_provider_env(name)
def use_keyless(name: str, api_key: str) -> bool:
from plugins.web.keyless_mcp import use_keyless as _use_keyless
return _use_keyless(name, api_key)
def _interrupted() -> bool:
from tools.interrupt import is_interrupted
return is_interrupted()
# --- Result shapes (key order is part of the contract — it reaches the model as JSON) ---
def search_ok(web_results: List[Dict[str, Any]]) -> Dict[str, Any]:
return {"success": True, "data": {"web": web_results}}
def search_fail(error: str) -> Dict[str, Any]:
return {"success": False, "error": error}
def web_hit(url: str, title: str, description: str, position: int) -> Dict[str, Any]:
return {"url": url, "title": title, "description": description, "position": position}
def document(url: str, title: str, content: str, *, source_url: Optional[str] = None) -> Dict[str, Any]:
"""Successful extract entry; ``raw_content`` mirrors ``content`` for the legacy pipeline."""
return {
"url": url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url if source_url is None else source_url, "title": title},
}
def extract_fail(urls: List[str], error: str) -> List[Dict[str, Any]]:
return [{"url": u, "title": "", "content": "", "error": error} for u in urls]
# --- Guarded execution: interrupt check + uniform failure classification ---
def _failure_message(
vendor: str, kind: str, exc: Exception, logger: logging.Logger, *, sdk: bool, verbatim_value_error: bool
) -> str:
"""Map an exception to the user-facing error string.
``verbatim_value_error``: ValueError carries a pre-formatted message (missing
key, HTTP body) and is returned as-is. ``sdk``: ImportError means the lazily
installed vendor SDK is missing. Anything else is logged and wrapped.
"""
if verbatim_value_error and isinstance(exc, ValueError):
return str(exc)
if sdk and isinstance(exc, ImportError):
return f"{vendor} SDK not installed: {exc}"
logger.warning("%s %s error: %s", vendor, kind, exc)
return f"{vendor} {kind} failed: {exc}"
def run_search(
vendor: str, logger: logging.Logger, body: Callable[[], Dict[str, Any]], *, sdk: bool = False, verbatim_value_error: bool = True
) -> Dict[str, Any]:
try:
if _interrupted():
return search_fail("Interrupted")
return body()
except Exception as exc: # noqa: BLE001 — surface as failure dict
return search_fail(_failure_message(vendor, "search", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error))
def _extract_interrupted(urls: List[str]) -> List[Dict[str, Any]]:
return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
def run_extract(
vendor: str, logger: logging.Logger, urls: List[str], body: Callable[[], List[Dict[str, Any]]],
*, sdk: bool = False, verbatim_value_error: bool = True,
) -> List[Dict[str, Any]]:
"""Per-URL failures are returned as entries with ``error`` — never raised."""
try:
if _interrupted():
return _extract_interrupted(urls)
return body()
except Exception as exc: # noqa: BLE001
return extract_fail(urls, _failure_message(vendor, "extract", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error))
async def run_extract_async(
vendor: str, logger: logging.Logger, urls: List[str], body: Callable[[], Awaitable[List[Dict[str, Any]]]],
*, sdk: bool = False, verbatim_value_error: bool = True,
) -> List[Dict[str, Any]]:
try:
if _interrupted():
return _extract_interrupted(urls)
return await body()
except Exception as exc: # noqa: BLE001
return extract_fail(urls, _failure_message(vendor, "extract", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error))
# --- HTTP + SDK client helpers ---
def http_status_detail(response: Any) -> str:
"""Response body text for a >=400 reply, or ``HTTP <code>`` when the body is empty."""
return (response.text or "").strip() or f"HTTP {response.status_code}"
def http_get_json(
label: str, url: str, *, params: Dict[str, Any], headers: Dict[str, str], timeout: int,
logger: logging.Logger, reach_target: Optional[str] = None,
) -> tuple[Any, Optional[Dict[str, Any]]]:
"""GET ``url`` and parse JSON. Returns ``(data, None)`` or ``(None, failure_dict)``.
``label`` names the vendor in error strings; ``reach_target`` overrides the
"Could not reach ..." subject (SearXNG includes its instance URL).
"""
try:
resp = httpx.get(url, params=params, headers=headers, timeout=timeout)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
logger.warning("%s HTTP error: %s", label, exc)
return None, search_fail(f"{label} returned HTTP {exc.response.status_code}")
except httpx.RequestError as exc:
logger.warning("%s request error: %s", label, exc)
return None, search_fail(f"Could not reach {reach_target or label}: {exc}")
try:
return resp.json(), None
except Exception as exc: # noqa: BLE001
logger.warning("%s response parse error: %s", label, exc)
return None, search_fail(f"Could not parse {label} response as JSON")
def cached_sdk_client(slot: str, env_var: str, missing_key_error: str, feature: str, factory: Callable[[str], Any]) -> Any:
"""Lazy-build + cache a vendor SDK client on ``tools.web_tools.<slot>``.
The cache slot lives on ``tools.web_tools`` so tests that reset
``tools.web_tools._<vendor>_client = None`` between cases see fresh state.
Raises ValueError when the key is unset; lazy_deps install hints are
re-raised as ImportError (its own ImportError is benign and swallowed).
"""
import tools.web_tools as _wt
cached = getattr(_wt, slot, None)
if cached is not None:
return cached
api_key = provider_env(env_var)
if not api_key:
raise ValueError(missing_key_error)
try:
from tools.lazy_deps import ensure as _lazy_ensure
_lazy_ensure(feature, prompt=False)
except ImportError:
pass
except Exception as exc: # noqa: BLE001
raise ImportError(str(exc))
client = factory(api_key)
setattr(_wt, slot, client)
return client
# --- Provider base ---
class BaseWebSearchProvider(WebSearchProvider):
"""Common surface for the bundled providers.
Subclasses set ``NAME`` / ``DISPLAY_NAME`` / ``KEY_ENV`` and flip
``EXTRACT`` / ``KEYLESS``. ``is_available`` deliberately ignores the
keyless tier: otherwise the legacy preference walk would route users
holding a key for a lower-priority backend onto this vendor's free tier.
``is_keyless_available`` is True for ring members and opt-in keyless
vendors unless the user pinned ``web.provider_tier.<name>: paid``.
"""
NAME: str = ""
DISPLAY_NAME: str = ""
KEY_ENV: str = ""
EXTRACT: bool = False
KEYLESS: bool = False
@property
def name(self) -> str:
return self.NAME
@property
def display_name(self) -> str:
return self.DISPLAY_NAME
def is_available(self) -> bool:
return bool(provider_env(self.KEY_ENV))
def is_keyless_available(self) -> bool:
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return self.KEYLESS and keyless_enabled() and provider_tier(self.NAME) != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return self.EXTRACT
def keyless_variant_schema(display: str, key_env: str, key_url: str, *, free_tag: str, paid_tag: str) -> Dict[str, Any]:
"""``hermes tools`` picker entry for a keyless-ring vendor with a paid variant."""
return {
"name": f"{display} · Free (keyless)",
"badge": "free · no key",
"tag": free_tag,
"env_vars": [],
"web_tier": "free",
"variants": [
{
"name": f"{display} · Paid (API key)",
"badge": "paid",
"tag": paid_tag,
"env_vars": [{"key": key_env, "prompt": f"{display} API key", "url": key_url}],
"web_tier": "paid",
},
],
}
plugins/web/brave_free/__init__.py
+1 -6
View File
@@ -1,8 +1,4 @@
"""Brave Search (free tier) plugin — bundled, auto-loaded.
Mirrors the ``plugins/image_gen/openai/`` layout: ``provider.py`` holds the
provider class, ``__init__.py::register(ctx)`` registers an instance.
"""
"""Brave Search (free tier) plugin — bundled, auto-loaded."""
from __future__ import annotations
@@ -10,5 +6,4 @@ from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
def register(ctx) -> None:
"""Register the Brave-free provider with the plugin context."""
ctx.register_web_search_provider(BraveFreeWebSearchProvider())
plugins/web/brave_free/provider.py
+25 -95
View File
@@ -1,111 +1,46 @@
"""Brave Search (free tier) — plugin form.
"""Brave Search (free-tier Data-for-Search API) — search only, 2,000 queries/month.
Subclasses :class:`agent.web_search_provider.WebSearchProvider` (the
plugin-facing ABC). The legacy in-tree module
``tools.web_providers.brave_free`` was removed in the same commit that
moved this code under ``plugins/``; this file is now the canonical
implementation.
Config keys this provider responds to::
web:
search_backend: "brave-free" # explicit per-capability
backend: "brave-free" # shared fallback
Auth env var::
BRAVE_SEARCH_API_KEY=... # https://brave.com/search/api/ (free tier)
Config: ``web.search_backend`` / ``web.backend: "brave-free"`` (hyphen form kept for
existing user configs). Env: ``BRAVE_SEARCH_API_KEY``. Pair with Firecrawl/Tavily/Exa
for ``web_extract``.
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import BaseWebSearchProvider, http_get_json, provider_env, search_fail, search_ok
logger = logging.getLogger(__name__)
_BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
class BraveFreeWebSearchProvider(WebSearchProvider):
"""Search-only Brave provider using the free-tier Data-for-Search API.
class BraveFreeWebSearchProvider(BaseWebSearchProvider):
"""Search-only Brave provider using the free-tier Data-for-Search API."""
Free tier is 2,000 queries/month (1 qps). No content-extraction capability —
users pair this with Firecrawl/Tavily/Exa for ``web_extract``.
"""
@property
def name(self) -> str:
# Hyphen form preserved for backward compat with the existing
# ``web.search_backend: "brave-free"`` config keys users have set.
return "brave-free"
@property
def display_name(self) -> str:
return "Brave Search (Free)"
def is_available(self) -> bool:
"""Return True when ``BRAVE_SEARCH_API_KEY`` is set to a non-empty value."""
from agent.web_search_provider import get_provider_env
return bool(get_provider_env("BRAVE_SEARCH_API_KEY"))
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return False
NAME = "brave-free"
DISPLAY_NAME = "Brave Search (Free)"
KEY_ENV = "BRAVE_SEARCH_API_KEY"
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a search against the Brave Search API.
Returns ``{"success": True, "data": {"web": [{"title", "url", "description", "position"}]}}``
on success, or ``{"success": False, "error": str}`` on failure.
"""
import httpx
from agent.web_search_provider import get_provider_env
api_key = get_provider_env("BRAVE_SEARCH_API_KEY")
api_key = provider_env("BRAVE_SEARCH_API_KEY")
if not api_key:
return {"success": False, "error": "BRAVE_SEARCH_API_KEY is not set"}
return search_fail("BRAVE_SEARCH_API_KEY is not set")
# Brave's `count` is capped at 20.
count = max(1, min(int(limit), 20))
try:
resp = httpx.get(
_BRAVE_ENDPOINT,
params={"q": query, "count": count},
headers={
"X-Subscription-Token": api_key,
"Accept": "application/json",
},
timeout=15,
)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
logger.warning("Brave Search HTTP error: %s", exc)
return {
"success": False,
"error": f"Brave Search returned HTTP {exc.response.status_code}",
}
except httpx.RequestError as exc:
logger.warning("Brave Search request error: %s", exc)
return {"success": False, "error": f"Could not reach Brave Search: {exc}"}
try:
data = resp.json()
except Exception as exc: # noqa: BLE001
logger.warning("Brave Search response parse error: %s", exc)
return {"success": False, "error": "Could not parse Brave Search response as JSON"}
data, failure = http_get_json(
"Brave Search",
_BRAVE_ENDPOINT,
params={"q": query, "count": max(1, min(int(limit), 20))}, # Brave caps count at 20
headers={"X-Subscription-Token": api_key, "Accept": "application/json"},
timeout=15,
logger=logger,
)
if failure is not None:
return failure
raw_results = (data.get("web") or {}).get("results", []) or []
truncated = raw_results[:limit]
web_results = [
{
"title": str(r.get("title", "")),
@@ -113,18 +48,13 @@ class BraveFreeWebSearchProvider(WebSearchProvider):
"description": str(r.get("description", "")),
"position": i + 1,
}
for i, r in enumerate(truncated)
for i, r in enumerate(raw_results[:limit])
]
logger.info(
"Brave Search '%s': %d results (from %d raw, limit %d)",
query,
len(web_results),
len(raw_results),
limit,
query, len(web_results), len(raw_results), limit,
)
return {"success": True, "data": {"web": web_results}}
return search_ok(web_results)
def get_setup_schema(self) -> Dict[str, Any]:
return {
plugins/web/ddgs/__init__.py
+1 -7
View File
@@ -1,9 +1,4 @@
"""DuckDuckGo search plugin — bundled, auto-loaded.
Backed by the community ``ddgs`` Python package which scrapes DDG's HTML
results page. No API key required, but the package itself must be installed
(it's an optional dep — gated via :meth:`is_available`).
"""
"""DuckDuckGo search plugin (``ddgs`` package, optional dep) — bundled, auto-loaded."""
from __future__ import annotations
@@ -11,5 +6,4 @@ from plugins.web.ddgs.provider import DDGSWebSearchProvider
def register(ctx) -> None:
"""Register the DDGS provider with the plugin context."""
ctx.register_web_search_provider(DDGSWebSearchProvider())
plugins/web/ddgs/_search_worker.py
+31 -49
View File
@@ -1,18 +1,9 @@
"""DDGS search child-process entrypoint (#68096).
"""DDGS search child-process entrypoint, run as ``python plugins/web/ddgs/_search_worker.py``.
Invoked as ``python plugins/web/ddgs/_search_worker.py`` (script path from the
parent provider). Reads one JSON request from stdin, writes one JSON envelope
to stdout, then exits.
Request::
{"query": str, "safe_limit": int}
Envelope::
{"ok": true, "results": [...]}
{"ok": false, "error": str}
Optional test hooks (only when ``HERMES_DDGS_ALLOW_TEST_HOOKS=1``)::
{"query": ..., "safe_limit": ..., "test_hook": "sleep"|"gil"|"success"|"error"|"empty"}
Reads one JSON request ``{"query": str, "safe_limit": int}`` from stdin, writes one
envelope ``{"ok": true, "results": [...]}`` / ``{"ok": false, "error": str}`` to
stdout, exits. Test hooks (``"test_hook": "sleep"|"gil"|"success"|"error"|"empty"``)
are honored only when ``HERMES_DDGS_ALLOW_TEST_HOOKS=1``.
"""
from __future__ import annotations
@@ -24,11 +15,8 @@ import time
def _hold_gil(secs: int) -> None:
"""Block in a foreign call that keeps the GIL (ctypes.PyDLL).
Mirrors native ``primp`` holding the interpreter lock. ``PyDLL`` (unlike
``CDLL``/``WinDLL``) does not release the GIL around the call.
"""
"""Block in a foreign call that keeps the GIL — mirrors native ``primp``.
``ctypes.PyDLL`` (unlike ``CDLL``/``WinDLL``) does not release the GIL."""
import ctypes
if sys.platform == "win32":
@@ -46,30 +34,26 @@ def _hold_gil(secs: int) -> None:
sleep(int(secs))
def _run_test_hook(hook: str) -> dict:
if hook == "sleep":
time.sleep(30)
return {"ok": False, "error": "sleep hook returned unexpectedly"}
if hook == "gil":
_hold_gil(30)
return {"ok": False, "error": "gil hook returned unexpectedly"}
if hook == "success":
return {
"ok": True,
"results": [
{
"title": "Hit",
"url": "https://example.com",
"description": "body",
"position": 1,
}
],
}
if hook == "empty":
return {"ok": True, "results": []}
if hook == "error":
return {"ok": False, "error": "RuntimeError: boom"}
return {"ok": False, "error": f"unknown test_hook: {hook!r}"}
def _hook_sleep() -> dict:
time.sleep(30)
return {"ok": False, "error": "sleep hook returned unexpectedly"}
def _hook_gil() -> dict:
_hold_gil(30)
return {"ok": False, "error": "gil hook returned unexpectedly"}
_TEST_HOOKS = {
"sleep": _hook_sleep,
"gil": _hook_gil,
"success": lambda: {
"ok": True,
"results": [{"title": "Hit", "url": "https://example.com", "description": "body", "position": 1}],
},
"empty": lambda: {"ok": True, "results": []},
"error": lambda: {"ok": False, "error": "RuntimeError: boom"},
}
def _write_envelope(envelope: dict) -> None:
@@ -87,19 +71,17 @@ def main() -> int:
hook = request.get("test_hook")
if hook:
if os.environ.get("HERMES_DDGS_ALLOW_TEST_HOOKS") != "1":
_write_envelope(
{"ok": False, "error": "test_hook refused (hooks not enabled)"}
)
_write_envelope({"ok": False, "error": "test_hook refused (hooks not enabled)"})
return 3
envelope = _run_test_hook(str(hook))
fn = _TEST_HOOKS.get(str(hook))
envelope = fn() if fn else {"ok": False, "error": f"unknown test_hook: {hook!r}"}
_write_envelope(envelope)
return 0 if envelope.get("ok") else 1
query = str(request.get("query") or "")
safe_limit = max(1, int(request.get("safe_limit") or 1))
try:
# Import inside main so script startup stays light / patchable.
from plugins.web.ddgs.provider import _run_ddgs_search
from plugins.web.ddgs.provider import _run_ddgs_search # lazy: light startup, patchable
results = _run_ddgs_search(query, safe_limit)
_write_envelope({"ok": True, "results": results})
plugins/web/ddgs/provider.py
+109 -196
View File
@@ -1,19 +1,12 @@
"""DuckDuckGo search — plugin form (via the ``ddgs`` package).
"""DuckDuckGo search via the optional ``ddgs`` package (search only, no key).
Subclasses the plugin-facing :class:`agent.web_search_provider.WebSearchProvider`.
The legacy in-tree module ``tools.web_providers.ddgs`` was removed in the
same commit that moved this code under ``plugins/``; this file is now the
canonical implementation.
``is_available()`` reflects package importability; the plugin registers either way
so ``hermes tools`` can offer to install it.
The ``ddgs`` package is an optional dependency. ``is_available()`` reflects
whether the package is importable; the plugin still registers either way so
``hermes tools`` can prompt the user to install it.
Isolation note (#68096): ``ddgs``/``primp`` can block inside native code while
holding the Python GIL. A ``ThreadPoolExecutor`` + ``future.result(timeout=…)``
cap (see #52118) cannot fire in that state — the waiter never reacquires the
GIL — so the whole Hermes process freezes through Ctrl+C/SIGTERM. Each search
therefore runs in a disposable child process the parent can terminate/kill.
Isolation: ``ddgs``/``primp`` can block inside native code while holding the GIL, so
a thread-pool ``future.result(timeout=…)`` cap can never fire and Ctrl+C/SIGTERM
freeze the whole process. Each search therefore runs in a disposable child process
the parent can terminate/kill.
"""
from __future__ import annotations
@@ -27,22 +20,22 @@ import sys
import time
from typing import Any, Dict, Optional
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import BaseWebSearchProvider, search_fail, search_ok
logger = logging.getLogger(__name__)
# Overall wall-clock cap for a single ddgs search. The DDGS constructor's
# ``timeout`` only bounds individual HTTP requests; ddgs's multi-engine retry
# loop has no overall cap, so a slow/rate-limited DuckDuckGo response can hang
# the (single, shared) agent loop indefinitely (#36776). Enforce a hard cap
# here by killing a disposable worker process (#68096).
# Hard wall-clock cap per search. ``DDGS(timeout=…)`` only bounds individual HTTP
# requests; ddgs's multi-engine retry loop has no overall cap, so a rate-limited
# response could otherwise hang the shared agent loop indefinitely.
_SEARCH_TIMEOUT_SECS = 30
_POLL_INTERVAL_SECS = 0.1 # parent stdout / interrupt-flag poll cadence
_TERMINATE_GRACE_SECS = 1.0 # wait after terminate() before escalating to kill()
# How often the parent polls stdout / interrupt flag while waiting.
_POLL_INTERVAL_SECS = 0.1
# Test-only hook name forwarded to the child (see _search_worker.py); never set in production.
_test_hook: Optional[str] = None
# After terminate(), wait this long before escalating to kill().
_TERMINATE_GRACE_SECS = 1.0
# Last worker Popen started by ``_run_ddgs_search_bounded`` (test reap checks).
_last_worker_proc: Optional[subprocess.Popen] = None
class _SearchInterrupted(Exception):
@@ -50,13 +43,8 @@ class _SearchInterrupted(Exception):
def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]:
"""Run the blocking ddgs query and return normalized hits.
Module-level (not a closure) so the child worker can import it and so
tests can patch it for in-process unit tests. ``DDGS(timeout=…)`` bounds
each individual HTTP request; the overall wall-clock cap is enforced by
the parent via process timeout (#68096).
"""
"""Blocking ddgs query → normalized hits. Module-level so the child worker can
import it and tests can patch it for in-process runs."""
from ddgs import DDGS # type: ignore
results: list[dict[str, Any]] = []
@@ -64,11 +52,10 @@ def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]:
for i, hit in enumerate(client.text(query, max_results=safe_limit)):
if i >= safe_limit:
break
url = str(hit.get("href") or hit.get("url") or "")
results.append(
{
"title": str(hit.get("title", "")),
"url": url,
"url": str(hit.get("href") or hit.get("url") or ""),
"description": str(hit.get("body", "")),
"position": i + 1,
}
@@ -76,20 +63,9 @@ def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]:
return results
# Optional test-only hook name forwarded to the child (see _search_worker.py).
# Production search() never sets this.
_test_hook: Optional[str] = None
# Last worker Popen started by ``_run_ddgs_search_bounded`` (test reap checks).
_last_worker_proc: Optional[subprocess.Popen] = None
def _plugins_path_entry() -> str:
"""Return the ``sys.path`` entry that makes ``import plugins`` work.
Prefer the live ``plugins`` package location over counting ``dirname``s from
this file — that stays correct for source checkouts and site-packages.
"""
"""``sys.path`` entry that makes ``import plugins`` work in the child (prefers
the live package location; correct for source checkouts and site-packages)."""
try:
import plugins as plugins_pkg
@@ -98,32 +74,24 @@ def _plugins_path_entry() -> str:
return os.path.dirname(os.path.dirname(os.path.abspath(pkg_file)))
except Exception: # noqa: BLE001 — fall through to path-walk fallback
pass
return os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
)
here = os.path.abspath(__file__)
for _ in range(4):
here = os.path.dirname(here)
return here
def _terminate_and_reap(
proc: Optional[subprocess.Popen],
*,
grace: float = _TERMINATE_GRACE_SECS,
) -> None:
def _terminate_and_reap(proc: Optional[subprocess.Popen], *, grace: float = _TERMINATE_GRACE_SECS) -> None:
"""Terminate a worker, escalate to kill, and wait so no orphan remains.
Does not close the parent's pipe ends — the caller must finish any
``communicate()``/reader first. Closing stdout while another thread is
blocked in ``read()`` deadlocks on some platforms.
Does not close the parent's pipe ends — closing stdout while another thread is
blocked in ``read()`` deadlocks on some platforms; the caller drains first.
"""
if proc is None:
return
def _wait_until_dead(seconds: float) -> bool:
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
if proc.poll() is not None:
return True
while proc.poll() is None and time.monotonic() < deadline:
time.sleep(0.05)
return proc.poll() is not None
@@ -139,57 +107,28 @@ def _terminate_and_reap(
logger.debug("DDGS worker reap error: %s", exc)
def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]]:
"""Run ``_run_ddgs_search`` in a disposable process with a hard deadline.
The parent never joins the child while it may be inside native code holding
*its* GIL — it only polls a communicator thread and, on timeout/interrupt,
terminates the child OS process. Raises ``TimeoutError``,
``_SearchInterrupted``, or ``RuntimeError``.
"""
# Imported lazily so plugin import stays light for ``hermes tools`` probes.
from tools.interrupt import is_interrupted
global _last_worker_proc
request: dict[str, Any] = {"query": query, "safe_limit": safe_limit}
if _test_hook:
request["test_hook"] = _test_hook
from tools.environments.local import _sanitize_subprocess_env
env = _sanitize_subprocess_env(dict(os.environ))
if _test_hook:
env["HERMES_DDGS_ALLOW_TEST_HOOKS"] = "1"
def _spawn_worker(env: dict[str, str]) -> subprocess.Popen:
"""Start ``_search_worker.py`` as a script with ``plugins`` importable."""
# Running the worker as a script puts ``plugins/web/ddgs/`` on ``sys.path[0]``,
# which breaks ``import plugins...``. Prepend the path entry that makes the
# live ``plugins`` package importable (source tree or site-packages).
# which breaks ``import plugins...``; prepend the real package location.
child_pythonpath = env.get("PYTHONPATH", "")
path_entry = _plugins_path_entry()
if path_entry and path_entry not in child_pythonpath.split(os.pathsep):
env["PYTHONPATH"] = (
path_entry + os.pathsep + child_pythonpath if child_pythonpath else path_entry
)
env["PYTHONPATH"] = path_entry + os.pathsep + child_pythonpath if child_pythonpath else path_entry
worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_search_worker.py")
# Platform-only spawn knobs — stdin/stdout/stderr must stay as explicit
# keyword args on the Popen call so scripts/check_subprocess_stdin.py can
# see them (TUI gateway inherits stdin; #14036).
extra_kwargs: dict[str, Any] = {}
if sys.platform == "win32":
# New process group so terminate/kill reach the worker cleanly on Windows.
extra_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
else:
# Own session so a hung primp/libcurl grandchild can be reaped with the worker.
extra_kwargs["start_new_session"] = True
proc = subprocess.Popen(
# stdin/stdout/stderr must stay explicit keyword args on the Popen call so
# scripts/check_subprocess_stdin.py can see them (TUI gateway inherits stdin).
extra_kwargs: dict[str, Any] = (
{"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} # so terminate/kill reach the worker
if sys.platform == "win32"
else {"start_new_session": True} # own session so a hung primp grandchild is reaped too
)
return subprocess.Popen(
[sys.executable, worker_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
# DEVNULL avoids the classic deadlock where a chatty child fills the
# stderr pipe buffer while the parent only drains stdout.
# DEVNULL: a chatty child filling the stderr pipe while we only drain stdout would deadlock.
stderr=subprocess.DEVNULL,
env=env,
text=True,
@@ -197,14 +136,53 @@ def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]
errors="replace",
**extra_kwargs,
)
def _parse_envelope(raw: str, proc: subprocess.Popen) -> list[dict[str, Any]]:
"""Decode the worker's stdout envelope; raise ``RuntimeError`` on any malformed shape."""
raw = raw.strip()
if not raw:
raise RuntimeError(f"DDGS worker exited without a result (code={proc.poll()})")
try:
envelope = json.loads(raw)
except json.JSONDecodeError as exc:
raise RuntimeError(f"DDGS worker returned invalid JSON: {raw[:200]!r}") from exc
if not isinstance(envelope, dict):
raise RuntimeError(f"DDGS worker returned an invalid envelope: {envelope!r}")
if envelope.get("ok"):
results = envelope.get("results") or []
if not isinstance(results, list):
raise RuntimeError("DDGS worker returned non-list results")
return results
raise RuntimeError(str(envelope.get("error") or "DDGS worker failed"))
def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]]:
"""Run ``_run_ddgs_search`` in a disposable process with a hard deadline.
The parent never joins a child that may be in native code holding *its* GIL —
it polls a communicator thread and, on timeout/interrupt, kills the OS process.
Raises ``TimeoutError``, ``_SearchInterrupted``, or ``RuntimeError``.
"""
from tools.interrupt import is_interrupted # lazy: keep plugin import light
from tools.environments.local import _sanitize_subprocess_env
global _last_worker_proc
request: dict[str, Any] = {"query": query, "safe_limit": safe_limit}
env = _sanitize_subprocess_env(dict(os.environ))
if _test_hook:
request["test_hook"] = _test_hook
env["HERMES_DDGS_ALLOW_TEST_HOOKS"] = "1"
proc = _spawn_worker(env)
_last_worker_proc = proc
# ``communicate`` runs in a side thread so the parent can poll interrupt /
# deadline without blocking. Killing the child unblocks communicate.
# deadline without blocking; killing the child unblocks it.
pool = cf.ThreadPoolExecutor(max_workers=1)
fut = pool.submit(proc.communicate, json.dumps(request))
timed_out = False
interrupted = False
timed_out = interrupted = False
raw = ""
try:
deadline = time.monotonic() + _SEARCH_TIMEOUT_SECS
@@ -237,56 +215,20 @@ def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]
if interrupted:
raise _SearchInterrupted("DuckDuckGo search interrupted")
if timed_out:
raise TimeoutError(
f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s"
)
raw = raw.strip()
if not raw:
raise RuntimeError(
f"DDGS worker exited without a result (code={proc.poll()})"
)
try:
envelope = json.loads(raw)
except json.JSONDecodeError as exc:
raise RuntimeError(
f"DDGS worker returned invalid JSON: {raw[:200]!r}"
) from exc
if not isinstance(envelope, dict):
raise RuntimeError(f"DDGS worker returned an invalid envelope: {envelope!r}")
if envelope.get("ok"):
results = envelope.get("results") or []
if not isinstance(results, list):
raise RuntimeError("DDGS worker returned non-list results")
return results
raise RuntimeError(str(envelope.get("error") or "DDGS worker failed"))
raise TimeoutError(f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s")
return _parse_envelope(raw, proc)
class DDGSWebSearchProvider(WebSearchProvider):
"""DuckDuckGo HTML-scrape search provider.
class DDGSWebSearchProvider(BaseWebSearchProvider):
"""DuckDuckGo HTML-scrape search provider (no API key; DDG rate-limits server-side).
ddgs errors surface as ``{"success": False, "error": ...}`` rather than raising."""
No API key needed. Rate limits are enforced server-side by DuckDuckGo;
the provider surfaces ``DuckDuckGoSearchException`` and other ddgs errors
as ``{"success": False, "error": ...}`` rather than raising.
"""
@property
def name(self) -> str:
return "ddgs"
@property
def display_name(self) -> str:
return "DuckDuckGo (ddgs)"
NAME = "ddgs"
DISPLAY_NAME = "DuckDuckGo (ddgs)"
def is_available(self) -> bool:
"""Return True when the ``ddgs`` package is importable.
Probes the import once; cheap because Python caches the import. Must
NOT perform network I/O — runs at tool-registration time and on every
``hermes tools`` paint.
"""
"""True when ``ddgs`` is importable. Must NOT do network I/O — runs at
tool-registration time and on every ``hermes tools`` paint."""
try:
import ddgs # noqa: F401
@@ -294,61 +236,33 @@ class DDGSWebSearchProvider(WebSearchProvider):
except ImportError:
return False
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return False
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a DuckDuckGo search and return normalized results.
"""Run the search in a disposable child with a hard wall-clock timeout so a
hung native ``primp`` call cannot freeze the Hermes process."""
if not self.is_available():
return search_fail("ddgs package is not installed — run `pip install ddgs`")
The synchronous ``ddgs`` call runs in a disposable child process with
a hard wall-clock timeout (``_SEARCH_TIMEOUT_SECS``) so a hung native
``primp`` call cannot freeze the Hermes process (#36776, #68096).
"""
try:
import ddgs # type: ignore # noqa: F401 — availability probe
except ImportError:
return {
"success": False,
"error": "ddgs package is not installed — run `pip install ddgs`",
}
# DDGS().text yields at most `max_results` items; we cap defensively
# in case the package ignores the hint.
# Defensive cap in case the package ignores its max_results hint.
safe_limit = max(1, int(limit))
try:
web_results = _run_ddgs_search_bounded(query, safe_limit)
except TimeoutError:
logger.warning(
"DDGS search timed out after %ds for query: %r",
_SEARCH_TIMEOUT_SECS,
query,
logger.warning("DDGS search timed out after %ds for query: %r", _SEARCH_TIMEOUT_SECS, query)
return search_fail(
f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — "
"DuckDuckGo may be rate-limiting or slow. Try again later "
"or switch to a different search provider."
)
return {
"success": False,
"error": (
f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — "
"DuckDuckGo may be rate-limiting or slow. Try again later "
"or switch to a different search provider."
),
}
except _SearchInterrupted:
logger.info("DDGS search interrupted for query: %r", query)
return {
"success": False,
"error": "DuckDuckGo search interrupted",
}
return search_fail("DuckDuckGo search interrupted")
except Exception as exc: # noqa: BLE001 — ddgs raises its own exceptions
logger.warning("DDGS search error: %s", exc)
return {"success": False, "error": f"DuckDuckGo search failed: {exc}"}
return search_fail(f"DuckDuckGo search failed: {exc}")
logger.info(
"DDGS search '%s': %d results (limit %d)", query, len(web_results), limit
)
return {"success": True, "data": {"web": web_results}}
logger.info("DDGS search '%s': %d results (limit %d)", query, len(web_results), limit)
return search_ok(web_results)
def get_setup_schema(self) -> Dict[str, Any]:
return {
@@ -356,7 +270,6 @@ class DDGSWebSearchProvider(WebSearchProvider):
"badge": "free · no key · search only",
"tag": "Search via the ddgs Python package — no API key (pair with any extract provider)",
"env_vars": [],
# Trigger `_run_post_setup("ddgs")` after the user picks this row
# so the ddgs Python package gets pip-installed on first selection.
# Triggers `_run_post_setup("ddgs")` so the package gets pip-installed on first pick.
"post_setup": "ddgs",
}
plugins/web/exa/__init__.py
+1 -7
View File
@@ -1,9 +1,4 @@
"""Exa web search + extract plugin — bundled, auto-loaded.
Backed by the official Exa SDK (``exa-py``). Both search and extract are
sync; the dispatcher in :mod:`tools.web_tools` handles the wrap when the
caller is async.
"""
"""Exa web search + extract plugin — bundled, auto-loaded (sync SDK; dispatcher wraps async callers)."""
from __future__ import annotations
@@ -11,5 +6,4 @@ from plugins.web.exa.provider import ExaWebSearchProvider
def register(ctx) -> None:
"""Register the Exa provider with the plugin context."""
ctx.register_web_search_provider(ExaWebSearchProvider())
plugins/web/exa/provider.py
+49 -231
View File
@@ -1,268 +1,86 @@
"""Exa web search + content extraction — plugin form.
"""Exa web search + content extraction via the ``exa-py`` SDK (lazy-installed).
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses the
official Exa SDK (``exa-py``) which is lazy-loaded via
:func:`tools.lazy_deps.ensure` so that cold-start CLI users don't pay the
SDK import cost when Exa isn't configured.
Config keys this provider responds to::
web:
search_backend: "exa" # explicit per-capability
extract_backend: "exa" # explicit per-capability
backend: "exa" # shared fallback for both
Env var::
EXA_API_KEY=... # https://exa.ai (paid tier; free trial available)
The previous in-tree implementation lived at
``tools.web_tools._exa_search`` / ``_exa_extract``; this file is the
canonical replacement. Behavior is bit-for-bit identical aside from the
ABC method-name change.
Env: ``EXA_API_KEY`` (https://exa.ai). Both methods are sync — Exa's SDK is
sync-only; the dispatcher threads extract when the caller is async.
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict, List
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import (
BaseWebSearchProvider,
cached_sdk_client,
document,
keyless_variant_schema,
provider_env,
run_extract,
run_search,
search_ok,
use_keyless,
web_hit,
)
logger = logging.getLogger(__name__)
# Module-level note: the canonical ``_exa_client`` cache slot lives on
# :mod:`tools.web_tools` so tests that do ``tools.web_tools._exa_client =
# None`` between cases see fresh state. The plugin reads/writes through
# that public module (see :func:`_get_exa_client`).
_MISSING_KEY = "EXA_API_KEY environment variable not set. Get your API key at https://exa.ai"
def _get_exa_client() -> Any:
"""Lazy-import and cache an Exa SDK client.
def _factory(api_key: str) -> Any:
from exa_py import Exa # deliberately lazy
Cache lives on :mod:`tools.web_tools` (as ``_exa_client``) so unit
tests that reset that name between cases keep working. Raises
``ValueError`` when ``EXA_API_KEY`` is unset.
"""
import tools.web_tools as _wt
client = Exa(api_key=api_key)
client.headers["x-exa-integration"] = "hermes-agent"
return client
cached = getattr(_wt, "_exa_client", None)
if cached is not None:
return cached
from agent.web_search_provider import get_provider_env
api_key = get_provider_env("EXA_API_KEY")
if not api_key:
raise ValueError(
"EXA_API_KEY environment variable not set. "
"Get your API key at https://exa.ai"
)
try:
from tools.lazy_deps import ensure as _lazy_ensure
_lazy_ensure("search.exa", prompt=False)
except ImportError:
pass
except Exception as exc: # noqa: BLE001 — lazy_deps surfaces install hints
raise ImportError(str(exc))
from exa_py import Exa # noqa: WPS433 — deliberately lazy
client = Exa(api_key=api_key)
client.headers["x-exa-integration"] = "hermes-agent"
_wt._exa_client = client
return client
return cached_sdk_client("_exa_client", "EXA_API_KEY", _MISSING_KEY, "search.exa", _factory)
def _reset_client_for_tests() -> None:
"""Drop the cached Exa client so tests can re-instantiate cleanly."""
import tools.web_tools as _wt
class ExaWebSearchProvider(BaseWebSearchProvider):
"""Exa search + extract provider."""
_wt._exa_client = None
class ExaWebSearchProvider(WebSearchProvider):
"""Exa search + extract provider.
Both methods are sync — Exa's SDK is sync-only. The web_extract_tool
dispatcher wraps sync extracts via ``asyncio.to_thread`` when it
needs to keep the event loop responsive.
"""
@property
def name(self) -> str:
return "exa"
@property
def display_name(self) -> str:
return "Exa"
def is_available(self) -> bool:
"""Return True when ``EXA_API_KEY`` is set to a non-empty value.
Deliberately does NOT consider the keyless free tier — that would
let the legacy preference walk route keyed users of lower-priority
backends onto Exa's anonymous tier. Keyless availability is a
separate, last-resort signal (:meth:`is_keyless_available`).
"""
from agent.web_search_provider import get_provider_env
return bool(get_provider_env("EXA_API_KEY"))
def is_keyless_available(self) -> bool:
"""Exa serves anonymous free-tier calls via its public MCP endpoint.
False when the user forced ``web.provider_tier.exa: paid`` — an
explicit paid selection must never silently resolve keyless.
"""
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return keyless_enabled() and provider_tier("exa") != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return True
NAME = "exa"
DISPLAY_NAME = "Exa"
KEY_ENV = "EXA_API_KEY"
EXTRACT = True
KEYLESS = True
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute an Exa search.
def _body() -> Dict[str, Any]:
from plugins.web.keyless_mcp import search_with_failover
Returns ``{"success": True, "data": {"web": [{...}, ...]}}`` on
success, ``{"success": False, "error": str}`` on failure (incl.
missing API key and SDK install errors).
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return {"success": False, "error": "Interrupted"}
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import search_with_failover, use_keyless
if use_keyless("exa", get_provider_env("EXA_API_KEY")):
# Keyless free tier — public MCP endpoint, no SDK needed.
logger.info(
"Exa keyless search: '%s' (limit=%d)", query, limit
)
if use_keyless("exa", provider_env("EXA_API_KEY")):
logger.info("Exa keyless search: '%s' (limit=%d)", query, limit)
return search_with_failover("exa", query, limit)
logger.info("Exa search: '%s' (limit=%d)", query, limit)
response = _get_exa_client().search(
query,
num_results=limit,
contents={"highlights": True},
)
response = _get_exa_client().search(query, num_results=limit, contents={"highlights": True})
return search_ok([
web_hit(r.url or "", r.title or "", " ".join(r.highlights or []), i + 1)
for i, r in enumerate(response.results or [])
])
web_results = []
for i, result in enumerate(response.results or []):
highlights = result.highlights or []
web_results.append(
{
"url": result.url or "",
"title": result.title or "",
"description": " ".join(highlights) if highlights else "",
"position": i + 1,
}
)
return {"success": True, "data": {"web": web_results}}
except ValueError as exc:
# Raised by _get_exa_client when EXA_API_KEY missing
return {"success": False, "error": str(exc)}
except ImportError as exc:
return {"success": False, "error": f"Exa SDK not installed: {exc}"}
except Exception as exc: # noqa: BLE001 — surface as failure
logger.warning("Exa search error: %s", exc)
return {"success": False, "error": f"Exa search failed: {exc}"}
return run_search("Exa", logger, _body, sdk=True)
def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
"""Extract content from one or more URLs via Exa.
def _body() -> List[Dict[str, Any]]:
from plugins.web.keyless_mcp import extract_with_failover
Returns a list of result dicts shaped for the legacy LLM
post-processing pipeline. On per-URL or whole-batch failure,
results carry an ``error`` field rather than raising.
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return [
{"url": u, "error": "Interrupted", "title": ""} for u in urls
]
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import extract_with_failover, use_keyless
if use_keyless("exa", get_provider_env("EXA_API_KEY")):
# Keyless free tier — public MCP endpoint, no SDK needed.
if use_keyless("exa", provider_env("EXA_API_KEY")):
logger.info("Exa keyless extract: %d URL(s)", len(urls))
return extract_with_failover("exa", list(urls))
logger.info("Exa extract: %d URL(s)", len(urls))
response = _get_exa_client().get_contents(urls, text=True)
return [document(r.url or "", r.title or "", r.text or "") for r in response.results or []]
results: List[Dict[str, Any]] = []
for result in response.results or []:
content = result.text or ""
url = result.url or ""
title = result.title or ""
results.append(
{
"url": url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url, "title": title},
}
)
return results
except ValueError as exc:
return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
except ImportError as exc:
return [
{"url": u, "title": "", "content": "", "error": f"Exa SDK not installed: {exc}"}
for u in urls
]
except Exception as exc: # noqa: BLE001
logger.warning("Exa extract error: %s", exc)
return [
{"url": u, "title": "", "content": "", "error": f"Exa extract failed: {exc}"}
for u in urls
]
return run_extract("Exa", logger, urls, _body, sdk=True)
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Exa · Free (keyless)",
"badge": "free · no key",
"tag": (
"Semantic + neural web search with content extraction on "
"Exa's anonymous free tier. Rate-limited under burst load."
),
"env_vars": [],
"web_tier": "free",
"variants": [
{
"name": "Exa · Paid (API key)",
"badge": "paid",
"tag": (
"Semantic + neural web search with content extraction "
"via the Exa SDK. Unthrottled, guaranteed service."
),
"env_vars": [
{
"key": "EXA_API_KEY",
"prompt": "Exa API key",
"url": "https://exa.ai",
},
],
"web_tier": "paid",
},
],
}
return keyless_variant_schema(
"Exa", "EXA_API_KEY", "https://exa.ai",
free_tag="Semantic + neural web search with content extraction on Exa's anonymous free tier. Rate-limited under burst load.",
paid_tag="Semantic + neural web search with content extraction via the Exa SDK. Unthrottled, guaranteed service.",
)
plugins/web/firecrawl/__init__.py
+1 -20
View File
@@ -1,22 +1,4 @@
"""Firecrawl web search + extract plugin — bundled, auto-loaded.
Largest single plugin in this PR. Captures everything the previous
inline implementation in tools/web_tools.py did:
- Lazy import of the firecrawl SDK (~200ms cold-start cost) via a
callable proxy that defers the actual import to first use.
- Dual client paths: direct (FIRECRAWL_API_KEY / FIRECRAWL_API_URL)
OR Nous-hosted tool-gateway routing for subscribers, with
web.use_gateway as the tie-breaker.
- Per-URL scrape loop with 60s timeout, SSRF re-check after redirect,
website-policy gating, and format-aware content selection.
- Robust response shape normalization across SDK / direct API /
gateway variants (search returns differ by transport).
The plugin re-exports ``Firecrawl`` (the lazy proxy) and
``check_firecrawl_api_key`` for backward-compatibility with tests and
external code that imports those names from ``tools.web_tools``.
"""
"""Firecrawl web search + extract plugin — bundled, auto-loaded."""
from __future__ import annotations
@@ -24,5 +6,4 @@ from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider
def register(ctx) -> None:
"""Register the Firecrawl provider with the plugin context."""
ctx.register_web_search_provider(FirecrawlWebSearchProvider())
plugins/web/firecrawl/provider.py
+189 -495
View File
@@ -1,85 +1,37 @@
"""Firecrawl web search + extract — plugin form.
"""Firecrawl web search + extract provider (direct SDK, keyless cloud, or Nous tool-gateway).
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. This is
the largest provider migrated in this PR; it captures the full inline
firecrawl implementation that previously lived in tools/web_tools.py:
- :data:`Firecrawl` lazy proxy that defers the ~200ms SDK import to
first use (re-exported by tools.web_tools for backward compat with
existing tests that mock that name).
- :func:`_get_firecrawl_client` with direct + managed-gateway dual
mode, controlled by ``web.use_gateway`` config when both are
configured.
- :func:`check_firecrawl_api_key` re-exported (tests + tools_config
setup hint depend on this name living in tools.web_tools).
- :func:`_extract_web_search_results` / :func:`_extract_scrape_payload`
response-shape normalizers that handle SDK / direct API / gateway
response variants.
- Per-URL extract loop with 60s timeout, redirect-aware SSRF re-check,
website-policy gating, and format-aware content selection.
Async note: the underlying SDK is sync. ``extract()`` is declared
``async def`` because it performs per-URL I/O that benefits from
running in an executor; the implementation wraps each scrape in
:func:`asyncio.to_thread` with :func:`asyncio.wait_for(timeout=60)` to
guard against hung fetches.
Config keys this provider responds to::
web:
search_backend: "firecrawl" # explicit per-capability
extract_backend: "firecrawl" # explicit per-capability
backend: "firecrawl" # shared fallback (default)
use_gateway: false # prefer managed gateway when both
# direct + gateway credentials exist
Env vars::
FIRECRAWL_API_KEY=... # direct cloud auth
FIRECRAWL_API_URL=... # self-hosted Firecrawl
FIRECRAWL_GATEWAY_URL=... # Nous tool-gateway (subscribers)
TOOL_GATEWAY_DOMAIN=... # alternate gateway env
TOOL_GATEWAY_SCHEME=...
TOOL_GATEWAY_USER_TOKEN=...
Config: ``web.backend`` / ``web.search_backend`` / ``web.extract_backend: firecrawl``.
Env: FIRECRAWL_API_KEY, FIRECRAWL_API_URL (self-hosted), FIRECRAWL_GATEWAY_URL / TOOL_GATEWAY_*.
"""
from __future__ import annotations
import asyncio
import logging
import os
from typing import Any, Dict, List, NoReturn, Optional, TYPE_CHECKING
from typing import Any, Dict, List, Optional
import httpx
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import BaseWebSearchProvider, search_fail, search_ok
from tools.url_safety import is_safe_url
# Module-level (cheap import) so tests can monkeypatch the policy gate on this module.
from tools.website_policy import check_website_access
logger = logging.getLogger(__name__)
_FIRECRAWL_CLOUD_API_URL = "https://api.firecrawl.dev"
_SELECTION_KEYS = ("backend", "search_backend", "extract_backend")
# ---------------------------------------------------------------------------
# Lazy Firecrawl SDK proxy
# ---------------------------------------------------------------------------
# The firecrawl SDK pulls ~200ms of imports (httpcore, firecrawl.v1/v2 type
# trees) on a cold CLI. We only need it when the backend is actually
# "firecrawl", so defer the import to first use via a callable proxy.
#
# Tests that do ``patch("tools.web_tools.Firecrawl", ...)`` continue to
# work because tools/web_tools.py re-exports ``Firecrawl`` from this
# module — so the patched name still references the same proxy instance.
if TYPE_CHECKING:
from firecrawl import Firecrawl as FirecrawlSDK # noqa: F401 — type hints only
# --- Lazy Firecrawl SDK proxy -------------------------------------------------
# The SDK costs ~200ms of imports on a cold CLI; defer to first use. tools.web_tools
# re-exports ``Firecrawl`` so ``patch("tools.web_tools.Firecrawl")`` keeps working.
_FIRECRAWL_CLS_CACHE: Optional[type] = None
def _load_firecrawl_cls() -> type:
"""Import and cache ``firecrawl.Firecrawl``."""
"""Import and cache ``firecrawl.Firecrawl`` (lazy_deps install hint → ImportError)."""
global _FIRECRAWL_CLS_CACHE
if _FIRECRAWL_CLS_CACHE is None:
try:
@@ -90,7 +42,7 @@ def _load_firecrawl_cls() -> type:
pass
except Exception as exc: # noqa: BLE001 — surface install hint
raise ImportError(str(exc))
from firecrawl import Firecrawl as _cls # noqa: WPS433 — deliberately lazy
from firecrawl import Firecrawl as _cls
_FIRECRAWL_CLS_CACHE = _cls
return _FIRECRAWL_CLS_CACHE
@@ -114,72 +66,53 @@ class _FirecrawlProxy:
Firecrawl = _FirecrawlProxy()
# ---------------------------------------------------------------------------
# Client construction (direct vs managed-gateway)
# ---------------------------------------------------------------------------
#
# The canonical cache slots live on :mod:`tools.web_tools` so tests that do
# ``tools.web_tools._firecrawl_client = None`` between cases see fresh
# state. The plugin reads/writes through that public module — see
# :func:`_get_firecrawl_client` below.
# --- Client construction (direct vs managed-gateway) ---------------------------
# Client cache slots and gateway/token helpers are read through tools.web_tools so
# tests that reset ``tools.web_tools._firecrawl_client`` or patch
# ``tools.web_tools._peek_nous_access_token`` see their changes.
def _env(name: str) -> str:
from hermes_cli.config import get_env_value
return (get_env_value(name) or "").strip()
def _get_direct_firecrawl_config() -> Optional[tuple]:
"""Return direct Firecrawl (mode, kwargs, cache key), or None when unavailable.
"""Return direct Firecrawl ``(mode, kwargs, cache_key)`` or None.
``mode`` is ``"sdk"`` (keyed / self-hosted via the Firecrawl SDK) or
``"keyless"`` (explicit Firecrawl selection with no credentials — served
by :class:`_KeylessFirecrawlClient` against the public cloud API, which
accepts anonymous rate-limited requests). Keyless requires the explicit
selection so an unconfigured install never silently routes to it.
``mode`` is ``"sdk"`` (keyed / self-hosted) or ``"keyless"`` (explicit
Firecrawl selection with no credentials — public cloud API, anonymous
rate-limited). Keyless requires the explicit selection so an unconfigured
install never silently routes to it.
"""
from hermes_cli.config import get_env_value
api_key = (get_env_value("FIRECRAWL_API_KEY") or "").strip()
api_url = (get_env_value("FIRECRAWL_API_URL") or "").strip().rstrip("/")
api_key = _env("FIRECRAWL_API_KEY")
api_url = _env("FIRECRAWL_API_URL").rstrip("/")
if not api_key and not api_url:
if _is_explicit_firecrawl_selection():
return (
"keyless",
{"api_url": _FIRECRAWL_CLOUD_API_URL},
("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None),
)
return "keyless", {"api_url": _FIRECRAWL_CLOUD_API_URL}, ("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None)
return None
kwargs: Dict[str, str] = {}
if api_key:
kwargs["api_key"] = api_key
if api_url:
kwargs["api_url"] = api_url
kwargs = {k: v for k, v in (("api_key", api_key), ("api_url", api_url)) if v}
return "sdk", kwargs, ("direct", api_url or None, api_key or None)
def _is_explicit_firecrawl_selection() -> bool:
"""Return True when config explicitly selects Firecrawl for web tools."""
"""True when config explicitly selects Firecrawl for web tools."""
import tools.web_tools as _wt
cfg = _wt._load_web_config()
return any(
(cfg.get(key) or "").lower().strip() == "firecrawl"
for key in ("backend", "search_backend", "extract_backend")
)
return any((cfg.get(key) or "").lower().strip() == "firecrawl" for key in _SELECTION_KEYS)
def _use_keyless_ring() -> bool:
"""True when Firecrawl calls should route via the keyless ring.
Ring dispatch applies when there are no direct credentials, the
managed Nous gateway isn't the selected path, and the keyless tier
isn't disabled or pinned paid. Keyed/self-hosted/gateway setups never
reach the ring.
Only when there are no direct credentials, the managed Nous gateway isn't
the selected path, and the keyless tier isn't disabled or pinned paid.
"""
from hermes_cli.config import get_env_value
if (get_env_value("FIRECRAWL_API_KEY") or "").strip():
return False
if (get_env_value("FIRECRAWL_API_URL") or "").strip():
if _env("FIRECRAWL_API_KEY") or _env("FIRECRAWL_API_URL"):
return False
import tools.web_tools as _wt
from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection
@@ -202,9 +135,7 @@ def _use_keyless_ring() -> bool:
class _KeylessFirecrawlClient:
"""Minimal REST client for Firecrawl's keyless cloud mode.
Duck-types the two SDK methods the provider calls (``search`` /
``scrape``) so the rest of the pipeline (result normalizers, caching)
is unchanged. No Authorization header is ever sent.
Duck-types the SDK's ``search`` / ``scrape``; never sends an Authorization header.
"""
def __init__(self, api_url: str = _FIRECRAWL_CLOUD_API_URL):
@@ -235,44 +166,24 @@ def _get_firecrawl_gateway_url() -> str:
def _is_tool_gateway_ready() -> bool:
"""Return True when gateway URL + Nous Subscriber token are available.
Reads ``peek_nous_access_token`` and ``resolve_managed_tool_gateway``
via :mod:`tools.web_tools` rather than direct imports, so unit tests
that ``patch("tools.web_tools._peek_nous_access_token", ...)`` see
their patches honored. The names are re-exported on
:mod:`tools.web_tools` for exactly this reason.
"""
"""True when gateway URL + Nous Subscriber token are available."""
import tools.web_tools as _wt
return _wt.resolve_managed_tool_gateway(
"firecrawl", token_reader=_wt._peek_nous_access_token
) is not None
def _has_direct_firecrawl_config() -> bool:
"""Return True when direct Firecrawl config is explicitly configured."""
return _get_direct_firecrawl_config() is not None
return _wt.resolve_managed_tool_gateway("firecrawl", token_reader=_wt._peek_nous_access_token) is not None
def check_firecrawl_api_key() -> bool:
"""Return True when the Firecrawl backend selected via `hermes tools`
(or, on a never-configured install, either route) is usable.
Re-exported by :mod:`tools.web_tools` for backward compatibility with
existing tests and the ``hermes tools`` setup flow.
"""
from tools.tool_backend_helpers import (
NOUS_MANAGED_PROVIDER,
read_selection,
)
"""True when the Firecrawl route selected via ``hermes tools`` (or, on a
never-configured install, either route) is usable. Re-exported by tools.web_tools."""
from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection
selected = read_selection("web")
if selected == NOUS_MANAGED_PROVIDER:
return _is_tool_gateway_ready()
has_direct = _get_direct_firecrawl_config() is not None
if selected is not None:
return _has_direct_firecrawl_config()
return _has_direct_firecrawl_config() or _is_tool_gateway_ready()
return has_direct
return has_direct or _is_tool_gateway_ready()
def _firecrawl_backend_help_suffix() -> str:
@@ -281,55 +192,16 @@ def _firecrawl_backend_help_suffix() -> str:
if not _wt.managed_nous_tools_enabled():
return ""
return (
", or use the Nous Tool Gateway via your subscription "
"(FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)"
)
def _raise_web_backend_configuration_error() -> "NoReturn":
"""Raise a clear error for unsupported web backend configuration."""
import tools.web_tools as _wt
message = (
"Web tools are not configured. "
"Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL "
"for a self-hosted Firecrawl instance."
)
if _wt.managed_nous_tools_enabled():
message += (
" With your Nous subscription you can also use the Tool Gateway. "
"run `hermes tools` and select Nous Subscription as the web provider."
)
else:
message += " " + _wt.nous_tool_gateway_unavailable_message(
"managed Firecrawl web tools",
)
raise ValueError(message)
return ", or use the Nous Tool Gateway via your subscription (FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)"
def _get_firecrawl_client() -> Any:
"""Get or create the cached Firecrawl client.
Strict selection semantics (switch on the stored ``web`` selection):
- ``"nous"`` (or legacy ``use_gateway: true``) → managed Tool Gateway
ONLY; unavailable is a selection-naming error (a present
FIRECRAWL_API_KEY does not reroute).
- any other stored web backend → direct Firecrawl ONLY; missing config
is a selection-naming error — never a silent managed fallback billed
to Nous.
- never-configured web section → legacy behavior: direct config when
present, else the managed gateway.
Raises ValueError when the resolved path is unusable.
The cached client is stored on :mod:`tools.web_tools` (as
``_firecrawl_client`` and ``_firecrawl_client_config``) rather than on
this plugin module so that unit tests that reset the cache via
``tools.web_tools._firecrawl_client = None`` keep working. Helper
functions (``resolve_managed_tool_gateway``, ``_read_nous_access_token``,
``Firecrawl``) are also looked up via :mod:`tools.web_tools` for the same
reason — see :func:`_is_tool_gateway_ready`.
Strict selection semantics on the stored ``web`` selection: ``"nous"`` →
managed Tool Gateway ONLY; any other stored backend → direct Firecrawl ONLY
(never a silent managed fallback billed to Nous); never-configured → direct
when present, else managed. Raises ValueError when the resolved path is unusable.
"""
import tools.web_tools as _wt
from tools.tool_backend_helpers import (
@@ -340,25 +212,16 @@ def _get_firecrawl_client() -> Any:
)
selected = read_selection("web")
direct_config = _get_direct_firecrawl_config()
def _managed_kwargs():
managed_gateway = _wt.resolve_managed_tool_gateway(
"firecrawl", token_reader=_wt._read_nous_access_token
)
if managed_gateway is None:
gw = _wt.resolve_managed_tool_gateway("firecrawl", token_reader=_wt._read_nous_access_token)
if gw is None:
return None
kwargs = {
"api_key": managed_gateway.nous_user_token,
"api_url": managed_gateway.gateway_origin,
}
return kwargs, (
"tool-gateway",
kwargs["api_url"],
managed_gateway.nous_user_token,
)
kwargs = {"api_key": gw.nous_user_token, "api_url": gw.gateway_origin}
return kwargs, ("tool-gateway", kwargs["api_url"], gw.nous_user_token)
client_mode = "sdk"
if selected == NOUS_MANAGED_PROVIDER:
managed = _managed_kwargs()
if managed is None:
@@ -368,16 +231,12 @@ def _get_firecrawl_client() -> Any:
"is unavailable."
)
raise ValueError(selection_error(
"web",
NOUS_MANAGED_PROVIDER,
"the Nous Tool Gateway is not available (not entitled or "
"unreachable)",
"web", NOUS_MANAGED_PROVIDER,
"the Nous Tool Gateway is not available (not entitled or unreachable)",
))
kwargs, client_config = managed
client_mode = "sdk"
elif selected is not None or selection_exists("web"):
# Stored vendor selection (or per-capability web keys routing to
# firecrawl): direct Firecrawl only. With no credentials, the
# Stored vendor selection: direct Firecrawl only. With no credentials the
# explicit selection unlocks keyless cloud mode instead of erroring.
if direct_config is None:
logger.error(
@@ -385,9 +244,7 @@ def _get_firecrawl_client() -> Any:
"selected but FIRECRAWL_API_KEY/FIRECRAWL_API_URL is not set."
)
raise ValueError(selection_error(
"web",
selected or "firecrawl",
"neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set",
"web", selected or "firecrawl", "neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set",
))
client_mode, kwargs, client_config = direct_config
elif direct_config is not None:
@@ -396,21 +253,26 @@ def _get_firecrawl_client() -> Any:
# Never-configured web section: legacy managed fallback.
managed = _managed_kwargs()
if managed is None:
logger.error(
"Firecrawl client initialization failed: "
"missing direct config and tool-gateway auth."
logger.error("Firecrawl client initialization failed: missing direct config and tool-gateway auth.")
message = (
"Web tools are not configured. "
"Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL "
"for a self-hosted Firecrawl instance."
)
_raise_web_backend_configuration_error()
if _wt.managed_nous_tools_enabled():
message += (
" With your Nous subscription you can also use the Tool Gateway. "
"run `hermes tools` and select Nous Subscription as the web provider."
)
else:
message += " " + _wt.nous_tool_gateway_unavailable_message("managed Firecrawl web tools")
raise ValueError(message)
kwargs, client_config = managed
client_mode = "sdk"
cached = getattr(_wt, "_firecrawl_client", None)
cached_config = getattr(_wt, "_firecrawl_client_config", None)
if cached is not None and cached_config == client_config:
if cached is not None and getattr(_wt, "_firecrawl_client_config", None) == client_config:
return cached
# Construct via the re-exported Firecrawl proxy on tools.web_tools so
# unit tests patching ``tools.web_tools.Firecrawl`` see their mock.
if client_mode == "keyless":
_wt._firecrawl_client = _KeylessFirecrawlClient(api_url=kwargs["api_url"])
else:
@@ -419,43 +281,23 @@ def _get_firecrawl_client() -> Any:
return _wt._firecrawl_client
def _reset_client_for_tests() -> None:
"""Drop the cached Firecrawl client so tests can re-instantiate cleanly.
Clears the canonical slots on :mod:`tools.web_tools` (where
:func:`_get_firecrawl_client` reads/writes them).
"""
import tools.web_tools as _wt
_wt._firecrawl_client = None
_wt._firecrawl_client_config = None
# ---------------------------------------------------------------------------
# Response shape normalization (SDK / direct / gateway differ)
# ---------------------------------------------------------------------------
# --- Response shape normalization (SDK / direct / gateway differ) --------------
def _to_plain_object(value: Any) -> Any:
"""Convert SDK objects to plain python data structures when possible."""
if value is None:
return None
if isinstance(value, (dict, list, str, int, float, bool)):
"""Convert SDK objects (pydantic ``model_dump`` / ``__dict__``) to plain data when possible."""
if value is None or isinstance(value, (dict, list, str, int, float, bool)):
return value
if hasattr(value, "model_dump"):
try:
return value.model_dump()
except Exception: # noqa: BLE001
pass
if hasattr(value, "__dict__"):
try:
return {k: v for k, v in value.__dict__.items() if not k.startswith("_")}
except Exception: # noqa: BLE001
pass
return value
@@ -463,13 +305,8 @@ def _normalize_result_list(values: Any) -> List[Dict[str, Any]]:
"""Normalize mixed SDK/list payloads into a list of dicts."""
if not isinstance(values, list):
return []
normalized: List[Dict[str, Any]] = []
for item in values:
plain = _to_plain_object(item)
if isinstance(plain, dict):
normalized.append(plain)
return normalized
plain = (_to_plain_object(item) for item in values)
return [p for p in plain if isinstance(p, dict)]
def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]:
@@ -480,26 +317,17 @@ def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]:
data = response_plain.get("data")
if isinstance(data, list):
return _normalize_result_list(data)
candidates = []
if isinstance(data, dict):
data_web = _normalize_result_list(data.get("web"))
if data_web:
return data_web
data_results = _normalize_result_list(data.get("results"))
if data_results:
return data_results
top_web = _normalize_result_list(response_plain.get("web"))
if top_web:
return top_web
top_results = _normalize_result_list(response_plain.get("results"))
if top_results:
return top_results
candidates += [data.get("web"), data.get("results")]
candidates += [response_plain.get("web"), response_plain.get("results")]
for candidate in candidates:
normalized = _normalize_result_list(candidate)
if normalized:
return normalized
if hasattr(response, "web"):
return _normalize_result_list(getattr(response, "web", []))
return []
@@ -508,292 +336,158 @@ def _extract_scrape_payload(scrape_result: Any) -> Dict[str, Any]:
result_plain = _to_plain_object(scrape_result)
if not isinstance(result_plain, dict):
return {}
nested = result_plain.get("data")
if isinstance(nested, dict):
return nested
return result_plain
return nested if isinstance(nested, dict) else result_plain
# ---------------------------------------------------------------------------
# Provider class
# ---------------------------------------------------------------------------
def _error_entry(
url: str,
error: str,
*,
title: str = "",
raw: bool = False,
blocked: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Per-URL extract failure item. ``raw`` adds the ``raw_content`` key (post-scrape
failures carry it, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``."""
entry: Dict[str, Any] = {"url": url, "title": title, "content": ""}
if raw:
entry["raw_content"] = ""
entry["error"] = error
if blocked:
entry["blocked_by_policy"] = {k: blocked[k] for k in ("host", "rule", "source")}
return entry
class FirecrawlWebSearchProvider(WebSearchProvider):
_SCRAPE_TIMEOUT_MSG = (
"Scrape timed out after 60s — page may be too large "
"or unresponsive. Try browser_navigate instead."
)
_UNSAFE_REDIRECT_MSG = "Blocked: URL targets a private or internal network address"
async def _scrape_one(url: str, formats: List[str], format: Optional[str]) -> Dict[str, Any]:
"""Scrape one URL (60s timeout) and re-check SSRF + website policy against the
post-redirect URL. Never raises for scrape errors; returns an error entry instead."""
blocked = check_website_access(url)
if blocked:
logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"])
return _error_entry(url, blocked["message"], blocked=blocked)
try:
logger.info("Firecrawl scraping: %s", url)
try:
scrape_result = await asyncio.wait_for(
asyncio.to_thread(_get_firecrawl_client().scrape, url=url, formats=formats),
timeout=60,
)
except asyncio.TimeoutError:
logger.warning("Firecrawl scrape timed out for %s", url)
return _error_entry(url, _SCRAPE_TIMEOUT_MSG)
scrape_payload = _extract_scrape_payload(scrape_result)
metadata = scrape_payload.get("metadata", {})
content_markdown = scrape_payload.get("markdown")
content_html = scrape_payload.get("html")
# SDK may return a typed object for metadata (raw __dict__ here, unlike _to_plain_object).
if not isinstance(metadata, dict):
if hasattr(metadata, "model_dump"):
metadata = metadata.model_dump()
elif hasattr(metadata, "__dict__"):
metadata = metadata.__dict__
else:
metadata = {}
title = metadata.get("title", "")
final_url = metadata.get("sourceURL", url)
if not is_safe_url(final_url):
logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)
return _error_entry(final_url, _UNSAFE_REDIRECT_MSG, title=title, raw=True)
final_blocked = check_website_access(final_url)
if final_blocked:
logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"])
return _error_entry(final_url, final_blocked["message"], title=title, raw=True, blocked=final_blocked)
if format == "markdown" or (format is None and content_markdown):
chosen_content = content_markdown
else:
chosen_content = content_html or content_markdown or ""
return {"url": final_url, "title": title, "content": chosen_content, "raw_content": chosen_content, "metadata": metadata}
except Exception as scrape_err: # noqa: BLE001
logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
return _error_entry(url, str(scrape_err), raw=True)
# --- Provider class ------------------------------------------------------------
class FirecrawlWebSearchProvider(BaseWebSearchProvider):
"""Firecrawl search + extract provider with dual auth paths."""
@property
def name(self) -> str:
return "firecrawl"
@property
def display_name(self) -> str:
return "Firecrawl"
NAME = "firecrawl"
DISPLAY_NAME = "Firecrawl"
EXTRACT = True
KEYLESS = True # default-on ring member unless pinned ``paid``
def is_available(self) -> bool:
"""Return True when direct Firecrawl OR managed-gateway path is configured."""
return check_firecrawl_api_key()
def is_keyless_available(self) -> bool:
"""Firecrawl serves keyless cloud requests (public API, no auth).
Default-on ring member of the keyless free tier: fresh installs
rotate across Exa/Parallel/Firecrawl/Keenable. False when
the user pinned ``web.provider_tier.firecrawl: paid``.
"""
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return keyless_enabled() and provider_tier("firecrawl") != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return True
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a Firecrawl search.
Sync; matches the legacy ``_get_firecrawl_client().search(...)``
call directly. Normalizes the response across SDK/direct/gateway
shapes via :func:`_extract_web_search_results`.
Pre-flight errors (``ValueError`` from configuration check,
``ImportError`` from missing SDK) propagate to the dispatcher's
top-level handler, which wraps them as ``tool_error(...)`` —
matching the legacy ``{"error": "Error searching web: ..."}``
envelope. Only in-flight errors are caught and surfaced as
``{"success": False, "error": ...}``.
"""
"""Sync search. Pre-flight errors (ValueError / ImportError) propagate so the
dispatcher emits the legacy ``tool_error`` envelope; in-flight errors are
returned as ``{"success": False, "error": ...}``."""
from tools.interrupt import is_interrupted
if is_interrupted():
return {"success": False, "error": "Interrupted"}
return search_fail("Interrupted")
if _use_keyless_ring():
# No credentials and no managed gateway: ring dispatch with
# next-in-line failover on rate limits (default-on free tier).
from plugins.web.keyless_mcp import search_with_failover
logger.info(
"Firecrawl keyless search: '%s' (limit=%d)", query, limit
)
logger.info("Firecrawl keyless search: '%s' (limit=%d)", query, limit)
return search_with_failover("firecrawl", query, limit)
logger.info("Firecrawl search: '%s' (limit=%d)", query, limit)
# _get_firecrawl_client() raises ValueError on unconfigured systems —
# let it propagate so the dispatcher emits the legacy envelope shape.
client = _get_firecrawl_client()
try:
response = client.search(query=query, limit=limit)
web_results = _extract_web_search_results(response)
logger.info("Firecrawl: found %d search results", len(web_results))
return {"success": True, "data": {"web": web_results}}
return search_ok(web_results)
except Exception as exc: # noqa: BLE001
logger.warning("Firecrawl search error: %s", exc)
return {"success": False, "error": f"Firecrawl search failed: {exc}"}
return search_fail(f"Firecrawl search failed: {exc}")
async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
"""Extract content from one or more URLs via Firecrawl.
Async; each URL is scraped in a background thread with a 60s
timeout. After scraping, the final URL (post-redirect) is
re-checked against website-access policy.
Accepted kwargs (others ignored for forward compat):
- ``format``: ``"markdown"`` or ``"html"``; default is both
(request both, return markdown when available).
Returns the legacy per-URL list-of-results shape. Per-URL failures
(timeout, SSRF block, scrape error, policy block) become items
with an ``error`` field rather than raising.
"""
"""Per-URL scrape via :func:`_scrape_one`; failures become items with an
``error`` field. ``format``: "markdown" | "html" | both (markdown preferred)."""
from tools.interrupt import is_interrupted as _is_interrupted
if _is_interrupted():
return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
if _use_keyless_ring():
# No credentials and no managed gateway: ring dispatch with
# next-in-line failover on rate limits (default-on free tier).
import asyncio as _asyncio
from plugins.web.keyless_mcp import extract_with_failover
logger.info("Firecrawl keyless extract: %d URL(s)", len(urls))
return await _asyncio.to_thread(
extract_with_failover, "firecrawl", list(urls)
)
return await asyncio.to_thread(extract_with_failover, "firecrawl", list(urls))
format = kwargs.get("format")
formats: List[str] = []
if format == "markdown":
formats = ["markdown"]
elif format == "html":
formats = ["html"]
else:
formats = ["markdown", "html"]
formats = [format] if format in ("markdown", "html") else ["markdown", "html"]
# check_website_access is the legacy policy gate; imported at
# module level (lazy-friendly because the website_policy import is
# cheap) so monkeypatching it in tests works as expected.
results: List[Dict[str, Any]] = []
for url in urls:
if _is_interrupted():
results.append({"url": url, "error": "Interrupted", "title": ""})
continue
# Pre-scrape website policy gate
blocked = check_website_access(url)
if blocked:
logger.info(
"Blocked web_extract for %s by rule %s",
blocked["host"],
blocked["rule"],
)
results.append(
{
"url": url,
"title": "",
"content": "",
"error": blocked["message"],
"blocked_by_policy": {
"host": blocked["host"],
"rule": blocked["rule"],
"source": blocked["source"],
},
}
)
continue
try:
logger.info("Firecrawl scraping: %s", url)
try:
scrape_result = await asyncio.wait_for(
asyncio.to_thread(
_get_firecrawl_client().scrape,
url=url,
formats=formats,
),
timeout=60,
)
except asyncio.TimeoutError:
logger.warning("Firecrawl scrape timed out for %s", url)
results.append(
{
"url": url,
"title": "",
"content": "",
"error": (
"Scrape timed out after 60s — page may be too large "
"or unresponsive. Try browser_navigate instead."
),
}
)
continue
scrape_payload = _extract_scrape_payload(scrape_result)
metadata = scrape_payload.get("metadata", {})
content_markdown = scrape_payload.get("markdown")
content_html = scrape_payload.get("html")
# Ensure metadata is a dict (SDK may return a typed object)
if not isinstance(metadata, dict):
if hasattr(metadata, "model_dump"):
metadata = metadata.model_dump()
elif hasattr(metadata, "__dict__"):
metadata = metadata.__dict__
else:
metadata = {}
title = metadata.get("title", "")
final_url = metadata.get("sourceURL", url)
# Re-check SSRF safety after any redirect reported by Firecrawl.
if not is_safe_url(final_url):
logger.info(
"Blocked redirected web_extract for unsafe final URL: %s",
final_url,
)
results.append(
{
"url": final_url,
"title": title,
"content": "",
"raw_content": "",
"error": (
"Blocked: URL targets a private or internal "
"network address"
),
}
)
continue
# Re-check website-access policy after any redirect
final_blocked = check_website_access(final_url)
if final_blocked:
logger.info(
"Blocked redirected web_extract for %s by rule %s",
final_blocked["host"],
final_blocked["rule"],
)
results.append(
{
"url": final_url,
"title": title,
"content": "",
"raw_content": "",
"error": final_blocked["message"],
"blocked_by_policy": {
"host": final_blocked["host"],
"rule": final_blocked["rule"],
"source": final_blocked["source"],
},
}
)
continue
# Choose markdown vs html according to the requested format
if format == "markdown" or (format is None and content_markdown):
chosen_content = content_markdown
else:
chosen_content = content_html or content_markdown or ""
results.append(
{
"url": final_url,
"title": title,
"content": chosen_content,
"raw_content": chosen_content,
"metadata": metadata,
}
)
except Exception as scrape_err: # noqa: BLE001
logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
results.append(
{
"url": url,
"title": "",
"content": "",
"raw_content": "",
"error": str(scrape_err),
}
)
return results
return [
{"url": url, "error": "Interrupted", "title": ""} if _is_interrupted() else await _scrape_one(url, formats, format)
for url in urls
]
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Firecrawl",
"badge": "keyless/paid · optional gateway",
"tag": (
"Full search + extract; supports keyless cloud, direct API, "
"and Nous tool-gateway routing."
),
"tag": "Full search + extract; supports keyless cloud, direct API, and Nous tool-gateway routing.",
"env_vars": [
{
"key": "FIRECRAWL_API_KEY",
plugins/web/keenable/__init__.py
+1 -5
View File
@@ -1,7 +1,4 @@
"""Keenable web search + extract plugin — bundled, auto-loaded.
Keyless-ring member (keyed via KEENABLE_API_KEY for higher limits).
"""
"""Keenable web search + extract plugin — bundled, auto-loaded; keyless-ring member."""
from __future__ import annotations
@@ -9,5 +6,4 @@ from plugins.web.keenable.provider import KeenableWebSearchProvider
def register(ctx) -> None:
"""Register the Keenable provider with the plugin context."""
ctx.register_web_search_provider(KeenableWebSearchProvider())
plugins/web/keenable/provider.py
+49 -171
View File
@@ -1,27 +1,7 @@
"""Keenable web search + content extraction — bundled plugin.
"""Keenable (https://keenable.ai) web search + fetch — keyless-ring member.
Keenable (https://keenable.ai) operates an independent web index for AI
apps with public keyless endpoints (rate-limited free tier; keyed access
via KEENABLE_API_KEY for higher limits). Integrated as a keyless-ring
member following the Exa/Parallel/Firecrawl pattern: fresh installs with
zero web credentials rotate across the ring vendors' free tiers.
Credit: Keenable integration originally proposed by Ilya Gusev (Keenable)
in PR #49758; the native provider form follows the salvage of that work
plus the keyless-ring design.
Config keys this provider responds to::
web:
search_backend: "keenable" # explicit per-capability
extract_backend: "keenable" # explicit per-capability
backend: "keenable" # shared fallback
provider_tier:
keenable: free|paid # pin the tier (unset = auto)
Env var::
KEENABLE_API_KEY=... # optional — keyless free tier works without it
Env: ``KEENABLE_API_KEY`` (optional; keyless free tier works without it).
Config: ``web.provider_tier.keenable: free|paid`` pins the tier (unset = auto).
"""
from __future__ import annotations
@@ -29,7 +9,20 @@ from __future__ import annotations
import logging
from typing import Any, Dict, List
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import (
SEARCH_LIMIT_CAP,
BaseWebSearchProvider,
document,
http_status_detail,
keyless_variant_schema,
provider_env,
run_extract,
run_search,
search_fail,
search_ok,
use_keyless,
web_hit,
)
logger = logging.getLogger(__name__)
@@ -37,67 +30,29 @@ _KEENABLE_API_URL = "https://api.keenable.ai"
def _keenable_headers(api_key: str) -> Dict[str, str]:
"""Build Keenable request headers for keyed or keyless access.
Their keyless tier structurally requires an app-identifier header
(X-Keenable-Title); no user identifiers are sent.
"""
# The keyless tier structurally requires an app-identifier header; no user identifiers are sent.
headers = {"X-Keenable-Title": "hermes-agent"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
return headers
class KeenableWebSearchProvider(WebSearchProvider):
class KeenableWebSearchProvider(BaseWebSearchProvider):
"""Keenable search + extract provider (keyed or keyless)."""
@property
def name(self) -> str:
return "keenable"
@property
def display_name(self) -> str:
return "Keenable"
def is_available(self) -> bool:
"""Return True when ``KEENABLE_API_KEY`` is set to a non-empty value."""
from agent.web_search_provider import get_provider_env
return bool(get_provider_env("KEENABLE_API_KEY"))
def is_keyless_available(self) -> bool:
"""Keenable serves anonymous free-tier calls via its public endpoints.
Default-on ring member of the keyless free tier. False when the
user pinned ``web.provider_tier.keenable: paid``.
"""
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return keyless_enabled() and provider_tier("keenable") != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return True
NAME = "keenable"
DISPLAY_NAME = "Keenable"
KEY_ENV = "KEENABLE_API_KEY"
EXTRACT = True
KEYLESS = True
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a Keenable search (keyed path or keyless ring)."""
try:
from tools.interrupt import is_interrupted
def _body() -> Dict[str, Any]:
from plugins.web.keyless_mcp import search_with_failover
if is_interrupted():
return {"success": False, "error": "Interrupted"}
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import search_with_failover, use_keyless
api_key = get_provider_env("KEENABLE_API_KEY")
api_key = provider_env("KEENABLE_API_KEY")
if use_keyless("keenable", api_key):
logger.info(
"Keenable keyless search: '%s' (limit=%d)", query, limit
)
logger.info("Keenable keyless search: '%s' (limit=%d)", query, limit)
return search_with_failover("keenable", query, limit)
import requests
@@ -105,52 +60,24 @@ class KeenableWebSearchProvider(WebSearchProvider):
logger.info("Keenable search: '%s' (limit=%d)", query, limit)
response = requests.post(
f"{_KEENABLE_API_URL}/v1/search",
json={"query": query, "max_results": min(max(1, int(limit)), 20)},
json={"query": query, "max_results": min(max(1, int(limit)), SEARCH_LIMIT_CAP)},
headers=_keenable_headers(api_key),
timeout=30,
)
if response.status_code >= 400:
detail = (response.text or "").strip() or f"HTTP {response.status_code}"
return {"success": False, "error": f"Keenable search failed: {detail}"}
data = response.json()
return search_fail(f"Keenable search failed: {http_status_detail(response)}")
return search_ok([
web_hit(r.get("url") or "", r.get("title") or "", r.get("snippet") or r.get("description") or "", i + 1)
for i, r in enumerate(response.json().get("results") or [])
])
web_results = []
for i, result in enumerate(data.get("results") or []):
web_results.append(
{
"url": result.get("url") or "",
"title": result.get("title") or "",
"description": result.get("snippet")
or result.get("description")
or "",
"position": i + 1,
}
)
return {"success": True, "data": {"web": web_results}}
except Exception as exc: # noqa: BLE001 — surface as failure
logger.warning("Keenable search error: %s", exc)
return {"success": False, "error": f"Keenable search failed: {exc}"}
return run_search("Keenable", logger, _body, verbatim_value_error=False)
def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
"""Extract content via Keenable's fetch endpoint (per-URL).
def _body() -> List[Dict[str, Any]]:
from plugins.web.keyless_mcp import extract_with_failover
Sync — the dispatcher wraps in a thread when the caller is async.
Returns the legacy list-of-results shape; per-URL failures become
items with an ``error`` field.
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return [
{"url": u, "error": "Interrupted", "title": ""} for u in urls
]
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import extract_with_failover, use_keyless
api_key = get_provider_env("KEENABLE_API_KEY")
api_key = provider_env("KEENABLE_API_KEY")
if use_keyless("keenable", api_key):
logger.info("Keenable keyless extract: %d URL(s)", len(urls))
return extract_with_failover("keenable", list(urls))
@@ -162,72 +89,23 @@ class KeenableWebSearchProvider(WebSearchProvider):
for url in urls:
try:
response = requests.get(
f"{_KEENABLE_API_URL}/v1/fetch",
params={"url": url},
headers=_keenable_headers(api_key),
timeout=30,
f"{_KEENABLE_API_URL}/v1/fetch", params={"url": url}, headers=_keenable_headers(api_key), timeout=30
)
if response.status_code >= 400:
raise ValueError(
(response.text or "").strip()
or f"HTTP {response.status_code}"
)
raise ValueError(http_status_detail(response))
data = response.json()
content = data.get("content") or ""
title = data.get("title") or ""
results.append(
{
"url": data.get("url") or url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url, "title": title},
}
document(data.get("url") or url, data.get("title") or "", data.get("content") or "", source_url=url)
)
except Exception as exc: # noqa: BLE001 — per-URL error entry
results.append(
{
"url": url,
"title": "",
"content": "",
"error": f"Keenable extract failed: {exc}",
}
)
results.append({"url": url, "title": "", "content": "", "error": f"Keenable extract failed: {exc}"})
return results
except Exception as exc: # noqa: BLE001
logger.warning("Keenable extract error: %s", exc)
return [
{"url": u, "title": "", "content": "",
"error": f"Keenable extract failed: {exc}"}
for u in urls
]
return run_extract("Keenable", logger, urls, _body, verbatim_value_error=False)
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Keenable · Free (keyless)",
"badge": "free · no key",
"tag": (
"Independent web index for AI apps — fast search + page "
"fetch on Keenable's anonymous free tier."
),
"env_vars": [],
"web_tier": "free",
"variants": [
{
"name": "Keenable · Paid (API key)",
"badge": "paid",
"tag": (
"Independent web index for AI apps. Keyed access "
"with higher limits and guaranteed service."
),
"env_vars": [
{
"key": "KEENABLE_API_KEY",
"prompt": "Keenable API key",
"url": "https://keenable.ai",
},
],
"web_tier": "paid",
},
],
}
return keyless_variant_schema(
"Keenable", "KEENABLE_API_KEY", "https://keenable.ai",
free_tag="Independent web index for AI apps — fast search + page fetch on Keenable's anonymous free tier.",
paid_tag="Independent web index for AI apps. Keyed access with higher limits and guaranteed service.",
)
plugins/web/keyless_mcp.py
+181 -463
View File
@@ -1,42 +1,30 @@
"""Keyless web search/extract via public MCP endpoints.
"""Keyless web search/extract via public free-tier endpoints (Exa, Parallel, Firecrawl, Keenable).
Exa and Parallel both operate public, anonymous MCP endpoints with a free
tier (the same endpoints the opencode CLI ships as its default search
path):
- Exa: https://mcp.exa.ai/mcp (tools: web_search_exa, web_fetch_exa)
- Parallel: https://search.parallel.ai/mcp (tools: web_search, web_fetch)
This module implements a minimal JSON-RPC ``tools/call`` client for those
two endpoints so a fresh Hermes install with **zero web credentials** still
gets working ``web_search`` / ``web_extract`` tools. The keyless tier is
resolved strictly LAST — after every keyed backend, the managed tool
gateway, ddgs, and custom plugin providers — so it never pre-empts a
deliberate setup (see ``tools.web_tools._get_backend`` and the registry's
``_KEYLESS_PREFERENCE`` walk).
Privacy: requests carry no user identifiers. Parallel's free tier asks for
a ``session_id`` used for rate limiting; we send a random per-process UUID
(rotates every restart, never persisted). Their optional ``model_name``
analytics field is deliberately omitted.
Disable the whole tier with ``web.keyless_fallback: false`` in config.yaml.
Resolved strictly LAST — after every keyed backend, the managed gateway, ddgs and
custom plugin providers — so it never pre-empts a deliberate setup. Privacy: no user
identifiers are sent; Parallel gets a random per-process ``session_id`` (rate
limiting only) and its optional ``model_name`` analytics field is deliberately
omitted. Disable the tier with ``web.keyless_fallback: false``.
"""
from __future__ import annotations
import json
import logging
import threading
import uuid
from typing import Any, Dict, List, Optional
from typing import Any, Callable, Dict, List, Optional
from plugins.web._common import document as _page, search_fail, search_ok, web_hit as _row
logger = logging.getLogger(__name__)
EXA_MCP_URL = "https://mcp.exa.ai/mcp"
PARALLEL_MCP_URL = "https://search.parallel.ai/mcp"
KEENABLE_API_URL = "https://api.keenable.ai"
_KEENABLE_TITLE = "hermes-agent"
# Free-tier rate-limit correlation id for Parallel — random per process,
# never persisted, not derived from any user/machine identifier.
# Parallel free-tier rate-limit correlation id — random per process, never persisted.
_SESSION_ID = uuid.uuid4().hex
_TIMEOUT_SECONDS = 30
@@ -46,30 +34,38 @@ class KeylessMCPError(RuntimeError):
"""A keyless MCP call failed (transport, rate limit, or tool error)."""
_RATE_LIMIT_MARKERS = (
"rate limit",
"rate-limit",
"ratelimit",
"too many requests",
"429",
"quota exceeded",
"slow down",
)
_RATE_LIMIT_MARKERS = ("rate limit", "rate-limit", "ratelimit", "too many requests", "429", "quota exceeded", "slow down")
# vendor -> (display label, env key, signup URL) for the standard failure hint.
_VENDOR_HINTS = {
"exa": ("Exa", "EXA_API_KEY", "https://exa.ai"),
"parallel": ("Parallel", "PARALLEL_API_KEY", "https://parallel.ai"),
"firecrawl": ("Firecrawl", "FIRECRAWL_API_KEY", "https://firecrawl.dev"),
"keenable": ("Keenable", "KEENABLE_API_KEY", "https://keenable.ai"),
}
def _is_rate_limitish(message: str) -> bool:
"""Heuristic: does an error message look like free-tier throttling?"""
lowered = (message or "").lower()
return any(marker in lowered for marker in _RATE_LIMIT_MARKERS)
return any(marker in (message or "").lower() for marker in _RATE_LIMIT_MARKERS)
def _fail_msg(vendor: str, kind: str, exc: Any, *, other_backends: bool = True) -> str:
label, env_key, site = _VENDOR_HINTS[vendor]
alt = " or another web backend via `hermes tools`" if other_backends else ""
return f"Keyless {label} {kind} failed: {exc}. Set {env_key} ({site}){alt} for reliable service."
def _page_error(url: str, message: str) -> Dict[str, Any]:
return {"url": url, "title": "", "content": "", "error": message}
# --- Tier / config ------------------------------------------------------------
def keyless_enabled() -> bool:
"""Return True when the keyless fallback tier is enabled.
Delegates to :func:`agent.web_search_registry._keyless_tier_enabled` so
the config chokepoint (``web.keyless_fallback``, default on) lives in
one place alongside the rest of backend resolution.
"""
"""True when the keyless tier is enabled. Delegates to the registry so the
``web.keyless_fallback`` (default on) chokepoint lives with backend resolution."""
try:
from agent.web_search_registry import _keyless_tier_enabled
@@ -80,16 +76,8 @@ def keyless_enabled() -> bool:
def provider_tier(name: str) -> str:
"""Return the user-selected tier for *name*: ``free``, ``paid``, or ``auto``.
Reads ``web.provider_tier.<name>`` from config.yaml (set by the
``hermes tools`` picker's Free/Paid rows). ``free`` forces the keyless
public endpoint even when the vendor API key is present; ``paid``
forces the keyed SDK path (missing key surfaces the standard
"X_API_KEY not set" error instead of silently downgrading to the free
tier). Anything else — including unset — is ``auto``: key present →
keyed, otherwise keyless when the tier is enabled.
"""
"""Return ``web.provider_tier.<name>`` (set by the ``hermes tools`` Free/Paid rows):
``free``, ``paid``, or ``auto`` for anything else including unset."""
try:
from hermes_cli.config import load_config
@@ -103,17 +91,10 @@ def provider_tier(name: str) -> str:
def use_keyless(name: str, api_key: str) -> bool:
"""Decide whether provider *name* should route via the keyless endpoint.
Single chokepoint shared by the Exa/Parallel search + extract paths so
tier semantics can't drift between capabilities:
- tier ``free`` → keyless, even when *api_key* is set
- tier ``paid`` → keyed, even when *api_key* is missing (the keyed
path then raises its usual missing-key error)
- tier ``auto`` → keyed when *api_key* is set; otherwise keyless when
``web.keyless_fallback`` is enabled
"""
"""Single chokepoint for search + extract so tier semantics can't drift:
``free`` → keyless even with a key; ``paid`` → keyed even without one (the keyed
path raises its usual missing-key error); ``auto`` → keyless only when no key
and the tier is enabled."""
tier = provider_tier(name)
if tier == "free":
return True
@@ -122,13 +103,15 @@ def use_keyless(name: str, api_key: str) -> bool:
return not api_key and keyless_enabled()
# --- MCP transport ------------------------------------------------------------
def _parse_mcp_body(body: str) -> str:
"""Extract the first text content item from an MCP tools/call response.
Handles both plain-JSON bodies and SSE (``data: {...}`` lines) — the
Exa endpoint answers as an event stream, Parallel as direct JSON.
Raises :class:`KeylessMCPError` for JSON-RPC errors and ``isError``
tool results (e.g. Exa's free-tier rate-limit message).
Handles plain-JSON bodies (Parallel) and SSE ``data: {...}`` lines (Exa). Raises
:class:`KeylessMCPError` for JSON-RPC errors and ``isError`` tool results (e.g.
Exa's free-tier rate-limit message).
"""
def _from_payload(payload: str) -> Optional[str]:
@@ -143,9 +126,7 @@ def _parse_mcp_body(body: str) -> str:
content = result.get("content") or []
if result.get("isError"):
texts = [c.get("text", "") for c in content if isinstance(c, dict)]
raise KeylessMCPError(
" ".join(t for t in texts if t) or "MCP tool call failed"
)
raise KeylessMCPError(" ".join(t for t in texts if t) or "MCP tool call failed")
for item in content:
if isinstance(item, dict) and item.get("text"):
return str(item["text"])
@@ -173,25 +154,15 @@ def _parse_mcp_body(body: str) -> str:
raise KeylessMCPError("Unrecognized MCP response shape")
def mcp_call(
url: str,
tool: str,
arguments: Dict[str, Any],
timeout: int = _TIMEOUT_SECONDS,
) -> str:
def mcp_call(url: str, tool: str, arguments: Dict[str, Any], timeout: int = _TIMEOUT_SECONDS) -> str:
"""POST a JSON-RPC ``tools/call`` to *url* and return the text payload.
Raises :class:`KeylessMCPError` on transport failures, non-2xx
statuses, JSON-RPC errors, and error-shaped tool results.
Raises :class:`KeylessMCPError` on transport failures, non-2xx statuses,
JSON-RPC errors, and error-shaped tool results.
"""
import requests
payload = {
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {"name": tool, "arguments": arguments},
}
payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": arguments}}
headers = {
"Content-Type": "application/json",
"Accept": "application/json, text/event-stream",
@@ -202,146 +173,74 @@ def mcp_call(
except requests.RequestException as exc:
raise KeylessMCPError(f"request failed: {exc}") from exc
if response.status_code >= 400:
raise KeylessMCPError(
f"HTTP {response.status_code}: {response.text[:300]}"
)
raise KeylessMCPError(f"HTTP {response.status_code}: {response.text[:300]}")
return _parse_mcp_body(response.text)
# ---------------------------------------------------------------------------
# Parallel (search.parallel.ai) — JSON text payloads
# ---------------------------------------------------------------------------
# --- Parallel (search.parallel.ai) — JSON text payloads -----------------------
def parallel_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
"""Keyless Parallel web search → legacy search response shape."""
try:
text = mcp_call(
PARALLEL_MCP_URL,
"web_search",
{
"objective": query,
"search_queries": [query],
"session_id": _SESSION_ID,
},
PARALLEL_MCP_URL, "web_search",
{"objective": query, "search_queries": [query], "session_id": _SESSION_ID},
)
data = json.loads(text)
web_results = []
for i, result in enumerate(data.get("results") or []):
if limit and i >= limit:
break
excerpts = result.get("excerpts") or []
web_results.append(
{
"url": result.get("url") or "",
"title": result.get("title") or "",
"description": " ".join(excerpts) if excerpts else "",
"position": i + 1,
}
)
return {"success": True, "data": {"web": web_results}}
web_results.append(_row(
result.get("url") or "", result.get("title") or "",
" ".join(result.get("excerpts") or []), i + 1,
))
return search_ok(web_results)
except KeylessMCPError as exc:
return {
"success": False,
"error": (
f"Keyless Parallel search failed: {exc}. "
"Set PARALLEL_API_KEY (https://parallel.ai) or another web "
"backend via `hermes tools` for reliable service."
),
}
return search_fail(_fail_msg("parallel", "search", exc))
except (json.JSONDecodeError, TypeError, KeyError) as exc:
return {"success": False, "error": f"Keyless Parallel search returned an unexpected payload: {exc}"}
return search_fail(f"Keyless Parallel search returned an unexpected payload: {exc}")
def parallel_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
"""Keyless Parallel web fetch → legacy extract result list."""
try:
text = mcp_call(
PARALLEL_MCP_URL,
"web_fetch",
{
"urls": list(urls),
"objective": "Full page content",
"session_id": _SESSION_ID,
},
PARALLEL_MCP_URL, "web_fetch",
{"urls": list(urls), "objective": "Full page content", "session_id": _SESSION_ID},
)
data = json.loads(text)
except (KeylessMCPError, json.JSONDecodeError, TypeError) as exc:
message = (
f"Keyless Parallel extract failed: {exc}. "
"Set PARALLEL_API_KEY (https://parallel.ai) or another web "
"backend via `hermes tools` for reliable service."
)
return [
{"url": u, "title": "", "content": "", "error": message}
for u in urls
]
message = _fail_msg("parallel", "extract", exc)
return [_page_error(u, message) for u in urls]
results: List[Dict[str, Any]] = []
seen = set()
for result in data.get("results") or []:
url = result.get("url") or ""
title = result.get("title") or ""
content = (
result.get("full_content")
or result.get("content")
or "\n\n".join(result.get("excerpts") or [])
)
content = result.get("full_content") or result.get("content") or "\n\n".join(result.get("excerpts") or [])
seen.add(url)
results.append(
{
"url": url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url, "title": title},
}
)
results.append(_page(url, result.get("title") or "", content))
for error in data.get("errors") or []:
url = error.get("url") or ""
seen.add(url)
results.append(
{
"url": url,
"title": "",
"content": "",
"error": str(
error.get("content") or error.get("error_type") or "extraction failed"
),
"metadata": {"sourceURL": url},
}
)
# Any URL the endpoint silently dropped still gets an error entry so the
# caller's per-URL contract holds.
for u in urls:
if u not in seen:
results.append(
{"url": u, "title": "", "content": "", "error": "no content returned"}
)
entry = _page_error(url, str(error.get("content") or error.get("error_type") or "extraction failed"))
entry["metadata"] = {"sourceURL": url}
results.append(entry)
# URLs the endpoint silently dropped still get an error entry (per-URL contract).
results.extend(_page_error(u, "no content returned") for u in urls if u not in seen)
return results
# ---------------------------------------------------------------------------
# Exa (mcp.exa.ai) — formatted plain-text payloads
# ---------------------------------------------------------------------------
# --- Exa (mcp.exa.ai) — formatted plain-text payloads -------------------------
def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]:
"""Parse Exa's formatted search text into result dicts.
The payload is blocks separated by ``---`` lines, each shaped like::
Title: <title>
URL: <url>
Published: ...
Author: ...
Highlights:
<free text>
"""
"""Parse Exa's ``---``-separated ``Title:/URL:/Published:/Author:/Highlights:`` blocks."""
results: List[Dict[str, Any]] = []
for block in text.split("\n---\n"):
title = ""
url = ""
title = url = ""
highlight_lines: List[str] = []
in_highlights = False
for line in block.splitlines():
@@ -359,14 +258,7 @@ def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]:
elif in_highlights and stripped:
highlight_lines.append(stripped)
if url:
results.append(
{
"url": url,
"title": title,
"description": " ".join(highlight_lines),
"position": len(results) + 1,
}
)
results.append(_row(url, title, " ".join(highlight_lines), len(results) + 1))
if limit and len(results) >= limit:
break
return results
@@ -375,46 +267,21 @@ def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]:
def exa_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
"""Keyless Exa web search → legacy search response shape."""
try:
text = mcp_call(
EXA_MCP_URL,
"web_search_exa",
{"query": query, "numResults": max(1, int(limit))},
)
text = mcp_call(EXA_MCP_URL, "web_search_exa", {"query": query, "numResults": max(1, int(limit))})
except KeylessMCPError as exc:
return {
"success": False,
"error": (
f"Keyless Exa search failed: {exc}. "
"Set EXA_API_KEY (https://exa.ai) or another web backend "
"via `hermes tools` for reliable service."
),
}
return {"success": True, "data": {"web": _parse_exa_search_text(text, limit)}}
return search_fail(_fail_msg("exa", "search", exc))
return search_ok(_parse_exa_search_text(text, limit))
def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
"""Keyless Exa web fetch → legacy extract result list.
``web_fetch_exa`` takes a ``urls`` array but returns one combined text
payload; we call it per-URL so each result maps cleanly.
"""
"""Keyless Exa web fetch → legacy extract result list (called per-URL; the
tool returns one combined text payload)."""
results: List[Dict[str, Any]] = []
for url in urls:
try:
text = mcp_call(EXA_MCP_URL, "web_fetch_exa", {"urls": [url]})
except KeylessMCPError as exc:
results.append(
{
"url": url,
"title": "",
"content": "",
"error": (
f"Keyless Exa extract failed: {exc}. "
"Set EXA_API_KEY (https://exa.ai) or another web "
"backend via `hermes tools` for reliable service."
),
}
)
results.append(_page_error(url, _fail_msg("exa", "extract", exc)))
continue
title = ""
for line in text.splitlines():
@@ -425,234 +292,107 @@ def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
if stripped.startswith("Title:"):
title = stripped[len("Title:"):].strip()
break
results.append(
{
"url": url,
"title": title,
"content": text,
"raw_content": text,
"metadata": {"sourceURL": url, "title": title},
}
)
results.append(_page(url, title, text))
return results
# ---------------------------------------------------------------------------
# Firecrawl keyless (public cloud API, no auth header)
# ---------------------------------------------------------------------------
# --- Firecrawl keyless (public cloud API, no auth header) ---------------------
def firecrawl_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
"""Keyless Firecrawl cloud search → legacy search response shape."""
from plugins.web.firecrawl.provider import (
_KeylessFirecrawlClient,
_extract_web_search_results,
)
from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_web_search_results
try:
response = _KeylessFirecrawlClient().search(query=query, limit=limit)
return {"success": True, "data": {"web": _extract_web_search_results(response)}}
return search_ok(_extract_web_search_results(response))
except Exception as exc: # noqa: BLE001 — normalized below
return {
"success": False,
"error": (
f"Keyless Firecrawl search failed: {exc}. "
"Set FIRECRAWL_API_KEY (https://firecrawl.dev) or another web "
"backend via `hermes tools` for reliable service."
),
}
return search_fail(_fail_msg("firecrawl", "search", exc))
def firecrawl_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
"""Keyless Firecrawl cloud scrape → legacy extract result list."""
from plugins.web.firecrawl.provider import (
_KeylessFirecrawlClient,
_extract_scrape_payload,
)
from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_scrape_payload
client = _KeylessFirecrawlClient()
results: List[Dict[str, Any]] = []
for url in urls:
try:
response = client.scrape(url=url, formats=["markdown"])
payload = _extract_scrape_payload(response) or {}
payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {}
metadata = payload.get("metadata") or {}
if not isinstance(metadata, dict):
metadata = {}
content = payload.get("markdown") or payload.get("html") or ""
title = metadata.get("title") or ""
results.append(
{
"url": url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url, "title": title},
}
)
results.append(_page(url, metadata.get("title") or "", content))
except Exception as exc: # noqa: BLE001 — per-URL error entry
results.append(
{
"url": url,
"title": "",
"content": "",
"error": (
f"Keyless Firecrawl extract failed: {exc}. "
"Set FIRECRAWL_API_KEY (https://firecrawl.dev) for "
"reliable service."
),
}
)
results.append(_page_error(url, _fail_msg("firecrawl", "extract", exc, other_backends=False)))
return results
# ---------------------------------------------------------------------------
# Keenable keyless (api.keenable.ai public endpoints)
# ---------------------------------------------------------------------------
# --- Keenable keyless (api.keenable.ai public endpoints) ----------------------
KEENABLE_API_URL = "https://api.keenable.ai"
_KEENABLE_TITLE = "hermes-agent"
def _keenable_request(method: str, path: str, **kwargs: Any) -> Dict[str, Any]:
"""Call a Keenable public endpoint with the mandatory X-Keenable-Title app id."""
import requests
headers = {"X-Keenable-Title": _KEENABLE_TITLE}
if method == "post":
headers["Content-Type"] = "application/json"
response = getattr(requests, method)(f"{KEENABLE_API_URL}{path}", headers=headers, timeout=_TIMEOUT_SECONDS, **kwargs)
if response.status_code >= 400:
raise KeylessMCPError((response.text or "").strip() or f"HTTP {response.status_code}")
return response.json()
def keenable_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
"""Keyless Keenable search → legacy search response shape.
POST /v1/search/public with the mandatory X-Keenable-Title app
identifier (their keyless tier requires an app name; no user
identifiers are sent). Response: {results: [{title, url, snippet}]}.
"""
import requests
"""Keyless Keenable search (POST /v1/search/public) → legacy search response shape."""
try:
response = requests.post(
f"{KEENABLE_API_URL}/v1/search/public",
json={"query": query, "max_results": max(1, int(limit))},
headers={
"Content-Type": "application/json",
"X-Keenable-Title": _KEENABLE_TITLE,
},
timeout=_TIMEOUT_SECONDS,
)
if response.status_code >= 400:
raise KeylessMCPError(
(response.text or "").strip() or f"HTTP {response.status_code}"
)
data = response.json()
data = _keenable_request("post", "/v1/search/public", json={"query": query, "max_results": max(1, int(limit))})
except KeylessMCPError as exc:
return {
"success": False,
"error": (
f"Keyless Keenable search failed: {exc}. "
"Set KEENABLE_API_KEY (https://keenable.ai) or another web "
"backend via `hermes tools` for reliable service."
),
}
return search_fail(_fail_msg("keenable", "search", exc))
except Exception as exc: # noqa: BLE001 — transport/JSON errors
return {
"success": False,
"error": f"Keyless Keenable search failed: {exc}.",
}
web_results = []
for i, result in enumerate(data.get("results") or []):
web_results.append(
{
"url": result.get("url") or "",
"title": result.get("title") or "",
"description": result.get("snippet")
or result.get("description")
or "",
"position": i + 1,
}
)
return {"success": True, "data": {"web": web_results}}
return search_fail(f"Keyless Keenable search failed: {exc}.")
return search_ok([
_row(r.get("url") or "", r.get("title") or "", r.get("snippet") or r.get("description") or "", i + 1)
for i, r in enumerate(data.get("results") or [])
])
def keenable_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
"""Keyless Keenable page fetch → legacy extract result list.
GET /v1/fetch/public?url=... returns {url, title, content} (markdown).
Called per-URL; failures become per-URL error entries.
"""
import requests
"""Keyless Keenable page fetch (GET /v1/fetch/public, per-URL) → legacy extract list."""
results: List[Dict[str, Any]] = []
for url in urls:
try:
response = requests.get(
f"{KEENABLE_API_URL}/v1/fetch/public",
params={"url": url},
headers={"X-Keenable-Title": _KEENABLE_TITLE},
timeout=_TIMEOUT_SECONDS,
)
if response.status_code >= 400:
raise KeylessMCPError(
(response.text or "").strip() or f"HTTP {response.status_code}"
)
data = response.json()
content = data.get("content") or ""
title = data.get("title") or ""
results.append(
{
"url": data.get("url") or url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url, "title": title},
}
)
data = _keenable_request("get", "/v1/fetch/public", params={"url": url})
results.append(_page(data.get("url") or url, data.get("title") or "", data.get("content") or "", source_url=url))
except Exception as exc: # noqa: BLE001 — per-URL error entry
results.append(
{
"url": url,
"title": "",
"content": "",
"error": (
f"Keyless Keenable extract failed: {exc}. "
"Set KEENABLE_API_KEY (https://keenable.ai) for "
"reliable service."
),
}
)
results.append(_page_error(url, _fail_msg("keenable", "extract", exc, other_backends=False)))
return results
# ---------------------------------------------------------------------------
# Round-robin ring + next-in-line failover (rate-limited free tiers)
# ---------------------------------------------------------------------------
# --- Round-robin ring + next-in-line failover (rate-limited free tiers) -------
_KEYLESS_RING = ("exa", "parallel", "firecrawl", "keenable")
_KEYLESS_SEARCHERS = {
"exa": lambda query, limit: exa_search_keyless(query, limit),
"parallel": lambda query, limit: parallel_search_keyless(query, limit),
"firecrawl": lambda query, limit: firecrawl_search_keyless(query, limit),
"keenable": lambda query, limit: keenable_search_keyless(query, limit),
# Late-bound lookups (not bare references) so ``patch.object(keyless_mcp, "<vendor>_search_keyless")``
# is honored at call time. Tests also ``setitem`` these dicts directly.
_KEYLESS_SEARCHERS: Dict[str, Callable[[str, int], Dict[str, Any]]] = {
v: (lambda query, limit, _v=v: globals()[f"{_v}_search_keyless"](query, limit)) for v in _KEYLESS_RING
}
_KEYLESS_EXTRACTORS: Dict[str, Callable[[List[str]], List[Dict[str, Any]]]] = {
v: (lambda urls, _v=v: globals()[f"{_v}_extract_keyless"](urls)) for v in _KEYLESS_RING
}
_KEYLESS_EXTRACTORS = {
"exa": lambda urls: exa_extract_keyless(urls),
"parallel": lambda urls: parallel_extract_keyless(urls),
"firecrawl": lambda urls: firecrawl_extract_keyless(urls),
"keenable": lambda urls: keenable_extract_keyless(urls),
}
# Per-process round-robin cursor, seeded by the random session id so the
# fleet spreads evenly across all five free tiers; advances once per
# unpinned keyless request so a single process also rotates.
_ring_lock = __import__("threading").Lock()
# Per-process round-robin cursor, seeded by the random session id so the fleet
# spreads across vendors; advances once per unpinned keyless request.
_ring_lock = threading.Lock()
_ring_cursor = int(_SESSION_ID, 16) % len(_KEYLESS_RING)
def _vendor_pinned(name: str) -> bool:
"""True when config explicitly routes web traffic to *name*.
A pinned vendor starts every keyless request (rotation off); the ring
is only walked past it on throttle. Pin signals: web.backend /
web.search_backend / web.extract_backend naming the vendor, or a
free-tier pin in web.provider_tier.
"""
"""True when config explicitly routes web traffic to *name* (backend keys or a
``free`` tier pin). A pinned vendor starts every keyless request."""
if provider_tier(name) == "free":
return True
try:
@@ -669,14 +409,9 @@ def _vendor_pinned(name: str) -> bool:
def _ring_order(name: str) -> List[str]:
"""Return the vendor walk order for a request entering via *name*.
Pinned vendor → start at it (its position in the ring determines the
failover succession). Unpinned → true round-robin: start at the next
cursor position, advancing the cursor per request. Vendors whose tier
is pinned ``paid`` are excluded entirely (an explicit paid selection
opts that vendor's free endpoint out).
"""
"""Vendor walk order: pinned → start at *name* (its ring position fixes the
failover succession); else round-robin from the cursor, advancing it per request.
Vendors pinned ``paid`` are excluded (explicit paid opts their free endpoint out)."""
global _ring_cursor
if _vendor_pinned(name):
start = _KEYLESS_RING.index(name) if name in _KEYLESS_RING else 0
@@ -684,77 +419,60 @@ def _ring_order(name: str) -> List[str]:
with _ring_lock:
start = _ring_cursor
_ring_cursor = (_ring_cursor + 1) % len(_KEYLESS_RING)
ordered = [
_KEYLESS_RING[(start + i) % len(_KEYLESS_RING)]
for i in range(len(_KEYLESS_RING))
]
ordered = _KEYLESS_RING[start:] + _KEYLESS_RING[:start]
return [v for v in ordered if provider_tier(v) != "paid"]
def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]:
"""Keyless search across the vendor ring with next-in-line failover.
_ALL_PAID_MSG = "All keyless web providers are pinned to paid tiers."
Starts at *name* when the user pinned it, otherwise at the round-robin
cursor. Rate-limit-shaped errors advance to the next ring vendor;
non-throttle errors stop the walk (a malformed query fails everywhere).
The result notes the serving vendor via ``data.served_by`` whenever it
differs from *name*.
"""
def _walk_ring(name: str, kind: str, call, throttled) -> tuple:
"""Shared ring walk: call each vendor from :func:`_ring_order` until a result is
not ``throttled``. Returns ``(order, vendor, result, exhausted)``; ``order`` is
empty (and result None) when every vendor is pinned paid."""
order = _ring_order(name)
if not order:
return {
"success": False,
"error": "All keyless web providers are pinned to paid tiers.",
}
last: Dict[str, Any] = {}
vendor = None
result: Any = None
for i, vendor in enumerate(order):
result = _KEYLESS_SEARCHERS[vendor](query, limit)
if result.get("success"):
if vendor != name:
result.setdefault("data", {})["served_by"] = vendor
return result
last = result
if not _is_rate_limitish(result.get("error", "")):
return result
nxt = order[i + 1] if i + 1 < len(order) else None
if nxt:
logger.info(
"keyless %s search throttled; failing over to %s", vendor, nxt
)
last["error"] = (
f"{last.get('error', '')} (all keyless vendors throttled: "
f"{', '.join(order)})"
result = call(vendor)
if not throttled(result):
return order, vendor, result, False
if i + 1 < len(order):
logger.info("keyless %s %s throttled; failing over to %s", vendor, kind, order[i + 1])
return order, vendor, result, True
def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]:
"""Keyless search across the ring; rate-limit-shaped errors advance to the
next vendor, other errors stop the walk (a malformed query fails everywhere).
``data.served_by`` is set when the serving vendor differs from *name*."""
def _throttled(result: Dict[str, Any]) -> bool:
return not result.get("success") and _is_rate_limitish(result.get("error", ""))
order, vendor, result, exhausted = _walk_ring(
name, "search", lambda v: _KEYLESS_SEARCHERS[v](query, limit), _throttled
)
return last
if not order:
return search_fail(_ALL_PAID_MSG)
if exhausted:
result["error"] = f"{result.get('error', '')} (all keyless vendors throttled: {', '.join(order)})"
elif result.get("success") and vendor != name:
result.setdefault("data", {})["served_by"] = vendor
return result
def extract_with_failover(name: str, urls: List[str]) -> List[Dict[str, Any]]:
"""Keyless extract across the vendor ring, failing over per-batch.
"""Keyless extract across the ring; fails over only when EVERY url in a batch
is rate-limit-shaped (partial failures are page problems, returned as-is)."""
Advances to the next ring vendor only when EVERY url in a batch comes
back with a rate-limit-shaped error — partial failures are page
problems, not throttling, and return as-is.
"""
order = _ring_order(name)
if not order:
return [
{"url": u, "title": "", "content": "",
"error": "All keyless web providers are pinned to paid tiers."}
for u in urls
]
last: List[Dict[str, Any]] = []
for i, vendor in enumerate(order):
results = _KEYLESS_EXTRACTORS[vendor](list(urls))
def _all_throttled(results: List[Dict[str, Any]]) -> bool:
errors = [r.get("error", "") for r in results]
all_throttled = bool(results) and all(
e and _is_rate_limitish(e) for e in errors
)
if not all_throttled:
return results
last = results
nxt = order[i + 1] if i + 1 < len(order) else None
if nxt:
logger.info(
"keyless %s extract throttled; failing over to %s", vendor, nxt
)
return last
return bool(results) and all(e and _is_rate_limitish(e) for e in errors)
order, _vendor, results, _exhausted = _walk_ring(
name, "extract", lambda v: _KEYLESS_EXTRACTORS[v](list(urls)), _all_throttled
)
if not order:
return [_page_error(u, _ALL_PAID_MSG) for u in urls]
return results
plugins/web/parallel/__init__.py
+1 -8
View File
@@ -1,10 +1,4 @@
"""Parallel.ai web search + extract plugin — bundled, auto-loaded.
First plugin in this repo to expose an async :meth:`extract` — Parallel's
SDK is async-native (``AsyncParallel.beta.extract``). The web_extract_tool
dispatcher detects coroutines via :func:`inspect.iscoroutinefunction` and
awaits.
"""
"""Parallel.ai web search + extract plugin — bundled, auto-loaded; async-native ``extract``."""
from __future__ import annotations
@@ -12,5 +6,4 @@ from plugins.web.parallel.provider import ParallelWebSearchProvider
def register(ctx) -> None:
"""Register the Parallel provider with the plugin context."""
ctx.register_web_search_provider(ParallelWebSearchProvider())
plugins/web/parallel/provider.py
+67 -300
View File
@@ -1,353 +1,120 @@
"""Parallel.ai web search + content extraction — plugin form.
"""Parallel.ai web search (sync ``Parallel`` SDK) + async extract (``AsyncParallel``).
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses two
distinct Parallel SDK clients:
- ``Parallel`` (sync) — for :meth:`search`
- ``AsyncParallel`` (async) — for :meth:`extract`
This is the first plugin to exercise the **async-extract** code path in
the ABC: :meth:`extract` is declared ``async def``, and the dispatcher
in :func:`tools.web_tools.web_extract_tool` detects coroutines via
:func:`inspect.iscoroutinefunction` and awaits.
Config keys this provider responds to::
web:
search_backend: "parallel" # explicit per-capability
extract_backend: "parallel" # explicit per-capability
backend: "parallel" # shared fallback
# Optional: search mode (default "agentic"; also "fast" or "one-shot")
# via the PARALLEL_SEARCH_MODE env var.
Env vars::
PARALLEL_API_KEY=... # https://parallel.ai (required)
PARALLEL_SEARCH_MODE=agentic # optional: agentic|fast|one-shot
Env: ``PARALLEL_API_KEY`` (https://parallel.ai), optional
``PARALLEL_SEARCH_MODE`` = agentic (default) | fast | one-shot.
"""
from __future__ import annotations
import asyncio
import logging
import os
from typing import Any, Dict, List
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import (
SEARCH_LIMIT_CAP,
BaseWebSearchProvider,
cached_sdk_client,
document,
keyless_variant_schema,
provider_env,
run_extract_async,
run_search,
search_ok,
use_keyless,
web_hit,
)
logger = logging.getLogger(__name__)
# Module-level note: the canonical cache slots ``_parallel_client`` and
# ``_async_parallel_client`` live on :mod:`tools.web_tools` so tests that do
# ``tools.web_tools._parallel_client = None`` between cases see fresh state.
# The plugin reads/writes through that public module (see
# :func:`_get_sync_client` / :func:`_get_async_client`).
_MISSING_KEY = "PARALLEL_API_KEY environment variable not set. Get your API key at https://parallel.ai"
def _ensure_parallel_sdk_installed() -> None:
"""Trigger lazy install of the parallel SDK if it isn't present.
def _client(slot: str, cls_name: str) -> Any:
def _factory(api_key: str) -> Any:
import parallel # deliberately lazy
Mirrors the lazy-deps pattern used by the legacy implementation.
Swallows benign ImportError from the lazy_deps helper itself; if the
SDK is genuinely missing the subsequent ``from parallel import ...``
raises ImportError that the caller can handle.
"""
try:
from tools.lazy_deps import ensure as _lazy_ensure
return getattr(parallel, cls_name)(api_key=api_key)
_lazy_ensure("search.parallel", prompt=False)
except ImportError:
pass
except Exception as exc: # noqa: BLE001 — surface install hint as ImportError
raise ImportError(str(exc))
return cached_sdk_client(slot, "PARALLEL_API_KEY", _MISSING_KEY, "search.parallel", _factory)
def _get_sync_client() -> Any:
"""Lazy-load + cache the sync Parallel client.
Cache lives on :mod:`tools.web_tools` (as ``_parallel_client``) so unit
tests that reset that name between cases keep working.
"""
import tools.web_tools as _wt
cached = getattr(_wt, "_parallel_client", None)
if cached is not None:
return cached
from agent.web_search_provider import get_provider_env
api_key = get_provider_env("PARALLEL_API_KEY")
if not api_key:
raise ValueError(
"PARALLEL_API_KEY environment variable not set. "
"Get your API key at https://parallel.ai"
)
_ensure_parallel_sdk_installed()
from parallel import Parallel # noqa: WPS433 — deliberately lazy
client = Parallel(api_key=api_key)
_wt._parallel_client = client
return client
return _client("_parallel_client", "Parallel")
def _get_async_client() -> Any:
"""Lazy-load + cache the async Parallel client.
Cache lives on :mod:`tools.web_tools` (as ``_async_parallel_client``).
"""
import tools.web_tools as _wt
cached = getattr(_wt, "_async_parallel_client", None)
if cached is not None:
return cached
from agent.web_search_provider import get_provider_env
api_key = get_provider_env("PARALLEL_API_KEY")
if not api_key:
raise ValueError(
"PARALLEL_API_KEY environment variable not set. "
"Get your API key at https://parallel.ai"
)
_ensure_parallel_sdk_installed()
from parallel import AsyncParallel # noqa: WPS433 — deliberately lazy
client = AsyncParallel(api_key=api_key)
_wt._async_parallel_client = client
return client
return _client("_async_parallel_client", "AsyncParallel")
def _reset_clients_for_tests() -> None:
"""Drop both cached clients so tests can re-instantiate cleanly.
Clears the canonical slots on :mod:`tools.web_tools` (where
:func:`_get_sync_client` / :func:`_get_async_client` read/write them).
"""
import tools.web_tools as _wt
_wt._parallel_client = None
_wt._async_parallel_client = None
# Backward-compatible aliases for the names that lived in tools.web_tools
# before the migration (matches existing tests + external callers).
# Names re-exported by tools.web_tools for existing tests/callers.
_get_parallel_client = _get_sync_client
_get_async_parallel_client = _get_async_client
def _resolve_search_mode() -> str:
"""Return the validated PARALLEL_SEARCH_MODE value (default "agentic")."""
mode = os.getenv("PARALLEL_SEARCH_MODE", "agentic").lower().strip()
if mode not in {"fast", "one-shot", "agentic"}:
mode = "agentic"
return mode
return mode if mode in {"fast", "one-shot", "agentic"} else "agentic"
class ParallelWebSearchProvider(WebSearchProvider):
class ParallelWebSearchProvider(BaseWebSearchProvider):
"""Parallel.ai search + async extract provider."""
@property
def name(self) -> str:
return "parallel"
@property
def display_name(self) -> str:
return "Parallel"
def is_available(self) -> bool:
"""Return True when ``PARALLEL_API_KEY`` is set to a non-empty value.
Deliberately does NOT consider the keyless free tier — that would
let the legacy preference walk route keyed users of lower-priority
backends onto Parallel's anonymous tier. Keyless availability is a
separate, last-resort signal (:meth:`is_keyless_available`).
"""
from agent.web_search_provider import get_provider_env
return bool(get_provider_env("PARALLEL_API_KEY"))
def is_keyless_available(self) -> bool:
"""Parallel serves anonymous free-tier calls via its public MCP endpoint.
False when the user forced ``web.provider_tier.parallel: paid`` —
an explicit paid selection must never silently resolve keyless.
"""
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return keyless_enabled() and provider_tier("parallel") != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return True
NAME = "parallel"
DISPLAY_NAME = "Parallel"
KEY_ENV = "PARALLEL_API_KEY"
EXTRACT = True
KEYLESS = True
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a Parallel search (sync).
def _body() -> Dict[str, Any]:
from plugins.web.keyless_mcp import search_with_failover
Uses the ``beta.search`` endpoint with the configured mode
(``PARALLEL_SEARCH_MODE`` env var, default "agentic"). Limit is
capped at 20 server-side.
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return {"success": False, "error": "Interrupted"}
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import search_with_failover, use_keyless
if use_keyless("parallel", get_provider_env("PARALLEL_API_KEY")):
# Keyless free tier — public MCP endpoint, no SDK needed.
logger.info(
"Parallel keyless search: '%s' (limit=%d)", query, limit
)
if use_keyless("parallel", provider_env("PARALLEL_API_KEY")):
logger.info("Parallel keyless search: '%s' (limit=%d)", query, limit)
return search_with_failover("parallel", query, limit)
mode = _resolve_search_mode()
logger.info(
"Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit
)
logger.info("Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit)
response = _get_sync_client().beta.search(
search_queries=[query],
objective=query,
mode=mode,
max_results=min(limit, 20),
search_queries=[query], objective=query, mode=mode, max_results=min(limit, SEARCH_LIMIT_CAP)
)
return search_ok([
web_hit(r.url or "", r.title or "", " ".join(r.excerpts or []), i + 1)
for i, r in enumerate(response.results or [])
])
web_results = []
for i, result in enumerate(response.results or []):
excerpts = result.excerpts or []
web_results.append(
{
"url": result.url or "",
"title": result.title or "",
"description": " ".join(excerpts) if excerpts else "",
"position": i + 1,
}
)
return run_search("Parallel", logger, _body, sdk=True)
return {"success": True, "data": {"web": web_results}}
except ValueError as exc:
return {"success": False, "error": str(exc)}
except ImportError as exc:
return {
"success": False,
"error": f"Parallel SDK not installed: {exc}",
}
except Exception as exc: # noqa: BLE001
logger.warning("Parallel search error: %s", exc)
return {"success": False, "error": f"Parallel search failed: {exc}"}
async def extract(
self, urls: List[str], **kwargs: Any
) -> List[Dict[str, Any]]:
"""Extract content from one or more URLs via the async SDK.
Returns the legacy list-of-results shape that
:func:`tools.web_tools.web_extract_tool` expects: one entry per
successful URL plus one entry per failed URL with an ``error``
field. Errors are not raised — they're returned as per-URL items.
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return [
{"url": u, "error": "Interrupted", "title": ""} for u in urls
]
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import extract_with_failover, use_keyless
if use_keyless("parallel", get_provider_env("PARALLEL_API_KEY")):
# Keyless free tier — blocking HTTP, so hop off the loop.
import asyncio
async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
async def _body() -> List[Dict[str, Any]]:
from plugins.web.keyless_mcp import extract_with_failover
if use_keyless("parallel", provider_env("PARALLEL_API_KEY")):
# Keyless ring is blocking HTTP — hop off the event loop.
logger.info("Parallel keyless extract: %d URL(s)", len(urls))
return await asyncio.to_thread(
extract_with_failover, "parallel", list(urls)
)
return await asyncio.to_thread(extract_with_failover, "parallel", list(urls))
logger.info("Parallel extract: %d URL(s)", len(urls))
response = await _get_async_client().beta.extract(
urls=urls,
full_content=True,
)
results: List[Dict[str, Any]] = []
for result in response.results or []:
content = result.full_content or ""
if not content:
content = "\n\n".join(result.excerpts or [])
url = result.url or ""
title = result.title or ""
results.append(
{
"url": url,
"title": title,
"content": content,
"raw_content": content,
"metadata": {"sourceURL": url, "title": title},
}
)
response = await _get_async_client().beta.extract(urls=urls, full_content=True)
results = [
document(r.url or "", r.title or "", r.full_content or "\n\n".join(r.excerpts or []))
for r in response.results or []
]
for error in response.errors or []:
results.append(
{
"url": error.url or "",
"title": "",
"content": "",
"error": error.content or error.error_type or "extraction failed",
"metadata": {"sourceURL": error.url or ""},
}
)
results.append({
"url": error.url or "", "title": "", "content": "",
"error": error.content or error.error_type or "extraction failed",
"metadata": {"sourceURL": error.url or ""},
})
return results
except ValueError as exc:
return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
except ImportError as exc:
return [
{"url": u, "title": "", "content": "", "error": f"Parallel SDK not installed: {exc}"}
for u in urls
]
except Exception as exc: # noqa: BLE001
logger.warning("Parallel extract error: %s", exc)
return [
{"url": u, "title": "", "content": "", "error": f"Parallel extract failed: {exc}"}
for u in urls
]
return await run_extract_async("Parallel", logger, urls, _body, sdk=True)
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Parallel · Free (keyless)",
"badge": "free · no key",
"tag": (
"Objective-tuned search + page extraction on Parallel's "
"anonymous free tier. Rate-limited under burst load."
),
"env_vars": [],
"web_tier": "free",
"variants": [
{
"name": "Parallel · Paid (API key)",
"badge": "paid",
"tag": (
"Objective-tuned search + parallel page extraction "
"via the Parallel SDK. Unthrottled, guaranteed service."
),
"env_vars": [
{
"key": "PARALLEL_API_KEY",
"prompt": "Parallel API key",
"url": "https://parallel.ai",
},
],
"web_tier": "paid",
},
],
}
return keyless_variant_schema(
"Parallel", "PARALLEL_API_KEY", "https://parallel.ai",
free_tag="Objective-tuned search + page extraction on Parallel's anonymous free tier. Rate-limited under burst load.",
paid_tag="Objective-tuned search + parallel page extraction via the Parallel SDK. Unthrottled, guaranteed service.",
)
plugins/web/searxng/__init__.py
+1 -7
View File
@@ -1,9 +1,4 @@
"""SearXNG search plugin — bundled, auto-loaded.
Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``).
Search-only — pair with an extract provider (firecrawl/tavily/exa) for
``web_extract`` calls.
"""
"""SearXNG search-only plugin — bundled, auto-loaded (``SEARXNG_URL``)."""
from __future__ import annotations
@@ -11,5 +6,4 @@ from plugins.web.searxng.provider import SearXNGWebSearchProvider
def register(ctx) -> None:
"""Register the SearXNG provider with the plugin context."""
ctx.register_web_search_provider(SearXNGWebSearchProvider())
plugins/web/searxng/provider.py
+24 -106
View File
@@ -1,123 +1,46 @@
"""SearXNG search — plugin form.
"""SearXNG search via a user-hosted instance (``/search?format=json``).
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Same JSON
API call (``/search?format=json``), same result normalization. The legacy
in-tree module ``tools.web_providers.searxng`` was removed in the same
commit that moved this code under ``plugins/``; this file is now the
canonical implementation.
Search-only — SearXNG aggregates results from upstream engines but does not
fetch/extract arbitrary URLs. ``supports_extract()`` returns False.
Config keys this provider responds to::
web:
search_backend: "searxng" # explicit per-capability
backend: "searxng" # shared fallback
Env var::
SEARXNG_URL=http://localhost:8080
Search-only — SearXNG aggregates upstream engines but does not fetch URLs.
Env: ``SEARXNG_URL=http://localhost:8080``.
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import BaseWebSearchProvider, http_get_json, provider_env, search_fail, search_ok
logger = logging.getLogger(__name__)
def _searxng_url() -> str:
"""Return SEARXNG_URL from Hermes config-aware env, falling back to process env."""
try:
from hermes_cli.config import get_env_value
val = get_env_value("SEARXNG_URL")
except Exception:
val = None
if val is None:
val = os.getenv("SEARXNG_URL", "")
return (val or "").strip()
class SearXNGWebSearchProvider(WebSearchProvider):
class SearXNGWebSearchProvider(BaseWebSearchProvider):
"""Search via a user-hosted SearXNG instance."""
@property
def name(self) -> str:
return "searxng"
@property
def display_name(self) -> str:
return "SearXNG"
def is_available(self) -> bool:
"""Return True when ``SEARXNG_URL`` is set."""
return bool(_searxng_url())
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return False
NAME = "searxng"
DISPLAY_NAME = "SearXNG"
KEY_ENV = "SEARXNG_URL"
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a search against the configured SearXNG instance."""
import httpx
base_url = _searxng_url().rstrip("/")
base_url = provider_env("SEARXNG_URL").rstrip("/")
if not base_url:
return {"success": False, "error": "SEARXNG_URL is not set"}
return search_fail("SEARXNG_URL is not set")
params: Dict[str, Any] = {
"q": query,
"format": "json",
"pageno": 1,
}
try:
resp = httpx.get(
f"{base_url}/search",
params=params,
timeout=15,
headers={"Accept": "application/json"},
)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
logger.warning("SearXNG HTTP error: %s", exc)
return {
"success": False,
"error": f"SearXNG returned HTTP {exc.response.status_code}",
}
except httpx.RequestError as exc:
logger.warning("SearXNG request error: %s", exc)
return {
"success": False,
"error": f"Could not reach SearXNG at {base_url}: {exc}",
}
try:
data = resp.json()
except Exception as exc: # noqa: BLE001
logger.warning("SearXNG response parse error: %s", exc)
return {
"success": False,
"error": "Could not parse SearXNG response as JSON",
}
data, failure = http_get_json(
"SearXNG",
f"{base_url}/search",
params={"q": query, "format": "json", "pageno": 1},
headers={"Accept": "application/json"},
timeout=15,
logger=logger,
reach_target=f"SearXNG at {base_url}",
)
if failure is not None:
return failure
raw_results = data.get("results", [])
# SearXNG may return a score field; sort descending and cap to limit.
sorted_results = sorted(
raw_results,
key=lambda r: float(r.get("score", 0)),
reverse=True,
)[:limit]
sorted_results = sorted(raw_results, key=lambda r: float(r.get("score", 0)), reverse=True)[:limit]
web_results = [
{
"title": str(r.get("title", "")),
@@ -127,16 +50,11 @@ class SearXNGWebSearchProvider(WebSearchProvider):
}
for i, r in enumerate(sorted_results)
]
logger.info(
"SearXNG search '%s': %d results (from %d raw, limit %d)",
query,
len(web_results),
len(raw_results),
limit,
query, len(web_results), len(raw_results), limit,
)
return {"success": True, "data": {"web": web_results}}
return search_ok(web_results)
def get_setup_schema(self) -> Dict[str, Any]:
return {
plugins/web/tavily/__init__.py
-1
View File
@@ -6,5 +6,4 @@ from plugins.web.tavily.provider import TavilyWebSearchProvider
def register(ctx) -> None:
"""Register the Tavily provider with the plugin context."""
ctx.register_web_search_provider(TavilyWebSearchProvider())
plugins/web/tavily/provider.py
+68 -233
View File
@@ -1,34 +1,9 @@
"""Tavily web search + content extraction — plugin form.
"""Tavily web search + content extraction (``/search``, ``/extract``; sync httpx).
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two
capabilities advertised:
- ``supports_search()`` -> True (Tavily ``/search``)
- ``supports_extract()`` -> True (Tavily ``/extract``)
Both are sync — the underlying call is ``httpx.post(...)``.
Config keys this provider responds to::
web:
search_backend: "tavily" # explicit per-capability
extract_backend: "tavily" # explicit per-capability
backend: "tavily" # shared fallback for both
Env vars::
TAVILY_API_KEY=... # https://app.tavily.com/home (optional)
TAVILY_BASE_URL=... # optional override of https://api.tavily.com
Auth is header-based. A key uses ``Authorization: Bearer``; without a
key the request is keyless (``X-Tavily-Access-Mode: keyless``). Both
paths send ``X-Client-Name: hermes-agent``.
Tavily is **not** a member of the zero-config keyless ring
(``plugins.web.keyless_mcp._KEYLESS_RING``). Keyless access is opt-in:
select Tavily in ``hermes tools`` (or set ``web.backend: tavily``).
Fresh installs with no web credentials rotate across Exa / Parallel /
Firecrawl / Keenable instead.
Env: ``TAVILY_API_KEY`` (https://app.tavily.com/home, optional), ``TAVILY_BASE_URL``.
Keyed requests use ``Authorization: Bearer``; without a key the request is
keyless (``X-Tavily-Access-Mode: keyless``). Tavily is NOT in the zero-config
keyless ring — keyless access is opt-in by selecting Tavily in ``hermes tools``.
"""
from __future__ import annotations
@@ -38,20 +13,28 @@ from typing import Any, Dict, List, Optional
import httpx
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import (
SEARCH_LIMIT_CAP,
BaseWebSearchProvider,
document,
extract_fail,
http_status_detail,
provider_env,
run_extract,
run_search,
search_fail,
search_ok,
use_keyless,
)
logger = logging.getLogger(__name__)
_CLIENT_NAME = "hermes-agent"
_SEARCH_PAYLOAD = {
"include_raw_content": False,
"include_images": False,
}
_SEARCH_PAYLOAD = {"include_raw_content": False, "include_images": False}
def _tavily_headers(api_key: str) -> Dict[str, str]:
"""Build Tavily request headers for keyed or keyless access."""
headers = {"X-Client-Name": _CLIENT_NAME}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
@@ -60,249 +43,101 @@ def _tavily_headers(api_key: str) -> Dict[str, str]:
return headers
def _tavily_request(
endpoint: str,
payload: Dict[str, Any],
*,
api_key: Optional[str] = None,
) -> Dict[str, Any]:
"""POST to the Tavily API and return the parsed JSON response.
def _tavily_request(endpoint: str, payload: Dict[str, Any], *, api_key: Optional[str] = None) -> Dict[str, Any]:
"""POST to Tavily and return parsed JSON.
Keyed when *api_key* (or ``TAVILY_API_KEY``) is set (Bearer auth);
otherwise keyless. Pass ``api_key=""`` to force the keyless header even
when a key is present (``web.provider_tier.tavily: free``). Non-2xx
responses raise ``ValueError`` with the response body so Tavily's
keyless rate-limit / upgrade text reaches the model.
``api_key=None`` reads ``TAVILY_API_KEY``; pass ``""`` to force the keyless
header even when a key exists (``web.provider_tier.tavily: free``). Non-2xx
raises ValueError with the body so Tavily's rate-limit/upgrade text reaches the model.
"""
from agent.web_search_provider import get_provider_env
if api_key is None:
api_key = get_provider_env("TAVILY_API_KEY")
base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
api_key = provider_env("TAVILY_API_KEY")
base_url = provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
url = f"{base_url}/{endpoint.lstrip('/')}"
logger.info("Tavily %s request to %s", endpoint, url)
response = httpx.post(
url,
json=payload,
timeout=60,
headers=_tavily_headers(api_key),
)
response = httpx.post(url, json=payload, timeout=60, headers=_tavily_headers(api_key))
if response.status_code >= 400:
body = (response.text or "").strip()
detail = body or f"HTTP {response.status_code}"
raise ValueError(detail)
raise ValueError(http_status_detail(response))
return response.json()
def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]:
"""Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``."""
web_results = []
for i, result in enumerate(response.get("results", [])):
web_results.append(
{
"title": result.get("title", ""),
"url": result.get("url", ""),
"description": result.get("content", ""),
"position": i + 1,
}
)
return {"success": True, "data": {"web": web_results}}
# title-first key order is Tavily's historical wire shape (differs from web_hit).
return search_ok([
{"title": r.get("title", ""), "url": r.get("url", ""), "description": r.get("content", ""), "position": i + 1}
for i, r in enumerate(response.get("results", []))
])
def _normalize_tavily_documents(
response: Dict[str, Any], fallback_url: str = ""
) -> List[Dict[str, Any]]:
"""Map Tavily ``/extract`` response to standard documents.
Documents follow the legacy LLM post-processing shape::
{"url", "title", "content", "raw_content", "metadata"}
Failures (``failed_results``, ``failed_urls``) become result entries
with an ``error`` field rather than raising.
"""
def _normalize_tavily_documents(response: Dict[str, Any], fallback_url: str = "") -> List[Dict[str, Any]]:
"""Map ``/extract`` to documents; ``failed_results`` / ``failed_urls`` become ``error`` entries."""
documents: List[Dict[str, Any]] = []
for result in response.get("results", []):
url = result.get("url", fallback_url)
raw = result.get("raw_content", "") or result.get("content", "")
documents.append(
{
"url": url,
"title": result.get("title", ""),
"content": raw,
"raw_content": raw,
"metadata": {"sourceURL": url, "title": result.get("title", "")},
}
)
documents.append(document(url, result.get("title", ""), raw))
for fail in response.get("failed_results", []):
documents.append(
{
"url": fail.get("url", fallback_url),
"title": "",
"content": "",
"raw_content": "",
"error": fail.get("error", "extraction failed"),
"metadata": {"sourceURL": fail.get("url", fallback_url)},
}
)
url = fail.get("url", fallback_url)
documents.append(_failed_document(url, fail.get("error", "extraction failed")))
for fail_url in response.get("failed_urls", []):
url_str = fail_url if isinstance(fail_url, str) else str(fail_url)
documents.append(
{
"url": url_str,
"title": "",
"content": "",
"raw_content": "",
"error": "extraction failed",
"metadata": {"sourceURL": url_str},
}
)
documents.append(_failed_document(str(fail_url), "extraction failed"))
return documents
def _failed_document(url: str, error: str) -> Dict[str, Any]:
return {"url": url, "title": "", "content": "", "raw_content": "", "error": error, "metadata": {"sourceURL": url}}
def _missing_key_error(action: str) -> str:
return (
f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home "
"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home "
f"or select Tavily in `hermes tools` for opt-in keyless {action}."
)
class TavilyWebSearchProvider(WebSearchProvider):
class TavilyWebSearchProvider(BaseWebSearchProvider):
"""Tavily search + extract provider (keyed, or opt-in keyless)."""
@property
def name(self) -> str:
return "tavily"
@property
def display_name(self) -> str:
return "Tavily"
def is_available(self) -> bool:
"""Return True when ``TAVILY_API_KEY`` is set to a non-empty value."""
from agent.web_search_provider import get_provider_env
return bool(get_provider_env("TAVILY_API_KEY"))
def is_keyless_available(self) -> bool:
"""Tavily serves anonymous keyless requests (X-Tavily-Access-Mode).
Opt-in only — Tavily is not a member of the zero-config keyless
ring. ``is_keyless_available`` is True so an explicit
``web.backend: tavily`` (or ``hermes tools`` pick) works without a
key. False when the user pinned ``web.provider_tier.tavily: paid``.
"""
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return keyless_enabled() and provider_tier("tavily") != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return True
NAME = "tavily"
DISPLAY_NAME = "Tavily"
KEY_ENV = "TAVILY_API_KEY"
EXTRACT = True
KEYLESS = True
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a Tavily search (keyed path or opt-in keyless)."""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return {"success": False, "error": "Interrupted"}
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import use_keyless
api_key = get_provider_env("TAVILY_API_KEY")
def _body() -> Dict[str, Any]:
api_key = provider_env("TAVILY_API_KEY")
force_keyless = use_keyless("tavily", api_key)
if not force_keyless and not api_key:
return {"success": False, "error": _missing_key_error("search")}
return search_fail(_missing_key_error("search"))
logger.info(
"Tavily %ssearch: '%s' (limit=%d)",
"keyless " if force_keyless else "",
query,
limit,
logger.info("Tavily %ssearch: '%s' (limit=%d)", "keyless " if force_keyless else "", query, limit)
payload = {"query": query, "max_results": min(limit, SEARCH_LIMIT_CAP), **_SEARCH_PAYLOAD}
return _normalize_tavily_search_results(
_tavily_request("search", payload, api_key="" if force_keyless else api_key)
)
raw = _tavily_request(
"search",
{
"query": query,
"max_results": min(limit, 20),
**_SEARCH_PAYLOAD,
},
api_key="" if force_keyless else api_key,
)
return _normalize_tavily_search_results(raw)
except ValueError as exc:
return {"success": False, "error": str(exc)}
except Exception as exc: # noqa: BLE001 — including httpx errors
logger.warning("Tavily search error: %s", exc)
return {"success": False, "error": f"Tavily search failed: {exc}"}
return run_search("Tavily", logger, _body)
def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
"""Extract content from one or more URLs via Tavily.
Sync — the underlying call is httpx.post(...). Returns the legacy
list-of-results shape; per-URL failures become items with ``error``.
Keyless uses Tavily's own endpoint, not the keyless ring.
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return [
{"url": u, "error": "Interrupted", "title": ""} for u in urls
]
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import use_keyless
api_key = get_provider_env("TAVILY_API_KEY")
def _body() -> List[Dict[str, Any]]:
api_key = provider_env("TAVILY_API_KEY")
force_keyless = use_keyless("tavily", api_key)
if not force_keyless and not api_key:
err = _missing_key_error("extract")
return [
{"url": u, "title": "", "content": "", "error": err}
for u in urls
]
return extract_fail(urls, _missing_key_error("extract"))
logger.info(
"Tavily %sextract: %d URL(s)",
"keyless " if force_keyless else "",
len(urls),
)
raw = _tavily_request(
"extract",
{
"urls": urls,
"include_images": False,
},
api_key="" if force_keyless else api_key,
)
return _normalize_tavily_documents(
raw, fallback_url=urls[0] if urls else ""
)
except ValueError as exc:
return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
except Exception as exc: # noqa: BLE001
logger.warning("Tavily extract error: %s", exc)
return [
{"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"}
for u in urls
]
logger.info("Tavily %sextract: %d URL(s)", "keyless " if force_keyless else "", len(urls))
raw = _tavily_request("extract", {"urls": urls, "include_images": False}, api_key="" if force_keyless else api_key)
return _normalize_tavily_documents(raw, fallback_url=urls[0] if urls else "")
return run_extract("Tavily", logger, urls, _body)
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Tavily",
"badge": "free · key optional",
"tag": (
"Search + extract. Opt-in keyless; "
"set TAVILY_API_KEY for higher limits."
),
"tag": "Search + extract. Opt-in keyless; set TAVILY_API_KEY for higher limits.",
"env_vars": [
{
"key": "TAVILY_API_KEY",
plugins/web/xai/__init__.py
+1 -6
View File
@@ -1,8 +1,4 @@
"""xAI web search plugin — bundled, auto-loaded.
Mirrors the ``plugins/web/brave_free/`` layout: ``provider.py`` holds the
provider class, ``__init__.py::register(ctx)`` registers an instance.
"""
"""xAI web search plugin — bundled, auto-loaded."""
from __future__ import annotations
@@ -10,5 +6,4 @@ from plugins.web.xai.provider import XAIWebSearchProvider
def register(ctx) -> None:
"""Register the xAI Web Search provider with the plugin context."""
ctx.register_web_search_provider(XAIWebSearchProvider())
plugins/web/xai/provider.py
+155 -319
View File
@@ -1,33 +1,13 @@
"""xAI Web Search — plugin form.
"""xAI Web Search — search-only provider backed by Grok's server-side ``web_search``
tool on the Responses API. Grok searches/browses server-side; we ask for structured
JSON so results match the ``{title, url, description, position}`` rows every other
Hermes web provider produces. Reference: https://docs.x.ai/developers/tools/web-search
Routes ``web_search`` tool calls through xAI's agentic Web Search tool
(server-side ``web_search`` on the Responses API). Grok runs the actual
searching and page-browsing server-side; we ask it to return the top
results as structured JSON so we can hand back the same
``{title, url, description, position}`` rows every other Hermes web
provider produces.
Reference: https://docs.x.ai/developers/tools/web-search
Config keys this provider responds to::
web:
search_backend: "xai" # explicit per-capability
backend: "xai" # shared fallback
Optional knobs (under ``web.xai`` in ``config.yaml``)::
web:
xai:
model: "grok-build-0.1" # reasoning model required by web_search
allowed_domains: ["x.ai"] # max 5 — mutually exclusive with excluded_domains
excluded_domains: ["bad.com"] # max 5 — mutually exclusive with allowed_domains
timeout: 90 # seconds (default 90)
Auth: reuses :func:`tools.xai_http.resolve_xai_http_credentials`, which
prefers Hermes-managed xAI Grok OAuth (via ``hermes auth``) and falls back
to ``XAI_API_KEY`` (resolved through ``~/.hermes/.env``, then
``os.environ``).
Config: ``web.search_backend`` / ``web.backend: "xai"``. Optional ``web.xai``:
``model`` (reasoning model, default grok-build-0.1), ``allowed_domains`` /
``excluded_domains`` (max 5, mutually exclusive), ``timeout`` (seconds, default 90).
Auth: :func:`tools.xai_http.resolve_xai_http_credentials` (Grok OAuth via
``hermes auth``, else ``XAI_API_KEY``).
"""
from __future__ import annotations
@@ -37,7 +17,7 @@ import logging
import re
from typing import Any, Dict, List, Optional
from agent.web_search_provider import WebSearchProvider
from plugins.web._common import BaseWebSearchProvider, search_fail as _fail, search_ok
from tools.xai_http import (
has_xai_credentials,
hermes_xai_user_agent,
@@ -50,17 +30,11 @@ DEFAULT_MODEL = "grok-build-0.1"
DEFAULT_TIMEOUT = 90
_MAX_DOMAIN_FILTERS = 5 # xAI hard cap on allowed_domains / excluded_domains
# Match the JSON object Grok is asked to emit. Tolerates leading/trailing
# prose since reasoning models occasionally narrate before the JSON block
# even when explicitly asked not to.
# The JSON object Grok is asked to emit; tolerates leading/trailing prose since
# reasoning models occasionally narrate before the JSON block.
_JSON_BLOCK_RE = re.compile(r"\{[\s\S]*\}", re.MULTILINE)
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
def _load_xai_web_config() -> Dict[str, Any]:
"""Read ``web.xai`` from config.yaml (returns {} on miss)."""
try:
@@ -79,83 +53,53 @@ def _coerce_domain_list(value: Any) -> List[str]:
"""Coerce a config value to a clean list of <=5 domain strings."""
if not isinstance(value, list):
return []
cleaned: List[str] = []
for item in value:
if isinstance(item, str) and item.strip():
cleaned.append(item.strip())
if len(cleaned) >= _MAX_DOMAIN_FILTERS:
break
return cleaned
cleaned = [item.strip() for item in value if isinstance(item, str) and item.strip()]
return cleaned[:_MAX_DOMAIN_FILTERS]
# ---------------------------------------------------------------------------
# Provider
# ---------------------------------------------------------------------------
def _coerce(cast, value: Any, default: Any) -> Any:
try:
return cast(value)
except (TypeError, ValueError):
return default
class XAIWebSearchProvider(WebSearchProvider):
def _row(title: str, url: str, description: str, position: int) -> Dict[str, Any]:
# Key order (title first) is this provider's wire shape; keep it distinct from web_hit.
return {"title": title, "url": url, "description": description, "position": position}
class XAIWebSearchProvider(BaseWebSearchProvider):
"""Search-only provider backed by xAI's agentic Web Search tool.
Sends a structured prompt to Grok with ``tools=[{"type": "web_search"}]``
enabled and asks it to return the top *limit* results as JSON. Falls
back to the Responses API ``citations`` list if Grok ignores the JSON
schema instruction (rare for grok-4.3 but cheap insurance).
Sends a structured prompt with ``tools=[{"type": "web_search"}]`` and parses the
JSON Grok returns; falls back to message annotations, then the ``citations``
list, if Grok ignores the schema. No extract capability.
No extract capability — pair with Firecrawl / Tavily / Exa for
``web_extract`` if you need page content.
Trust model
-----------
Unlike index-backed providers (Brave / Tavily / Exa) which return
verbatim search-engine results, this backend is an LLM in a trench
coat: Grok decides which URLs to surface, generates the titles and
descriptions itself, and is influenced by the *content of the query*.
A maliciously crafted query (e.g. injected via untrusted upstream
input the agent picked up) can in principle steer Grok into emitting
attacker-chosen URLs. Callers that pipe untrusted text directly into
``web_search`` should treat returned URLs the same way they would
treat any model-generated link — validate before fetching.
Trust model: unlike index-backed providers, Grok *generates* the URLs, titles
and descriptions and is steerable by the query text itself — treat returned
URLs like any model-generated link and validate before fetching.
"""
@property
def name(self) -> str:
return "xai"
@property
def display_name(self) -> str:
return "xAI Web Search (Grok)"
NAME = "xai"
DISPLAY_NAME = "xAI Web Search (Grok)"
def is_available(self) -> bool:
"""Cheap availability probe — env var OR auth-store has OAuth tokens.
Delegates to :func:`tools.xai_http.has_xai_credentials`, which is
deliberately *not* the same as :func:`resolve_xai_http_credentials`:
it never triggers OAuth token refresh or acquires the auth-store
lock. The ABC contract requires this method to be safe to call on
every ``hermes tools`` repaint and at tool-registration time.
Token freshness / refresh is handled inside :meth:`search`.
"""
"""Cheap probe (env var OR auth-store tokens). Deliberately NOT
``resolve_xai_http_credentials``: must never refresh tokens or take the
auth-store lock, since this runs on every ``hermes tools`` repaint."""
return has_xai_credentials()
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return False
# -- Search -----------------------------------------------------------
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a Grok-backed web search.
Returns ``{"success": True, "data": {"web": [{title, url, description, position}, ...]}}``
on success, ``{"success": False, "error": str}`` on failure.
"""
"""Grok-backed web search → ``{"success": True, "data": {"web": [...]}}``
or ``{"success": False, "error": str}``."""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return {"success": False, "error": "Interrupted"}
return _fail("Interrupted")
except Exception: # noqa: BLE001 — interrupt module is best-effort
pass
@@ -163,103 +107,105 @@ class XAIWebSearchProvider(WebSearchProvider):
api_key = str(creds.get("api_key") or "").strip()
base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
if not api_key:
return {
"success": False,
"error": (
"No xAI credentials found. Run `hermes auth` to sign in with "
"xAI Grok OAuth, or set XAI_API_KEY."
),
}
return _fail(
"No xAI credentials found. Run `hermes auth` to sign in with "
"xAI Grok OAuth, or set XAI_API_KEY."
)
# Clamp limit to the same range the caller (web_search_tool) accepts,
# so we don't silently downgrade explicit limits. Grok happily
# produces longer lists; cost scales linearly with the requested
# count via reasoning tokens, but that's the caller's call to make.
try:
limit = int(limit)
except (TypeError, ValueError):
limit = 5
limit = max(1, min(limit, 100))
# Same clamp range as web_search_tool so explicit limits aren't downgraded;
# cost scales with the count via reasoning tokens, but that's the caller's call.
limit = max(1, min(_coerce(int, limit, 5), 100))
cfg = _load_xai_web_config()
model = cfg.get("model") if isinstance(cfg.get("model"), str) else DEFAULT_MODEL
model = model.strip() or DEFAULT_MODEL
timeout = _coerce(float, cfg.get("timeout", DEFAULT_TIMEOUT), DEFAULT_TIMEOUT)
try:
timeout = float(cfg.get("timeout", DEFAULT_TIMEOUT))
except (TypeError, ValueError):
timeout = DEFAULT_TIMEOUT
allowed = _coerce_domain_list(cfg.get("allowed_domains"))
excluded = _coerce_domain_list(cfg.get("excluded_domains"))
if allowed and excluded:
# xAI explicitly rejects this combo — surface a clear error
# rather than a 400 from the API.
return {
"success": False,
"error": (
"web.xai.allowed_domains and web.xai.excluded_domains "
"cannot both be set (xAI restriction)."
),
}
web_search_tool: Dict[str, Any] = {"type": "web_search"}
if allowed:
web_search_tool["filters"] = {"allowed_domains": allowed}
elif excluded:
web_search_tool["filters"] = {"excluded_domains": excluded}
prompt = self._build_prompt(query, limit)
web_search_tool = self._web_search_tool(cfg)
if web_search_tool is None:
# xAI rejects this combo — surface a clear error rather than an API 400.
return _fail(
"web.xai.allowed_domains and web.xai.excluded_domains "
"cannot both be set (xAI restriction)."
)
payload: Dict[str, Any] = {
"model": model,
"input": [{"role": "user", "content": prompt}],
"input": [{"role": "user", "content": self._build_prompt(query, limit)}],
"tools": [web_search_tool],
# Drop inline citation markdown — we want the JSON block clean,
# and we read URLs from annotations / citations separately.
# Keep the JSON block clean; URLs are read from annotations/citations.
"include": ["no_inline_citations"],
}
try:
import httpx # noqa: F401 — availability probe
except ImportError:
return _fail("httpx is not installed (required for xAI web search)")
logger.info("xAI web search via %s: '%s' (limit=%d, model=%s)", base_url, query, limit, model)
data, error = self._post_responses(
base_url, payload, api_key, timeout,
is_oauth_path=(creds.get("provider") == "xai-oauth"),
)
if error:
return error
# xAI sometimes returns HTTP 200 with an error envelope (overloaded, refusal);
# without this check we'd report success-with-no-rows and mask a real failure.
api_error = data.get("error") if isinstance(data, dict) else None
if isinstance(api_error, dict):
err_msg = api_error.get("message") or api_error.get("code") or "unknown error"
logger.warning("xAI web search returned error envelope: %s", err_msg)
return _fail(f"xAI returned an error: {err_msg}")
# Empty list on 0 hits is a success (matches brave-free / exa) so the model
# can decide whether to retry.
return search_ok(self._extract_results(data, limit=limit))
@staticmethod
def _web_search_tool(cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Build the ``web_search`` tool spec with optional domain filters; None when
both allowed and excluded are set (xAI rejects the combination)."""
allowed = _coerce_domain_list(cfg.get("allowed_domains"))
excluded = _coerce_domain_list(cfg.get("excluded_domains"))
if allowed and excluded:
return None
tool: Dict[str, Any] = {"type": "web_search"}
if allowed:
tool["filters"] = {"allowed_domains": allowed}
elif excluded:
tool["filters"] = {"excluded_domains": excluded}
return tool
@staticmethod
def _post_responses(
base_url: str,
payload: Dict[str, Any],
api_key: str,
timeout: float,
*,
is_oauth_path: bool,
) -> tuple[Any, Optional[Dict[str, Any]]]:
"""POST to ``/responses`` → ``(parsed_json, None)`` or ``(None, failure_envelope)``
on transport/HTTP/JSON failure.
Two attempts: on a first-call 401 with OAuth creds, force-refresh once and
retry. Covers opaque (non-JWT) tokens the resolver can't pre-check and
mid-window revocation/rotation. XAI_API_KEY creds can't be refreshed, so
they skip the retry rather than burn quota.
"""
import httpx
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"User-Agent": hermes_xai_user_agent(),
}
try:
import httpx
except ImportError:
return {
"success": False,
"error": "httpx is not installed (required for xAI web search)",
}
logger.info(
"xAI web search via %s: '%s' (limit=%d, model=%s)",
base_url, query, limit, model,
)
# Two-attempt loop: if the first call returns 401 and our creds came
# from the OAuth path, force-refresh the token once and retry. This
# closes two gaps the proactive resolver check doesn't cover:
# (1) opaque (non-JWT) access tokens — `_xai_access_token_is_expiring`
# can't decode them and returns False, so refresh never fires
# until the server hands us a 401.
# (2) mid-window revocation — admin revoke, refresh-token rotation,
# or clock skew can produce 401s on a token whose JWT `exp` claim
# is still in the future.
# Env-var (`XAI_API_KEY`) credentials skip the retry entirely — we
# can't refresh those and an immediate retry would just burn quota.
is_oauth_path = (creds.get("provider") == "xai-oauth")
resp = None
for attempt in range(2):
try:
resp = httpx.post(
f"{base_url}/responses",
headers=headers,
json=payload,
timeout=timeout,
)
resp = httpx.post(f"{base_url}/responses", headers=headers, json=payload, timeout=timeout)
resp.raise_for_status()
break
except httpx.HTTPStatusError as exc:
@@ -270,84 +216,40 @@ class XAIWebSearchProvider(WebSearchProvider):
"refresh and retrying once.",
)
try:
refreshed = resolve_xai_http_credentials(
force_refresh=True,
api_key_hint=api_key,
)
refreshed = resolve_xai_http_credentials(force_refresh=True, api_key_hint=api_key)
refreshed_key = str(refreshed.get("api_key") or "").strip()
if refreshed_key and refreshed_key != api_key:
api_key = refreshed_key
headers["Authorization"] = f"Bearer {api_key}"
continue
# Refresh returned the same (or empty) token — no point
# in retrying. Fall through to the error return below.
# Same/empty token back — retrying is pointless; fall through.
except Exception as refresh_exc: # noqa: BLE001
logger.warning(
"xAI web search OAuth refresh after 401 failed: %s",
refresh_exc,
)
body = ""
logger.warning("xAI web search OAuth refresh after 401 failed: %s", refresh_exc)
try:
body = exc.response.text[:300] if exc.response is not None else ""
except Exception:
body = ""
logger.warning("xAI web search HTTP %d: %s", status, body)
return {
"success": False,
"error": f"xAI web search returned HTTP {status}: {body}".rstrip(),
}
return None, _fail(f"xAI web search returned HTTP {status}: {body}".rstrip())
except httpx.RequestError as exc:
logger.warning("xAI web search request error: %s", exc)
return {"success": False, "error": f"Could not reach xAI: {exc}"}
return None, _fail(f"Could not reach xAI: {exc}")
if resp is None:
# Defensive — both attempts somehow exited the loop without resp.
return {"success": False, "error": "xAI web search produced no response"}
return None, _fail("xAI web search produced no response")
try:
data = resp.json()
return resp.json(), None
except Exception as exc: # noqa: BLE001
logger.warning("xAI web search bad JSON: %s", exc)
return {
"success": False,
"error": "Could not parse xAI Responses API reply as JSON",
}
# xAI's Responses surface sometimes returns HTTP 200 with an error
# envelope (model overloaded, content-policy refusal, etc.). Without
# this check, ``_extract_results`` would silently produce an empty
# list and we'd report success-with-no-rows — masking a real failure
# the agent should see and decide whether to retry.
api_error = data.get("error") if isinstance(data, dict) else None
if isinstance(api_error, dict):
err_msg = (
api_error.get("message")
or api_error.get("code")
or "unknown error"
)
logger.warning("xAI web search returned error envelope: %s", err_msg)
return {"success": False, "error": f"xAI returned an error: {err_msg}"}
web_results = self._extract_results(data, limit=limit)
if not web_results:
# Successful call, just no usable rows — return success with an
# empty list so the model can decide whether to retry. Matches
# what brave-free / exa do when the upstream API returns 0 hits.
return {"success": True, "data": {"web": []}}
return {"success": True, "data": {"web": web_results}}
return None, _fail("Could not parse xAI Responses API reply as JSON")
# -- Prompt + parsing -------------------------------------------------
@staticmethod
def _build_prompt(query: str, limit: int) -> str:
"""Compose the prompt that asks Grok to act as a search engine.
We deliberately ask for a JSON object (not bare array) so we can
match it cheaply with ``_JSON_BLOCK_RE``; we explicitly forbid
prose, markdown fences, and inline-citation links to keep the
payload parseable.
"""
"""Ask Grok for a JSON *object* (cheap to match with ``_JSON_BLOCK_RE``) and
forbid prose/fences/inline citations to keep the payload parseable."""
return (
"Use the web_search tool to find current information for the query below, "
"then respond with ONLY a single JSON object — no prose, no markdown "
@@ -361,56 +263,28 @@ class XAIWebSearchProvider(WebSearchProvider):
)
@classmethod
def _extract_results(
cls,
response_data: Dict[str, Any],
*,
limit: int,
) -> List[Dict[str, Any]]:
"""Pull a ``[{title, url, description, position}, ...]`` list out of a
Responses-API reply.
Strategy:
1. Walk ``output[*].content[*].text`` for ``output_text`` blocks and
try to parse the first JSON object that has a ``results`` list.
2. If the JSON path fails, fall back to the message annotations
(``url_citation`` entries) — every annotation carries a URL and
a ``title`` (citation number); we pair those URLs with surrounding
text from the message body as a best-effort description.
"""
def _extract_results(cls, response_data: Dict[str, Any], *, limit: int) -> List[Dict[str, Any]]:
"""Result rows from a Responses-API reply, in order of preference:
(1) the JSON object in ``output_text`` blocks, (2) ``url_citation``
annotations paired with surrounding text, (3) the raw ``citations`` list.
Only short-circuit on (2) when it yields rows, so future annotation types
don't mask real data in ``citations``."""
text_blocks, annotations = cls._collect_output_text(response_data)
# Primary path: parse the JSON object Grok was asked for.
for block in text_blocks:
parsed = cls._try_parse_json_results(block, limit=limit)
if parsed:
return parsed
# Secondary path: derive results from message annotations + raw text.
# Only short-circuit when annotations actually yielded usable rows;
# otherwise fall through to the citations list. (xAI currently only
# emits ``url_citation`` annotations, but future annotation types
# would silently produce an empty result set if we returned here
# unconditionally — masking real data in ``citations``.)
if annotations:
joined_text = "\n".join(text_blocks)
annotation_results = cls._results_from_annotations(
annotations, joined_text, limit=limit,
)
annotation_results = cls._results_from_annotations(annotations, "\n".join(text_blocks), limit=limit)
if annotation_results:
return annotation_results
# Last-ditch: raw citations list (no titles or descriptions).
citations = response_data.get("citations") or []
if isinstance(citations, list):
return [
{
"title": "",
"url": str(u),
"description": "",
"position": i + 1,
}
_row("", str(u), "", i + 1)
for i, u in enumerate(citations[:limit])
if isinstance(u, str) and u.strip()
]
@@ -418,10 +292,8 @@ class XAIWebSearchProvider(WebSearchProvider):
return []
@staticmethod
def _collect_output_text(
response_data: Dict[str, Any],
) -> tuple[List[str], List[Dict[str, Any]]]:
"""Return (text_blocks, annotations) extracted from ``response.output``."""
def _collect_output_text(response_data: Dict[str, Any]) -> tuple[List[str], List[Dict[str, Any]]]:
"""Return (text_blocks, annotations) from ``response.output`` message chunks."""
text_blocks: List[str] = []
annotations: List[Dict[str, Any]] = []
output = response_data.get("output")
@@ -442,25 +314,14 @@ class XAIWebSearchProvider(WebSearchProvider):
text_blocks.append(text)
chunk_annotations = chunk.get("annotations")
if isinstance(chunk_annotations, list):
for ann in chunk_annotations:
if isinstance(ann, dict):
annotations.append(ann)
annotations.extend(a for a in chunk_annotations if isinstance(a, dict))
return text_blocks, annotations
@staticmethod
def _try_parse_json_results(
text: str,
*,
limit: int,
) -> Optional[List[Dict[str, Any]]]:
"""Parse a JSON object with a ``results`` array out of ``text``.
Returns the normalized result list on success, ``None`` when the
block has no valid JSON object or no ``results`` key. Tolerates
leading/trailing prose because reasoning models sometimes prefix a
short narration even when told not to.
"""
# Try the whole string first — cheapest path when Grok obeys.
def _try_parse_json_results(text: str, *, limit: int) -> Optional[List[Dict[str, Any]]]:
"""Parse a JSON object with a ``results`` array out of ``text``; None when
absent. Tries the whole string first, then the regex-matched block, since
reasoning models sometimes prefix narration."""
candidates = [text]
match = _JSON_BLOCK_RE.search(text)
if match and match.group(0) != text:
@@ -483,34 +344,21 @@ class XAIWebSearchProvider(WebSearchProvider):
url = str(row.get("url", "")).strip()
if not url:
continue
normalized.append(
{
"title": str(row.get("title", "")).strip(),
"url": url,
"description": str(row.get("description", "")).strip(),
# Renumber from the kept results, not the raw input
# index, so a dropped malformed row doesn't leave a
# gap in the positions handed back to the agent.
"position": len(normalized) + 1,
}
)
# Renumber from kept rows so a dropped malformed row leaves no gap.
normalized.append(_row(
str(row.get("title", "")).strip(), url,
str(row.get("description", "")).strip(), len(normalized) + 1,
))
if normalized:
return normalized
return None
@staticmethod
def _results_from_annotations(
annotations: List[Dict[str, Any]],
joined_text: str,
*,
limit: int,
annotations: List[Dict[str, Any]], joined_text: str, *, limit: int
) -> List[Dict[str, Any]]:
"""Best-effort fallback when JSON parsing fails.
Uses each ``url_citation`` annotation's ``url`` (the citation
title is just the integer label, so we don't surface it) and
slices ~200 characters of surrounding text as the description.
"""
"""Fallback rows from ``url_citation`` annotations: URL plus ~200 chars of
preceding text as the description (the annotation title is just a number)."""
seen: set[str] = set()
results: List[Dict[str, Any]] = []
for ann in annotations:
@@ -525,19 +373,11 @@ class XAIWebSearchProvider(WebSearchProvider):
start = ann.get("start_index")
end = ann.get("end_index")
if isinstance(start, int) and isinstance(end, int) and 0 <= start < end <= len(joined_text):
window_start = max(0, start - 200)
description = joined_text[window_start:start].strip()
description = joined_text[max(0, start - 200):start].strip()
if len(description) > 200:
description = description[-200:].strip()
results.append(
{
"title": "",
"url": url,
"description": description,
"position": len(results) + 1,
}
)
results.append(_row("", url, description, len(results) + 1))
if len(results) >= limit:
break
return results
@@ -545,16 +385,12 @@ class XAIWebSearchProvider(WebSearchProvider):
# -- Setup picker -----------------------------------------------------
def get_setup_schema(self) -> Dict[str, Any]:
# Auth resolution is delegated to the shared ``xai_grok`` post_setup
# hook (same one image_gen.xai and tts.xai use) so users see the
# familiar OAuth-or-API-key prompt for every xAI service.
# Auth resolution is delegated to the shared ``xai_grok`` post_setup hook
# (same one image_gen.xai / tts.xai use) for a consistent OAuth-or-key prompt.
return {
"name": "xAI Web Search (Grok)",
"badge": "paid",
"tag": (
"Agentic web search via Grok's web_search tool — uses xAI "
"Grok OAuth or XAI_API_KEY."
),
"tag": "Agentic web search via Grok's web_search tool — uses xAI Grok OAuth or XAI_API_KEY.",
"env_vars": [],
"post_setup": "xai_grok",
}