diff --git a/agent/web_search_registry.py b/agent/web_search_registry.py index c4aa5ea62e..112fadc6df 100644 --- a/agent/web_search_registry.py +++ b/agent/web_search_registry.py @@ -168,36 +168,40 @@ _LEGACY_PREFERENCE = ( # Keyless free-tier walk — strictly LAST-resort, tried only after the # availability-filtered legacy walk finds nothing (i.e. the user has zero -# web credentials and no importable ddgs). These providers expose public -# anonymous MCP endpoints (see plugins/web/keyless_mcp.py). Like opencode, -# unpinned keyless traffic is split 50/50 between Exa and Parallel per -# process (see _keyless_preference()); an explicit `hermes tools` pick -# (web.backend / web._backend) bypasses this walk entirely. +# web credentials and no importable ddgs). All five vendors expose public +# anonymous free tiers (see plugins/web/keyless_mcp.py). Unpinned keyless +# traffic round-robins across the ring per request (the ring cursor lives +# in keyless_mcp; an explicit `hermes tools` pick bypasses this walk +# entirely, and rate-limited requests fail over to the next ring vendor). # Disable the tier with ``web.keyless_fallback: false``. _KEYLESS_PREFERENCE = ( "exa", "parallel", + "tavily", + "firecrawl", + "keenable", ) def _keyless_preference() -> tuple: - """Return the keyless walk order, split 50/50 per process. + """Return the keyless walk order for resolution. - Mirrors opencode's session-checksum A/B split between Exa and - Parallel: the per-process random session id (also used as Parallel's - free-tier rate-limit token) picks which vendor goes first, so keyless - load spreads evenly across both free tiers fleet-wide while staying - stable within one process. The runner-up stays in the walk as a - fallback if the first isn't registered. Explicit user selection never - reaches this function — configured names resolve in step 1. + Delegates the entry-vendor choice to the ring cursor in + :mod:`plugins.web.keyless_mcp` (round-robin per request, seeded by the + per-process random session id) so resolution and dispatch agree on + which vendor a fresh install starts at. The remaining vendors follow + in ring order as fallbacks for registration gaps. """ try: - from plugins.web.keyless_mcp import _SESSION_ID + from plugins.web.keyless_mcp import _KEYLESS_RING, _ring_cursor - if int(_SESSION_ID, 16) % 2: - return ("parallel", "exa") - except Exception as exc: # noqa: BLE001 — split is best-effort - logger.debug("keyless 50/50 split unavailable: %s", exc) + start = _ring_cursor % len(_KEYLESS_RING) + return tuple( + _KEYLESS_RING[(start + i) % len(_KEYLESS_RING)] + for i in range(len(_KEYLESS_RING)) + ) + except Exception as exc: # noqa: BLE001 — ring optional in stripped envs + logger.debug("keyless ring order unavailable: %s", exc) return _KEYLESS_PREFERENCE diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 448dd0c071..4b82f5bd9d 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -492,17 +492,20 @@ DEFAULT_CONFIG = { "search_backend": "", # per-capability override for web_search (e.g. "searxng") "extract_backend": "", # per-capability override for web_extract (e.g. "native") "extract_char_limit": 15000, # per-page char budget for web_extract; larger pages truncate + store full text in cache/web - # Keyless free-tier fallback: with NO web backend configured or keyed, - # web_search/web_extract fall back to Parallel's / Exa's public - # anonymous MCP endpoints (rate-limited free tiers). Never pre-empts - # a configured or keyed backend. Set false to disable entirely. + # Keyless free-tier ring: with NO web backend configured or keyed, + # web_search/web_extract rotate round-robin across five vendors' + # public free tiers (exa, parallel, tavily, firecrawl, keenable), + # failing over to the next ring vendor on rate limits. Never + # pre-empts a configured or keyed backend. Set false to disable. "keyless_fallback": True, - # Per-provider tier selection for providers with both a keyless free - # endpoint and a keyed paid SDK path (exa, parallel). Set by the - # `hermes tools` picker's "Free (keyless)" / "Paid (API key)" rows. + # Per-provider tier selection for ring vendors with both a keyless + # free endpoint and a keyed paid path (exa, parallel, tavily, + # firecrawl, keenable). Set by the `hermes tools` picker's + # "Free (keyless)" / "Paid (API key)" rows. # free — always use the anonymous free endpoint (even with a key) - # paid — always use the keyed SDK path (missing key = error) - # unset — auto: keyed when the API key is present, else keyless + # paid — always use the keyed path (missing key = error; vendor + # is also excluded from the keyless ring) + # unset — auto: keyed when the API key is present, else the ring "provider_tier": {}, }, @@ -4120,6 +4123,14 @@ OPTIONAL_ENV_VARS = { "password": True, "category": "tool", }, + "KEENABLE_API_KEY": { + "description": "Keenable API key for fast independent-index web search and page fetch (optional — keyless free tier works without it)", + "prompt": "Keenable API key", + "url": "https://keenable.ai", + "tools": ["web_search", "web_extract"], + "password": True, + "category": "tool", + }, "SEARXNG_URL": { "description": "URL of your SearXNG instance for free self-hosted web search", "prompt": "SearXNG URL (e.g. http://localhost:8080)", diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.py index 69f9c84fe2..0bc88cddcb 100644 --- a/plugins/web/firecrawl/provider.py +++ b/plugins/web/firecrawl/provider.py @@ -167,6 +167,38 @@ def _is_explicit_firecrawl_selection() -> bool: ) +def _use_keyless_ring() -> bool: + """True when Firecrawl calls should route via the keyless ring. + + Ring dispatch applies when there are no direct credentials, the + managed Nous gateway isn't the selected path, and the keyless tier + isn't disabled or pinned paid. Keyed/self-hosted/gateway setups never + reach the ring. + """ + from hermes_cli.config import get_env_value + + if (get_env_value("FIRECRAWL_API_KEY") or "").strip(): + return False + if (get_env_value("FIRECRAWL_API_URL") or "").strip(): + return False + import tools.web_tools as _wt + from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection + + try: + if read_selection("web") == NOUS_MANAGED_PROVIDER: + return False + except Exception: # noqa: BLE001 — selection helpers optional + pass + try: + if _wt._is_tool_gateway_ready() and not _is_explicit_firecrawl_selection(): + return False + except Exception: # noqa: BLE001 — probe optional + pass + from plugins.web.keyless_mcp import use_keyless + + return use_keyless("firecrawl", "") + + class _KeylessFirecrawlClient: """Minimal REST client for Firecrawl's keyless cloud mode. @@ -505,17 +537,15 @@ class FirecrawlWebSearchProvider(WebSearchProvider): return check_firecrawl_api_key() def is_keyless_available(self) -> bool: - """Firecrawl serves keyless cloud requests when explicitly selected. + """Firecrawl serves keyless cloud requests (public API, no auth). - Mirrors :func:`_is_explicit_firecrawl_selection` — keyless cloud - mode is opt-in by selection, never part of the automatic - zero-config fallback. Keeps doctor/readiness gates (#78412) from - flagging a working selected-keyless Firecrawl setup as unconfigured. + Default-on ring member of the keyless free tier: fresh installs + rotate across Exa/Parallel/Tavily/Firecrawl/Keenable. False when + the user pinned ``web.provider_tier.firecrawl: paid``. """ - try: - return _is_explicit_firecrawl_selection() - except Exception: # noqa: BLE001 — config layer optional - return False + from plugins.web.keyless_mcp import keyless_enabled, provider_tier + + return keyless_enabled() and provider_tier("firecrawl") != "paid" def supports_search(self) -> bool: return True @@ -542,6 +572,16 @@ class FirecrawlWebSearchProvider(WebSearchProvider): if is_interrupted(): return {"success": False, "error": "Interrupted"} + if _use_keyless_ring(): + # No credentials and no managed gateway: ring dispatch with + # next-in-line failover on rate limits (default-on free tier). + from plugins.web.keyless_mcp import search_with_failover + + logger.info( + "Firecrawl keyless search: '%s' (limit=%d)", query, limit + ) + return search_with_failover("firecrawl", query, limit) + logger.info("Firecrawl search: '%s' (limit=%d)", query, limit) # _get_firecrawl_client() raises ValueError on unconfigured systems — # let it propagate so the dispatcher emits the legacy envelope shape. @@ -575,6 +615,18 @@ class FirecrawlWebSearchProvider(WebSearchProvider): if _is_interrupted(): return [{"url": u, "error": "Interrupted", "title": ""} for u in urls] + if _use_keyless_ring(): + # No credentials and no managed gateway: ring dispatch with + # next-in-line failover on rate limits (default-on free tier). + import asyncio as _asyncio + + from plugins.web.keyless_mcp import extract_with_failover + + logger.info("Firecrawl keyless extract: %d URL(s)", len(urls)) + return await _asyncio.to_thread( + extract_with_failover, "firecrawl", list(urls) + ) + format = kwargs.get("format") formats: List[str] = [] if format == "markdown": diff --git a/plugins/web/keenable/__init__.py b/plugins/web/keenable/__init__.py new file mode 100644 index 0000000000..1897ca0926 --- /dev/null +++ b/plugins/web/keenable/__init__.py @@ -0,0 +1,13 @@ +"""Keenable web search + extract plugin — bundled, auto-loaded. + +Keyless-ring member (keyed via KEENABLE_API_KEY for higher limits). +""" + +from __future__ import annotations + +from plugins.web.keenable.provider import KeenableWebSearchProvider + + +def register(ctx) -> None: + """Register the Keenable provider with the plugin context.""" + ctx.register_web_search_provider(KeenableWebSearchProvider()) diff --git a/plugins/web/keenable/plugin.yaml b/plugins/web/keenable/plugin.yaml new file mode 100644 index 0000000000..a11b7ca73d --- /dev/null +++ b/plugins/web/keenable/plugin.yaml @@ -0,0 +1,7 @@ +name: web-keenable +version: 1.0.0 +description: "Keenable web search + page fetch (independent web index for AI apps). Works keyless on Keenable's free tier as part of the default rotation; set KEENABLE_API_KEY for higher limits — https://keenable.ai." +author: NousResearch +kind: backend +provides_web_providers: + - keenable diff --git a/plugins/web/keenable/provider.py b/plugins/web/keenable/provider.py new file mode 100644 index 0000000000..d185b5941e --- /dev/null +++ b/plugins/web/keenable/provider.py @@ -0,0 +1,233 @@ +"""Keenable web search + content extraction — bundled plugin. + +Keenable (https://keenable.ai) operates an independent web index for AI +apps with public keyless endpoints (rate-limited free tier; keyed access +via KEENABLE_API_KEY for higher limits). Integrated as a keyless-ring +member following the Exa/Parallel/Tavily/Firecrawl pattern: fresh installs +with zero web credentials rotate across all five vendors' free tiers. + +Credit: Keenable integration originally proposed by Ilya Gusev (Keenable) +in PR #49758; the native provider form follows the salvage of that work +plus the keyless-ring design. + +Config keys this provider responds to:: + + web: + search_backend: "keenable" # explicit per-capability + extract_backend: "keenable" # explicit per-capability + backend: "keenable" # shared fallback + provider_tier: + keenable: free|paid # pin the tier (unset = auto) + +Env var:: + + KEENABLE_API_KEY=... # optional — keyless free tier works without it +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + +_KEENABLE_API_URL = "https://api.keenable.ai" + + +def _keenable_headers(api_key: str) -> Dict[str, str]: + """Build Keenable request headers for keyed or keyless access. + + Their keyless tier structurally requires an app-identifier header + (X-Keenable-Title); no user identifiers are sent. + """ + headers = {"X-Keenable-Title": "hermes-agent"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + +class KeenableWebSearchProvider(WebSearchProvider): + """Keenable search + extract provider (keyed or keyless).""" + + @property + def name(self) -> str: + return "keenable" + + @property + def display_name(self) -> str: + return "Keenable" + + def is_available(self) -> bool: + """Return True when ``KEENABLE_API_KEY`` is set to a non-empty value.""" + from agent.web_search_provider import get_provider_env + + return bool(get_provider_env("KEENABLE_API_KEY")) + + def is_keyless_available(self) -> bool: + """Keenable serves anonymous free-tier calls via its public endpoints. + + Default-on ring member of the keyless free tier. False when the + user pinned ``web.provider_tier.keenable: paid``. + """ + from plugins.web.keyless_mcp import keyless_enabled, provider_tier + + return keyless_enabled() and provider_tier("keenable") != "paid" + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return True + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + """Execute a Keenable search (keyed path or keyless ring).""" + try: + from tools.interrupt import is_interrupted + + if is_interrupted(): + return {"success": False, "error": "Interrupted"} + + from agent.web_search_provider import get_provider_env + + from plugins.web.keyless_mcp import search_with_failover, use_keyless + + api_key = get_provider_env("KEENABLE_API_KEY") + if use_keyless("keenable", api_key): + logger.info( + "Keenable keyless search: '%s' (limit=%d)", query, limit + ) + return search_with_failover("keenable", query, limit) + + import requests + + logger.info("Keenable search: '%s' (limit=%d)", query, limit) + response = requests.post( + f"{_KEENABLE_API_URL}/v1/search", + json={"query": query, "max_results": min(max(1, int(limit)), 20)}, + headers=_keenable_headers(api_key), + timeout=30, + ) + if response.status_code >= 400: + detail = (response.text or "").strip() or f"HTTP {response.status_code}" + return {"success": False, "error": f"Keenable search failed: {detail}"} + data = response.json() + + web_results = [] + for i, result in enumerate(data.get("results") or []): + web_results.append( + { + "url": result.get("url") or "", + "title": result.get("title") or "", + "description": result.get("snippet") + or result.get("description") + or "", + "position": i + 1, + } + ) + return {"success": True, "data": {"web": web_results}} + except Exception as exc: # noqa: BLE001 — surface as failure + logger.warning("Keenable search error: %s", exc) + return {"success": False, "error": f"Keenable search failed: {exc}"} + + def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: + """Extract content via Keenable's fetch endpoint (per-URL). + + Sync — the dispatcher wraps in a thread when the caller is async. + Returns the legacy list-of-results shape; per-URL failures become + items with an ``error`` field. + """ + try: + from tools.interrupt import is_interrupted + + if is_interrupted(): + return [ + {"url": u, "error": "Interrupted", "title": ""} for u in urls + ] + + from agent.web_search_provider import get_provider_env + + from plugins.web.keyless_mcp import extract_with_failover, use_keyless + + api_key = get_provider_env("KEENABLE_API_KEY") + if use_keyless("keenable", api_key): + logger.info("Keenable keyless extract: %d URL(s)", len(urls)) + return extract_with_failover("keenable", list(urls)) + + import requests + + logger.info("Keenable extract: %d URL(s)", len(urls)) + results: List[Dict[str, Any]] = [] + for url in urls: + try: + response = requests.get( + f"{_KEENABLE_API_URL}/v1/fetch", + params={"url": url}, + headers=_keenable_headers(api_key), + timeout=30, + ) + if response.status_code >= 400: + raise ValueError( + (response.text or "").strip() + or f"HTTP {response.status_code}" + ) + data = response.json() + content = data.get("content") or "" + title = data.get("title") or "" + results.append( + { + "url": data.get("url") or url, + "title": title, + "content": content, + "raw_content": content, + "metadata": {"sourceURL": url, "title": title}, + } + ) + except Exception as exc: # noqa: BLE001 — per-URL error entry + results.append( + { + "url": url, + "title": "", + "content": "", + "error": f"Keenable extract failed: {exc}", + } + ) + return results + except Exception as exc: # noqa: BLE001 + logger.warning("Keenable extract error: %s", exc) + return [ + {"url": u, "title": "", "content": "", + "error": f"Keenable extract failed: {exc}"} + for u in urls + ] + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Keenable · Free (keyless)", + "badge": "free · no key", + "tag": ( + "Independent web index for AI apps — fast search + page " + "fetch on Keenable's anonymous free tier." + ), + "env_vars": [], + "web_tier": "free", + "variants": [ + { + "name": "Keenable · Paid (API key)", + "badge": "paid", + "tag": ( + "Independent web index for AI apps. Keyed access " + "with higher limits and guaranteed service." + ), + "env_vars": [ + { + "key": "KEENABLE_API_KEY", + "prompt": "Keenable API key", + "url": "https://keenable.ai", + }, + ], + "web_tier": "paid", + }, + ], + } diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.py index 462625ee84..c3c1947fd3 100644 --- a/plugins/web/keyless_mcp.py +++ b/plugins/web/keyless_mcp.py @@ -437,90 +437,435 @@ def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: return results + # --------------------------------------------------------------------------- -# Cross-vendor failover (rate-limited free tiers) +# Tavily keyless (api.tavily.com — X-Tavily-Access-Mode: keyless) # --------------------------------------------------------------------------- + +TAVILY_API_URL = "https://api.tavily.com" + + +def _tavily_keyless_post(endpoint: str, payload: Dict[str, Any]) -> Dict[str, Any]: + """POST to Tavily with keyless headers; raise KeylessMCPError on failure.""" + import requests + + try: + response = requests.post( + f"{TAVILY_API_URL}/{endpoint.lstrip('/')}", + json=payload, + headers={ + "Content-Type": "application/json", + "X-Client-Name": "hermes-agent", + "X-Tavily-Access-Mode": "keyless", + }, + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + raise KeylessMCPError(f"request failed: {exc}") from exc + if response.status_code >= 400: + raise KeylessMCPError( + (response.text or "").strip() or f"HTTP {response.status_code}" + ) + return response.json() + + +def tavily_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: + """Keyless Tavily search → legacy search response shape.""" + try: + data = _tavily_keyless_post( + "search", {"query": query, "max_results": max(1, int(limit))} + ) + except KeylessMCPError as exc: + return { + "success": False, + "error": ( + f"Keyless Tavily search failed: {exc}. " + "Set TAVILY_API_KEY (https://app.tavily.com) or another web " + "backend via `hermes tools` for reliable service." + ), + } + web_results = [] + for i, result in enumerate(data.get("results") or []): + web_results.append( + { + "url": result.get("url") or "", + "title": result.get("title") or "", + "description": result.get("content") or "", + "position": i + 1, + } + ) + return {"success": True, "data": {"web": web_results}} + + +def tavily_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: + """Keyless Tavily extract → legacy extract result list.""" + try: + data = _tavily_keyless_post("extract", {"urls": list(urls)}) + except KeylessMCPError as exc: + message = ( + f"Keyless Tavily extract failed: {exc}. " + "Set TAVILY_API_KEY (https://app.tavily.com) or another web " + "backend via `hermes tools` for reliable service." + ) + return [ + {"url": u, "title": "", "content": "", "error": message} + for u in urls + ] + results: List[Dict[str, Any]] = [] + seen = set() + for result in data.get("results") or []: + url = result.get("url") or "" + raw = result.get("raw_content") or result.get("content") or "" + seen.add(url) + results.append( + { + "url": url, + "title": result.get("title") or "", + "content": raw, + "raw_content": raw, + "metadata": {"sourceURL": url, "title": result.get("title") or ""}, + } + ) + for fail in data.get("failed_results") or []: + url = (fail.get("url") if isinstance(fail, dict) else str(fail)) or "" + seen.add(url) + results.append( + { + "url": url, + "title": "", + "content": "", + "error": (fail.get("error") if isinstance(fail, dict) else None) + or "extraction failed", + } + ) + for u in urls: + if u not in seen: + results.append( + {"url": u, "title": "", "content": "", "error": "no content returned"} + ) + return results + + +# --------------------------------------------------------------------------- +# Firecrawl keyless (public cloud API, no auth header) +# --------------------------------------------------------------------------- + + +def firecrawl_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: + """Keyless Firecrawl cloud search → legacy search response shape.""" + from plugins.web.firecrawl.provider import ( + _KeylessFirecrawlClient, + _extract_web_search_results, + ) + + try: + response = _KeylessFirecrawlClient().search(query=query, limit=limit) + return {"success": True, "data": {"web": _extract_web_search_results(response)}} + except Exception as exc: # noqa: BLE001 — normalized below + return { + "success": False, + "error": ( + f"Keyless Firecrawl search failed: {exc}. " + "Set FIRECRAWL_API_KEY (https://firecrawl.dev) or another web " + "backend via `hermes tools` for reliable service." + ), + } + + +def firecrawl_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: + """Keyless Firecrawl cloud scrape → legacy extract result list.""" + from plugins.web.firecrawl.provider import ( + _KeylessFirecrawlClient, + _extract_scrape_payload, + ) + + client = _KeylessFirecrawlClient() + results: List[Dict[str, Any]] = [] + for url in urls: + try: + response = client.scrape(url=url, formats=["markdown"]) + payload = _extract_scrape_payload(response) or {} + metadata = payload.get("metadata") or {} + if not isinstance(metadata, dict): + metadata = {} + content = payload.get("markdown") or payload.get("html") or "" + title = metadata.get("title") or "" + results.append( + { + "url": url, + "title": title, + "content": content, + "raw_content": content, + "metadata": {"sourceURL": url, "title": title}, + } + ) + except Exception as exc: # noqa: BLE001 — per-URL error entry + results.append( + { + "url": url, + "title": "", + "content": "", + "error": ( + f"Keyless Firecrawl extract failed: {exc}. " + "Set FIRECRAWL_API_KEY (https://firecrawl.dev) for " + "reliable service." + ), + } + ) + return results + + +# --------------------------------------------------------------------------- +# Keenable keyless (api.keenable.ai public endpoints) +# --------------------------------------------------------------------------- + + +KEENABLE_API_URL = "https://api.keenable.ai" +_KEENABLE_TITLE = "hermes-agent" + + +def keenable_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: + """Keyless Keenable search → legacy search response shape. + + POST /v1/search/public with the mandatory X-Keenable-Title app + identifier (their keyless tier requires an app name; no user + identifiers are sent). Response: {results: [{title, url, snippet}]}. + """ + import requests + + try: + response = requests.post( + f"{KEENABLE_API_URL}/v1/search/public", + json={"query": query, "max_results": max(1, int(limit))}, + headers={ + "Content-Type": "application/json", + "X-Keenable-Title": _KEENABLE_TITLE, + }, + timeout=_TIMEOUT_SECONDS, + ) + if response.status_code >= 400: + raise KeylessMCPError( + (response.text or "").strip() or f"HTTP {response.status_code}" + ) + data = response.json() + except KeylessMCPError as exc: + return { + "success": False, + "error": ( + f"Keyless Keenable search failed: {exc}. " + "Set KEENABLE_API_KEY (https://keenable.ai) or another web " + "backend via `hermes tools` for reliable service." + ), + } + except Exception as exc: # noqa: BLE001 — transport/JSON errors + return { + "success": False, + "error": f"Keyless Keenable search failed: {exc}.", + } + web_results = [] + for i, result in enumerate(data.get("results") or []): + web_results.append( + { + "url": result.get("url") or "", + "title": result.get("title") or "", + "description": result.get("snippet") + or result.get("description") + or "", + "position": i + 1, + } + ) + return {"success": True, "data": {"web": web_results}} + + +def keenable_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: + """Keyless Keenable page fetch → legacy extract result list. + + GET /v1/fetch/public?url=... returns {url, title, content} (markdown). + Called per-URL; failures become per-URL error entries. + """ + import requests + + results: List[Dict[str, Any]] = [] + for url in urls: + try: + response = requests.get( + f"{KEENABLE_API_URL}/v1/fetch/public", + params={"url": url}, + headers={"X-Keenable-Title": _KEENABLE_TITLE}, + timeout=_TIMEOUT_SECONDS, + ) + if response.status_code >= 400: + raise KeylessMCPError( + (response.text or "").strip() or f"HTTP {response.status_code}" + ) + data = response.json() + content = data.get("content") or "" + title = data.get("title") or "" + results.append( + { + "url": data.get("url") or url, + "title": title, + "content": content, + "raw_content": content, + "metadata": {"sourceURL": url, "title": title}, + } + ) + except Exception as exc: # noqa: BLE001 — per-URL error entry + results.append( + { + "url": url, + "title": "", + "content": "", + "error": ( + f"Keyless Keenable extract failed: {exc}. " + "Set KEENABLE_API_KEY (https://keenable.ai) for " + "reliable service." + ), + } + ) + return results + + +# --------------------------------------------------------------------------- +# Round-robin ring + next-in-line failover (rate-limited free tiers) +# --------------------------------------------------------------------------- + +_KEYLESS_RING = ("exa", "parallel", "tavily", "firecrawl", "keenable") + _KEYLESS_SEARCHERS = { "exa": lambda query, limit: exa_search_keyless(query, limit), "parallel": lambda query, limit: parallel_search_keyless(query, limit), + "tavily": lambda query, limit: tavily_search_keyless(query, limit), + "firecrawl": lambda query, limit: firecrawl_search_keyless(query, limit), + "keenable": lambda query, limit: keenable_search_keyless(query, limit), } _KEYLESS_EXTRACTORS = { "exa": lambda urls: exa_extract_keyless(urls), "parallel": lambda urls: parallel_extract_keyless(urls), + "tavily": lambda urls: tavily_extract_keyless(urls), + "firecrawl": lambda urls: firecrawl_extract_keyless(urls), + "keenable": lambda urls: keenable_extract_keyless(urls), } +# Per-process round-robin cursor, seeded by the random session id so the +# fleet spreads evenly across all five free tiers; advances once per +# unpinned keyless request so a single process also rotates. +_ring_lock = __import__("threading").Lock() +_ring_cursor = int(_SESSION_ID, 16) % len(_KEYLESS_RING) -def _failover_peer(name: str) -> Optional[str]: - """Return the OTHER keyless vendor, or None when failover is off. - Only fires in auto/free tiers: a user who explicitly pinned a paid - tier for the peer (``web.provider_tier.: paid``) has opted the - peer's free endpoint out, so we respect that and don't route to it. +def _vendor_pinned(name: str) -> bool: + """True when config explicitly routes web traffic to *name*. + + A pinned vendor starts every keyless request (rotation off); the ring + is only walked past it on throttle. Pin signals: web.backend / + web.search_backend / web.extract_backend naming the vendor, or a + free-tier pin in web.provider_tier. """ - peer = {"exa": "parallel", "parallel": "exa"}.get(name) - if peer is None or not keyless_enabled(): - return None - if provider_tier(peer) == "paid": - return None - return peer + if provider_tier(name) == "free": + return True + try: + import tools.web_tools as _wt + + web_cfg = _wt._load_web_config() + return any( + (web_cfg.get(key) or "").lower().strip() == name + for key in ("backend", "search_backend", "extract_backend") + ) + except Exception as exc: # noqa: BLE001 — config layer optional + logger.debug("_vendor_pinned(%r) config read failed: %s", name, exc) + return False + + +def _ring_order(name: str) -> List[str]: + """Return the vendor walk order for a request entering via *name*. + + Pinned vendor → start at it (its position in the ring determines the + failover succession). Unpinned → true round-robin: start at the next + cursor position, advancing the cursor per request. Vendors whose tier + is pinned ``paid`` are excluded entirely (an explicit paid selection + opts that vendor's free endpoint out). + """ + global _ring_cursor + if _vendor_pinned(name): + start = _KEYLESS_RING.index(name) if name in _KEYLESS_RING else 0 + else: + with _ring_lock: + start = _ring_cursor + _ring_cursor = (_ring_cursor + 1) % len(_KEYLESS_RING) + ordered = [ + _KEYLESS_RING[(start + i) % len(_KEYLESS_RING)] + for i in range(len(_KEYLESS_RING)) + ] + return [v for v in ordered if provider_tier(v) != "paid"] def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]: - """Keyless search via *name*, failing over to the peer vendor on throttle. + """Keyless search across the vendor ring with next-in-line failover. - When the primary's free tier returns a rate-limit-shaped error, retry - once on the other vendor's free endpoint (Exa <-> Parallel). Non-throttle - errors are returned as-is (a malformed-query error on vendor A would - just fail identically on vendor B). The failover result notes which - vendor actually served the request via ``data.served_by``. + Starts at *name* when the user pinned it, otherwise at the round-robin + cursor. Rate-limit-shaped errors advance to the next ring vendor; + non-throttle errors stop the walk (a malformed query fails everywhere). + The result notes the serving vendor via ``data.served_by`` whenever it + differs from *name*. """ - primary = _KEYLESS_SEARCHERS[name] - result = primary(query, limit) - if result.get("success") or not _is_rate_limitish(result.get("error", "")): - return result - - peer = _failover_peer(name) - if peer is None: - return result - logger.info("keyless %s search throttled; failing over to %s", name, peer) - fallback = _KEYLESS_SEARCHERS[peer](query, limit) - if fallback.get("success"): - fallback.setdefault("data", {})["served_by"] = peer - return fallback - # Both throttled: surface the primary's error (it names the pinned - # vendor's key), with a note that the peer was tried too. - result["error"] = ( - f"{result.get('error', '')} Failover to {peer} also failed: " - f"{fallback.get('error', 'unknown error')}" + order = _ring_order(name) + if not order: + return { + "success": False, + "error": "All keyless web providers are pinned to paid tiers.", + } + last: Dict[str, Any] = {} + for i, vendor in enumerate(order): + result = _KEYLESS_SEARCHERS[vendor](query, limit) + if result.get("success"): + if vendor != name: + result.setdefault("data", {})["served_by"] = vendor + return result + last = result + if not _is_rate_limitish(result.get("error", "")): + return result + nxt = order[i + 1] if i + 1 < len(order) else None + if nxt: + logger.info( + "keyless %s search throttled; failing over to %s", vendor, nxt + ) + last["error"] = ( + f"{last.get('error', '')} (all keyless vendors throttled: " + f"{', '.join(order)})" ) - return result + return last def extract_with_failover(name: str, urls: List[str]) -> List[Dict[str, Any]]: - """Keyless extract via *name*, failing over per-batch on throttle. + """Keyless extract across the vendor ring, failing over per-batch. - If EVERY url in the primary's result carries a rate-limit-shaped - error, retry the whole batch on the peer vendor. Partial failures - (some URLs fine, some broken) are returned as-is — those are page - problems, not throttling. + Advances to the next ring vendor only when EVERY url in a batch comes + back with a rate-limit-shaped error — partial failures are page + problems, not throttling, and return as-is. """ - primary = _KEYLESS_EXTRACTORS[name] - results = primary(list(urls)) - errors = [r.get("error", "") for r in results] - all_throttled = bool(results) and all( - e and _is_rate_limitish(e) for e in errors - ) - if not all_throttled: - return results - - peer = _failover_peer(name) - if peer is None: - return results - logger.info("keyless %s extract throttled; failing over to %s", name, peer) - fallback = _KEYLESS_EXTRACTORS[peer](list(urls)) - fallback_errors = [r.get("error", "") for r in fallback] - if all(e and _is_rate_limitish(e) for e in fallback_errors): - return results # both throttled: keep primary's key guidance - return fallback + order = _ring_order(name) + if not order: + return [ + {"url": u, "title": "", "content": "", + "error": "All keyless web providers are pinned to paid tiers."} + for u in urls + ] + last: List[Dict[str, Any]] = [] + for i, vendor in enumerate(order): + results = _KEYLESS_EXTRACTORS[vendor](list(urls)) + errors = [r.get("error", "") for r in results] + all_throttled = bool(results) and all( + e and _is_rate_limitish(e) for e in errors + ) + if not all_throttled: + return results + last = results + nxt = order[i + 1] if i + 1 < len(order) else None + if nxt: + logger.info( + "keyless %s extract throttled; failing over to %s", vendor, nxt + ) + return last diff --git a/plugins/web/tavily/provider.py b/plugins/web/tavily/provider.py index 71badedad5..096558f9a3 100644 --- a/plugins/web/tavily/provider.py +++ b/plugins/web/tavily/provider.py @@ -160,24 +160,16 @@ class TavilyWebSearchProvider(WebSearchProvider): return bool(get_provider_env("TAVILY_API_KEY")) def is_keyless_available(self) -> bool: - """Tavily serves keyless requests when explicitly selected. + """Tavily serves anonymous keyless requests (X-Tavily-Access-Mode). - Keyless mode is opt-in by selection (X-Tavily-Access-Mode header), - not part of the automatic zero-config fallback — so this only - reports True when config actually routes a capability to Tavily. - Keeps doctor/readiness gates (#78412) from flagging a working - selected-keyless Tavily setup as unconfigured. + Default-on ring member of the keyless free tier: fresh installs + rotate across Exa/Parallel/Tavily/Firecrawl/Keenable. False when + the user pinned ``web.provider_tier.tavily: paid`` — an explicit + paid selection opts the free endpoint out. """ - import tools.web_tools as _wt + from plugins.web.keyless_mcp import keyless_enabled, provider_tier - try: - cfg = _wt._load_web_config() - except Exception: # noqa: BLE001 — config layer optional - return False - return any( - (cfg.get(key) or "").lower().strip() == "tavily" - for key in ("backend", "search_backend", "extract_backend") - ) + return keyless_enabled() and provider_tier("tavily") != "paid" def supports_search(self) -> bool: return True @@ -193,6 +185,18 @@ class TavilyWebSearchProvider(WebSearchProvider): if is_interrupted(): return {"success": False, "error": "Interrupted"} + from agent.web_search_provider import get_provider_env + + from plugins.web.keyless_mcp import search_with_failover, use_keyless + + if use_keyless("tavily", get_provider_env("TAVILY_API_KEY")): + # Keyless free tier — ring dispatch with next-in-line + # failover on rate limits. + logger.info( + "Tavily keyless search: '%s' (limit=%d)", query, limit + ) + return search_with_failover("tavily", query, limit) + logger.info("Tavily search: '%s' (limit=%d)", query, limit) raw = _tavily_request( "search", @@ -224,6 +228,16 @@ class TavilyWebSearchProvider(WebSearchProvider): {"url": u, "error": "Interrupted", "title": ""} for u in urls ] + from agent.web_search_provider import get_provider_env + + from plugins.web.keyless_mcp import extract_with_failover, use_keyless + + if use_keyless("tavily", get_provider_env("TAVILY_API_KEY")): + # Keyless free tier — ring dispatch with next-in-line + # failover on rate limits. + logger.info("Tavily keyless extract: %d URL(s)", len(urls)) + return extract_with_failover("tavily", list(urls)) + logger.info("Tavily extract: %d URL(s)", len(urls)) raw = _tavily_request( "extract", diff --git a/tests/plugins/web/test_web_search_provider_plugins.py b/tests/plugins/web/test_web_search_provider_plugins.py index d462c0a933..200ce9878a 100644 --- a/tests/plugins/web/test_web_search_provider_plugins.py +++ b/tests/plugins/web/test_web_search_provider_plugins.py @@ -70,7 +70,7 @@ def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None: class TestBundledPluginsRegister: """All eight bundled web plugins discover and register correctly.""" - def test_all_seven_plugins_present_in_registry(self) -> None: + def test_all_bundled_plugins_present_in_registry(self) -> None: _ensure_plugins_loaded() from agent.web_search_registry import list_providers @@ -80,6 +80,7 @@ class TestBundledPluginsRegister: "ddgs", "exa", "firecrawl", + "keenable", "parallel", "searxng", "tavily", diff --git a/tests/tools/test_web_keyless_fallback.py b/tests/tools/test_web_keyless_fallback.py index 74c6e5e51e..d2a959c626 100644 --- a/tests/tools/test_web_keyless_fallback.py +++ b/tests/tools/test_web_keyless_fallback.py @@ -172,15 +172,16 @@ class TestKeylessCalls: class TestProviderRouting: - def test_parallel_keyless_path_when_no_key(self): + def test_parallel_keyless_path_when_no_key(self, monkeypatch): + # Pin parallel so the ring deterministically starts there. + monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda n: n == "parallel") provider = ParallelWebSearchProvider() - with patch.object( - keyless_mcp, "parallel_search_keyless", - return_value={"success": True, "data": {"web": []}}, - ) as keyless: + with patch.dict( + keyless_mcp._KEYLESS_SEARCHERS, + {"parallel": lambda q, l: {"success": True, "data": {"web": []}}}, + ): out = provider.search("q", limit=3) assert out["success"] is True - keyless.assert_called_once_with("q", 3) def test_exa_keyless_path_when_no_key(self): provider = ExaWebSearchProvider() @@ -279,23 +280,28 @@ class TestResolutionOrder: monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) provider = registry.get_active_search_provider() assert provider is not None - # 50/50 split: either keyless vendor is valid; it must match the - # process-stable preference order. - assert provider.name == registry._keyless_preference()[0] - assert provider.name in ("exa", "parallel") - - def test_keyless_split_is_process_stable_and_covers_both(self, fresh_registry, monkeypatch): - monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) - # Stable within a process: repeated resolution never flip-flops. - first = registry.get_active_search_provider().name - assert all( - registry.get_active_search_provider().name == first for _ in range(5) + # Ring: resolution picks the first REGISTERED vendor in ring order + # (only exa/parallel are registered in this fixture). + expected = next( + v for v in registry._keyless_preference() if v in ("exa", "parallel") ) - # Both split outcomes route correctly (simulate the two parities). - monkeypatch.setattr(keyless_mcp, "_SESSION_ID", "0" * 32) # even - assert registry._keyless_preference() == ("exa", "parallel") - monkeypatch.setattr(keyless_mcp, "_SESSION_ID", "1" * 32) # odd - assert registry._keyless_preference() == ("parallel", "exa") + assert provider.name == expected + + def test_keyless_ring_rotates_and_covers_all_vendors(self, fresh_registry, monkeypatch): + monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) + # The ring order always contains all five vendors, starting at the + # current cursor and wrapping. + order = registry._keyless_preference() + assert sorted(order) == sorted(keyless_mcp._KEYLESS_RING) + # Unpinned dispatch rotates: consecutive _ring_order calls start at + # successive vendors (round-robin cursor advances per request). + monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda name: False) + starts = [keyless_mcp._ring_order("exa")[0] for _ in range(len(keyless_mcp._KEYLESS_RING))] + assert sorted(starts) == sorted(keyless_mcp._KEYLESS_RING) # full cycle + # Pinned dispatch starts at the pinned vendor every time. + monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda name: name == "tavily") + assert keyless_mcp._ring_order("tavily")[0] == "tavily" + assert keyless_mcp._ring_order("tavily")[0] == "tavily" def test_registry_keyless_disabled_returns_none(self, fresh_registry, monkeypatch): monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) @@ -322,7 +328,10 @@ class TestResolutionOrder: ) monkeypatch.setattr(web_tools, "_list_registered_web_providers", list) from agent.web_search_registry import _keyless_preference - assert web_tools._get_backend() == _keyless_preference()[0] + expected = next( + v for v in _keyless_preference() if v in ("exa", "parallel") + ) + assert web_tools._get_backend() == expected def test_get_backend_key_beats_keyless(self, monkeypatch): monkeypatch.setattr( @@ -428,50 +437,84 @@ class TestKeylessFailover: def _throttled(self, vendor): return {"success": False, "error": f"Keyless {vendor} search failed: free MCP rate limit."} + def _pin(self, monkeypatch, name): + """Pin *name* so the ring starts there deterministically.""" + monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda n: n == name) + def test_search_fails_over_on_rate_limit(self, monkeypatch): - monkeypatch.setattr(keyless_mcp, "exa_search_keyless", lambda q, l: self._throttled("Exa")) - monkeypatch.setattr(keyless_mcp, "parallel_search_keyless", lambda q, l: self._ok("parallel")) + self._pin(monkeypatch, "exa") + monkeypatch.setitem(keyless_mcp._KEYLESS_SEARCHERS, "exa", lambda q, l: self._throttled("Exa")) + monkeypatch.setitem(keyless_mcp._KEYLESS_SEARCHERS, "parallel", lambda q, l: self._ok("parallel")) out = keyless_mcp.search_with_failover("exa", "q", 3) assert out["success"] is True assert out["data"]["served_by"] == "parallel" def test_search_no_failover_on_non_throttle_error(self, monkeypatch): - monkeypatch.setattr( - keyless_mcp, "exa_search_keyless", + self._pin(monkeypatch, "exa") + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, "exa", lambda q, l: {"success": False, "error": "Unrecognized MCP response shape"}, ) called = [] - monkeypatch.setattr( - keyless_mcp, "parallel_search_keyless", + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, "parallel", lambda q, l: called.append(1) or self._ok("parallel"), ) out = keyless_mcp.search_with_failover("exa", "q") assert out["success"] is False assert not called # peer never tried - def test_search_both_throttled_reports_both(self, monkeypatch): - monkeypatch.setattr(keyless_mcp, "exa_search_keyless", lambda q, l: self._throttled("Exa")) - monkeypatch.setattr(keyless_mcp, "parallel_search_keyless", lambda q, l: self._throttled("Parallel")) + def test_search_all_throttled_reports_ring(self, monkeypatch): + self._pin(monkeypatch, "exa") + for vendor in keyless_mcp._KEYLESS_RING: + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, vendor, + lambda q, l, v=vendor: self._throttled(v), + ) out = keyless_mcp.search_with_failover("exa", "q") assert out["success"] is False - assert "Failover to parallel also failed" in out["error"] + assert "all keyless vendors throttled" in out["error"] + + def test_search_walks_ring_past_multiple_throttles(self, monkeypatch): + # exa -> parallel -> tavily all throttled; firecrawl serves. + self._pin(monkeypatch, "exa") + for vendor in ("exa", "parallel", "tavily"): + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, vendor, + lambda q, l, v=vendor: self._throttled(v), + ) + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, "firecrawl", + lambda q, l: self._ok("firecrawl"), + ) + out = keyless_mcp.search_with_failover("exa", "q") + assert out["success"] is True + assert out["data"]["served_by"] == "firecrawl" def test_failover_respects_peer_paid_pin(self, monkeypatch): - monkeypatch.setattr(keyless_mcp, "parallel_search_keyless", lambda q, l: self._throttled("Parallel")) + # Every vendor except exa throttles; exa is pinned paid so its free + # endpoint must never be used. monkeypatch.setattr( keyless_mcp, "provider_tier", lambda name: "paid" if name == "exa" else "auto", ) + monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda n: n == "parallel") called = [] - monkeypatch.setattr( - keyless_mcp, "exa_search_keyless", + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, "exa", lambda q, l: called.append(1) or self._ok("exa"), ) + for vendor in ("parallel", "tavily", "firecrawl", "keenable"): + monkeypatch.setitem( + keyless_mcp._KEYLESS_SEARCHERS, vendor, + lambda q, l, v=vendor: self._throttled(v), + ) out = keyless_mcp.search_with_failover("parallel", "q") assert out["success"] is False assert not called # exa pinned paid: its free tier is opted out def test_extract_fails_over_when_all_urls_throttled(self, monkeypatch): + self._pin(monkeypatch, "exa") throttled = [ {"url": "https://a", "title": "", "content": "", "error": "rate limit hit"}, {"url": "https://b", "title": "", "content": "", "error": "429 too many requests"}, @@ -480,20 +523,21 @@ class TestKeylessFailover: {"url": "https://a", "title": "A", "content": "x"}, {"url": "https://b", "title": "B", "content": "y"}, ] - monkeypatch.setattr(keyless_mcp, "exa_extract_keyless", lambda urls: throttled) - monkeypatch.setattr(keyless_mcp, "parallel_extract_keyless", lambda urls: good) + monkeypatch.setitem(keyless_mcp._KEYLESS_EXTRACTORS, "exa", lambda urls: throttled) + monkeypatch.setitem(keyless_mcp._KEYLESS_EXTRACTORS, "parallel", lambda urls: good) out = keyless_mcp.extract_with_failover("exa", ["https://a", "https://b"]) assert out == good def test_extract_partial_failure_stays_on_primary(self, monkeypatch): + self._pin(monkeypatch, "exa") partial = [ {"url": "https://a", "title": "A", "content": "x"}, {"url": "https://b", "title": "", "content": "", "error": "rate limit"}, ] called = [] - monkeypatch.setattr(keyless_mcp, "exa_extract_keyless", lambda urls: partial) - monkeypatch.setattr( - keyless_mcp, "parallel_extract_keyless", + monkeypatch.setitem(keyless_mcp._KEYLESS_EXTRACTORS, "exa", lambda urls: partial) + monkeypatch.setitem( + keyless_mcp._KEYLESS_EXTRACTORS, "parallel", lambda urls: called.append(1) or [], ) out = keyless_mcp.extract_with_failover("exa", ["https://a", "https://b"]) diff --git a/tests/tools/test_web_providers.py b/tests/tools/test_web_providers.py index 9a07f4390a..31b498605c 100644 --- a/tests/tools/test_web_providers.py +++ b/tests/tools/test_web_providers.py @@ -232,6 +232,11 @@ class TestUnconfiguredErrorEnvelopeParity: monkeypatch.setattr(fc, "_load_web_config", lambda: {"backend": "firecrawl"}, raising=False) monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False) monkeypatch.setattr(web_tools, "check_firecrawl_api_key", lambda: False) + # Developer machines may carry FIRECRAWL_* in ~/.hermes/.env — the + # config-aware lookup must see a truly keyless environment here. + monkeypatch.setattr( + "hermes_cli.config.get_env_value", lambda name: None, raising=True + ) calls = {} diff --git a/tests/tools/test_web_tools_tavily.py b/tests/tools/test_web_tools_tavily.py index 46849ed2d8..73ee8ebd4c 100644 --- a/tests/tools/test_web_tools_tavily.py +++ b/tests/tools/test_web_tools_tavily.py @@ -256,12 +256,17 @@ class TestWebSearchTavily: assert result["data"]["web"][0]["title"] == "Result" def test_search_keyless_dispatch(self): + """Keyless Tavily routes through the ring; pinned tavily starts at + tavily and the ring searcher sends the keyless headers.""" + from plugins.web import keyless_mcp + mock_response = _ok_response({ "results": [{"title": "Result", "url": "https://r.com", "content": "desc"}] }) with patch("tools.web_tools._get_backend", return_value="tavily"), \ - patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post, \ + patch.object(keyless_mcp, "_vendor_pinned", lambda n: n == "tavily"), \ + patch("requests.post", return_value=mock_response) as mock_post, \ patch("tools.interrupt.is_interrupted", return_value=False): os.environ.pop("TAVILY_API_KEY", None) from tools.web_tools import web_search_tool diff --git a/tools/web_tools.py b/tools/web_tools.py index b0804b341f..290fbbb73b 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -169,7 +169,7 @@ def _load_web_config() -> dict: # WebSearchProvider. Keep the two sets aligned by hand: if xai ever ships as # a registered provider, drop it here so the registry path takes over. _LEGACY_WEB_BACKENDS = frozenset( - {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai"} + {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"} ) @@ -262,6 +262,7 @@ def _get_backend() -> str: ("tavily", _has_env("TAVILY_API_KEY")), ("exa", _has_env("EXA_API_KEY")), ("parallel", _has_env("PARALLEL_API_KEY")), + ("keenable", _has_env("KEENABLE_API_KEY")), ("firecrawl", _has_env("FIRECRAWL_API_KEY") or _has_env("FIRECRAWL_API_URL")), ("firecrawl", _is_tool_gateway_ready()), ("searxng", _has_env("SEARXNG_URL")), @@ -387,6 +388,8 @@ def _is_backend_available(backend: str) -> bool: return _has_env("EXA_API_KEY") if backend == "parallel": return _has_env("PARALLEL_API_KEY") + if backend == "keenable": + return _has_env("KEENABLE_API_KEY") if backend == "firecrawl": return check_firecrawl_api_key() if backend == "tavily": @@ -450,6 +453,7 @@ def _web_requires_env() -> list[str]: "EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", + "KEENABLE_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 4787fc8093..7bfcce21e5 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -2290,7 +2290,7 @@ web: | **Tavily** | `TAVILY_API_KEY` (optional — keyless when selected) | ✔ | ✔ | | **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ | -**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel. With **no selection and no credentials at all**, Hermes falls back to the Exa/Parallel keyless free tier (unpinned installs split 50/50 between the vendors) so web tools work on a fresh install — see the [Web Search guide](/user-guide/features/web-search) for details and limits. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily in `hermes tools` (or `web.backend: tavily`) also works without a key. +**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Tavily / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key. **SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](/user-guide/features/web-search) for Docker setup instructions. diff --git a/website/docs/user-guide/features/web-search.md b/website/docs/user-guide/features/web-search.md index e9cd8dc699..7e0df647eb 100644 --- a/website/docs/user-guide/features/web-search.md +++ b/website/docs/user-guide/features/web-search.md @@ -22,22 +22,21 @@ Both are configured through a single backend selection. Providers are chosen via | **SearXNG** | `SEARXNG_URL` | ✔ | — | ✔ Free (self-hosted) | | **Brave Search (free tier)** | `BRAVE_SEARCH_API_KEY` | ✔ | — | 2 000 queries/mo | | **DDGS (DuckDuckGo)** | — (no key) | ✔ | — | ✔ Free | -| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless when selected · 1 000 searches/mo with a free key | -| **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless free tier · 1 000 searches/mo with key | -| **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless free tier · paid with key | +| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with a free key | +| **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with key | +| **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key | +| **Keenable** | `KEENABLE_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key | | **xAI (Grok)** | `XAI_API_KEY` or `hermes auth add xai-oauth` | ✔ | — | Paid (SuperGrok or per-token) | Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below). **Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below. -:::info Works out of the box — keyless free tier -A fresh install with **no web credentials at all** still gets working `web_search` and `web_extract`: Hermes falls back to Exa's and Parallel's public anonymous endpoints (rate-limited free tiers), splitting unpinned installs 50/50 between the two vendors — the pick is random per process and stable within it. If one vendor's free tier throttles a request, Hermes automatically retries it once on the other vendor's free tier. No signup, no key. This tier is strictly last-resort — any configured backend or present API key always wins — and requests carry no user identifiers (only a random per-process session id, rotated on restart). For reliable, unthrottled service, set up a keyed provider. Disable the keyless tier entirely with `web.keyless_fallback: false`. - -Tavily and Firecrawl also offer keyless access when **explicitly selected** (`web.backend: tavily` / `firecrawl` or via `hermes tools`) — they are not part of the automatic zero-config fallback, but picking them without entering a key now works instead of erroring. +:::info Works out of the box — keyless free-tier rotation +A fresh install with **no web credentials at all** gets working `web_search` and `web_extract` out of the box: requests rotate round-robin across five vendors' public free tiers — **Exa, Parallel, Tavily, Firecrawl, and Keenable** — spreading load evenly, and a rate-limited request automatically retries on the next vendor in the ring (multi-hop, until one serves or all are throttled). No signup, no key. This tier is strictly last-resort — any configured backend or present API key always wins — and requests carry no user identifiers (only a random per-process session id, rotated on restart). For guaranteed, unthrottled service, set up a keyed provider. Disable the keyless tier entirely with `web.keyless_fallback: false`. ::: -**Choosing free vs paid explicitly:** in `hermes tools`, Exa and Parallel each appear as two rows — **Free (keyless)** and **Paid (API key)**. Picking Free pins the anonymous endpoint (even if you later add a key); picking Paid pins the keyed SDK path (a missing key then errors instead of silently downgrading to the free tier). The selection is stored as `web.provider_tier.: free|paid`; leave it unset for auto (key present → paid, otherwise free). +**Choosing free vs paid explicitly:** in `hermes tools`, Exa, Parallel, and Keenable each appear as two rows — **Free (keyless)** and **Paid (API key)**. Picking Free pins that vendor's anonymous endpoint (even if you later add a key); picking Paid pins the keyed path (a missing key then errors instead of silently downgrading to the free tier). The selection is stored as `web.provider_tier.: free|paid`; leave it unset for auto (key present → paid, otherwise the keyless ring). :::tip Nous Subscribers If you have a paid [Nous Portal](https://portal.nousresearch.com) subscription, web search and extract are available through the **[Tool Gateway](tool-gateway.md)** via managed Firecrawl — no API key needed. New installs can run `hermes setup --portal` to log in and turn on all gateway tools at once; existing installs can flip just web via `hermes tools`. @@ -371,9 +370,9 @@ If no backend has **ever** been selected (no `web.backend` / per-capability key | `SEARXNG_URL` | searxng | | `BRAVE_SEARCH_API_KEY` | brave-free | | `ddgs` package importable | ddgs | -| *(nothing set at all)* | exa / parallel keyless free tier (50/50 split) | +| *(nothing set at all)* | keyless ring: exa / parallel / tavily / firecrawl / keenable (round-robin) | -**Keyless free tier:** when *no* credential above is present, Hermes falls back to Exa's and Parallel's public anonymous endpoints so web tools work on a fresh install with zero setup — unpinned installs split 50/50 between the two vendors (random per process, stable within it); pick one explicitly in `hermes tools` to pin it. Both free tiers are rate-limited by the vendors under burst load; in practice sustained normal usage goes through fine. On throttling, the tool returns an error suggesting the matching API key. Set `web.keyless_fallback: false` to turn this tier off — with it off and no credentials, web tools are unavailable until a provider is configured. +**Keyless free-tier ring:** when *no* credential above is present, requests rotate across five vendors' public free tiers (Exa, Parallel, Tavily, Firecrawl, Keenable) so web tools work on a fresh install with zero setup — and a rate-limited request fails over to the next vendor in the ring automatically. Pin one vendor in `hermes tools` to stop the rotation (the ring is then only used as failover succession on throttles). All free tiers are vendor-rate-limited under burst load; sustained normal usage goes through fine. Set `web.keyless_fallback: false` to turn the tier off — with it off and no credentials, web tools are unavailable until a provider is configured. xAI Web Search is **not** in the auto-detection chain — having `XAI_API_KEY` set (or being signed in via xAI Grok OAuth) does not automatically route web traffic through xAI, since those credentials are also used for inference / TTS / image gen and the user may want a different backend for web. Opt in explicitly with `web.backend: "xai"`.