diff --git a/agent/transports/codex.py b/agent/transports/codex.py index eff74dca1c..e64cfc2a70 100644 --- a/agent/transports/codex.py +++ b/agent/transports/codex.py @@ -59,7 +59,7 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]: # A function literally named ``web_search`` collides with Grok's native # server-side tool (incomplete hang or HTTP 400 duplicate names); this alias # avoids that while still dispatching through Hermes's configured provider -# (Firecrawl / Tavily / …). Mapped back to ``web_search`` in normalize_response. +# (Firecrawl / Exa / …). Mapped back to ``web_search`` in normalize_response. _XAI_CLIENT_WEB_SEARCH_ALIAS = "hermes_web_search" # OpenCode's /v1/responses endpoints (Zen and Go, including custom providers @@ -601,7 +601,7 @@ class ResponsesApiTransport(ProviderTransport): # fails): drop the client ``web_search`` function and declare # xAI's built-in instead. 1:1 swap only when client ``web_search`` # was already present — never an additive grant. - # 2. **Client** (Firecrawl / Tavily / Exa / … configured or resolved): + # 2. **Client** (Firecrawl / Keenable / Exa / … configured or resolved): # keep Hermes dispatch so ``web.backend`` / ``web.search_backend`` # is honored, but rename the wire tool to # ``hermes_web_search`` so Grok cannot hijack the name. The alias diff --git a/agent/web_search_provider.py b/agent/web_search_provider.py index 0f7f706c86..f1abcc9b63 100644 --- a/agent/web_search_provider.py +++ b/agent/web_search_provider.py @@ -13,7 +13,7 @@ Providers live in ``/plugins/web//`` (built-in, auto-loaded as ``plugins.enabled``). This ABC is the SINGLE plugin-facing surface for web providers — every -provider in the tree (brave-free, ddgs, searxng, exa, parallel, tavily, +provider in the tree (brave-free, ddgs, searxng, exa, parallel, keenable, firecrawl) implements it. The legacy in-tree ``tools.web_providers.base`` ABCs were deleted in PR #25182 along with the per-vendor inline helpers in ``tools/web_tools.py``; the response-shape contract documented below @@ -93,7 +93,7 @@ class WebSearchProvider(abc.ABC): :meth:`search` / :meth:`extract`. The :meth:`supports_search` / :meth:`supports_extract` capability flags let the registry route each tool call to the right provider, and let multi-capability providers - (Firecrawl, Tavily, Exa, …) advertise multiple capabilities from a + (Firecrawl, Keenable, Exa, …) advertise multiple capabilities from a single class. """ diff --git a/agent/web_search_registry.py b/agent/web_search_registry.py index 112fadc6df..5260272133 100644 --- a/agent/web_search_registry.py +++ b/agent/web_search_registry.py @@ -16,7 +16,7 @@ The active provider is chosen by configuration with this precedence: 2. ``web.backend`` (shared fallback). 3. If exactly one capability-eligible provider is registered AND available, use it. -4. Legacy preference order — ``firecrawl`` → ``parallel`` → ``tavily`` → +4. Legacy preference order — ``firecrawl`` → ``parallel`` → ``exa`` → ``searxng`` → ``brave-free`` → ``ddgs`` — filtered by availability. Matches the historic ``tools.web_tools._get_backend()`` candidate order so installs that never set a config key keep landing @@ -159,7 +159,6 @@ def _read_config_key(*path: str) -> Optional[str]: _LEGACY_PREFERENCE = ( "firecrawl", "parallel", - "tavily", "exa", "searxng", "brave-free", @@ -177,7 +176,6 @@ _LEGACY_PREFERENCE = ( _KEYLESS_PREFERENCE = ( "exa", "parallel", - "tavily", "firecrawl", "keenable", ) @@ -222,7 +220,7 @@ def _resolve(configured: Optional[str], *, capability: str) -> Optional[WebSearc supports *capability* AND ``is_available()`` reports True, return it. 3. **Legacy preference walk, filtered by availability.** Walk the - :data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → tavily → + :data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → exa → searxng → brave-free → ddgs) looking for a provider whose ``supports_()`` is True AND whose ``is_available()`` is True. Matches the historic ``tools.web_tools._get_backend()`` diff --git a/evals/browser_use/single_run.py b/evals/browser_use/single_run.py index 6015656fb1..346c425de7 100644 --- a/evals/browser_use/single_run.py +++ b/evals/browser_use/single_run.py @@ -63,7 +63,7 @@ with open(os.path.join(hh, "config.yaml"), "w", encoding="utf-8") as f: os.environ["HERMES_HOME"] = hh # Strip web-fetch shortcuts: every arm must drive the browser. os.environ.pop("BROWSER_USE_API_KEY", None) -for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "TAVILY_API_KEY", "SERPER_API_KEY"): +for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "SERPER_API_KEY"): os.environ.pop(k, None) os.environ["BU_CDP_URL"] = cdp os.environ["PATH"] = ( diff --git a/hermes_cli/config.py b/hermes_cli/config.py index f05cf90ad6..051472ad90 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1021,7 +1021,6 @@ ENV_VARS_BY_VERSION: Dict[int, List[str]] = { 4: ["VOICE_TOOLS_OPENAI_KEY", "ELEVENLABS_API_KEY"], 5: ["WHATSAPP_ENABLED", "WHATSAPP_MODE", "WHATSAPP_ALLOWED_USERS", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN", "SLACK_ALLOWED_USERS"], - 10: ["TAVILY_API_KEY"], 11: ["TERMINAL_MODAL_MODE"], } @@ -1226,7 +1225,7 @@ def _is_env_config_key(key: str) -> bool: 'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY', 'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL', 'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME', - 'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', + 'TOOL_GATEWAY_USER_TOKEN', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY', 'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN', 'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY', @@ -4827,7 +4826,6 @@ def show_config(): ("EXA_API_KEY", "Exa"), ("PARALLEL_API_KEY", "Parallel"), ("FIRECRAWL_API_KEY", "Firecrawl"), - ("TAVILY_API_KEY", "Tavily"), ("BROWSERBASE_API_KEY", "Browserbase"), ("BROWSER_USE_API_KEY", "Browser Use"), ("FAL_KEY", "FAL"), diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 0fabb45884..4882108c17 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -521,7 +521,7 @@ DEFAULT_CONFIG = { "extract_char_limit": 15000, # per-page char budget for web_extract; larger pages truncate + store full text in cache/web # Keyless free-tier ring: with NO web backend configured or keyed, # web_search/web_extract rotate round-robin across five vendors' - # public free tiers (exa, parallel, tavily, firecrawl, keenable), + # public free tiers (exa, parallel, firecrawl, keenable), # failing over to the next ring vendor on rate limits. Never # pre-empts a configured or keyed backend. Set false to disable. "keyless_fallback": True, @@ -531,7 +531,7 @@ DEFAULT_CONFIG = { # (no sticky failover). Off when keyless_fallback is false. "keyless_rescue": True, # Per-provider tier selection for ring vendors with both a keyless - # free endpoint and a keyed paid path (exa, parallel, tavily, + # free endpoint and a keyed paid path (exa, parallel, # firecrawl, keenable). Set by the `hermes tools` picker's # "Free (keyless)" / "Paid (API key)" rows. # free — always use the anonymous free endpoint (even with a key) @@ -4438,14 +4438,6 @@ OPTIONAL_ENV_VARS = { "category": "tool", "advanced": True, }, - "TAVILY_API_KEY": { - "description": "Tavily API key for AI-native web search and extract (optional — keyless works without it)", - "prompt": "Tavily API key", - "url": "https://app.tavily.com/home", - "tools": ["web_search", "web_extract"], - "password": True, - "category": "tool", - }, "KEENABLE_API_KEY": { "description": "Keenable API key for fast independent-index web search and page fetch (optional — keyless free tier works without it)", "prompt": "Keenable API key", diff --git a/hermes_cli/dump.py b/hermes_cli/dump.py index e29675ce13..c00d55f12f 100644 --- a/hermes_cli/dump.py +++ b/hermes_cli/dump.py @@ -387,7 +387,7 @@ def run_dump(args): ("COMMANDCODE_API_KEY", "commandcode"), ("KILOCODE_API_KEY", "kilocode"), ("FIRECRAWL_API_KEY", "firecrawl"), - ("TAVILY_API_KEY", "tavily"), + ("KEENABLE_API_KEY", "keenable"), ("BROWSERBASE_API_KEY", "browserbase"), ("FAL_KEY", "fal"), ("ELEVENLABS_API_KEY", "elevenlabs"), diff --git a/hermes_cli/nous_subscription.py b/hermes_cli/nous_subscription.py index bdd6f9532e..f9ca7ef35f 100644 --- a/hermes_cli/nous_subscription.py +++ b/hermes_cli/nous_subscription.py @@ -505,10 +505,6 @@ def get_nous_subscription_features( direct_exa = bool(get_env_value("EXA_API_KEY")) direct_firecrawl = bool(get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL")) direct_parallel = bool(get_env_value("PARALLEL_API_KEY")) - direct_tavily = bool(get_env_value("TAVILY_API_KEY")) - # Keyless Tavily is opt-in: selecting it in `hermes tools` / setup writes - # web.backend (or a per-capability override) without requiring a key. - tavily_selected = "tavily" in {web_backend, web_search_backend, web_extract_backend} direct_searxng = bool(get_env_value("SEARXNG_URL")) direct_fal = fal_key_is_configured() direct_fal_video = direct_fal # same FAL_KEY; separate var so use_gateway is independent @@ -540,8 +536,6 @@ def get_nous_subscription_features( direct_firecrawl = False direct_exa = False direct_parallel = False - direct_tavily = False - tavily_selected = False if image_use_gateway: direct_fal = False if video_use_gateway: @@ -630,7 +624,6 @@ def get_nous_subscription_features( direct_camofox = False - tavily_ready = direct_tavily or tavily_selected web_managed = web_backend == "firecrawl" and managed_web_available and not direct_firecrawl web_active = bool( web_tool_enabled @@ -639,7 +632,6 @@ def get_nous_subscription_features( or (web_backend == "exa" and direct_exa) or (web_backend == "firecrawl" and direct_firecrawl) or (web_backend == "parallel" and direct_parallel) - or (web_backend == "tavily" and tavily_ready) or (web_backend == "searxng" and direct_searxng) # Per-capability overrides: search_backend or extract_backend may be set # without web.backend (using the new split config from #20061) @@ -647,8 +639,6 @@ def get_nous_subscription_features( or (web_search_backend == "exa" and direct_exa) or (web_search_backend == "firecrawl" and direct_firecrawl) or (web_search_backend == "parallel" and direct_parallel) - or (web_search_backend == "tavily" and tavily_ready) - or (web_extract_backend == "tavily" and tavily_ready) ) ) web_available = bool( @@ -656,7 +646,6 @@ def get_nous_subscription_features( or direct_exa or direct_firecrawl or direct_parallel - or tavily_ready or direct_searxng ) @@ -900,7 +889,6 @@ def apply_nous_managed_defaults( if "web" in selected_toolsets and not features.web.explicit_configured and not ( get_env_value("PARALLEL_API_KEY") - or get_env_value("TAVILY_API_KEY") or get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL") ): @@ -998,7 +986,6 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]: get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL") or get_env_value("PARALLEL_API_KEY") - or get_env_value("TAVILY_API_KEY") or get_env_value("EXA_API_KEY") # Env-configured keyless local backend: a reachable self-hosted # SearXNG is a working web setup even with no stored selection @@ -1034,7 +1021,7 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]: _GATEWAY_DIRECT_LABELS = { - "web": "Firecrawl/Exa/Parallel/Tavily key or SearXNG", + "web": "Firecrawl/Exa/Parallel/Keenable key or SearXNG", "image_gen": "FAL key", "video_gen": "FAL key", "tts": "OpenAI/ElevenLabs key", diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 0b64de2944..390d71669a 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -513,7 +513,7 @@ def _print_setup_summary(config: dict, hermes_home): tool_status.append(("Vision (image analysis)", False, "run 'hermes setup' to configure")) - # Web tools (Exa, Parallel, Firecrawl, or Tavily) + # Web tools (Exa, Parallel, Firecrawl, or Keenable) if subscription_features.web.managed_by_nous: tool_status.append(("Web Search & Extract (Nous subscription)", True, None)) elif subscription_features.web.available: @@ -522,7 +522,7 @@ def _print_setup_summary(config: dict, hermes_home): label = f"Web Search & Extract ({subscription_features.web.current_provider})" tool_status.append((label, True, None)) else: - tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, TAVILY_API_KEY, or SEARXNG_URL")) + tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, KEENABLE_API_KEY, or SEARXNG_URL")) # Browser tools (local Chromium, Camofox, Browserbase, Browser Use, or Firecrawl) browser_provider = subscription_features.browser.current_provider diff --git a/hermes_cli/status.py b/hermes_cli/status.py index 981bb3a387..569c759835 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -184,7 +184,7 @@ def show_status(args): "MiniMax-CN": "MINIMAX_CN_API_KEY", "DeepInfra": "DEEPINFRA_API_KEY", "Firecrawl": "FIRECRAWL_API_KEY", - "Tavily": "TAVILY_API_KEY", + "Keenable": "KEENABLE_API_KEY", "Browser Use": "BROWSER_USE_API_KEY", # Optional — local browser works without this "Browserbase": "BROWSERBASE_API_KEY", # Optional — direct credentials only "FAL": "FAL_KEY", diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 018178d916..831a8ac105 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -3314,7 +3314,7 @@ def _plugin_video_gen_providers() -> list[dict]: # Mirror of _plugin_image_gen_providers for web search backends. Surfaces # every plugin-registered web provider so it appears in the # "Web Search & Extract" picker. All seven providers (brave-free, ddgs, -# searxng, exa, parallel, tavily, firecrawl) live as plugins after +# searxng, exa, parallel, firecrawl, keenable) live as plugins after # PR #25182 — this helper is the sole source of truth for the category's # provider rows. The hardcoded entries that used to drive the category # were deleted in the same PR; only the two non-provider UX rows @@ -3331,7 +3331,7 @@ def _plugin_web_search_providers() -> list[dict]: hardcoded or plugin-registered. After PR #25182, all seven web providers (brave-free, ddgs, searxng, - exa, parallel, tavily, firecrawl) are plugins; this helper is the sole + exa, parallel, firecrawl, keenable) are plugins; this helper is the sole source of provider rows for the Web Search & Extract category. """ try: diff --git a/plugins/web/brave_free/provider.py b/plugins/web/brave_free/provider.py index 0da8d11c99..769a850587 100644 --- a/plugins/web/brave_free/provider.py +++ b/plugins/web/brave_free/provider.py @@ -34,7 +34,7 @@ class BraveFreeWebSearchProvider(WebSearchProvider): """Search-only Brave provider using the free-tier Data-for-Search API. Free tier is 2,000 queries/month (1 qps). No content-extraction capability — - users pair this with Firecrawl/Tavily/Exa for ``web_extract``. + users pair this with Firecrawl/Keenable/Exa for ``web_extract``. """ @property diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.py index 0bc88cddcb..34784a034a 100644 --- a/plugins/web/firecrawl/provider.py +++ b/plugins/web/firecrawl/provider.py @@ -540,7 +540,7 @@ class FirecrawlWebSearchProvider(WebSearchProvider): """Firecrawl serves keyless cloud requests (public API, no auth). Default-on ring member of the keyless free tier: fresh installs - rotate across Exa/Parallel/Tavily/Firecrawl/Keenable. False when + rotate across Exa/Parallel/Firecrawl/Keenable. False when the user pinned ``web.provider_tier.firecrawl: paid``. """ from plugins.web.keyless_mcp import keyless_enabled, provider_tier diff --git a/plugins/web/keenable/provider.py b/plugins/web/keenable/provider.py index d185b5941e..2795189107 100644 --- a/plugins/web/keenable/provider.py +++ b/plugins/web/keenable/provider.py @@ -3,8 +3,8 @@ Keenable (https://keenable.ai) operates an independent web index for AI apps with public keyless endpoints (rate-limited free tier; keyed access via KEENABLE_API_KEY for higher limits). Integrated as a keyless-ring -member following the Exa/Parallel/Tavily/Firecrawl pattern: fresh installs -with zero web credentials rotate across all five vendors' free tiers. +member following the Exa/Parallel/Firecrawl pattern: fresh installs with +zero web credentials rotate across the ring vendors' free tiers. Credit: Keenable integration originally proposed by Ilya Gusev (Keenable) in PR #49758; the native provider form follows the salvage of that work diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.py index c3c1947fd3..a2f39a7c05 100644 --- a/plugins/web/keyless_mcp.py +++ b/plugins/web/keyless_mcp.py @@ -438,115 +438,6 @@ def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: -# --------------------------------------------------------------------------- -# Tavily keyless (api.tavily.com — X-Tavily-Access-Mode: keyless) -# --------------------------------------------------------------------------- - - -TAVILY_API_URL = "https://api.tavily.com" - - -def _tavily_keyless_post(endpoint: str, payload: Dict[str, Any]) -> Dict[str, Any]: - """POST to Tavily with keyless headers; raise KeylessMCPError on failure.""" - import requests - - try: - response = requests.post( - f"{TAVILY_API_URL}/{endpoint.lstrip('/')}", - json=payload, - headers={ - "Content-Type": "application/json", - "X-Client-Name": "hermes-agent", - "X-Tavily-Access-Mode": "keyless", - }, - timeout=_TIMEOUT_SECONDS, - ) - except requests.RequestException as exc: - raise KeylessMCPError(f"request failed: {exc}") from exc - if response.status_code >= 400: - raise KeylessMCPError( - (response.text or "").strip() or f"HTTP {response.status_code}" - ) - return response.json() - - -def tavily_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]: - """Keyless Tavily search → legacy search response shape.""" - try: - data = _tavily_keyless_post( - "search", {"query": query, "max_results": max(1, int(limit))} - ) - except KeylessMCPError as exc: - return { - "success": False, - "error": ( - f"Keyless Tavily search failed: {exc}. " - "Set TAVILY_API_KEY (https://app.tavily.com) or another web " - "backend via `hermes tools` for reliable service." - ), - } - web_results = [] - for i, result in enumerate(data.get("results") or []): - web_results.append( - { - "url": result.get("url") or "", - "title": result.get("title") or "", - "description": result.get("content") or "", - "position": i + 1, - } - ) - return {"success": True, "data": {"web": web_results}} - - -def tavily_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: - """Keyless Tavily extract → legacy extract result list.""" - try: - data = _tavily_keyless_post("extract", {"urls": list(urls)}) - except KeylessMCPError as exc: - message = ( - f"Keyless Tavily extract failed: {exc}. " - "Set TAVILY_API_KEY (https://app.tavily.com) or another web " - "backend via `hermes tools` for reliable service." - ) - return [ - {"url": u, "title": "", "content": "", "error": message} - for u in urls - ] - results: List[Dict[str, Any]] = [] - seen = set() - for result in data.get("results") or []: - url = result.get("url") or "" - raw = result.get("raw_content") or result.get("content") or "" - seen.add(url) - results.append( - { - "url": url, - "title": result.get("title") or "", - "content": raw, - "raw_content": raw, - "metadata": {"sourceURL": url, "title": result.get("title") or ""}, - } - ) - for fail in data.get("failed_results") or []: - url = (fail.get("url") if isinstance(fail, dict) else str(fail)) or "" - seen.add(url) - results.append( - { - "url": url, - "title": "", - "content": "", - "error": (fail.get("error") if isinstance(fail, dict) else None) - or "extraction failed", - } - ) - for u in urls: - if u not in seen: - results.append( - {"url": u, "title": "", "content": "", "error": "no content returned"} - ) - return results - - # --------------------------------------------------------------------------- # Firecrawl keyless (public cloud API, no auth header) # --------------------------------------------------------------------------- @@ -731,12 +622,11 @@ def keenable_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]: # Round-robin ring + next-in-line failover (rate-limited free tiers) # --------------------------------------------------------------------------- -_KEYLESS_RING = ("exa", "parallel", "tavily", "firecrawl", "keenable") +_KEYLESS_RING = ("exa", "parallel", "firecrawl", "keenable") _KEYLESS_SEARCHERS = { "exa": lambda query, limit: exa_search_keyless(query, limit), "parallel": lambda query, limit: parallel_search_keyless(query, limit), - "tavily": lambda query, limit: tavily_search_keyless(query, limit), "firecrawl": lambda query, limit: firecrawl_search_keyless(query, limit), "keenable": lambda query, limit: keenable_search_keyless(query, limit), } @@ -744,7 +634,6 @@ _KEYLESS_SEARCHERS = { _KEYLESS_EXTRACTORS = { "exa": lambda urls: exa_extract_keyless(urls), "parallel": lambda urls: parallel_extract_keyless(urls), - "tavily": lambda urls: tavily_extract_keyless(urls), "firecrawl": lambda urls: firecrawl_extract_keyless(urls), "keenable": lambda urls: keenable_extract_keyless(urls), } diff --git a/plugins/web/searxng/__init__.py b/plugins/web/searxng/__init__.py index cea8eabb18..62e12a5c7d 100644 --- a/plugins/web/searxng/__init__.py +++ b/plugins/web/searxng/__init__.py @@ -1,7 +1,7 @@ """SearXNG search plugin — bundled, auto-loaded. Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``). -Search-only — pair with an extract provider (firecrawl/tavily/exa) for +Search-only — pair with an extract provider (firecrawl/keenable/exa) for ``web_extract`` calls. """ diff --git a/plugins/web/tavily/__init__.py b/plugins/web/tavily/__init__.py deleted file mode 100644 index 1e0ced61d1..0000000000 --- a/plugins/web/tavily/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Tavily web search + extract plugin — bundled, auto-loaded.""" - -from __future__ import annotations - -from plugins.web.tavily.provider import TavilyWebSearchProvider - - -def register(ctx) -> None: - """Register the Tavily provider with the plugin context.""" - ctx.register_web_search_provider(TavilyWebSearchProvider()) diff --git a/plugins/web/tavily/plugin.yaml b/plugins/web/tavily/plugin.yaml deleted file mode 100644 index ae1676f211..0000000000 --- a/plugins/web/tavily/plugin.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: web-tavily -version: 1.0.0 -description: "Tavily web search + content extraction. Works keyless (rate-limited); set TAVILY_API_KEY for higher limits — https://app.tavily.com/home." -author: NousResearch -kind: backend -provides_web_providers: - - tavily diff --git a/plugins/web/tavily/provider.py b/plugins/web/tavily/provider.py deleted file mode 100644 index 096558f9a3..0000000000 --- a/plugins/web/tavily/provider.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Tavily web search + content extraction — plugin form. - -Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two -capabilities advertised: - -- ``supports_search()`` -> True (Tavily ``/search``) -- ``supports_extract()`` -> True (Tavily ``/extract``) - -Both are sync — the underlying call is ``httpx.post(...)``. - -Config keys this provider responds to:: - - web: - search_backend: "tavily" # explicit per-capability - extract_backend: "tavily" # explicit per-capability - backend: "tavily" # shared fallback for both - -Env vars:: - - TAVILY_API_KEY=... # https://app.tavily.com/home (optional) - TAVILY_BASE_URL=... # optional override of https://api.tavily.com - -Auth is header-based. A key uses ``Authorization: Bearer``; without a key -the request is keyless (``X-Tavily-Access-Mode: keyless``). Both paths -send ``X-Client-Name: hermes-agent``. -""" - -from __future__ import annotations - -import logging -from typing import Any, Dict, List - -import httpx - -from agent.web_search_provider import WebSearchProvider - -logger = logging.getLogger(__name__) - -_CLIENT_NAME = "hermes-agent" - - -def _tavily_headers(api_key: str) -> Dict[str, str]: - """Build Tavily request headers for keyed or keyless access.""" - headers = {"X-Client-Name": _CLIENT_NAME} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - else: - headers["X-Tavily-Access-Mode"] = "keyless" - return headers - - -def _tavily_request(endpoint: str, payload: Dict[str, Any]) -> Dict[str, Any]: - """POST to the Tavily API and return the parsed JSON response. - - Keyed when ``TAVILY_API_KEY`` is set (Bearer auth); otherwise keyless. - Non-2xx responses raise ``ValueError`` with the response body so Tavily's - keyless rate-limit / upgrade text reaches the model. - """ - from agent.web_search_provider import get_provider_env - - api_key = get_provider_env("TAVILY_API_KEY") - base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com" - url = f"{base_url}/{endpoint.lstrip('/')}" - logger.info("Tavily %s request to %s", endpoint, url) - - response = httpx.post( - url, - json=payload, - timeout=60, - headers=_tavily_headers(api_key), - ) - if response.status_code >= 400: - body = (response.text or "").strip() - detail = body or f"HTTP {response.status_code}" - raise ValueError(detail) - return response.json() - - -def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]: - """Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``.""" - web_results = [] - for i, result in enumerate(response.get("results", [])): - web_results.append( - { - "title": result.get("title", ""), - "url": result.get("url", ""), - "description": result.get("content", ""), - "position": i + 1, - } - ) - return {"success": True, "data": {"web": web_results}} - - -def _normalize_tavily_documents( - response: Dict[str, Any], fallback_url: str = "" -) -> List[Dict[str, Any]]: - """Map Tavily ``/extract`` response to standard documents. - - Documents follow the legacy LLM post-processing shape:: - - {"url", "title", "content", "raw_content", "metadata"} - - Failures (``failed_results``, ``failed_urls``) become result entries - with an ``error`` field rather than raising. - """ - documents: List[Dict[str, Any]] = [] - for result in response.get("results", []): - url = result.get("url", fallback_url) - raw = result.get("raw_content", "") or result.get("content", "") - documents.append( - { - "url": url, - "title": result.get("title", ""), - "content": raw, - "raw_content": raw, - "metadata": {"sourceURL": url, "title": result.get("title", "")}, - } - ) - for fail in response.get("failed_results", []): - documents.append( - { - "url": fail.get("url", fallback_url), - "title": "", - "content": "", - "raw_content": "", - "error": fail.get("error", "extraction failed"), - "metadata": {"sourceURL": fail.get("url", fallback_url)}, - } - ) - for fail_url in response.get("failed_urls", []): - url_str = fail_url if isinstance(fail_url, str) else str(fail_url) - documents.append( - { - "url": url_str, - "title": "", - "content": "", - "raw_content": "", - "error": "extraction failed", - "metadata": {"sourceURL": url_str}, - } - ) - return documents - - -class TavilyWebSearchProvider(WebSearchProvider): - """Tavily search + extract provider.""" - - @property - def name(self) -> str: - return "tavily" - - @property - def display_name(self) -> str: - return "Tavily" - - def is_available(self) -> bool: - """Return True when ``TAVILY_API_KEY`` is set to a non-empty value.""" - from agent.web_search_provider import get_provider_env - - return bool(get_provider_env("TAVILY_API_KEY")) - - def is_keyless_available(self) -> bool: - """Tavily serves anonymous keyless requests (X-Tavily-Access-Mode). - - Default-on ring member of the keyless free tier: fresh installs - rotate across Exa/Parallel/Tavily/Firecrawl/Keenable. False when - the user pinned ``web.provider_tier.tavily: paid`` — an explicit - paid selection opts the free endpoint out. - """ - from plugins.web.keyless_mcp import keyless_enabled, provider_tier - - return keyless_enabled() and provider_tier("tavily") != "paid" - - def supports_search(self) -> bool: - return True - - def supports_extract(self) -> bool: - return True - - def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a Tavily search.""" - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return {"success": False, "error": "Interrupted"} - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import search_with_failover, use_keyless - - if use_keyless("tavily", get_provider_env("TAVILY_API_KEY")): - # Keyless free tier — ring dispatch with next-in-line - # failover on rate limits. - logger.info( - "Tavily keyless search: '%s' (limit=%d)", query, limit - ) - return search_with_failover("tavily", query, limit) - - logger.info("Tavily search: '%s' (limit=%d)", query, limit) - raw = _tavily_request( - "search", - { - "query": query, - "max_results": min(limit, 20), - "include_raw_content": False, - "include_images": False, - }, - ) - return _normalize_tavily_search_results(raw) - except ValueError as exc: - return {"success": False, "error": str(exc)} - except Exception as exc: # noqa: BLE001 — including httpx errors - logger.warning("Tavily search error: %s", exc) - return {"success": False, "error": f"Tavily search failed: {exc}"} - - def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: - """Extract content from one or more URLs via Tavily. - - Sync — the underlying call is httpx.post(...). Returns the legacy - list-of-results shape; per-URL failures become items with ``error``. - """ - try: - from tools.interrupt import is_interrupted - - if is_interrupted(): - return [ - {"url": u, "error": "Interrupted", "title": ""} for u in urls - ] - - from agent.web_search_provider import get_provider_env - - from plugins.web.keyless_mcp import extract_with_failover, use_keyless - - if use_keyless("tavily", get_provider_env("TAVILY_API_KEY")): - # Keyless free tier — ring dispatch with next-in-line - # failover on rate limits. - logger.info("Tavily keyless extract: %d URL(s)", len(urls)) - return extract_with_failover("tavily", list(urls)) - - logger.info("Tavily extract: %d URL(s)", len(urls)) - raw = _tavily_request( - "extract", - { - "urls": urls, - "include_images": False, - }, - ) - return _normalize_tavily_documents( - raw, fallback_url=urls[0] if urls else "" - ) - except ValueError as exc: - return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls] - except Exception as exc: # noqa: BLE001 - logger.warning("Tavily extract error: %s", exc) - return [ - {"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"} - for u in urls - ] - - def get_setup_schema(self) -> Dict[str, Any]: - return { - "name": "Tavily", - "badge": "free · key optional", - "tag": "Search + extract. Works keyless; set TAVILY_API_KEY for higher limits.", - "env_vars": [ - { - "key": "TAVILY_API_KEY", - "prompt": "Tavily API key (optional — keyless works without it)", - "url": "https://app.tavily.com/home", - }, - ], - } diff --git a/plugins/web/xai/provider.py b/plugins/web/xai/provider.py index 77d80a4398..922b9856be 100644 --- a/plugins/web/xai/provider.py +++ b/plugins/web/xai/provider.py @@ -101,12 +101,12 @@ class XAIWebSearchProvider(WebSearchProvider): back to the Responses API ``citations`` list if Grok ignores the JSON schema instruction (rare for grok-4.3 but cheap insurance). - No extract capability — pair with Firecrawl / Tavily / Exa for + No extract capability — pair with Firecrawl / Keenable / Exa for ``web_extract`` if you need page content. Trust model ----------- - Unlike index-backed providers (Brave / Tavily / Exa) which return + Unlike index-backed providers (Brave / Keenable / Exa) which return verbatim search-engine results, this backend is an LLM in a trench coat: Grok decides which URLs to surface, generates the titles and descriptions itself, and is influenced by the *content of the query*. diff --git a/tests/conftest.py b/tests/conftest.py index e4dfb0ed06..ffa5e7b928 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -186,7 +186,7 @@ _CREDENTIAL_NAMES = frozenset({ "FIRECRAWL_API_KEY", "PARALLEL_API_KEY", "EXA_API_KEY", - "TAVILY_API_KEY", + "TAVILY_API_KEY", # removed backend; still blanked for hermeticity "WANDB_API_KEY", "ELEVENLABS_API_KEY", "HONCHO_API_KEY", diff --git a/tests/hermes_cli/test_config.py b/tests/hermes_cli/test_config.py index faac5aed26..968649e58a 100644 --- a/tests/hermes_cli/test_config.py +++ b/tests/hermes_cli/test_config.py @@ -655,24 +655,24 @@ class TestSanitizeEnvLines: class TestOptionalEnvVarsRegistry: """Verify that key env vars are registered in OPTIONAL_ENV_VARS.""" - def test_tavily_api_key_registered(self): - """TAVILY_API_KEY is listed in OPTIONAL_ENV_VARS.""" + def test_keenable_api_key_registered(self): + """KEENABLE_API_KEY is listed in OPTIONAL_ENV_VARS.""" from hermes_cli.config import OPTIONAL_ENV_VARS - assert "TAVILY_API_KEY" in OPTIONAL_ENV_VARS + assert "KEENABLE_API_KEY" in OPTIONAL_ENV_VARS - def test_tavily_api_key_has_url(self): - """TAVILY_API_KEY has a URL.""" + def test_keenable_api_key_has_url(self): + """KEENABLE_API_KEY has a URL.""" from hermes_cli.config import OPTIONAL_ENV_VARS - assert OPTIONAL_ENV_VARS["TAVILY_API_KEY"]["url"] == "https://app.tavily.com/home" + assert OPTIONAL_ENV_VARS["KEENABLE_API_KEY"]["url"] == "https://keenable.ai" - def test_tavily_in_env_vars_by_version(self): - """TAVILY_API_KEY is listed in ENV_VARS_BY_VERSION.""" + def test_removed_tavily_var_not_in_env_vars_by_version(self): + """TAVILY_API_KEY was removed with the Tavily backend.""" from hermes_cli.config import ENV_VARS_BY_VERSION all_vars = [] for vars_list in ENV_VARS_BY_VERSION.values(): all_vars.extend(vars_list) - assert "TAVILY_API_KEY" in all_vars + assert "TAVILY_API_KEY" not in all_vars def test_max_iterations_not_offered_as_env_var(self): """HERMES_MAX_ITERATIONS must NOT be in OPTIONAL_ENV_VARS (issue #17534). diff --git a/tests/hermes_cli/test_dump_env_visibility.py b/tests/hermes_cli/test_dump_env_visibility.py index 7908242fa1..40feba0cec 100644 --- a/tests/hermes_cli/test_dump_env_visibility.py +++ b/tests/hermes_cli/test_dump_env_visibility.py @@ -46,7 +46,7 @@ def test_dump_leaves_unset_key_untouched(monkeypatch, capsys, tmp_path): from hermes_cli.config import get_hermes_home monkeypatch.setattr(dump, "get_project_root", lambda: tmp_path / "noproject") - monkeypatch.delenv("TAVILY_API_KEY", raising=False) + monkeypatch.delenv("KEENABLE_API_KEY", raising=False) home = get_hermes_home() home.mkdir(parents=True, exist_ok=True) @@ -54,6 +54,6 @@ def test_dump_leaves_unset_key_untouched(monkeypatch, capsys, tmp_path): dump.run_dump(SimpleNamespace(show_keys=False)) - line = _api_key_line(capsys.readouterr().out, "tavily") + line = _api_key_line(capsys.readouterr().out, "keenable") assert "not set" in line assert "shell only" not in line diff --git a/tests/hermes_cli/test_nous_subscription.py b/tests/hermes_cli/test_nous_subscription.py index c9ffaa931a..d9f71a9af1 100644 --- a/tests/hermes_cli/test_nous_subscription.py +++ b/tests/hermes_cli/test_nous_subscription.py @@ -58,52 +58,6 @@ def test_get_nous_subscription_features_recognizes_direct_exa_backend(monkeypatc assert features.web.current_provider == "exa" -def test_get_nous_subscription_features_recognizes_keyless_tavily_backend(monkeypatch): - """Selecting Tavily in setup/tools counts as available with no API key. - - Mirrors tools.web_tools._is_backend_available('tavily'): keyless is - opt-in via web.backend / search_backend / extract_backend, not a - silent empty-install default. The setup summary previously required - TAVILY_API_KEY and printed a false 'missing' after a skipped key prompt. - """ - monkeypatch.setattr(ns, "get_env_value", lambda name: "") - monkeypatch.setattr( - ns, "get_nous_portal_account_info", lambda: _account(logged_in=False) - ) - monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web") - monkeypatch.setattr(ns, "_has_agent_browser", lambda: False) - monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "") - monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False) - - features = ns.get_nous_subscription_features({"web": {"backend": "tavily"}}) - - assert features.web.available is True - assert features.web.active is True - assert features.web.managed_by_nous is False - assert features.web.direct_override is True - assert features.web.current_provider == "tavily" - assert features.web.explicit_configured is True - - -def test_keyless_tavily_search_backend_without_shared_backend(monkeypatch): - monkeypatch.setattr(ns, "get_env_value", lambda name: "") - monkeypatch.setattr( - ns, "get_nous_portal_account_info", lambda: _account(logged_in=False) - ) - monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web") - monkeypatch.setattr(ns, "_has_agent_browser", lambda: False) - monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "") - monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False) - - features = ns.get_nous_subscription_features( - {"web": {"search_backend": "tavily"}} - ) - - assert features.web.available is True - assert features.web.active is True - assert features.web.current_provider == "tavily" - - def test_unconfigured_web_without_keys_is_unavailable(monkeypatch): monkeypatch.setattr(ns, "get_env_value", lambda name: "") monkeypatch.setattr( diff --git a/tests/hermes_cli/test_plugins_cmd_category_discovery.py b/tests/hermes_cli/test_plugins_cmd_category_discovery.py index 21591b90ed..43c3a77156 100644 --- a/tests/hermes_cli/test_plugins_cmd_category_discovery.py +++ b/tests/hermes_cli/test_plugins_cmd_category_discovery.py @@ -105,8 +105,8 @@ class TestDiscoverAllPlugins: _make_plugin_dir(tmp_path, "disk-cleanup", { "name": "disk-cleanup", "version": "1.0.0" }) - _make_category_plugin(tmp_path, "web", "tavily", { - "name": "web-tavily", "version": "1.0.0" + _make_category_plugin(tmp_path, "web", "keenable", { + "name": "web-keenable", "version": "1.0.0" }) _make_category_plugin(tmp_path, "web", "exa", { "name": "web-exa", "version": "1.0.0" @@ -117,7 +117,7 @@ class TestDiscoverAllPlugins: entries = _discover_all_plugins() keys = [e[5] for e in entries] assert "disk-cleanup" in keys - assert "web/tavily" in keys + assert "web/keenable" in keys assert "web/exa" in keys assert len(entries) == 3 @@ -128,8 +128,8 @@ class TestDiscoverAllPlugins: from hermes_cli.plugins_cmd import _discover_all_plugins # 2 levels: should be found - _make_category_plugin(tmp_path, "web", "tavily", { - "name": "web-tavily", "version": "1.0.0" + _make_category_plugin(tmp_path, "web", "keenable", { + "name": "web-keenable", "version": "1.0.0" }) # 3 levels: should NOT be found deep = tmp_path / "a" / "b" / "c" @@ -143,7 +143,7 @@ class TestDiscoverAllPlugins: entries = _discover_all_plugins() keys = [e[5] for e in entries] - assert "web/tavily" in keys + assert "web/keenable" in keys assert "a/b/c" not in keys @patch("hermes_cli.plugins.get_bundled_plugins_dir") @@ -219,7 +219,7 @@ class TestPluginStatus: def test_key_in_disabled(self): from hermes_cli.plugins_cmd import _plugin_status - assert _plugin_status("web-tavily", set(), {"web/tavily"}, key="web/tavily") == "disabled" + assert _plugin_status("web-keenable", set(), {"web/keenable"}, key="web/keenable") == "disabled" def test_neither_name_nor_key(self): from hermes_cli.plugins_cmd import _plugin_status @@ -236,7 +236,7 @@ class TestFilterPluginEntries: from hermes_cli.plugins_cmd import _filter_plugin_entries entries = [ - ("web-tavily", "1.0.0", "search", "user", Path("/tmp"), "web/tavily"), + ("web-keenable", "1.0.0", "search", "user", Path("/tmp"), "web/keenable"), ("disk-cleanup", "1.0.0", "cleanup", "bundled", Path("/tmp"), "disk-cleanup"), ] args = MagicMock() @@ -244,9 +244,9 @@ class TestFilterPluginEntries: args.user = False args.enabled = True - result = _filter_plugin_entries(entries, args, {"web/tavily"}, set()) + result = _filter_plugin_entries(entries, args, {"web/keenable"}, set()) assert len(result) == 1 - assert result[0][5] == "web/tavily" + assert result[0][5] == "web/keenable" # --------------------------------------------------------------------------- @@ -260,8 +260,8 @@ class TestCmdListJson: def test_json_output_includes_category_plugins(self, mock_user_dir, mock_bundled_dir, tmp_path, capsys): from hermes_cli.plugins_cmd import cmd_list - _make_category_plugin(tmp_path, "web", "tavily", { - "name": "web-tavily", "version": "1.0.0", "description": "search" + _make_category_plugin(tmp_path, "web", "keenable", { + "name": "web-keenable", "version": "1.0.0", "description": "search" }) _make_plugin_dir(tmp_path, "disk-cleanup", { "name": "disk-cleanup", "version": "2.0.0", "description": "cleanup" @@ -280,7 +280,7 @@ class TestCmdListJson: captured = capsys.readouterr() payload = json.loads(captured.out) names = [p["name"] for p in payload] - assert "web-tavily" in names + assert "web-keenable" in names assert "disk-cleanup" in names @patch("hermes_cli.plugins.get_bundled_plugins_dir") @@ -288,14 +288,14 @@ class TestCmdListJson: def test_json_status_uses_key(self, mock_user_dir, mock_bundled_dir, tmp_path, capsys): from hermes_cli.plugins_cmd import cmd_list - _make_category_plugin(tmp_path, "web", "tavily", { - "name": "web-tavily", "version": "1.0.0" + _make_category_plugin(tmp_path, "web", "keenable", { + "name": "web-keenable", "version": "1.0.0" }) mock_user_dir.return_value = tmp_path mock_bundled_dir.return_value = tmp_path / "nonexistent" - # Patch config to return web/tavily as enabled - with patch("hermes_cli.plugins_cmd._get_enabled_set", return_value={"web/tavily"}): + # Patch config to return web/keenable as enabled + with patch("hermes_cli.plugins_cmd._get_enabled_set", return_value={"web/keenable"}): args = MagicMock() args.json = True args.plain = False diff --git a/tests/hermes_cli/test_status.py b/tests/hermes_cli/test_status.py index 750ff60021..4a5746b94d 100644 --- a/tests/hermes_cli/test_status.py +++ b/tests/hermes_cli/test_status.py @@ -3,15 +3,15 @@ from types import SimpleNamespace from hermes_cli.status import show_status -def test_show_status_all_does_not_print_tavily_key_value(monkeypatch, capsys, tmp_path): +def test_show_status_all_does_not_print_keenable_key_value(monkeypatch, capsys, tmp_path): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) sentinel = "NONSECRET_SENTINEL_VALUE_DO_NOT_PRINT_123456" - monkeypatch.setenv("TAVILY_API_KEY", sentinel) + monkeypatch.setenv("KEENABLE_API_KEY", sentinel) show_status(SimpleNamespace(all=True, deep=False)) output = capsys.readouterr().out - assert "Tavily" in output + assert "Keenable" in output assert sentinel not in output diff --git a/tests/hermes_cli/test_tools_config.py b/tests/hermes_cli/test_tools_config.py index e11d6e96d7..0ff5e99c44 100644 --- a/tests/hermes_cli/test_tools_config.py +++ b/tests/hermes_cli/test_tools_config.py @@ -220,7 +220,7 @@ def test_first_install_nous_auto_configures_video_gen(monkeypatch): "ELEVENLABS_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", - "TAVILY_API_KEY", + "KEENABLE_API_KEY", "PARALLEL_API_KEY", "BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", diff --git a/tests/plugins/web/test_web_search_provider_plugins.py b/tests/plugins/web/test_web_search_provider_plugins.py index 200ce9878a..117733e045 100644 --- a/tests/plugins/web/test_web_search_provider_plugins.py +++ b/tests/plugins/web/test_web_search_provider_plugins.py @@ -2,8 +2,8 @@ Covers: -- All eight bundled plugins (brave-free, ddgs, searxng, exa, parallel, - tavily, firecrawl, xai) instantiate and self-report the expected +- All bundled plugins (brave-free, ddgs, searxng, exa, parallel, + firecrawl, keenable, xai) instantiate and self-report the expected capabilities + ABC-derived defaults. - Each plugin's ``is_available()`` correctly reflects env-var presence. - The web_search_registry resolves an active provider in the documented @@ -34,8 +34,7 @@ def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None: for k in ( "BRAVE_SEARCH_API_KEY", "SEARXNG_URL", - "TAVILY_API_KEY", - "TAVILY_BASE_URL", + "KEENABLE_API_KEY", "EXA_API_KEY", "PARALLEL_API_KEY", "PARALLEL_SEARCH_MODE", @@ -68,7 +67,7 @@ def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None: class TestBundledPluginsRegister: - """All eight bundled web plugins discover and register correctly.""" + """All bundled web plugins discover and register correctly.""" def test_all_bundled_plugins_present_in_registry(self) -> None: _ensure_plugins_loaded() @@ -83,7 +82,6 @@ class TestBundledPluginsRegister: "keenable", "parallel", "searxng", - "tavily", "xai", ] @@ -95,7 +93,7 @@ class TestBundledPluginsRegister: ("searxng", True, False), ("exa", True, True), ("parallel", True, True), - ("tavily", True, True), + ("keenable", True, True), ("firecrawl", True, True), # xai: search-only via Grok's agentic web_search tool. ("xai", True, False), @@ -117,7 +115,7 @@ class TestBundledPluginsRegister: @pytest.mark.parametrize( "plugin_name", - ["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"], + ["brave-free", "ddgs", "searxng", "exa", "parallel", "firecrawl", "keenable", "xai"], ) def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None: _ensure_plugins_loaded() @@ -157,14 +155,14 @@ class TestIsAvailable: monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") assert p.is_available() is True - def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_keenable_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider - p = get_provider("tavily") + p = get_provider("keenable") assert p is not None assert p.is_available() is False - monkeypatch.setenv("TAVILY_API_KEY", "real") + monkeypatch.setenv("KEENABLE_API_KEY", "real") assert p.is_available() is True def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index df6e0d82e5..cefe4584dc 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -82,8 +82,8 @@ def register_all_web_providers(): from plugins.web.exa.provider import ExaWebSearchProvider from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider from plugins.web.parallel.provider import ParallelWebSearchProvider + from plugins.web.keenable.provider import KeenableWebSearchProvider from plugins.web.searxng.provider import SearXNGWebSearchProvider - from plugins.web.tavily.provider import TavilyWebSearchProvider from plugins.web.xai.provider import XAIWebSearchProvider _reset_for_tests() @@ -93,8 +93,8 @@ def register_all_web_providers(): ExaWebSearchProvider, FirecrawlWebSearchProvider, ParallelWebSearchProvider, + KeenableWebSearchProvider, SearXNGWebSearchProvider, - TavilyWebSearchProvider, XAIWebSearchProvider, ): register_provider(cls()) diff --git a/tests/tools/test_strict_provider_selection.py b/tests/tools/test_strict_provider_selection.py index aed8ae9e32..17cead6052 100644 --- a/tests/tools/test_strict_provider_selection.py +++ b/tests/tools/test_strict_provider_selection.py @@ -334,9 +334,9 @@ class TestWriteProviderConfig: from hermes_cli.tools_config import _write_provider_config config = {"web": {"backend": "nous", "use_gateway": True}} - provider = {"name": "Tavily", "web_backend": "tavily"} + provider = {"name": "Keenable", "web_backend": "keenable"} _write_provider_config(provider, config, managed_feature=None) - assert config["web"]["backend"] == "tavily" + assert config["web"]["backend"] == "keenable" assert "use_gateway" not in config["web"] def test_managed_image_row_persists_nous_provider(self): diff --git a/tests/tools/test_web_keyless_fallback.py b/tests/tools/test_web_keyless_fallback.py index f2c12ec21b..1721d3f98d 100644 --- a/tests/tools/test_web_keyless_fallback.py +++ b/tests/tools/test_web_keyless_fallback.py @@ -25,7 +25,7 @@ from plugins.web.parallel.provider import ParallelWebSearchProvider def _no_web_env(monkeypatch): """Blank every web credential and neutralize config lookups.""" for var in ( - "EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", + "EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "BRAVE_SEARCH_API_KEY", "SEARXNG_URL", "TOOL_GATEWAY_USER_TOKEN", ): @@ -256,7 +256,7 @@ class TestProviderRouting: ) assert keyless_mcp.provider_tier("exa") == "free" assert keyless_mcp.provider_tier("parallel") == "auto" # invalid → auto - assert keyless_mcp.provider_tier("tavily") == "auto" # unset → auto + assert keyless_mcp.provider_tier("keenable") == "auto" # unset → auto @pytest.mark.asyncio async def test_parallel_keyless_extract(self, monkeypatch): @@ -299,9 +299,9 @@ class TestResolutionOrder: starts = [keyless_mcp._ring_order("exa")[0] for _ in range(len(keyless_mcp._KEYLESS_RING))] assert sorted(starts) == sorted(keyless_mcp._KEYLESS_RING) # full cycle # Pinned dispatch starts at the pinned vendor every time. - monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda name: name == "tavily") - assert keyless_mcp._ring_order("tavily")[0] == "tavily" - assert keyless_mcp._ring_order("tavily")[0] == "tavily" + monkeypatch.setattr(keyless_mcp, "_vendor_pinned", lambda name: name == "keenable") + assert keyless_mcp._ring_order("keenable")[0] == "keenable" + assert keyless_mcp._ring_order("keenable")[0] == "keenable" def test_registry_keyless_disabled_returns_none(self, fresh_registry, monkeypatch): monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) @@ -336,9 +336,9 @@ class TestResolutionOrder: def test_get_backend_key_beats_keyless(self, monkeypatch): monkeypatch.setattr( web_tools, "_env_value", - lambda name: "sk-x" if name == "TAVILY_API_KEY" else "", + lambda name: "sk-x" if name == "EXA_API_KEY" else "", ) - assert web_tools._get_backend() == "tavily" + assert web_tools._get_backend() == "exa" def test_get_backend_keyless_disabled(self, monkeypatch): monkeypatch.setattr( @@ -476,9 +476,9 @@ class TestKeylessFailover: assert "all keyless vendors throttled" in out["error"] def test_search_walks_ring_past_multiple_throttles(self, monkeypatch): - # exa -> parallel -> tavily all throttled; firecrawl serves. + # exa -> parallel all throttled; firecrawl serves. self._pin(monkeypatch, "exa") - for vendor in ("exa", "parallel", "tavily"): + for vendor in ("exa", "parallel"): monkeypatch.setitem( keyless_mcp._KEYLESS_SEARCHERS, vendor, lambda q, l, v=vendor: self._throttled(v), @@ -504,7 +504,7 @@ class TestKeylessFailover: keyless_mcp._KEYLESS_SEARCHERS, "exa", lambda q, l: called.append(1) or self._ok("exa"), ) - for vendor in ("parallel", "tavily", "firecrawl", "keenable"): + for vendor in ("parallel", "firecrawl", "keenable"): monkeypatch.setitem( keyless_mcp._KEYLESS_SEARCHERS, vendor, lambda q, l, v=vendor: self._throttled(v), diff --git a/tests/tools/test_web_keyless_rescue.py b/tests/tools/test_web_keyless_rescue.py index aac14fce8b..408200006f 100644 --- a/tests/tools/test_web_keyless_rescue.py +++ b/tests/tools/test_web_keyless_rescue.py @@ -20,14 +20,14 @@ import pytest import tools.web_tools as web_tools from plugins.web import keyless_mcp -from plugins.web.tavily.provider import TavilyWebSearchProvider +from plugins.web.keenable.provider import KeenableWebSearchProvider class _KeyedBoomProvider: """Minimal keyed provider double that always fails.""" - name = "tavily" - display_name = "Tavily" + name = "keenable" + display_name = "Keenable" def supports_search(self): return True @@ -54,13 +54,13 @@ class _RaisingProvider(_KeyedBoomProvider): @pytest.fixture(autouse=True) -def _keyed_tavily_env(monkeypatch): - """Simulate a keyed Tavily setup with rescue enabled.""" +def _keyed_keenable_env(monkeypatch): + """Simulate a keyed Keenable setup with rescue enabled.""" monkeypatch.setattr( "agent.web_search_provider.get_provider_env", - lambda name: "tvly-real" if name == "TAVILY_API_KEY" else "", + lambda name: "kn-real" if name == "KEENABLE_API_KEY" else "", ) - monkeypatch.setattr(web_tools, "_load_web_config", lambda: {"backend": "tavily"}) + monkeypatch.setattr(web_tools, "_load_web_config", lambda: {"backend": "keenable"}) monkeypatch.setattr( "agent.web_search_registry._keyless_tier_enabled", lambda: True ) @@ -76,11 +76,11 @@ class TestEligibility: assert web_tools._rescue_eligible(_KeyedBoomProvider()) is True def test_keyless_mode_ring_vendor_not_eligible(self, monkeypatch): - # No key: the tavily call already rode the ring; no double-walk. + # No key: the keenable call already rode the ring; no double-walk. monkeypatch.setattr( "agent.web_search_provider.get_provider_env", lambda name: "" ) - assert web_tools._rescue_eligible(TavilyWebSearchProvider()) is False + assert web_tools._rescue_eligible(KeenableWebSearchProvider()) is False def test_non_ring_backend_is_eligible(self): class _SearxProvider(_KeyedBoomProvider): @@ -91,7 +91,7 @@ class TestEligibility: def test_config_gate_disables(self, monkeypatch): monkeypatch.setattr( web_tools, "_load_web_config", - lambda: {"backend": "tavily", "keyless_rescue": False}, + lambda: {"backend": "keenable", "keyless_rescue": False}, ) assert web_tools._rescue_eligible(_KeyedBoomProvider()) is False @@ -116,7 +116,7 @@ class TestSearchRescue: ) as ring: out = self._dispatch(monkeypatch, _KeyedBoomProvider()) assert out["success"] is True - assert out["data"]["rescued_from"] == "tavily" + assert out["data"]["rescued_from"] == "keenable" assert "HTTP 500" in out["data"]["backend_error"] assert "next call" in out["data"]["backend_error"].lower() ring.assert_called_once() @@ -161,7 +161,7 @@ class TestSearchRescue: def test_no_rescue_when_disabled(self, monkeypatch): monkeypatch.setattr( web_tools, "_load_web_config", - lambda: {"backend": "tavily", "keyless_rescue": False}, + lambda: {"backend": "keenable", "keyless_rescue": False}, ) with patch.object(keyless_mcp, "search_with_failover") as ring: out = self._dispatch(monkeypatch, _KeyedBoomProvider()) @@ -211,8 +211,8 @@ class TestExtractRescue: with patch.object( keyless_mcp, "extract_with_failover", return_value=good ): - out = web_tools._rescue_extract("tavily", ["https://a"], failed) - assert out[0]["metadata"]["rescued_from"] == "tavily" + out = web_tools._rescue_extract("keenable", ["https://a"], failed) + assert out[0]["metadata"]["rescued_from"] == "keenable" assert "HTTP 500" in out[0]["metadata"]["backend_error"] @pytest.mark.asyncio diff --git a/tests/tools/test_web_providers.py b/tests/tools/test_web_providers.py index 31b498605c..775e343f48 100644 --- a/tests/tools/test_web_providers.py +++ b/tests/tools/test_web_providers.py @@ -82,22 +82,22 @@ class TestPerCapabilityBackendSelection: monkeypatch.setattr(web_tools, "_load_web_config", lambda: { "backend": "firecrawl", - "search_backend": "tavily", + "search_backend": "keenable", }) - monkeypatch.setenv("TAVILY_API_KEY", "test-key") - assert web_tools._get_search_backend() == "tavily" + monkeypatch.setenv("KEENABLE_API_KEY", "test-key") + assert web_tools._get_search_backend() == "keenable" def test_fully_backward_compatible_with_web_backend_only(self, monkeypatch): from tools import web_tools monkeypatch.setattr(web_tools, "_load_web_config", lambda: { - "backend": "tavily", + "backend": "keenable", }) - monkeypatch.setenv("TAVILY_API_KEY", "test-key") + monkeypatch.setenv("KEENABLE_API_KEY", "test-key") # No search_backend or extract_backend set — both fall through - assert web_tools._get_search_backend() == "tavily" - assert web_tools._get_extract_backend() == "tavily" + assert web_tools._get_search_backend() == "keenable" + assert web_tools._get_extract_backend() == "keenable" # --------------------------------------------------------------------------- @@ -180,7 +180,7 @@ class TestUnconfiguredErrorEnvelopeParity: for k in ( "BRAVE_SEARCH_API_KEY", "SEARXNG_URL", - "TAVILY_API_KEY", + "KEENABLE_API_KEY", "EXA_API_KEY", "PARALLEL_API_KEY", "FIRECRAWL_API_KEY", @@ -218,7 +218,7 @@ class TestUnconfiguredErrorEnvelopeParity: def test_explicit_firecrawl_unconfigured_uses_firecrawl_keyless(self, monkeypatch): """``web.backend: firecrawl`` with no creds routes through Firecrawl's - keyless cloud client (PR #50659 salvage) — keyless Tavily must not + keyless cloud client (PR #50659 salvage) — a keyless ring peer must not silently take over, and the request must hit api.firecrawl.dev. """ from tools import web_tools diff --git a/tests/tools/test_web_providers_brave_free.py b/tests/tools/test_web_providers_brave_free.py index e00c09646a..9a9de6daa5 100644 --- a/tests/tools/test_web_providers_brave_free.py +++ b/tests/tools/test_web_providers_brave_free.py @@ -126,15 +126,15 @@ class TestBraveFreeBackendWiring: def test_brave_free_does_not_override_paid_provider(self, monkeypatch): - """Tavily (higher priority) should win in auto-detect.""" + """Exa (higher priority) should win in auto-detect.""" from tools import web_tools monkeypatch.setattr(web_tools, "_load_web_config", lambda: {}) for key in ("FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY", "EXA_API_KEY", "SEARXNG_URL"): monkeypatch.delenv(key, raising=False) - monkeypatch.setenv("TAVILY_API_KEY", "tvly") + monkeypatch.setenv("EXA_API_KEY", "exa_test") monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False) - assert web_tools._get_backend() == "tavily" + assert web_tools._get_backend() == "exa" def test_check_web_api_key_true_when_brave_free_configured(self, monkeypatch): from tools import web_tools diff --git a/tests/tools/test_web_providers_ddgs.py b/tests/tools/test_web_providers_ddgs.py index 959f3c703b..b5dd4d6270 100644 --- a/tests/tools/test_web_providers_ddgs.py +++ b/tests/tools/test_web_providers_ddgs.py @@ -243,7 +243,7 @@ class TestDDGSBackendWiring: from tools import web_tools monkeypatch.setattr(web_tools, "_load_web_config", lambda: {}) for key in ("FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY", - "TAVILY_API_KEY", "EXA_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY"): + "EXA_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY", "KEENABLE_API_KEY"): monkeypatch.delenv(key, raising=False) monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False) monkeypatch.setattr(web_tools, "_ddgs_package_importable", lambda: True) diff --git a/tests/tools/test_web_providers_searxng.py b/tests/tools/test_web_providers_searxng.py index be9d438c68..415de09f77 100644 --- a/tests/tools/test_web_providers_searxng.py +++ b/tests/tools/test_web_providers_searxng.py @@ -141,16 +141,16 @@ class TestGetBackendSearXNG: def test_searxng_does_not_override_higher_priority_provider(self, monkeypatch): - """Tavily (higher priority than searxng) should win in auto-detect.""" + """Exa (higher priority than searxng) should win in auto-detect.""" from tools import web_tools monkeypatch.setattr(web_tools, "_load_web_config", lambda: {}) monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) monkeypatch.delenv("FIRECRAWL_API_URL", raising=False) monkeypatch.delenv("PARALLEL_API_KEY", raising=False) - monkeypatch.setenv("TAVILY_API_KEY", "tvly-key") + monkeypatch.setenv("EXA_API_KEY", "exa_test_key") monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False) - assert web_tools._get_backend() == "tavily" + assert web_tools._get_backend() == "exa" def test_auto_detect_picks_searxng_when_url_only_in_hermes_config(self, monkeypatch): """#34290 follow-up: a config-only SEARXNG_URL (absent from process env) @@ -161,7 +161,7 @@ class TestGetBackendSearXNG: monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) monkeypatch.delenv("FIRECRAWL_API_URL", raising=False) monkeypatch.delenv("PARALLEL_API_KEY", raising=False) - monkeypatch.delenv("TAVILY_API_KEY", raising=False) + monkeypatch.delenv("EXA_API_KEY", raising=False) monkeypatch.delenv("EXA_API_KEY", raising=False) monkeypatch.delenv("SEARXNG_URL", raising=False) monkeypatch.setattr( @@ -205,7 +205,7 @@ class TestCheckWebApiKey: monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) monkeypatch.delenv("FIRECRAWL_API_URL", raising=False) monkeypatch.delenv("PARALLEL_API_KEY", raising=False) - monkeypatch.delenv("TAVILY_API_KEY", raising=False) + monkeypatch.delenv("EXA_API_KEY", raising=False) monkeypatch.delenv("EXA_API_KEY", raising=False) monkeypatch.delenv("SEARXNG_URL", raising=False) monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False) diff --git a/tests/tools/test_web_providers_xai.py b/tests/tools/test_web_providers_xai.py index a57c66620b..6f42456136 100644 --- a/tests/tools/test_web_providers_xai.py +++ b/tests/tools/test_web_providers_xai.py @@ -442,7 +442,7 @@ class TestXAIBackendWiring: monkeypatch.setattr(web_tools, "_load_web_config", lambda: {}) for key in ( "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY", - "TAVILY_API_KEY", "EXA_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY", + "EXA_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY", "KEENABLE_API_KEY", ): monkeypatch.delenv(key, raising=False) monkeypatch.setenv("XAI_API_KEY", "xai-test-key") diff --git a/tests/tools/test_web_result_cache.py b/tests/tools/test_web_result_cache.py index 8cb2cf3a23..b9a40be73a 100644 --- a/tests/tools/test_web_result_cache.py +++ b/tests/tools/test_web_result_cache.py @@ -78,7 +78,7 @@ def test_search_memo_hit_within_ttl(): def test_search_memo_miss_across_providers_and_buckets(): memo = SearchMemo() memo.store("firecrawl", "q", 5, _ok_response()) - assert memo.lookup("tavily", "q", 5) is None # different provider + assert memo.lookup("keenable", "q", 5) is None # different provider assert memo.lookup("firecrawl", "q", 15) is None # different bucket assert memo.lookup("firecrawl", "other", 5) is None # different query @@ -184,7 +184,7 @@ def test_extract_cache_provider_participates_in_key(_isolated_cache): """Switching extract backends within the TTL must not serve the old backend's rendering (#94618 review, additional risk 3).""" extract_cache_put("https://e.com/p", "firecrawl version", provider="firecrawl") - assert extract_cache_get("https://e.com/p", provider="tavily") is None + assert extract_cache_get("https://e.com/p", provider="keenable") is None hit = extract_cache_get("https://e.com/p", provider="firecrawl") assert hit is not None and hit["content"] == "firecrawl version" diff --git a/tests/tools/test_web_tools_config.py b/tests/tools/test_web_tools_config.py index 25819e69af..29c5f3b8cf 100644 --- a/tests/tools/test_web_tools_config.py +++ b/tests/tools/test_web_tools_config.py @@ -209,7 +209,7 @@ class TestBackendSelection: "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME", "TOOL_GATEWAY_USER_TOKEN", - "TAVILY_API_KEY", + "KEENABLE_API_KEY", ) def setup_method(self): @@ -254,26 +254,26 @@ class TestBackendSelection: assert _get_backend() == "exa" def test_fallback_exa_takes_priority_over_parallel(self): - """Direct-credential backends are tried in the order tavily > exa > parallel + """Direct-credential backends are tried in the order exa > parallel > keenable so an explicit Exa key wins when both Exa and Parallel are configured.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={}), \ patch.dict(os.environ, {"EXA_API_KEY": "exa-test", "PARALLEL_API_KEY": "par-test"}): assert _get_backend() == "exa" - def test_fallback_tavily_only_key(self): - """Only TAVILY_API_KEY set → 'tavily'.""" + def test_fallback_keenable_only_key(self): + """Only KEENABLE_API_KEY set → 'keenable'.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={}), \ - patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}): - assert _get_backend() == "tavily" + patch.dict(os.environ, {"KEENABLE_API_KEY": "kn-test"}): + assert _get_backend() == "keenable" - def test_fallback_tavily_beats_firecrawl_direct(self): - """Tavily ranks above firecrawl in the explicit-credential block.""" + def test_fallback_exa_beats_firecrawl_direct(self): + """Exa ranks above firecrawl in the explicit-credential block.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={}), \ - patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "FIRECRAWL_API_KEY": "fc-test"}): - assert _get_backend() == "tavily" + patch.dict(os.environ, {"EXA_API_KEY": "exa-test", "FIRECRAWL_API_KEY": "fc-test"}): + assert _get_backend() == "exa" def test_fallback_parallel_beats_firecrawl_direct(self): @@ -320,7 +320,7 @@ class TestBackendSelection: vendors — no availability probe, no credential override.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={"backend": "firecrawl"}), \ - patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}): + patch.dict(os.environ, {"EXA_API_KEY": "exa-test"}): assert _get_backend() == "firecrawl" def test_nous_backend_maps_to_firecrawl(self): @@ -330,17 +330,17 @@ class TestBackendSelection: with patch("tools.web_tools._load_web_config", return_value={"backend": "nous"}): assert _get_backend() == "firecrawl" - def test_managed_gateway_does_not_preempt_explicit_tavily(self): + def test_managed_gateway_does_not_preempt_explicit_exa(self): """Regression: a Nous OAuth token (managed gateway "ready") must NOT - beat an explicitly configured TAVILY_API_KEY in the fallback path. + beat an explicitly configured EXA_API_KEY in the fallback path. Free Nous tiers don't include web search, so the user's deliberate - Tavily setup would fail at runtime with "no subscription" if the + Exa setup would fail at runtime with "no subscription" if the gateway pre-empted it.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={}), \ patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \ - patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}): - assert _get_backend() == "tavily" + patch.dict(os.environ, {"EXA_API_KEY": "exa-test"}): + assert _get_backend() == "exa" def test_managed_gateway_only_falls_through_to_firecrawl(self): """When no explicit-credential backend is configured, a Nous-managed @@ -493,7 +493,7 @@ class TestCheckWebApiKey: "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME", "TOOL_GATEWAY_USER_TOKEN", - "TAVILY_API_KEY", + "KEENABLE_API_KEY", ) def setup_method(self): @@ -624,7 +624,7 @@ class TestNonBuiltinProviderAvailability: "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME", "TOOL_GATEWAY_USER_TOKEN", - "TAVILY_API_KEY", + "KEENABLE_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY", "XAI_API_KEY", @@ -764,7 +764,7 @@ class TestSiblingProvidersEnvResolution: _CASES = [ ("plugins.web.exa.provider", "ExaWebSearchProvider", "EXA_API_KEY"), ("plugins.web.parallel.provider", "ParallelWebSearchProvider", "PARALLEL_API_KEY"), - ("plugins.web.tavily.provider", "TavilyWebSearchProvider", "TAVILY_API_KEY"), + ("plugins.web.keenable.provider", "KeenableWebSearchProvider", "KEENABLE_API_KEY"), ("plugins.web.brave_free.provider", "BraveFreeWebSearchProvider", "BRAVE_SEARCH_API_KEY"), ] @@ -790,9 +790,9 @@ class TestSiblingProvidersEnvResolution: "config-aware env layer (get_env_value)" ) - def test_tavily_request_reads_key_via_get_env_value(self, monkeypatch): - """Keyed Tavily must Bearer-auth with a key that lives only in .env.""" - monkeypatch.delenv("TAVILY_API_KEY", raising=False) + def test_keenable_search_reads_key_via_get_env_value(self, monkeypatch): + """Keyed Keenable must Bearer-auth with a key that lives only in .env.""" + monkeypatch.delenv("KEENABLE_API_KEY", raising=False) mock_response = MagicMock() mock_response.status_code = 200 mock_response.json.return_value = {"results": []} @@ -800,17 +800,16 @@ class TestSiblingProvidersEnvResolution: with patch( "hermes_cli.config.get_env_value", - side_effect=lambda k: "tvly-from-dotenv" if k == "TAVILY_API_KEY" else None, + side_effect=lambda k: "kn-from-dotenv" if k == "KEENABLE_API_KEY" else None, ), patch( - "plugins.web.tavily.provider.httpx.post", return_value=mock_response + "requests.post", return_value=mock_response ) as mock_post: - from plugins.web.tavily.provider import _tavily_request + from plugins.web.keenable.provider import KeenableWebSearchProvider - _tavily_request("search", {"query": "q"}) + KeenableWebSearchProvider().search("q", limit=2) headers = mock_post.call_args.kwargs["headers"] - assert headers["Authorization"] == "Bearer tvly-from-dotenv" - assert headers["X-Client-Name"] == "hermes-agent" - assert "X-Tavily-Access-Mode" not in headers + assert headers["Authorization"] == "Bearer kn-from-dotenv" + assert headers["X-Keenable-Title"] == "hermes-agent" def test_get_provider_env_unset_returns_empty(self, monkeypatch): diff --git a/tests/tools/test_web_tools_tavily.py b/tests/tools/test_web_tools_tavily.py deleted file mode 100644 index 73ee8ebd4c..0000000000 --- a/tests/tools/test_web_tools_tavily.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Tests for Tavily web backend integration. - -Coverage: - _tavily_request() — keyed Bearer vs keyless header, attribution, error bodies. - _normalize_tavily_search_results() — search response normalization. - _normalize_tavily_documents() — extract response normalization, failed_results. - web_search_tool / web_extract_tool — Tavily dispatch paths. - auto-detect ranking — keyed paid-band; keyless only when Tavily is selected. -""" - -import json -import os -import asyncio -import pytest -from unittest.mock import patch, MagicMock - -from tests.tools.conftest import register_all_web_providers - - -def _ok_response(payload=None): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = payload if payload is not None else {"results": []} - mock_response.text = json.dumps(mock_response.json.return_value) - return mock_response - - -# ─── _tavily_request ───────────────────────────────────────────────────────── - -class TestTavilyRequest: - """Test suite for the _tavily_request helper.""" - - def test_keyless_when_no_api_key(self): - """No TAVILY_API_KEY → keyless header, no Authorization, no body key.""" - mock_response = _ok_response() - - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("TAVILY_API_KEY", None) - with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post: - from plugins.web.tavily.provider import _tavily_request - _tavily_request("search", {"query": "test"}) - - mock_post.assert_called_once() - headers = mock_post.call_args.kwargs["headers"] - payload = mock_post.call_args.kwargs["json"] - assert headers["X-Client-Name"] == "hermes-agent" - assert headers["X-Tavily-Access-Mode"] == "keyless" - assert "Authorization" not in headers - assert "api_key" not in payload - assert payload["query"] == "test" - assert "api.tavily.com/search" in mock_post.call_args.args[0] - - def test_keyed_uses_bearer_not_body(self): - """TAVILY_API_KEY → Bearer auth, attribution, no body api_key.""" - mock_response = _ok_response() - - with patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test-key"}): - with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post: - from plugins.web.tavily.provider import _tavily_request - _tavily_request("search", {"query": "hello"}) - - mock_post.assert_called_once() - headers = mock_post.call_args.kwargs["headers"] - payload = mock_post.call_args.kwargs["json"] - assert headers == { - "X-Client-Name": "hermes-agent", - "Authorization": "Bearer tvly-test-key", - } - assert "X-Tavily-Access-Mode" not in headers - assert "api_key" not in payload - assert payload["query"] == "hello" - assert "api.tavily.com/search" in mock_post.call_args.args[0] - - def test_http_error_surfaces_response_body(self): - """Non-2xx responses raise ValueError with Tavily's response body.""" - mock_response = MagicMock() - mock_response.status_code = 429 - mock_response.text = "Rate limit hit. Sign up for a free API key at https://app.tavily.com" - mock_response.json.return_value = {} - - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("TAVILY_API_KEY", None) - with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response): - from plugins.web.tavily.provider import _tavily_request - with pytest.raises(ValueError, match="Rate limit hit"): - _tavily_request("search", {"query": "test"}) - - -# ─── _normalize_tavily_search_results ───────────────────────────────────────── - -class TestNormalizeTavilySearchResults: - """Test search result normalization.""" - - def test_basic_normalization(self): - from tools.web_tools import _normalize_tavily_search_results - raw = { - "results": [ - {"title": "Python Docs", "url": "https://docs.python.org", "content": "Official docs", "score": 0.9}, - {"title": "Tutorial", "url": "https://example.com", "content": "A tutorial", "score": 0.8}, - ] - } - result = _normalize_tavily_search_results(raw) - assert result["success"] is True - web = result["data"]["web"] - assert len(web) == 2 - assert web[0]["title"] == "Python Docs" - assert web[0]["url"] == "https://docs.python.org" - assert web[0]["description"] == "Official docs" - assert web[0]["position"] == 1 - assert web[1]["position"] == 2 - - - def test_missing_fields(self): - from tools.web_tools import _normalize_tavily_search_results - result = _normalize_tavily_search_results({"results": [{}]}) - web = result["data"]["web"] - assert web[0]["title"] == "" - assert web[0]["url"] == "" - assert web[0]["description"] == "" - - -# ─── _normalize_tavily_documents ────────────────────────────────────────────── - -class TestNormalizeTavilyDocuments: - """Test extract document normalization.""" - - def test_basic_document(self): - from tools.web_tools import _normalize_tavily_documents - raw = { - "results": [{ - "url": "https://example.com", - "title": "Example", - "raw_content": "Full page content here", - }] - } - docs = _normalize_tavily_documents(raw) - assert len(docs) == 1 - assert docs[0]["url"] == "https://example.com" - assert docs[0]["title"] == "Example" - assert docs[0]["content"] == "Full page content here" - assert docs[0]["raw_content"] == "Full page content here" - assert docs[0]["metadata"]["sourceURL"] == "https://example.com" - - - def test_fallback_url(self): - from tools.web_tools import _normalize_tavily_documents - raw = {"results": [{"content": "data"}]} - docs = _normalize_tavily_documents(raw, fallback_url="https://fallback.com") - assert docs[0]["url"] == "https://fallback.com" - - -# ─── availability / auto-detect ─────────────────────────────────────────────── - -class TestTavilyAvailability: - """Keyed Tavily stays in the paid band; keyless only when selected.""" - - def test_is_available_without_key(self): - from plugins.web.tavily.provider import TavilyWebSearchProvider - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("TAVILY_API_KEY", None) - assert TavilyWebSearchProvider().is_available() is False - - def test_is_backend_available_without_key(self): - from tools.web_tools import _is_backend_available - with patch("tools.web_tools._load_web_config", return_value={}), \ - patch.dict(os.environ, {}, clear=False): - os.environ.pop("TAVILY_API_KEY", None) - assert _is_backend_available("tavily") is False - - def test_is_backend_available_when_configured_without_key(self): - from tools.web_tools import _is_backend_available - with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \ - patch.dict(os.environ, {}, clear=False): - os.environ.pop("TAVILY_API_KEY", None) - assert _is_backend_available("tavily") is True - - def test_keyless_does_not_preempt_managed_firecrawl(self): - """No TAVILY_API_KEY + Nous gateway ready → firecrawl, not keyless tavily.""" - from tools.web_tools import _get_backend - with patch("tools.web_tools._load_web_config", return_value={}), \ - patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \ - patch("tools.web_tools._ddgs_package_importable", return_value=False): - os.environ.pop("TAVILY_API_KEY", None) - assert _get_backend() == "firecrawl" - - def test_keyless_does_not_preempt_ddgs(self): - from tools.web_tools import _get_backend - with patch("tools.web_tools._load_web_config", return_value={}), \ - patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \ - patch("tools.web_tools._ddgs_package_importable", return_value=True): - os.environ.pop("TAVILY_API_KEY", None) - assert _get_backend() == "ddgs" - - def test_no_keys_defaults_to_firecrawl(self): - """Keyless tier disabled: zero-credential resolve hits the legacy - firecrawl sentinel. (With the tier on — the default — it resolves - to the Exa/Parallel keyless split; see test_web_keyless_fallback.py.) - """ - from tools.web_tools import _get_backend - with patch("tools.web_tools._load_web_config", return_value={}), \ - patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \ - patch("tools.web_tools._ddgs_package_importable", return_value=False), \ - patch("tools.web_tools._list_registered_web_providers", return_value=[]), \ - patch("agent.web_search_registry._keyless_tier_enabled", return_value=False): - os.environ.pop("TAVILY_API_KEY", None) - assert _get_backend() == "firecrawl" - - def test_explicit_search_backend_tavily_without_key(self): - """web.search_backend=tavily sticks even with no TAVILY_API_KEY.""" - from tools.web_tools import _get_search_backend - with patch("tools.web_tools._load_web_config", - return_value={"backend": "firecrawl", "search_backend": "tavily"}), \ - patch("tools.web_tools._is_tool_gateway_ready", return_value=True): - os.environ.pop("TAVILY_API_KEY", None) - assert _get_search_backend() == "tavily" - - def test_check_web_api_key_when_tavily_configured_without_key(self): - from tools.web_tools import check_web_api_key - with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \ - patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \ - patch("tools.web_tools.check_firecrawl_api_key", return_value=False), \ - patch("tools.web_tools._ddgs_package_importable", return_value=False), \ - patch("agent.web_search_registry.get_active_search_provider", return_value=None), \ - patch("agent.web_search_registry.get_active_extract_provider", return_value=None): - os.environ.pop("TAVILY_API_KEY", None) - assert check_web_api_key() is True - - -# ─── web_search_tool (Tavily dispatch) ──────────────────────────────────────── - -class TestWebSearchTavily: - """Test web_search_tool dispatch to Tavily.""" - - _register_providers = staticmethod(register_all_web_providers) - - @pytest.fixture(autouse=True) - def _populate_web_registry(self): - self._register_providers() - yield - from agent.web_search_registry import _reset_for_tests - _reset_for_tests() - - def test_search_dispatches_to_tavily(self): - mock_response = _ok_response({ - "results": [{"title": "Result", "url": "https://r.com", "content": "desc", "score": 0.9}] - }) - - with patch("tools.web_tools._get_backend", return_value="tavily"), \ - patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \ - patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \ - patch("tools.interrupt.is_interrupted", return_value=False): - from tools.web_tools import web_search_tool - result = json.loads(web_search_tool("test query", limit=3)) - assert result["success"] is True - assert len(result["data"]["web"]) == 1 - assert result["data"]["web"][0]["title"] == "Result" - - def test_search_keyless_dispatch(self): - """Keyless Tavily routes through the ring; pinned tavily starts at - tavily and the ring searcher sends the keyless headers.""" - from plugins.web import keyless_mcp - - mock_response = _ok_response({ - "results": [{"title": "Result", "url": "https://r.com", "content": "desc"}] - }) - - with patch("tools.web_tools._get_backend", return_value="tavily"), \ - patch.object(keyless_mcp, "_vendor_pinned", lambda n: n == "tavily"), \ - patch("requests.post", return_value=mock_response) as mock_post, \ - patch("tools.interrupt.is_interrupted", return_value=False): - os.environ.pop("TAVILY_API_KEY", None) - from tools.web_tools import web_search_tool - result = json.loads(web_search_tool("test query")) - assert result["success"] is True - headers = mock_post.call_args.kwargs["headers"] - assert headers["X-Tavily-Access-Mode"] == "keyless" - assert headers["X-Client-Name"] == "hermes-agent" - - -# ─── web_extract_tool (Tavily dispatch) ─────────────────────────────────────── - -class TestWebExtractTavily: - """Test web_extract_tool dispatch to Tavily.""" - - _register_providers = staticmethod(register_all_web_providers) - - @pytest.fixture(autouse=True) - def _populate_web_registry(self): - self._register_providers() - yield - from agent.web_search_registry import _reset_for_tests - _reset_for_tests() - - def test_extract_dispatches_to_tavily(self): - mock_response = _ok_response({ - "results": [{"url": "https://example.com", "raw_content": "Extracted content", "title": "Page"}] - }) - - async def _allow_ssrf(_url: str) -> bool: - return True - - with patch("tools.web_tools._get_backend", return_value="tavily"), \ - patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \ - patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \ - patch("tools.web_tools.async_is_safe_url", _allow_ssrf): - from tools.web_tools import web_extract_tool - result = json.loads(asyncio.get_event_loop().run_until_complete( - web_extract_tool(["https://example.com"]) - )) - assert "results" in result - assert len(result["results"]) == 1 - assert result["results"][0]["url"] == "https://example.com" - assert "Extracted content" in result["results"][0]["content"] diff --git a/tools/url_safety.py b/tools/url_safety.py index 6442fe4bf5..e9b230ac68 100644 --- a/tools/url_safety.py +++ b/tools/url_safety.py @@ -21,7 +21,7 @@ Limitations: connects to the validated IP while preserving Host/SNI semantics. - Redirect-based bypass is mitigated by httpx event hooks that re-validate each redirect target in vision_tools, gateway platform adapters, and - media cache helpers. Web tools use third-party SDKs (Firecrawl/Tavily) + media cache helpers. Web tools use third-party SDKs (Firecrawl/Exa) where redirect handling is on their servers. """ diff --git a/tools/web_tools.py b/tools/web_tools.py index 0d9aca14f3..93b8e5e9d0 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -15,7 +15,6 @@ Backend compatibility: - Exa: https://exa.ai (search, extract) - Firecrawl: https://docs.firecrawl.dev/introduction (search, extract; direct or derived firecrawl-gateway. for Nous Subscribers) - Parallel: https://docs.parallel.ai (search, extract) -- Tavily: https://tavily.com (search, extract) LLM Processing: - Uses OpenRouter API with Gemini 3 Flash Preview for intelligent content extraction @@ -58,13 +57,6 @@ from plugins.web.firecrawl.provider import ( _is_tool_gateway_ready, check_firecrawl_api_key, ) -# Tavily helpers re-exported for backward-compat with existing unit tests -# (tests/tools/test_web_tools_tavily.py imports these names directly). -from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names - _normalize_tavily_documents, - _normalize_tavily_search_results, - _tavily_request, -) # Parallel + Exa clients re-exported for backward-compat with existing # unit tests (tests/tools/test_web_tools_config.py imports _get_parallel_client # / _get_async_parallel_client / _get_exa_client directly). @@ -169,7 +161,7 @@ def _load_web_config() -> dict: # WebSearchProvider. Keep the two sets aligned by hand: if xai ever ships as # a registered provider, drop it here so the registry path takes over. _LEGACY_WEB_BACKENDS = frozenset( - {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"} + {"parallel", "firecrawl", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"} ) @@ -252,14 +244,13 @@ def _get_backend() -> str: return "firecrawl" # Never-configured install — pick the highest-priority available - # backend. Explicit user credentials (TAVILY_API_KEY etc.) + # backend. Explicit user credentials (EXA_API_KEY etc.) # beat the managed-tool-gateway probe so a deliberate setup is not # pre-empted by a Nous OAuth token whose subscription tier may not # actually grant web-search access (the gateway then fails at runtime # with "no subscription" and the tool returns an error to the agent # without falling back). Free-tier backends trail the paid ones. backend_candidates = ( - ("tavily", _has_env("TAVILY_API_KEY")), ("exa", _has_env("EXA_API_KEY")), ("parallel", _has_env("PARALLEL_API_KEY")), ("keenable", _has_env("KEENABLE_API_KEY")), @@ -359,13 +350,6 @@ def _get_capability_backend(capability: str) -> str: return _get_backend() -def _tavily_explicitly_configured() -> bool: - cfg = _load_web_config() - return any( - (cfg.get(key) or "").lower().strip() == "tavily" - for key in ("backend", "search_backend", "extract_backend") - ) - def _is_backend_available(backend: str) -> bool: """Return True when the selected backend is currently usable. @@ -392,8 +376,6 @@ def _is_backend_available(backend: str) -> bool: return _has_env("KEENABLE_API_KEY") if backend == "firecrawl": return check_firecrawl_api_key() - if backend == "tavily": - return _has_env("TAVILY_API_KEY") or _tavily_explicitly_configured() if backend == "searxng": return _has_env("SEARXNG_URL") if backend == "brave-free": @@ -428,6 +410,7 @@ def _ddgs_package_importable() -> bool: return False + # ─── One-shot keyless rescue (keyed/configured backend failed) ─────────────── def _keyless_rescue_enabled() -> bool: @@ -469,7 +452,6 @@ def _rescue_eligible(provider) -> bool: key_var = { "exa": "EXA_API_KEY", "parallel": "PARALLEL_API_KEY", - "tavily": "TAVILY_API_KEY", "firecrawl": "FIRECRAWL_API_KEY", "keenable": "KEENABLE_API_KEY", }.get(name, "") @@ -603,7 +585,6 @@ def _web_requires_env() -> list[str]: return [ "EXA_API_KEY", "PARALLEL_API_KEY", - "TAVILY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", @@ -614,11 +595,10 @@ def _web_requires_env() -> list[str]: ] -# ─── Parallel / Tavily / Firecrawl helpers — moved into plugins ────────────── +# ─── Parallel / Firecrawl helpers — moved into plugins ─────────────────────── # After PR #25182, the per-vendor client construction, request helpers, and # response normalizers all live in plugins.web..provider: # - parallel: plugins/web/parallel/provider.py -# - tavily: plugins/web/tavily/provider.py # - firecrawl: plugins/web/firecrawl/provider.py # The names from the firecrawl plugin (Firecrawl proxy, _get_firecrawl_client, # _to_plain_object, _normalize_result_list, _extract_web_search_results, @@ -810,7 +790,7 @@ def _ensure_web_plugins_loaded() -> None: """Idempotently trigger plugin discovery so the web registry is populated. Every bundled web provider (brave-free, ddgs, searxng, exa, parallel, - tavily, firecrawl) registers itself via ``plugins/web//__init__.py`` + firecrawl, keenable) registers itself via ``plugins/web//__init__.py`` during plugin discovery. Tool dispatch can be reached from contexts that haven't already triggered discovery — subprocess agent runs, delegate children, standalone scripts, certain test paths — and without it the @@ -892,7 +872,7 @@ def web_search_tool(query: str, limit: int = 5) -> str: return tool_error("Interrupted", success=False) # Dispatch through the web search registry. All 7 providers - # (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl) + # (brave-free, ddgs, searxng, exa, parallel, firecrawl, keenable) # now live as plugins; the dispatcher is just a registry lookup + # delegation. Sync only — every provider's search() is sync. _ensure_web_plugins_loaded() @@ -1054,7 +1034,7 @@ async def web_extract_tool( Extract content from specific web pages using available extraction API backend. Returns clean page content (markdown/text) with NO LLM summarization. The - extract backends (Firecrawl, Tavily, Exa, Parallel) already return clean, + extract backends (Firecrawl, Exa, Parallel, Keenable) already return clean, boilerplate-stripped content, so we return it directly and fast. Pages over ``char_limit`` are head+tail truncated with an explicit footer; the full text is stored under cache/web and the footer tells the model how to @@ -1163,9 +1143,9 @@ async def web_extract_tool( backend = _get_extract_backend() # All seven providers (brave-free, ddgs, searxng, exa, parallel, - # tavily, firecrawl) now live as plugins. The dispatcher is a + # firecrawl, keenable) now live as plugins. The dispatcher is a # registry lookup + delegation. Some providers' extract() is - # async (parallel, firecrawl), others sync (exa, tavily) — we + # async (parallel, firecrawl), others sync (exa, keenable) — we # detect coroutine functions and await; sync functions run # inline (the policy gate, SSRF re-check, etc. live inside the # provider itself for the firecrawl per-URL loop). @@ -1192,7 +1172,7 @@ async def web_extract_tool( f"{provider.display_name} is a search-only " "backend and cannot extract URL content. " "Set web.extract_backend to firecrawl, " - "tavily, exa, or parallel." + "keenable, exa, or parallel." ), }, ensure_ascii=False, @@ -1255,7 +1235,7 @@ async def web_extract_tool( "error": ( "No web extract provider configured. " "Set web.extract_backend to firecrawl, " - "tavily, exa, or parallel." + "keenable, exa, or parallel." ), }, ensure_ascii=False, @@ -1304,7 +1284,7 @@ async def web_extract_tool( ) # Async-or-sync dispatch: parallel + firecrawl have async - # extract(); exa + tavily are sync. + # extract(); exa + keenable are sync. import inspect _extract_rescued = False try: @@ -1588,11 +1568,6 @@ if __name__ == "__main__": print(" Using Exa API (https://exa.ai)") elif backend == "parallel": print(" Using Parallel API (https://parallel.ai)") - elif backend == "tavily": - if _has_env("TAVILY_API_KEY"): - print(" Using Tavily API (https://tavily.com)") - else: - print(" Using Tavily keyless (https://docs.tavily.com/documentation/keyless)") elif backend == "searxng": print(f" Using SearXNG (search only): {_env_value('SEARXNG_URL')}") elif backend == "brave-free": @@ -1610,7 +1585,7 @@ if __name__ == "__main__": else: print("❌ No web search backend configured") print( - "Set EXA_API_KEY, PARALLEL_API_KEY, TAVILY_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL" + "Set EXA_API_KEY, PARALLEL_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL" f"{_firecrawl_backend_help_suffix()}" ) diff --git a/website/docs/developer-guide/web-search-provider-plugin.md b/website/docs/developer-guide/web-search-provider-plugin.md index 4f880866c6..98bd98f174 100644 --- a/website/docs/developer-guide/web-search-provider-plugin.md +++ b/website/docs/developer-guide/web-search-provider-plugin.md @@ -6,7 +6,7 @@ description: "How to build a web-search/extract/crawl backend plugin for Hermes # Building a Web Search Provider Plugin -Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Tavily, Exa, Parallel, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web//`. You can add a new one, or override a bundled one, by dropping a directory next to them. +Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web//`. You can add a new one, or override a bundled one, by dropping a directory next to them. :::tip Web search is one of several **backend plugins** Hermes supports. The others (with their own ABCs) are [Image Generation Provider Plugins](/developer-guide/image-gen-provider-plugin), [Video Generation Provider Plugins](/developer-guide/video-gen-provider-plugin), [Memory Provider Plugins](/developer-guide/memory-provider-plugin), [Context Engine Plugins](/developer-guide/context-engine-plugin), and [Model Provider Plugins](/developer-guide/model-provider-plugin). General tool/hook/CLI plugins live in [Build a Hermes Plugin](/developer-guide/plugins). @@ -157,7 +157,7 @@ Full contract in `agent/web_search_provider.py`. Methods you may override: | `search(query, limit)` | conditional | raises | Required when `supports_search()` returns `True` | | `extract(urls, **kwargs)` | conditional | raises | Required when `supports_extract()` returns `True` | -Providers can advertise multiple capabilities from a single class — Firecrawl, Tavily, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow. +Providers can advertise multiple capabilities from a single class — Firecrawl, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow. ## Response shape diff --git a/website/docs/integrations/index.md b/website/docs/integrations/index.md index 8225c21d37..51555976ee 100644 --- a/website/docs/integrations/index.md +++ b/website/docs/integrations/index.md @@ -34,7 +34,6 @@ The `web_search` and `web_extract` tools support eight backend providers, config | **SearXNG** | `SEARXNG_URL` | ✔ | — | — | | **Brave** (free tier) | `BRAVE_SEARCH_API_KEY` | ✔ | — | — | | **DuckDuckGo** (ddgs) | _(none)_ | ✔ | — | — | -| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | — | | **Exa** | `EXA_API_KEY` | ✔ | ✔ | — | | **Parallel** | `PARALLEL_API_KEY` | ✔ | ✔ | — | | **xAI** | `XAI_API_KEY` | ✔ | — | — | @@ -43,10 +42,10 @@ Quick setup example: ```yaml web: - backend: firecrawl # firecrawl | searxng | brave-free | ddgs | tavily | exa | parallel | xai + backend: firecrawl # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai ``` -If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`. Selecting Tavily in `hermes tools` works without a key. +If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`. ## Browser Automation diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index dd8ef3bffb..aaf226d897 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -151,9 +151,7 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe | `PARALLEL_API_KEY` | AI-native web search ([parallel.ai](https://parallel.ai/)) | | `FIRECRAWL_API_KEY` | Web scraping and cloud browser ([firecrawl.dev](https://firecrawl.dev/)) | | `FIRECRAWL_API_URL` | Custom Firecrawl API endpoint for self-hosted instances (optional) | -| `TAVILY_API_KEY` | Optional Tavily API key for higher search/extract limits. After selecting Tavily as the web backend, keyless access works without it ([app.tavily.com](https://app.tavily.com/home), [keyless docs](https://docs.tavily.com/documentation/keyless)) | | `SEARXNG_URL` | SearXNG instance URL for free self-hosted web search — no API key required ([searxng.github.io](https://searxng.github.io/searxng/)) | -| `TAVILY_BASE_URL` | Override the Tavily API endpoint. Useful for corporate proxies and self-hosted Tavily-compatible search backends. Same pattern as `GROQ_BASE_URL`. | | `EXA_API_KEY` | Exa API key for AI-native web search and contents ([exa.ai](https://exa.ai/)) | | `BRAVE_SEARCH_API_KEY` | Brave Search API subscription token for web search (free tier available) ([brave.com/search/api](https://brave.com/search/api/)) | | `BROWSERBASE_API_KEY` | Browser automation ([browserbase.com](https://browserbase.com/)) | diff --git a/website/docs/reference/tools-reference.md b/website/docs/reference/tools-reference.md index 706f718a23..a0775b8f61 100644 --- a/website/docs/reference/tools-reference.md +++ b/website/docs/reference/tools-reference.md @@ -315,8 +315,8 @@ The single `video_generate` tool covers both modalities — pass `image_url` to | Tool | Description | Requires environment | |------|-------------|----------------------| -| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY | -| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY | +| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY | +| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY | ## `x_search` toolset diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 9c3cdb3bd8..89f6e78e8f 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -2340,7 +2340,7 @@ The `web_search` and `web_extract` tools support five backend providers. Configu ```yaml web: - backend: firecrawl # firecrawl | searxng | parallel | tavily | exa + backend: firecrawl # firecrawl | searxng | parallel | keenable | exa # Or use per-capability keys to mix providers (e.g. free search + paid extract): search_backend: "searxng" @@ -2348,7 +2348,7 @@ web: # Keyless free-tier fallback (default: true). With no backend configured # and no API keys present, web tools rotate across the Exa/Parallel/ - # Tavily/Firecrawl/Keenable free tiers. Set false to disable. + # Firecrawl/Keenable free tiers. Set false to disable. keyless_fallback: true # One-shot keyless rescue (default: true). When the chosen/keyed backend @@ -2369,10 +2369,9 @@ web: | **Firecrawl** (default) | `FIRECRAWL_API_KEY` | ✔ | ✔ | | **SearXNG** | `SEARXNG_URL` | ✔ | — | | **Parallel** | `PARALLEL_API_KEY` (optional — keyless free tier) | ✔ | ✔ | -| **Tavily** | `TAVILY_API_KEY` (optional — keyless when selected) | ✔ | ✔ | | **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ | -**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Tavily / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key. +**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Firecrawl or Keenable in `hermes tools` also works without a key. **SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](/user-guide/features/web-search) for Docker setup instructions. diff --git a/website/docs/user-guide/features/web-dashboard.md b/website/docs/user-guide/features/web-dashboard.md index be4ed441d7..17eb492b49 100644 --- a/website/docs/user-guide/features/web-dashboard.md +++ b/website/docs/user-guide/features/web-dashboard.md @@ -235,7 +235,7 @@ Config changes take effect on the next agent session or gateway restart. The web Manage the `.env` file where API keys and credentials are stored. Keys are grouped by category: - **LLM Providers** — OpenRouter, Anthropic, OpenAI, DeepSeek, etc. -- **Tool API Keys** — Browserbase, Firecrawl, Tavily, ElevenLabs, etc. +- **Tool API Keys** — Browserbase, Firecrawl, Keenable, ElevenLabs, etc. - **Messaging Platforms** — Telegram, Discord, Slack bot tokens, etc. - **Agent Settings** — non-secret env vars like `API_SERVER_ENABLED` diff --git a/website/docs/user-guide/features/web-search.md b/website/docs/user-guide/features/web-search.md index ddacf7f4a8..44911dc5ab 100644 --- a/website/docs/user-guide/features/web-search.md +++ b/website/docs/user-guide/features/web-search.md @@ -22,18 +22,17 @@ Both are configured through a single backend selection. Providers are chosen via | **SearXNG** | `SEARXNG_URL` | ✔ | — | ✔ Free (self-hosted) | | **Brave Search (free tier)** | `BRAVE_SEARCH_API_KEY` | ✔ | — | 2 000 queries/mo | | **DDGS (DuckDuckGo)** | — (no key) | ✔ | — | ✔ Free | -| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with a free key | | **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with key | | **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key | | **Keenable** | `KEENABLE_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key | | **xAI (Grok)** | `XAI_API_KEY` or `hermes auth add xai-oauth` | ✔ | — | Paid (SuperGrok or per-token) | -Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below). +Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below). **Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below. :::info Works out of the box — keyless free-tier rotation -A fresh install with **no web credentials at all** gets working `web_search` and `web_extract` out of the box: requests rotate round-robin across five vendors' public free tiers — **Exa, Parallel, Tavily, Firecrawl, and Keenable** — spreading load evenly, and a rate-limited request automatically retries on the next vendor in the ring (multi-hop, until one serves or all are throttled). No signup, no key. This tier is strictly last-resort — any configured backend or present API key always wins — and requests carry no user identifiers (only a random per-process session id, rotated on restart). For guaranteed, unthrottled service, set up a keyed provider. Disable the keyless tier entirely with `web.keyless_fallback: false`. +A fresh install with **no web credentials at all** gets working `web_search` and `web_extract` out of the box: requests rotate round-robin across the ring vendors' public free tiers — **Exa, Parallel, Firecrawl, and Keenable** — spreading load evenly, and a rate-limited request automatically retries on the next vendor in the ring (multi-hop, until one serves or all are throttled). No signup, no key. This tier is strictly last-resort — any configured backend or present API key always wins — and requests carry no user identifiers (only a random per-process session id, rotated on restart). For guaranteed, unthrottled service, set up a keyed provider. Disable the keyless tier entirely with `web.keyless_fallback: false`. ::: **Choosing free vs paid explicitly:** in `hermes tools`, Exa, Parallel, and Keenable each appear as two rows — **Free (keyless)** and **Paid (API key)**. Picking Free pins that vendor's anonymous endpoint (even if you later add a key); picking Paid pins the keyed path (a missing key then errors instead of silently downgrading to the free tier). The selection is stored as `web.provider_tier.: free|paid`; leave it unset for auto (key present → paid, otherwise the keyless ring). @@ -267,29 +266,13 @@ SearXNG handles search; you need a separate provider for `web_extract`. Use the # ~/.hermes/config.yaml web: search_backend: "searxng" - extract_backend: "firecrawl" # or tavily, exa, parallel + extract_backend: "firecrawl" # or keenable, exa, parallel ``` With this config, Hermes uses SearXNG for all search queries and Firecrawl for URL extraction — combining free search with high-quality extraction. --- -### Tavily - -AI-optimised search and extract. Select Tavily in `hermes tools` (or set `web.backend: tavily`) to use it **keyless** with no account (rate-limited). Set an API key when you want higher limits. - -```bash -# optional — skip this for keyless access after selecting Tavily -# ~/.hermes/.env -TAVILY_API_KEY=tvly-your-key-here -``` - -Get a key at [app.tavily.com](https://app.tavily.com/home). See [Tavily keyless](https://docs.tavily.com/documentation/keyless). - -Empty installs keep Firecrawl as the named default. Keyless Tavily is not auto-selected. - ---- - ### Exa Neural search with semantic understanding. Good for research and finding conceptually related content. @@ -355,10 +338,10 @@ web: timeout: 90 # seconds (default) ``` -**Search-only** — pair with Firecrawl / Tavily / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry. +**Search-only** — pair with Firecrawl / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry. :::caution Trust model -Unlike index-backed providers (Brave, Tavily, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input. +Unlike index-backed providers (Brave, Keenable, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input. ::: --- @@ -372,7 +355,7 @@ Set one provider for all web capabilities: ```yaml # ~/.hermes/config.yaml web: - backend: "searxng" # firecrawl | searxng | brave-free | ddgs | tavily | exa | parallel | xai + backend: "searxng" # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai ``` ### Per-capability configuration @@ -399,16 +382,15 @@ If no backend has **ever** been selected (no `web.backend` / per-capability key | Credential present | Auto-selected backend | |--------------------|-----------------------| -| `TAVILY_API_KEY` | tavily | | `EXA_API_KEY` | exa | | `PARALLEL_API_KEY` | parallel | | `FIRECRAWL_API_KEY` or `FIRECRAWL_API_URL` (or the Nous Tool Gateway is ready) | firecrawl | | `SEARXNG_URL` | searxng | | `BRAVE_SEARCH_API_KEY` | brave-free | | `ddgs` package importable | ddgs | -| *(nothing set at all)* | keyless ring: exa / parallel / tavily / firecrawl / keenable (round-robin) | +| *(nothing set at all)* | keyless ring: exa / parallel / firecrawl / keenable (round-robin) | -**Keyless free-tier ring:** when *no* credential above is present, requests rotate across five vendors' public free tiers (Exa, Parallel, Tavily, Firecrawl, Keenable) so web tools work on a fresh install with zero setup — and a rate-limited request fails over to the next vendor in the ring automatically. Pin one vendor in `hermes tools` to stop the rotation (the ring is then only used as failover succession on throttles). All free tiers are vendor-rate-limited under burst load; sustained normal usage goes through fine. Set `web.keyless_fallback: false` to turn the tier off — with it off and no credentials, web tools are unavailable until a provider is configured. +**Keyless free-tier ring:** when *no* credential above is present, requests rotate across the ring vendors' public free tiers (Exa, Parallel, Firecrawl, Keenable) so web tools work on a fresh install with zero setup — and a rate-limited request fails over to the next vendor in the ring automatically. Pin one vendor in `hermes tools` to stop the rotation (the ring is then only used as failover succession on throttles). All free tiers are vendor-rate-limited under burst load; sustained normal usage goes through fine. Set `web.keyless_fallback: false` to turn the tier off — with it off and no credentials, web tools are unavailable until a provider is configured. **One-shot keyless rescue for keyed backends:** when your chosen/keyed backend fails a call (bad key, outage, upstream 5xx), that single call automatically retries on the keyless free-tier ring instead of erroring — the result notes which vendor served it and why (`rescued_from` / `backend_error`). The failover is never sticky: the very next `web_search`/`web_extract` call attempts your chosen backend again. Disable with `web.keyless_rescue: false` (also off whenever `keyless_fallback` is off). @@ -456,7 +438,7 @@ SearXNG cannot extract URL content. Set `web.extract_backend` to a provider that ```yaml web: search_backend: "searxng" - extract_backend: "firecrawl" # or tavily / exa / parallel + extract_backend: "firecrawl" # or keenable / exa / parallel ``` ### SearXNG returns 0 results