diff --git a/agent/transports/codex.py b/agent/transports/codex.py index ff8979edce..99bd2f65e6 100644 --- a/agent/transports/codex.py +++ b/agent/transports/codex.py @@ -59,7 +59,7 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]: # A function literally named ``web_search`` collides with Grok's native # server-side tool (incomplete hang or HTTP 400 duplicate names); this alias # avoids that while still dispatching through Hermes's configured provider -# (Firecrawl / Exa / …). Mapped back to ``web_search`` in normalize_response. +# (Firecrawl / Tavily / …). Mapped back to ``web_search`` in normalize_response. _XAI_CLIENT_WEB_SEARCH_ALIAS = "hermes_web_search" # OpenCode's /v1/responses endpoints (Zen and Go, including custom providers @@ -661,7 +661,7 @@ class ResponsesApiTransport(ProviderTransport): # fails): drop the client ``web_search`` function and declare # xAI's built-in instead. 1:1 swap only when client ``web_search`` # was already present — never an additive grant. - # 2. **Client** (Firecrawl / Keenable / Exa / … configured or resolved): + # 2. **Client** (Firecrawl / Tavily / Exa / … configured or resolved): # keep Hermes dispatch so ``web.backend`` / ``web.search_backend`` # is honored, but rename the wire tool to # ``hermes_web_search`` so Grok cannot hijack the name. The alias diff --git a/agent/web_search_provider.py b/agent/web_search_provider.py index f1abcc9b63..66cab340b6 100644 --- a/agent/web_search_provider.py +++ b/agent/web_search_provider.py @@ -13,8 +13,8 @@ Providers live in ``/plugins/web//`` (built-in, auto-loaded as ``plugins.enabled``). This ABC is the SINGLE plugin-facing surface for web providers — every -provider in the tree (brave-free, ddgs, searxng, exa, parallel, keenable, -firecrawl) implements it. The legacy in-tree ``tools.web_providers.base`` +provider in the tree (brave-free, ddgs, searxng, exa, parallel, tavily, +keenable, firecrawl) implements it. The legacy in-tree ``tools.web_providers.base`` ABCs were deleted in PR #25182 along with the per-vendor inline helpers in ``tools/web_tools.py``; the response-shape contract documented below is preserved bit-for-bit so the tool wrapper does not have to translate. @@ -93,7 +93,7 @@ class WebSearchProvider(abc.ABC): :meth:`search` / :meth:`extract`. The :meth:`supports_search` / :meth:`supports_extract` capability flags let the registry route each tool call to the right provider, and let multi-capability providers - (Firecrawl, Keenable, Exa, …) advertise multiple capabilities from a + (Firecrawl, Tavily, Exa, …) advertise multiple capabilities from a single class. """ diff --git a/agent/web_search_registry.py b/agent/web_search_registry.py index 5260272133..7f60ea838e 100644 --- a/agent/web_search_registry.py +++ b/agent/web_search_registry.py @@ -16,7 +16,7 @@ The active provider is chosen by configuration with this precedence: 2. ``web.backend`` (shared fallback). 3. If exactly one capability-eligible provider is registered AND available, use it. -4. Legacy preference order — ``firecrawl`` → ``parallel`` → +4. Legacy preference order — ``firecrawl`` → ``parallel`` → ``tavily`` → ``exa`` → ``searxng`` → ``brave-free`` → ``ddgs`` — filtered by availability. Matches the historic ``tools.web_tools._get_backend()`` candidate order so installs that never set a config key keep landing @@ -159,6 +159,7 @@ def _read_config_key(*path: str) -> Optional[str]: _LEGACY_PREFERENCE = ( "firecrawl", "parallel", + "tavily", "exa", "searxng", "brave-free", @@ -167,7 +168,7 @@ _LEGACY_PREFERENCE = ( # Keyless free-tier walk — strictly LAST-resort, tried only after the # availability-filtered legacy walk finds nothing (i.e. the user has zero -# web credentials and no importable ddgs). All five vendors expose public +# web credentials and no importable ddgs). Ring vendors expose public # anonymous free tiers (see plugins/web/keyless_mcp.py). Unpinned keyless # traffic round-robins across the ring per request (the ring cursor lives # in keyless_mcp; an explicit `hermes tools` pick bypasses this walk @@ -220,7 +221,7 @@ def _resolve(configured: Optional[str], *, capability: str) -> Optional[WebSearc supports *capability* AND ``is_available()`` reports True, return it. 3. **Legacy preference walk, filtered by availability.** Walk the - :data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → + :data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → tavily → exa → searxng → brave-free → ddgs) looking for a provider whose ``supports_()`` is True AND whose ``is_available()`` is True. Matches the historic ``tools.web_tools._get_backend()`` diff --git a/evals/browser_use/single_run.py b/evals/browser_use/single_run.py index 346c425de7..6015656fb1 100644 --- a/evals/browser_use/single_run.py +++ b/evals/browser_use/single_run.py @@ -63,7 +63,7 @@ with open(os.path.join(hh, "config.yaml"), "w", encoding="utf-8") as f: os.environ["HERMES_HOME"] = hh # Strip web-fetch shortcuts: every arm must drive the browser. os.environ.pop("BROWSER_USE_API_KEY", None) -for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "SERPER_API_KEY"): +for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "TAVILY_API_KEY", "SERPER_API_KEY"): os.environ.pop(k, None) os.environ["BU_CDP_URL"] = cdp os.environ["PATH"] = ( diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 33e0d58274..af532c1199 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1111,6 +1111,7 @@ ENV_VARS_BY_VERSION: Dict[int, List[str]] = { 4: ["VOICE_TOOLS_OPENAI_KEY", "ELEVENLABS_API_KEY"], 5: ["WHATSAPP_ENABLED", "WHATSAPP_MODE", "WHATSAPP_ALLOWED_USERS", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN", "SLACK_ALLOWED_USERS"], + 10: ["TAVILY_API_KEY"], 11: ["TERMINAL_MODAL_MODE"], } @@ -1456,7 +1457,7 @@ def _is_env_config_key(key: str) -> bool: 'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY', 'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL', 'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME', - 'TOOL_GATEWAY_USER_TOKEN', + 'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY', 'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN', 'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY', @@ -5090,6 +5091,7 @@ def show_config(): ("EXA_API_KEY", "Exa"), ("PARALLEL_API_KEY", "Parallel"), ("FIRECRAWL_API_KEY", "Firecrawl"), + ("TAVILY_API_KEY", "Tavily"), ("BROWSERBASE_API_KEY", "Browserbase"), ("BROWSER_USE_API_KEY", "Browser Use"), ("FAL_KEY", "FAL"), diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index c6a6a977b5..b6fb967cf3 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -555,7 +555,7 @@ DEFAULT_CONFIG = { "extract_backend": "", # per-capability override for web_extract (e.g. "native") "extract_char_limit": 15000, # per-page char budget for web_extract; larger pages truncate + store full text in cache/web # Keyless free-tier ring: with NO web backend configured or keyed, - # web_search/web_extract rotate round-robin across five vendors' + # web_search/web_extract rotate round-robin across four vendors' # public free tiers (exa, parallel, firecrawl, keenable), # failing over to the next ring vendor on rate limits. Never # pre-empts a configured or keyed backend. Set false to disable. @@ -565,10 +565,11 @@ DEFAULT_CONFIG = { # free-tier ring — the next call attempts the chosen backend again # (no sticky failover). Off when keyless_fallback is false. "keyless_rescue": True, - # Per-provider tier selection for ring vendors with both a keyless + # Per-provider tier selection for vendors with both a keyless # free endpoint and a keyed paid path (exa, parallel, - # firecrawl, keenable). Set by the `hermes tools` picker's - # "Free (keyless)" / "Paid (API key)" rows. + # firecrawl, keenable on the ring; tavily is opt-in keyless via + # `hermes tools`, not a ring member). Set by the `hermes tools` + # picker's "Free (keyless)" / "Paid (API key)" rows. # free — always use the anonymous free endpoint (even with a key) # paid — always use the keyed path (missing key = error; vendor # is also excluded from the keyless ring) @@ -4513,6 +4514,14 @@ OPTIONAL_ENV_VARS = { "category": "tool", "advanced": True, }, + "TAVILY_API_KEY": { + "description": "Tavily API key for AI-native web search and extract (optional — keyless works when Tavily is selected)", + "prompt": "Tavily API key", + "url": "https://app.tavily.com/home", + "tools": ["web_search", "web_extract"], + "password": True, + "category": "tool", + }, "KEENABLE_API_KEY": { "description": "Keenable API key for fast independent-index web search and page fetch (optional — keyless free tier works without it)", "prompt": "Keenable API key", diff --git a/hermes_cli/dump.py b/hermes_cli/dump.py index c7399f39f8..fa27044f43 100644 --- a/hermes_cli/dump.py +++ b/hermes_cli/dump.py @@ -388,6 +388,7 @@ def run_dump(args): ("COMMANDCODE_API_KEY", "commandcode"), ("KILOCODE_API_KEY", "kilocode"), ("FIRECRAWL_API_KEY", "firecrawl"), + ("TAVILY_API_KEY", "tavily"), ("KEENABLE_API_KEY", "keenable"), ("BROWSERBASE_API_KEY", "browserbase"), ("FAL_KEY", "fal"), diff --git a/hermes_cli/nous_subscription.py b/hermes_cli/nous_subscription.py index f9ca7ef35f..a930989e60 100644 --- a/hermes_cli/nous_subscription.py +++ b/hermes_cli/nous_subscription.py @@ -505,6 +505,10 @@ def get_nous_subscription_features( direct_exa = bool(get_env_value("EXA_API_KEY")) direct_firecrawl = bool(get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL")) direct_parallel = bool(get_env_value("PARALLEL_API_KEY")) + direct_tavily = bool(get_env_value("TAVILY_API_KEY")) + # Keyless Tavily is opt-in: selecting it in `hermes tools` / setup writes + # web.backend (or a per-capability override) without requiring a key. + tavily_selected = "tavily" in {web_backend, web_search_backend, web_extract_backend} direct_searxng = bool(get_env_value("SEARXNG_URL")) direct_fal = fal_key_is_configured() direct_fal_video = direct_fal # same FAL_KEY; separate var so use_gateway is independent @@ -536,6 +540,8 @@ def get_nous_subscription_features( direct_firecrawl = False direct_exa = False direct_parallel = False + direct_tavily = False + tavily_selected = False if image_use_gateway: direct_fal = False if video_use_gateway: @@ -624,6 +630,7 @@ def get_nous_subscription_features( direct_camofox = False + tavily_ready = direct_tavily or tavily_selected web_managed = web_backend == "firecrawl" and managed_web_available and not direct_firecrawl web_active = bool( web_tool_enabled @@ -632,6 +639,7 @@ def get_nous_subscription_features( or (web_backend == "exa" and direct_exa) or (web_backend == "firecrawl" and direct_firecrawl) or (web_backend == "parallel" and direct_parallel) + or (web_backend == "tavily" and tavily_ready) or (web_backend == "searxng" and direct_searxng) # Per-capability overrides: search_backend or extract_backend may be set # without web.backend (using the new split config from #20061) @@ -639,6 +647,8 @@ def get_nous_subscription_features( or (web_search_backend == "exa" and direct_exa) or (web_search_backend == "firecrawl" and direct_firecrawl) or (web_search_backend == "parallel" and direct_parallel) + or (web_search_backend == "tavily" and tavily_ready) + or (web_extract_backend == "tavily" and tavily_ready) ) ) web_available = bool( @@ -646,6 +656,7 @@ def get_nous_subscription_features( or direct_exa or direct_firecrawl or direct_parallel + or tavily_ready or direct_searxng ) @@ -889,6 +900,7 @@ def apply_nous_managed_defaults( if "web" in selected_toolsets and not features.web.explicit_configured and not ( get_env_value("PARALLEL_API_KEY") + or get_env_value("TAVILY_API_KEY") or get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL") ): @@ -986,6 +998,7 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]: get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL") or get_env_value("PARALLEL_API_KEY") + or get_env_value("TAVILY_API_KEY") or get_env_value("EXA_API_KEY") # Env-configured keyless local backend: a reachable self-hosted # SearXNG is a working web setup even with no stored selection diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 390d71669a..4445eb8812 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -513,7 +513,7 @@ def _print_setup_summary(config: dict, hermes_home): tool_status.append(("Vision (image analysis)", False, "run 'hermes setup' to configure")) - # Web tools (Exa, Parallel, Firecrawl, or Keenable) + # Web tools (Exa, Parallel, Firecrawl, Tavily, or Keenable) if subscription_features.web.managed_by_nous: tool_status.append(("Web Search & Extract (Nous subscription)", True, None)) elif subscription_features.web.available: @@ -522,7 +522,7 @@ def _print_setup_summary(config: dict, hermes_home): label = f"Web Search & Extract ({subscription_features.web.current_provider})" tool_status.append((label, True, None)) else: - tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, KEENABLE_API_KEY, or SEARXNG_URL")) + tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, TAVILY_API_KEY, KEENABLE_API_KEY, or SEARXNG_URL")) # Browser tools (local Chromium, Camofox, Browserbase, Browser Use, or Firecrawl) browser_provider = subscription_features.browser.current_provider diff --git a/hermes_cli/status.py b/hermes_cli/status.py index 569c759835..f5c435d912 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -184,6 +184,7 @@ def show_status(args): "MiniMax-CN": "MINIMAX_CN_API_KEY", "DeepInfra": "DEEPINFRA_API_KEY", "Firecrawl": "FIRECRAWL_API_KEY", + "Tavily": "TAVILY_API_KEY", "Keenable": "KEENABLE_API_KEY", "Browser Use": "BROWSER_USE_API_KEY", # Optional — local browser works without this "Browserbase": "BROWSERBASE_API_KEY", # Optional — direct credentials only diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 10d433db7c..77bae34105 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -3331,8 +3331,8 @@ def _plugin_video_gen_providers() -> list[dict]: # Mirror of _plugin_image_gen_providers for web search backends. Surfaces # every plugin-registered web provider so it appears in the -# "Web Search & Extract" picker. All seven providers (brave-free, ddgs, -# searxng, exa, parallel, firecrawl, keenable) live as plugins after +# "Web Search & Extract" picker. All bundled providers (brave-free, ddgs, +# searxng, exa, parallel, tavily, firecrawl, keenable) live as plugins after # PR #25182 — this helper is the sole source of truth for the category's # provider rows. The hardcoded entries that used to drive the category # were deleted in the same PR; only the two non-provider UX rows @@ -3348,8 +3348,8 @@ def _plugin_web_search_providers() -> list[dict]: marker) so the picker behaves identically whether a provider is hardcoded or plugin-registered. - After PR #25182, all seven web providers (brave-free, ddgs, searxng, - exa, parallel, firecrawl, keenable) are plugins; this helper is the sole + After PR #25182, all bundled web providers (brave-free, ddgs, searxng, + exa, parallel, tavily, firecrawl, keenable) are plugins; this helper is the sole source of provider rows for the Web Search & Extract category. """ try: diff --git a/plugins/web/brave_free/provider.py b/plugins/web/brave_free/provider.py index 769a850587..0da8d11c99 100644 --- a/plugins/web/brave_free/provider.py +++ b/plugins/web/brave_free/provider.py @@ -34,7 +34,7 @@ class BraveFreeWebSearchProvider(WebSearchProvider): """Search-only Brave provider using the free-tier Data-for-Search API. Free tier is 2,000 queries/month (1 qps). No content-extraction capability — - users pair this with Firecrawl/Keenable/Exa for ``web_extract``. + users pair this with Firecrawl/Tavily/Exa for ``web_extract``. """ @property diff --git a/plugins/web/searxng/__init__.py b/plugins/web/searxng/__init__.py index 62e12a5c7d..cea8eabb18 100644 --- a/plugins/web/searxng/__init__.py +++ b/plugins/web/searxng/__init__.py @@ -1,7 +1,7 @@ """SearXNG search plugin — bundled, auto-loaded. Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``). -Search-only — pair with an extract provider (firecrawl/keenable/exa) for +Search-only — pair with an extract provider (firecrawl/tavily/exa) for ``web_extract`` calls. """ diff --git a/plugins/web/tavily/__init__.py b/plugins/web/tavily/__init__.py new file mode 100644 index 0000000000..1e0ced61d1 --- /dev/null +++ b/plugins/web/tavily/__init__.py @@ -0,0 +1,10 @@ +"""Tavily web search + extract plugin — bundled, auto-loaded.""" + +from __future__ import annotations + +from plugins.web.tavily.provider import TavilyWebSearchProvider + + +def register(ctx) -> None: + """Register the Tavily provider with the plugin context.""" + ctx.register_web_search_provider(TavilyWebSearchProvider()) diff --git a/plugins/web/tavily/plugin.yaml b/plugins/web/tavily/plugin.yaml new file mode 100644 index 0000000000..3ac90594e5 --- /dev/null +++ b/plugins/web/tavily/plugin.yaml @@ -0,0 +1,7 @@ +name: web-tavily +version: 1.0.0 +description: "Tavily web search + extract. Opt-in keyless via hermes tools; set TAVILY_API_KEY for higher limits — https://app.tavily.com/home." +author: NousResearch +kind: backend +provides_web_providers: + - tavily diff --git a/plugins/web/tavily/provider.py b/plugins/web/tavily/provider.py new file mode 100644 index 0000000000..df7f21a3f6 --- /dev/null +++ b/plugins/web/tavily/provider.py @@ -0,0 +1,313 @@ +"""Tavily web search + content extraction — plugin form. + +Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two +capabilities advertised: + +- ``supports_search()`` -> True (Tavily ``/search``) +- ``supports_extract()`` -> True (Tavily ``/extract``) + +Both are sync — the underlying call is ``httpx.post(...)``. + +Config keys this provider responds to:: + + web: + search_backend: "tavily" # explicit per-capability + extract_backend: "tavily" # explicit per-capability + backend: "tavily" # shared fallback for both + +Env vars:: + + TAVILY_API_KEY=... # https://app.tavily.com/home (optional) + TAVILY_BASE_URL=... # optional override of https://api.tavily.com + +Auth is header-based. A key uses ``Authorization: Bearer``; without a +key the request is keyless (``X-Tavily-Access-Mode: keyless``). Both +paths send ``X-Client-Name: hermes-agent``. + +Tavily is **not** a member of the zero-config keyless ring +(``plugins.web.keyless_mcp._KEYLESS_RING``). Keyless access is opt-in: +select Tavily in ``hermes tools`` (or set ``web.backend: tavily``). +Fresh installs with no web credentials rotate across Exa / Parallel / +Firecrawl / Keenable instead. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional + +import httpx + +from agent.web_search_provider import WebSearchProvider + +logger = logging.getLogger(__name__) + +_CLIENT_NAME = "hermes-agent" + +_SEARCH_PAYLOAD = { + "include_raw_content": False, + "include_images": False, +} + + +def _tavily_headers(api_key: str) -> Dict[str, str]: + """Build Tavily request headers for keyed or keyless access.""" + headers = {"X-Client-Name": _CLIENT_NAME} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + else: + headers["X-Tavily-Access-Mode"] = "keyless" + return headers + + +def _tavily_request( + endpoint: str, + payload: Dict[str, Any], + *, + api_key: Optional[str] = None, +) -> Dict[str, Any]: + """POST to the Tavily API and return the parsed JSON response. + + Keyed when *api_key* (or ``TAVILY_API_KEY``) is set (Bearer auth); + otherwise keyless. Pass ``api_key=""`` to force the keyless header even + when a key is present (``web.provider_tier.tavily: free``). Non-2xx + responses raise ``ValueError`` with the response body so Tavily's + keyless rate-limit / upgrade text reaches the model. + """ + from agent.web_search_provider import get_provider_env + + if api_key is None: + api_key = get_provider_env("TAVILY_API_KEY") + base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com" + url = f"{base_url}/{endpoint.lstrip('/')}" + logger.info("Tavily %s request to %s", endpoint, url) + + response = httpx.post( + url, + json=payload, + timeout=60, + headers=_tavily_headers(api_key), + ) + if response.status_code >= 400: + body = (response.text or "").strip() + detail = body or f"HTTP {response.status_code}" + raise ValueError(detail) + return response.json() + + +def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]: + """Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``.""" + web_results = [] + for i, result in enumerate(response.get("results", [])): + web_results.append( + { + "title": result.get("title", ""), + "url": result.get("url", ""), + "description": result.get("content", ""), + "position": i + 1, + } + ) + return {"success": True, "data": {"web": web_results}} + + +def _normalize_tavily_documents( + response: Dict[str, Any], fallback_url: str = "" +) -> List[Dict[str, Any]]: + """Map Tavily ``/extract`` response to standard documents. + + Documents follow the legacy LLM post-processing shape:: + + {"url", "title", "content", "raw_content", "metadata"} + + Failures (``failed_results``, ``failed_urls``) become result entries + with an ``error`` field rather than raising. + """ + documents: List[Dict[str, Any]] = [] + for result in response.get("results", []): + url = result.get("url", fallback_url) + raw = result.get("raw_content", "") or result.get("content", "") + documents.append( + { + "url": url, + "title": result.get("title", ""), + "content": raw, + "raw_content": raw, + "metadata": {"sourceURL": url, "title": result.get("title", "")}, + } + ) + for fail in response.get("failed_results", []): + documents.append( + { + "url": fail.get("url", fallback_url), + "title": "", + "content": "", + "raw_content": "", + "error": fail.get("error", "extraction failed"), + "metadata": {"sourceURL": fail.get("url", fallback_url)}, + } + ) + for fail_url in response.get("failed_urls", []): + url_str = fail_url if isinstance(fail_url, str) else str(fail_url) + documents.append( + { + "url": url_str, + "title": "", + "content": "", + "raw_content": "", + "error": "extraction failed", + "metadata": {"sourceURL": url_str}, + } + ) + return documents + + +def _missing_key_error(action: str) -> str: + return ( + f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home " + f"or select Tavily in `hermes tools` for opt-in keyless {action}." + ) + + +class TavilyWebSearchProvider(WebSearchProvider): + """Tavily search + extract provider (keyed, or opt-in keyless).""" + + @property + def name(self) -> str: + return "tavily" + + @property + def display_name(self) -> str: + return "Tavily" + + def is_available(self) -> bool: + """Return True when ``TAVILY_API_KEY`` is set to a non-empty value.""" + from agent.web_search_provider import get_provider_env + + return bool(get_provider_env("TAVILY_API_KEY")) + + def is_keyless_available(self) -> bool: + """Tavily serves anonymous keyless requests (X-Tavily-Access-Mode). + + Opt-in only — Tavily is not a member of the zero-config keyless + ring. ``is_keyless_available`` is True so an explicit + ``web.backend: tavily`` (or ``hermes tools`` pick) works without a + key. False when the user pinned ``web.provider_tier.tavily: paid``. + """ + from plugins.web.keyless_mcp import keyless_enabled, provider_tier + + return keyless_enabled() and provider_tier("tavily") != "paid" + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return True + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + """Execute a Tavily search (keyed path or opt-in keyless).""" + try: + from tools.interrupt import is_interrupted + + if is_interrupted(): + return {"success": False, "error": "Interrupted"} + + from agent.web_search_provider import get_provider_env + + from plugins.web.keyless_mcp import use_keyless + + api_key = get_provider_env("TAVILY_API_KEY") + force_keyless = use_keyless("tavily", api_key) + if not force_keyless and not api_key: + return {"success": False, "error": _missing_key_error("search")} + + logger.info( + "Tavily %ssearch: '%s' (limit=%d)", + "keyless " if force_keyless else "", + query, + limit, + ) + raw = _tavily_request( + "search", + { + "query": query, + "max_results": min(limit, 20), + **_SEARCH_PAYLOAD, + }, + api_key="" if force_keyless else api_key, + ) + return _normalize_tavily_search_results(raw) + except ValueError as exc: + return {"success": False, "error": str(exc)} + except Exception as exc: # noqa: BLE001 — including httpx errors + logger.warning("Tavily search error: %s", exc) + return {"success": False, "error": f"Tavily search failed: {exc}"} + + def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: + """Extract content from one or more URLs via Tavily. + + Sync — the underlying call is httpx.post(...). Returns the legacy + list-of-results shape; per-URL failures become items with ``error``. + Keyless uses Tavily's own endpoint, not the keyless ring. + """ + try: + from tools.interrupt import is_interrupted + + if is_interrupted(): + return [ + {"url": u, "error": "Interrupted", "title": ""} for u in urls + ] + + from agent.web_search_provider import get_provider_env + + from plugins.web.keyless_mcp import use_keyless + + api_key = get_provider_env("TAVILY_API_KEY") + force_keyless = use_keyless("tavily", api_key) + if not force_keyless and not api_key: + err = _missing_key_error("extract") + return [ + {"url": u, "title": "", "content": "", "error": err} + for u in urls + ] + + logger.info( + "Tavily %sextract: %d URL(s)", + "keyless " if force_keyless else "", + len(urls), + ) + raw = _tavily_request( + "extract", + { + "urls": urls, + "include_images": False, + }, + api_key="" if force_keyless else api_key, + ) + return _normalize_tavily_documents( + raw, fallback_url=urls[0] if urls else "" + ) + except ValueError as exc: + return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls] + except Exception as exc: # noqa: BLE001 + logger.warning("Tavily extract error: %s", exc) + return [ + {"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"} + for u in urls + ] + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Tavily", + "badge": "free · key optional", + "tag": ( + "Search + extract. Opt-in keyless (not in the free-tier ring); " + "set TAVILY_API_KEY for higher limits." + ), + "env_vars": [ + { + "key": "TAVILY_API_KEY", + "prompt": "Tavily API key (optional — keyless works when Tavily is selected)", + "url": "https://app.tavily.com/home", + }, + ], + } diff --git a/plugins/web/xai/provider.py b/plugins/web/xai/provider.py index 922b9856be..77d80a4398 100644 --- a/plugins/web/xai/provider.py +++ b/plugins/web/xai/provider.py @@ -101,12 +101,12 @@ class XAIWebSearchProvider(WebSearchProvider): back to the Responses API ``citations`` list if Grok ignores the JSON schema instruction (rare for grok-4.3 but cheap insurance). - No extract capability — pair with Firecrawl / Keenable / Exa for + No extract capability — pair with Firecrawl / Tavily / Exa for ``web_extract`` if you need page content. Trust model ----------- - Unlike index-backed providers (Brave / Keenable / Exa) which return + Unlike index-backed providers (Brave / Tavily / Exa) which return verbatim search-engine results, this backend is an LLM in a trench coat: Grok decides which URLs to surface, generates the titles and descriptions itself, and is influenced by the *content of the query*. diff --git a/tests/conftest.py b/tests/conftest.py index ffa5e7b928..e4dfb0ed06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -186,7 +186,7 @@ _CREDENTIAL_NAMES = frozenset({ "FIRECRAWL_API_KEY", "PARALLEL_API_KEY", "EXA_API_KEY", - "TAVILY_API_KEY", # removed backend; still blanked for hermeticity + "TAVILY_API_KEY", "WANDB_API_KEY", "ELEVENLABS_API_KEY", "HONCHO_API_KEY", diff --git a/tests/hermes_cli/test_config.py b/tests/hermes_cli/test_config.py index 968649e58a..366782d76a 100644 --- a/tests/hermes_cli/test_config.py +++ b/tests/hermes_cli/test_config.py @@ -666,13 +666,23 @@ class TestOptionalEnvVarsRegistry: from hermes_cli.config import OPTIONAL_ENV_VARS assert OPTIONAL_ENV_VARS["KEENABLE_API_KEY"]["url"] == "https://keenable.ai" - def test_removed_tavily_var_not_in_env_vars_by_version(self): - """TAVILY_API_KEY was removed with the Tavily backend.""" + def test_tavily_api_key_registered(self): + """TAVILY_API_KEY is listed in OPTIONAL_ENV_VARS.""" + from hermes_cli.config import OPTIONAL_ENV_VARS + assert "TAVILY_API_KEY" in OPTIONAL_ENV_VARS + + def test_tavily_api_key_has_url(self): + """TAVILY_API_KEY has a URL.""" + from hermes_cli.config import OPTIONAL_ENV_VARS + assert OPTIONAL_ENV_VARS["TAVILY_API_KEY"]["url"] == "https://app.tavily.com/home" + + def test_tavily_in_env_vars_by_version(self): + """TAVILY_API_KEY is listed in ENV_VARS_BY_VERSION.""" from hermes_cli.config import ENV_VARS_BY_VERSION all_vars = [] for vars_list in ENV_VARS_BY_VERSION.values(): all_vars.extend(vars_list) - assert "TAVILY_API_KEY" not in all_vars + assert "TAVILY_API_KEY" in all_vars def test_max_iterations_not_offered_as_env_var(self): """HERMES_MAX_ITERATIONS must NOT be in OPTIONAL_ENV_VARS (issue #17534). diff --git a/tests/hermes_cli/test_dump_env_visibility.py b/tests/hermes_cli/test_dump_env_visibility.py index 40feba0cec..ba98cfa3a5 100644 --- a/tests/hermes_cli/test_dump_env_visibility.py +++ b/tests/hermes_cli/test_dump_env_visibility.py @@ -47,6 +47,7 @@ def test_dump_leaves_unset_key_untouched(monkeypatch, capsys, tmp_path): monkeypatch.setattr(dump, "get_project_root", lambda: tmp_path / "noproject") monkeypatch.delenv("KEENABLE_API_KEY", raising=False) + monkeypatch.delenv("TAVILY_API_KEY", raising=False) home = get_hermes_home() home.mkdir(parents=True, exist_ok=True) diff --git a/tests/hermes_cli/test_nous_subscription.py b/tests/hermes_cli/test_nous_subscription.py index d9f71a9af1..c9ffaa931a 100644 --- a/tests/hermes_cli/test_nous_subscription.py +++ b/tests/hermes_cli/test_nous_subscription.py @@ -58,6 +58,52 @@ def test_get_nous_subscription_features_recognizes_direct_exa_backend(monkeypatc assert features.web.current_provider == "exa" +def test_get_nous_subscription_features_recognizes_keyless_tavily_backend(monkeypatch): + """Selecting Tavily in setup/tools counts as available with no API key. + + Mirrors tools.web_tools._is_backend_available('tavily'): keyless is + opt-in via web.backend / search_backend / extract_backend, not a + silent empty-install default. The setup summary previously required + TAVILY_API_KEY and printed a false 'missing' after a skipped key prompt. + """ + monkeypatch.setattr(ns, "get_env_value", lambda name: "") + monkeypatch.setattr( + ns, "get_nous_portal_account_info", lambda: _account(logged_in=False) + ) + monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web") + monkeypatch.setattr(ns, "_has_agent_browser", lambda: False) + monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "") + monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False) + + features = ns.get_nous_subscription_features({"web": {"backend": "tavily"}}) + + assert features.web.available is True + assert features.web.active is True + assert features.web.managed_by_nous is False + assert features.web.direct_override is True + assert features.web.current_provider == "tavily" + assert features.web.explicit_configured is True + + +def test_keyless_tavily_search_backend_without_shared_backend(monkeypatch): + monkeypatch.setattr(ns, "get_env_value", lambda name: "") + monkeypatch.setattr( + ns, "get_nous_portal_account_info", lambda: _account(logged_in=False) + ) + monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web") + monkeypatch.setattr(ns, "_has_agent_browser", lambda: False) + monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "") + monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False) + + features = ns.get_nous_subscription_features( + {"web": {"search_backend": "tavily"}} + ) + + assert features.web.available is True + assert features.web.active is True + assert features.web.current_provider == "tavily" + + def test_unconfigured_web_without_keys_is_unavailable(monkeypatch): monkeypatch.setattr(ns, "get_env_value", lambda name: "") monkeypatch.setattr( diff --git a/tests/hermes_cli/test_status.py b/tests/hermes_cli/test_status.py index 4a5746b94d..37d0c0bb24 100644 --- a/tests/hermes_cli/test_status.py +++ b/tests/hermes_cli/test_status.py @@ -15,6 +15,18 @@ def test_show_status_all_does_not_print_keenable_key_value(monkeypatch, capsys, assert sentinel not in output +def test_show_status_all_does_not_print_tavily_key_value(monkeypatch, capsys, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + sentinel = "NONSECRET_SENTINEL_VALUE_DO_NOT_PRINT_TAVILY_123456" + monkeypatch.setenv("TAVILY_API_KEY", sentinel) + + show_status(SimpleNamespace(all=True, deep=False)) + + output = capsys.readouterr().out + assert "Tavily" in output + assert sentinel not in output + + def test_show_status_termux_gateway_section_skips_systemctl(monkeypatch, capsys, tmp_path): from hermes_cli import status as status_mod import hermes_cli.auth as auth_mod diff --git a/tests/hermes_cli/test_tools_config.py b/tests/hermes_cli/test_tools_config.py index 8523bf3f94..ce19955cc8 100644 --- a/tests/hermes_cli/test_tools_config.py +++ b/tests/hermes_cli/test_tools_config.py @@ -245,6 +245,7 @@ def test_first_install_nous_auto_configures_video_gen(monkeypatch): "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "KEENABLE_API_KEY", + "TAVILY_API_KEY", "PARALLEL_API_KEY", "BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID", diff --git a/tests/plugins/web/test_web_search_provider_plugins.py b/tests/plugins/web/test_web_search_provider_plugins.py index 117733e045..9a0f253147 100644 --- a/tests/plugins/web/test_web_search_provider_plugins.py +++ b/tests/plugins/web/test_web_search_provider_plugins.py @@ -3,7 +3,7 @@ Covers: - All bundled plugins (brave-free, ddgs, searxng, exa, parallel, - firecrawl, keenable, xai) instantiate and self-report the expected + tavily, firecrawl, keenable, xai) instantiate and self-report the expected capabilities + ABC-derived defaults. - Each plugin's ``is_available()`` correctly reflects env-var presence. - The web_search_registry resolves an active provider in the documented @@ -35,6 +35,8 @@ def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None: "BRAVE_SEARCH_API_KEY", "SEARXNG_URL", "KEENABLE_API_KEY", + "TAVILY_API_KEY", + "TAVILY_BASE_URL", "EXA_API_KEY", "PARALLEL_API_KEY", "PARALLEL_SEARCH_MODE", @@ -82,6 +84,7 @@ class TestBundledPluginsRegister: "keenable", "parallel", "searxng", + "tavily", "xai", ] @@ -94,6 +97,7 @@ class TestBundledPluginsRegister: ("exa", True, True), ("parallel", True, True), ("keenable", True, True), + ("tavily", True, True), ("firecrawl", True, True), # xai: search-only via Grok's agentic web_search tool. ("xai", True, False), @@ -115,7 +119,7 @@ class TestBundledPluginsRegister: @pytest.mark.parametrize( "plugin_name", - ["brave-free", "ddgs", "searxng", "exa", "parallel", "firecrawl", "keenable", "xai"], + ["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "keenable", "xai"], ) def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None: _ensure_plugins_loaded() @@ -165,6 +169,16 @@ class TestIsAvailable: monkeypatch.setenv("KEENABLE_API_KEY", "real") assert p.is_available() is True + def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: + _ensure_plugins_loaded() + from agent.web_search_registry import get_provider + + p = get_provider("tavily") + assert p is not None + assert p.is_available() is False + monkeypatch.setenv("TAVILY_API_KEY", "real") + assert p.is_available() is True + def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index cefe4584dc..e8fec5cf5c 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -83,6 +83,7 @@ def register_all_web_providers(): from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider from plugins.web.parallel.provider import ParallelWebSearchProvider from plugins.web.keenable.provider import KeenableWebSearchProvider + from plugins.web.tavily.provider import TavilyWebSearchProvider from plugins.web.searxng.provider import SearXNGWebSearchProvider from plugins.web.xai.provider import XAIWebSearchProvider @@ -94,6 +95,7 @@ def register_all_web_providers(): FirecrawlWebSearchProvider, ParallelWebSearchProvider, KeenableWebSearchProvider, + TavilyWebSearchProvider, SearXNGWebSearchProvider, XAIWebSearchProvider, ): diff --git a/tests/tools/test_web_keyless_fallback.py b/tests/tools/test_web_keyless_fallback.py index 1721d3f98d..1de9fba14b 100644 --- a/tests/tools/test_web_keyless_fallback.py +++ b/tests/tools/test_web_keyless_fallback.py @@ -25,7 +25,7 @@ from plugins.web.parallel.provider import ParallelWebSearchProvider def _no_web_env(monkeypatch): """Blank every web credential and neutralize config lookups.""" for var in ( - "EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY", + "EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY", "TAVILY_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "BRAVE_SEARCH_API_KEY", "SEARXNG_URL", "TOOL_GATEWAY_USER_TOKEN", ): @@ -289,7 +289,7 @@ class TestResolutionOrder: def test_keyless_ring_rotates_and_covers_all_vendors(self, fresh_registry, monkeypatch): monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) - # The ring order always contains all five vendors, starting at the + # The ring order always contains all four vendors, starting at the # current cursor and wrapping. order = registry._keyless_preference() assert sorted(order) == sorted(keyless_mcp._KEYLESS_RING) @@ -303,6 +303,15 @@ class TestResolutionOrder: assert keyless_mcp._ring_order("keenable")[0] == "keenable" assert keyless_mcp._ring_order("keenable")[0] == "keenable" + def test_tavily_is_not_a_ring_member(self): + """Tavily is opt-in keyless; zero-config rotation must not include it.""" + from plugins.web import keyless_mcp + + assert "tavily" not in keyless_mcp._KEYLESS_RING + assert "tavily" not in keyless_mcp._KEYLESS_SEARCHERS + assert "tavily" not in keyless_mcp._KEYLESS_EXTRACTORS + assert "tavily" not in registry._KEYLESS_PREFERENCE + def test_registry_keyless_disabled_returns_none(self, fresh_registry, monkeypatch): monkeypatch.setattr(registry, "_read_config_key", lambda *p: None) monkeypatch.setattr(registry, "_keyless_tier_enabled", lambda: False) diff --git a/tests/tools/test_web_tools_config.py b/tests/tools/test_web_tools_config.py index 29c5f3b8cf..92f30ae57b 100644 --- a/tests/tools/test_web_tools_config.py +++ b/tests/tools/test_web_tools_config.py @@ -210,6 +210,7 @@ class TestBackendSelection: "TOOL_GATEWAY_SCHEME", "TOOL_GATEWAY_USER_TOKEN", "KEENABLE_API_KEY", + "TAVILY_API_KEY", ) def setup_method(self): @@ -254,7 +255,7 @@ class TestBackendSelection: assert _get_backend() == "exa" def test_fallback_exa_takes_priority_over_parallel(self): - """Direct-credential backends are tried in the order exa > parallel > keenable + """Direct-credential backends are tried in the order tavily > exa > parallel > keenable so an explicit Exa key wins when both Exa and Parallel are configured.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={}), \ @@ -275,6 +276,27 @@ class TestBackendSelection: patch.dict(os.environ, {"EXA_API_KEY": "exa-test", "FIRECRAWL_API_KEY": "fc-test"}): assert _get_backend() == "exa" + def test_fallback_tavily_only_key(self): + """Only TAVILY_API_KEY set → 'tavily'.""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}): + assert _get_backend() == "tavily" + + def test_fallback_tavily_beats_firecrawl_direct(self): + """Tavily ranks above firecrawl in the explicit-credential block.""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "FIRECRAWL_API_KEY": "fc-test"}): + assert _get_backend() == "tavily" + + def test_fallback_tavily_beats_exa(self): + """Tavily ranks above Exa in the explicit-credential block.""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "EXA_API_KEY": "exa-test"}): + assert _get_backend() == "tavily" + def test_fallback_parallel_beats_firecrawl_direct(self): """Parallel + Firecrawl-direct → parallel (parallel is the higher-priority @@ -342,6 +364,14 @@ class TestBackendSelection: patch.dict(os.environ, {"EXA_API_KEY": "exa-test"}): assert _get_backend() == "exa" + def test_managed_gateway_does_not_preempt_explicit_tavily(self): + """A Nous OAuth token must not beat an explicit TAVILY_API_KEY.""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}): + assert _get_backend() == "tavily" + def test_managed_gateway_only_falls_through_to_firecrawl(self): """When no explicit-credential backend is configured, a Nous-managed gateway token still selects firecrawl — the convenience path is @@ -494,6 +524,7 @@ class TestCheckWebApiKey: "TOOL_GATEWAY_SCHEME", "TOOL_GATEWAY_USER_TOKEN", "KEENABLE_API_KEY", + "TAVILY_API_KEY", ) def setup_method(self): @@ -597,7 +628,9 @@ class TestCheckWebApiKey: def test_web_requires_env_includes_exa_key(): from tools.web_tools import _web_requires_env - assert "EXA_API_KEY" in _web_requires_env() + env = _web_requires_env() + assert "EXA_API_KEY" in env + assert "TAVILY_API_KEY" in env class TestNonBuiltinProviderAvailability: @@ -625,6 +658,7 @@ class TestNonBuiltinProviderAvailability: "TOOL_GATEWAY_SCHEME", "TOOL_GATEWAY_USER_TOKEN", "KEENABLE_API_KEY", + "TAVILY_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY", "XAI_API_KEY", @@ -765,6 +799,7 @@ class TestSiblingProvidersEnvResolution: ("plugins.web.exa.provider", "ExaWebSearchProvider", "EXA_API_KEY"), ("plugins.web.parallel.provider", "ParallelWebSearchProvider", "PARALLEL_API_KEY"), ("plugins.web.keenable.provider", "KeenableWebSearchProvider", "KEENABLE_API_KEY"), + ("plugins.web.tavily.provider", "TavilyWebSearchProvider", "TAVILY_API_KEY"), ("plugins.web.brave_free.provider", "BraveFreeWebSearchProvider", "BRAVE_SEARCH_API_KEY"), ] @@ -811,6 +846,28 @@ class TestSiblingProvidersEnvResolution: assert headers["Authorization"] == "Bearer kn-from-dotenv" assert headers["X-Keenable-Title"] == "hermes-agent" + def test_tavily_request_reads_key_via_get_env_value(self, monkeypatch): + """Keyed Tavily must Bearer-auth with a key that lives only in .env.""" + monkeypatch.delenv("TAVILY_API_KEY", raising=False) + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"results": []} + mock_response.text = "{}" + + with patch( + "hermes_cli.config.get_env_value", + side_effect=lambda k: "tvly-from-dotenv" if k == "TAVILY_API_KEY" else None, + ), patch( + "plugins.web.tavily.provider.httpx.post", return_value=mock_response + ) as mock_post: + from plugins.web.tavily.provider import _tavily_request + + _tavily_request("search", {"query": "q"}) + headers = mock_post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer tvly-from-dotenv" + assert headers["X-Client-Name"] == "hermes-agent" + assert "X-Tavily-Access-Mode" not in headers + def test_get_provider_env_unset_returns_empty(self, monkeypatch): monkeypatch.delenv("WSP_TEST_UNSET_KEY", raising=False) diff --git a/tests/tools/test_web_tools_tavily.py b/tests/tools/test_web_tools_tavily.py new file mode 100644 index 0000000000..b6fe37c59b --- /dev/null +++ b/tests/tools/test_web_tools_tavily.py @@ -0,0 +1,317 @@ +"""Tests for Tavily web backend integration. + +Coverage: + _tavily_request() — keyed Bearer vs keyless header, attribution, error bodies. + _normalize_tavily_search_results() — search response normalization. + _normalize_tavily_documents() — extract response normalization, failed_results. + web_search_tool / web_extract_tool — Tavily dispatch paths. + auto-detect ranking — keyed paid-band; keyless only when Tavily is selected. +""" + +import json +import os +import asyncio +import pytest +from unittest.mock import patch, MagicMock + +from tests.tools.conftest import register_all_web_providers + + +def _ok_response(payload=None): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = payload if payload is not None else {"results": []} + mock_response.text = json.dumps(mock_response.json.return_value) + return mock_response + + +# ─── _tavily_request ───────────────────────────────────────────────────────── + +class TestTavilyRequest: + """Test suite for the _tavily_request helper.""" + + def test_keyless_when_no_api_key(self): + """No TAVILY_API_KEY → keyless header, no Authorization, no body key.""" + mock_response = _ok_response() + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("TAVILY_API_KEY", None) + with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post: + from plugins.web.tavily.provider import _tavily_request + _tavily_request("search", {"query": "test"}) + + mock_post.assert_called_once() + headers = mock_post.call_args.kwargs["headers"] + payload = mock_post.call_args.kwargs["json"] + assert headers["X-Client-Name"] == "hermes-agent" + assert headers["X-Tavily-Access-Mode"] == "keyless" + assert "Authorization" not in headers + assert "api_key" not in payload + assert payload["query"] == "test" + assert "api.tavily.com/search" in mock_post.call_args.args[0] + + def test_keyed_uses_bearer_not_body(self): + """TAVILY_API_KEY → Bearer auth, attribution, no body api_key.""" + mock_response = _ok_response() + + with patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test-key"}): + with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post: + from plugins.web.tavily.provider import _tavily_request + _tavily_request("search", {"query": "hello"}) + + mock_post.assert_called_once() + headers = mock_post.call_args.kwargs["headers"] + payload = mock_post.call_args.kwargs["json"] + assert headers == { + "X-Client-Name": "hermes-agent", + "Authorization": "Bearer tvly-test-key", + } + assert "X-Tavily-Access-Mode" not in headers + assert "api_key" not in payload + assert payload["query"] == "hello" + assert "api.tavily.com/search" in mock_post.call_args.args[0] + + def test_http_error_surfaces_response_body(self): + """Non-2xx responses raise ValueError with Tavily's response body.""" + mock_response = MagicMock() + mock_response.status_code = 429 + mock_response.text = "Rate limit hit. Sign up for a free API key at https://app.tavily.com" + mock_response.json.return_value = {} + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("TAVILY_API_KEY", None) + with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response): + from plugins.web.tavily.provider import _tavily_request + with pytest.raises(ValueError, match="Rate limit hit"): + _tavily_request("search", {"query": "test"}) + + +# ─── _normalize_tavily_search_results ───────────────────────────────────────── + +class TestNormalizeTavilySearchResults: + """Test search result normalization.""" + + def test_basic_normalization(self): + from tools.web_tools import _normalize_tavily_search_results + raw = { + "results": [ + {"title": "Python Docs", "url": "https://docs.python.org", "content": "Official docs", "score": 0.9}, + {"title": "Tutorial", "url": "https://example.com", "content": "A tutorial", "score": 0.8}, + ] + } + result = _normalize_tavily_search_results(raw) + assert result["success"] is True + web = result["data"]["web"] + assert len(web) == 2 + assert web[0]["title"] == "Python Docs" + assert web[0]["url"] == "https://docs.python.org" + assert web[0]["description"] == "Official docs" + assert web[0]["position"] == 1 + assert web[1]["position"] == 2 + + + def test_missing_fields(self): + from tools.web_tools import _normalize_tavily_search_results + result = _normalize_tavily_search_results({"results": [{}]}) + web = result["data"]["web"] + assert web[0]["title"] == "" + assert web[0]["url"] == "" + assert web[0]["description"] == "" + + +# ─── _normalize_tavily_documents ────────────────────────────────────────────── + +class TestNormalizeTavilyDocuments: + """Test extract document normalization.""" + + def test_basic_document(self): + from tools.web_tools import _normalize_tavily_documents + raw = { + "results": [{ + "url": "https://example.com", + "title": "Example", + "raw_content": "Full page content here", + }] + } + docs = _normalize_tavily_documents(raw) + assert len(docs) == 1 + assert docs[0]["url"] == "https://example.com" + assert docs[0]["title"] == "Example" + assert docs[0]["content"] == "Full page content here" + assert docs[0]["raw_content"] == "Full page content here" + assert docs[0]["metadata"]["sourceURL"] == "https://example.com" + + + def test_fallback_url(self): + from tools.web_tools import _normalize_tavily_documents + raw = {"results": [{"content": "data"}]} + docs = _normalize_tavily_documents(raw, fallback_url="https://fallback.com") + assert docs[0]["url"] == "https://fallback.com" + + +# ─── availability / auto-detect ─────────────────────────────────────────────── + +class TestTavilyAvailability: + """Keyed Tavily stays in the paid band; keyless only when selected.""" + + def test_is_available_without_key(self): + from plugins.web.tavily.provider import TavilyWebSearchProvider + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("TAVILY_API_KEY", None) + assert TavilyWebSearchProvider().is_available() is False + + def test_is_backend_available_without_key(self): + from tools.web_tools import _is_backend_available + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch.dict(os.environ, {}, clear=False): + os.environ.pop("TAVILY_API_KEY", None) + assert _is_backend_available("tavily") is False + + def test_is_backend_available_when_configured_without_key(self): + from tools.web_tools import _is_backend_available + with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \ + patch.dict(os.environ, {}, clear=False): + os.environ.pop("TAVILY_API_KEY", None) + assert _is_backend_available("tavily") is True + + def test_keyless_does_not_preempt_managed_firecrawl(self): + """No TAVILY_API_KEY + Nous gateway ready → firecrawl, not keyless tavily.""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \ + patch("tools.web_tools._ddgs_package_importable", return_value=False): + os.environ.pop("TAVILY_API_KEY", None) + assert _get_backend() == "firecrawl" + + def test_keyless_does_not_preempt_ddgs(self): + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \ + patch("tools.web_tools._ddgs_package_importable", return_value=True): + os.environ.pop("TAVILY_API_KEY", None) + assert _get_backend() == "ddgs" + + def test_no_keys_defaults_to_firecrawl(self): + """Keyless tier disabled: zero-credential resolve hits the legacy + firecrawl sentinel. (With the tier on — the default — it resolves + to the Exa/Parallel keyless split; see test_web_keyless_fallback.py.) + """ + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={}), \ + patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \ + patch("tools.web_tools._ddgs_package_importable", return_value=False), \ + patch("tools.web_tools._list_registered_web_providers", return_value=[]), \ + patch("agent.web_search_registry._keyless_tier_enabled", return_value=False): + os.environ.pop("TAVILY_API_KEY", None) + assert _get_backend() == "firecrawl" + + def test_explicit_search_backend_tavily_without_key(self): + """web.search_backend=tavily sticks even with no TAVILY_API_KEY.""" + from tools.web_tools import _get_search_backend + with patch("tools.web_tools._load_web_config", + return_value={"backend": "firecrawl", "search_backend": "tavily"}), \ + patch("tools.web_tools._is_tool_gateway_ready", return_value=True): + os.environ.pop("TAVILY_API_KEY", None) + assert _get_search_backend() == "tavily" + + def test_check_web_api_key_when_tavily_configured_without_key(self): + from tools.web_tools import check_web_api_key + with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \ + patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \ + patch("tools.web_tools.check_firecrawl_api_key", return_value=False), \ + patch("tools.web_tools._ddgs_package_importable", return_value=False), \ + patch("agent.web_search_registry.get_active_search_provider", return_value=None), \ + patch("agent.web_search_registry.get_active_extract_provider", return_value=None): + os.environ.pop("TAVILY_API_KEY", None) + assert check_web_api_key() is True + + +# ─── web_search_tool (Tavily dispatch) ──────────────────────────────────────── + +class TestWebSearchTavily: + """Test web_search_tool dispatch to Tavily.""" + + _register_providers = staticmethod(register_all_web_providers) + + @pytest.fixture(autouse=True) + def _populate_web_registry(self): + self._register_providers() + yield + from agent.web_search_registry import _reset_for_tests + _reset_for_tests() + + def test_search_dispatches_to_tavily(self): + mock_response = _ok_response({ + "results": [{"title": "Result", "url": "https://r.com", "content": "desc", "score": 0.9}] + }) + + with patch("tools.web_tools._get_backend", return_value="tavily"), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \ + patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \ + patch("tools.interrupt.is_interrupted", return_value=False): + from tools.web_tools import web_search_tool + result = json.loads(web_search_tool("test query", limit=3)) + assert result["success"] is True + assert len(result["data"]["web"]) == 1 + assert result["data"]["web"][0]["title"] == "Result" + + def test_search_keyless_dispatch(self): + """Opt-in keyless Tavily hits Tavily's own endpoint, not the ring.""" + mock_response = _ok_response({ + "results": [{"title": "Result", "url": "https://r.com", "content": "desc"}] + }) + + with patch("tools.web_tools._get_backend", return_value="tavily"), \ + patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post, \ + patch("tools.interrupt.is_interrupted", return_value=False): + os.environ.pop("TAVILY_API_KEY", None) + from tools.web_tools import web_search_tool + result = json.loads(web_search_tool("test query")) + assert result["success"] is True + headers = mock_post.call_args.kwargs["headers"] + assert headers["X-Tavily-Access-Mode"] == "keyless" + assert headers["X-Client-Name"] == "hermes-agent" + assert "Authorization" not in headers + assert "api.tavily.com/search" in mock_post.call_args.args[0] + + def test_tavily_is_not_in_keyless_ring(self): + from plugins.web.keyless_mcp import _KEYLESS_RING, _KEYLESS_SEARCHERS, _KEYLESS_EXTRACTORS + assert "tavily" not in _KEYLESS_RING + assert "tavily" not in _KEYLESS_SEARCHERS + assert "tavily" not in _KEYLESS_EXTRACTORS + + +# ─── web_extract_tool (Tavily dispatch) ─────────────────────────────────────── + +class TestWebExtractTavily: + """Test web_extract_tool dispatch to Tavily.""" + + _register_providers = staticmethod(register_all_web_providers) + + @pytest.fixture(autouse=True) + def _populate_web_registry(self): + self._register_providers() + yield + from agent.web_search_registry import _reset_for_tests + _reset_for_tests() + + def test_extract_dispatches_to_tavily(self): + mock_response = _ok_response({ + "results": [{"url": "https://example.com", "raw_content": "Extracted content", "title": "Page"}] + }) + + async def _allow_ssrf(_url: str) -> bool: + return True + + with patch("tools.web_tools._get_backend", return_value="tavily"), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \ + patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \ + patch("tools.web_tools.async_is_safe_url", _allow_ssrf): + from tools.web_tools import web_extract_tool + result = json.loads(asyncio.get_event_loop().run_until_complete( + web_extract_tool(["https://example.com"]) + )) + assert "results" in result + assert len(result["results"]) == 1 + assert result["results"][0]["url"] == "https://example.com" + assert "Extracted content" in result["results"][0]["content"] diff --git a/tools/url_safety.py b/tools/url_safety.py index e9b230ac68..6442fe4bf5 100644 --- a/tools/url_safety.py +++ b/tools/url_safety.py @@ -21,7 +21,7 @@ Limitations: connects to the validated IP while preserving Host/SNI semantics. - Redirect-based bypass is mitigated by httpx event hooks that re-validate each redirect target in vision_tools, gateway platform adapters, and - media cache helpers. Web tools use third-party SDKs (Firecrawl/Exa) + media cache helpers. Web tools use third-party SDKs (Firecrawl/Tavily) where redirect handling is on their servers. """ diff --git a/tools/web_tools.py b/tools/web_tools.py index 93b8e5e9d0..e34b54b7f5 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -15,6 +15,7 @@ Backend compatibility: - Exa: https://exa.ai (search, extract) - Firecrawl: https://docs.firecrawl.dev/introduction (search, extract; direct or derived firecrawl-gateway. for Nous Subscribers) - Parallel: https://docs.parallel.ai (search, extract) +- Tavily: https://tavily.com (search, extract; keyed or opt-in keyless, not in the free-tier ring) LLM Processing: - Uses OpenRouter API with Gemini 3 Flash Preview for intelligent content extraction @@ -57,6 +58,13 @@ from plugins.web.firecrawl.provider import ( _is_tool_gateway_ready, check_firecrawl_api_key, ) +# Tavily helpers re-exported for backward-compat with existing unit tests +# (tests/tools/test_web_tools_tavily.py imports these names directly). +from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names + _normalize_tavily_documents, + _normalize_tavily_search_results, + _tavily_request, +) # Parallel + Exa clients re-exported for backward-compat with existing # unit tests (tests/tools/test_web_tools_config.py imports _get_parallel_client # / _get_async_parallel_client / _get_exa_client directly). @@ -161,7 +169,7 @@ def _load_web_config() -> dict: # WebSearchProvider. Keep the two sets aligned by hand: if xai ever ships as # a registered provider, drop it here so the registry path takes over. _LEGACY_WEB_BACKENDS = frozenset( - {"parallel", "firecrawl", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"} + {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"} ) @@ -244,13 +252,14 @@ def _get_backend() -> str: return "firecrawl" # Never-configured install — pick the highest-priority available - # backend. Explicit user credentials (EXA_API_KEY etc.) + # backend. Explicit user credentials (TAVILY_API_KEY etc.) # beat the managed-tool-gateway probe so a deliberate setup is not # pre-empted by a Nous OAuth token whose subscription tier may not # actually grant web-search access (the gateway then fails at runtime # with "no subscription" and the tool returns an error to the agent # without falling back). Free-tier backends trail the paid ones. backend_candidates = ( + ("tavily", _has_env("TAVILY_API_KEY")), ("exa", _has_env("EXA_API_KEY")), ("parallel", _has_env("PARALLEL_API_KEY")), ("keenable", _has_env("KEENABLE_API_KEY")), @@ -350,6 +359,13 @@ def _get_capability_backend(capability: str) -> str: return _get_backend() +def _tavily_explicitly_configured() -> bool: + cfg = _load_web_config() + return any( + (cfg.get(key) or "").lower().strip() == "tavily" + for key in ("backend", "search_backend", "extract_backend") + ) + def _is_backend_available(backend: str) -> bool: """Return True when the selected backend is currently usable. @@ -376,6 +392,8 @@ def _is_backend_available(backend: str) -> bool: return _has_env("KEENABLE_API_KEY") if backend == "firecrawl": return check_firecrawl_api_key() + if backend == "tavily": + return _has_env("TAVILY_API_KEY") or _tavily_explicitly_configured() if backend == "searxng": return _has_env("SEARXNG_URL") if backend == "brave-free": @@ -585,6 +603,7 @@ def _web_requires_env() -> list[str]: return [ "EXA_API_KEY", "PARALLEL_API_KEY", + "TAVILY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", @@ -595,10 +614,11 @@ def _web_requires_env() -> list[str]: ] -# ─── Parallel / Firecrawl helpers — moved into plugins ─────────────────────── +# ─── Parallel / Tavily / Firecrawl helpers — moved into plugins ────────────── # After PR #25182, the per-vendor client construction, request helpers, and # response normalizers all live in plugins.web..provider: # - parallel: plugins/web/parallel/provider.py +# - tavily: plugins/web/tavily/provider.py # - firecrawl: plugins/web/firecrawl/provider.py # The names from the firecrawl plugin (Firecrawl proxy, _get_firecrawl_client, # _to_plain_object, _normalize_result_list, _extract_web_search_results, @@ -790,7 +810,7 @@ def _ensure_web_plugins_loaded() -> None: """Idempotently trigger plugin discovery so the web registry is populated. Every bundled web provider (brave-free, ddgs, searxng, exa, parallel, - firecrawl, keenable) registers itself via ``plugins/web//__init__.py`` + tavily, firecrawl, keenable) registers itself via ``plugins/web//__init__.py`` during plugin discovery. Tool dispatch can be reached from contexts that haven't already triggered discovery — subprocess agent runs, delegate children, standalone scripts, certain test paths — and without it the @@ -871,9 +891,9 @@ def web_search_tool(query: str, limit: int = 5) -> str: if is_interrupted(): return tool_error("Interrupted", success=False) - # Dispatch through the web search registry. All 7 providers - # (brave-free, ddgs, searxng, exa, parallel, firecrawl, keenable) - # now live as plugins; the dispatcher is just a registry lookup + + # Dispatch through the web search registry. All bundled providers + # (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl, + # keenable) now live as plugins; the dispatcher is just a registry lookup + # delegation. Sync only — every provider's search() is sync. _ensure_web_plugins_loaded() from agent.web_search_registry import ( @@ -1034,7 +1054,7 @@ async def web_extract_tool( Extract content from specific web pages using available extraction API backend. Returns clean page content (markdown/text) with NO LLM summarization. The - extract backends (Firecrawl, Exa, Parallel, Keenable) already return clean, + extract backends (Firecrawl, Tavily, Exa, Parallel, Keenable) already return clean, boilerplate-stripped content, so we return it directly and fast. Pages over ``char_limit`` are head+tail truncated with an explicit footer; the full text is stored under cache/web and the footer tells the model how to @@ -1142,10 +1162,10 @@ async def web_extract_tool( else: backend = _get_extract_backend() - # All seven providers (brave-free, ddgs, searxng, exa, parallel, - # firecrawl, keenable) now live as plugins. The dispatcher is a + # All bundled providers (brave-free, ddgs, searxng, exa, parallel, + # tavily, firecrawl, keenable) now live as plugins. The dispatcher is a # registry lookup + delegation. Some providers' extract() is - # async (parallel, firecrawl), others sync (exa, keenable) — we + # async (parallel, firecrawl), others sync (exa, tavily, keenable) — we # detect coroutine functions and await; sync functions run # inline (the policy gate, SSRF re-check, etc. live inside the # provider itself for the firecrawl per-URL loop). @@ -1172,7 +1192,7 @@ async def web_extract_tool( f"{provider.display_name} is a search-only " "backend and cannot extract URL content. " "Set web.extract_backend to firecrawl, " - "keenable, exa, or parallel." + "tavily, keenable, exa, or parallel." ), }, ensure_ascii=False, @@ -1235,7 +1255,7 @@ async def web_extract_tool( "error": ( "No web extract provider configured. " "Set web.extract_backend to firecrawl, " - "keenable, exa, or parallel." + "tavily, keenable, exa, or parallel." ), }, ensure_ascii=False, @@ -1284,7 +1304,7 @@ async def web_extract_tool( ) # Async-or-sync dispatch: parallel + firecrawl have async - # extract(); exa + keenable are sync. + # extract(); exa + tavily + keenable are sync. import inspect _extract_rescued = False try: @@ -1568,6 +1588,11 @@ if __name__ == "__main__": print(" Using Exa API (https://exa.ai)") elif backend == "parallel": print(" Using Parallel API (https://parallel.ai)") + elif backend == "tavily": + if _has_env("TAVILY_API_KEY"): + print(" Using Tavily API (https://tavily.com)") + else: + print(" Using Tavily keyless (https://docs.tavily.com/documentation/keyless)") elif backend == "searxng": print(f" Using SearXNG (search only): {_env_value('SEARXNG_URL')}") elif backend == "brave-free": @@ -1585,7 +1610,7 @@ if __name__ == "__main__": else: print("❌ No web search backend configured") print( - "Set EXA_API_KEY, PARALLEL_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL" + "Set EXA_API_KEY, PARALLEL_API_KEY, TAVILY_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL" f"{_firecrawl_backend_help_suffix()}" ) diff --git a/website/docs/developer-guide/web-search-provider-plugin.md b/website/docs/developer-guide/web-search-provider-plugin.md index 98bd98f174..257df89548 100644 --- a/website/docs/developer-guide/web-search-provider-plugin.md +++ b/website/docs/developer-guide/web-search-provider-plugin.md @@ -6,7 +6,7 @@ description: "How to build a web-search/extract/crawl backend plugin for Hermes # Building a Web Search Provider Plugin -Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web//`. You can add a new one, or override a bundled one, by dropping a directory next to them. +Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Tavily, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web//`. You can add a new one, or override a bundled one, by dropping a directory next to them. :::tip Web search is one of several **backend plugins** Hermes supports. The others (with their own ABCs) are [Image Generation Provider Plugins](/developer-guide/image-gen-provider-plugin), [Video Generation Provider Plugins](/developer-guide/video-gen-provider-plugin), [Memory Provider Plugins](/developer-guide/memory-provider-plugin), [Context Engine Plugins](/developer-guide/context-engine-plugin), and [Model Provider Plugins](/developer-guide/model-provider-plugin). General tool/hook/CLI plugins live in [Build a Hermes Plugin](/developer-guide/plugins). @@ -157,7 +157,7 @@ Full contract in `agent/web_search_provider.py`. Methods you may override: | `search(query, limit)` | conditional | raises | Required when `supports_search()` returns `True` | | `extract(urls, **kwargs)` | conditional | raises | Required when `supports_extract()` returns `True` | -Providers can advertise multiple capabilities from a single class — Firecrawl, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow. +Providers can advertise multiple capabilities from a single class — Firecrawl, Tavily, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow. ## Response shape diff --git a/website/docs/integrations/index.md b/website/docs/integrations/index.md index 51555976ee..37bac9d8bf 100644 --- a/website/docs/integrations/index.md +++ b/website/docs/integrations/index.md @@ -42,7 +42,7 @@ Quick setup example: ```yaml web: - backend: firecrawl # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai + backend: firecrawl # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai ``` If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`. diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 8fb15ecadc..d79bda4892 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -151,6 +151,8 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe | `PARALLEL_API_KEY` | AI-native web search ([parallel.ai](https://parallel.ai/)) | | `FIRECRAWL_API_KEY` | Web scraping and cloud browser ([firecrawl.dev](https://firecrawl.dev/)) | | `FIRECRAWL_API_URL` | Custom Firecrawl API endpoint for self-hosted instances (optional) | +| `TAVILY_API_KEY` | Optional Tavily API key for higher search/extract limits. After selecting Tavily as the web backend, keyless access works without it ([app.tavily.com](https://app.tavily.com/home), [keyless docs](https://docs.tavily.com/documentation/keyless)) | +| `TAVILY_BASE_URL` | Override the Tavily API endpoint. Useful for corporate proxies and self-hosted Tavily-compatible search backends. Same pattern as `GROQ_BASE_URL`. | | `SEARXNG_URL` | SearXNG instance URL for free self-hosted web search — no API key required ([searxng.github.io](https://searxng.github.io/searxng/)) | | `EXA_API_KEY` | Exa API key for AI-native web search and contents ([exa.ai](https://exa.ai/)) | | `BRAVE_SEARCH_API_KEY` | Brave Search API subscription token for web search (free tier available) ([brave.com/search/api](https://brave.com/search/api/)) | diff --git a/website/docs/reference/tools-reference.md b/website/docs/reference/tools-reference.md index 3a3c725c68..1775066553 100644 --- a/website/docs/reference/tools-reference.md +++ b/website/docs/reference/tools-reference.md @@ -320,8 +320,8 @@ The single `video_generate` tool covers both modalities — pass `image_url` to | Tool | Description | Requires environment | |------|-------------|----------------------| -| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY | -| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY | +| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY | +| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY | ## `x_search` toolset diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index ef001c8a17..abbcc01620 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -2353,7 +2353,7 @@ The `web_search` and `web_extract` tools support five backend providers. Configu ```yaml web: - backend: firecrawl # firecrawl | searxng | parallel | keenable | exa + backend: firecrawl # firecrawl | searxng | parallel | tavily | keenable | exa # Or use per-capability keys to mix providers (e.g. free search + paid extract): search_backend: "searxng" @@ -2382,9 +2382,10 @@ web: | **Firecrawl** (default) | `FIRECRAWL_API_KEY` | ✔ | ✔ | | **SearXNG** | `SEARXNG_URL` | ✔ | — | | **Parallel** | `PARALLEL_API_KEY` (optional — keyless free tier) | ✔ | ✔ | +| **Tavily** | `TAVILY_API_KEY` (optional — keyless when selected; not in the free-tier ring) | ✔ | ✔ | | **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ | -**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Firecrawl or Keenable in `hermes tools` also works without a key. +**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key. **SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](/user-guide/features/web-search) for Docker setup instructions. diff --git a/website/docs/user-guide/features/web-dashboard.md b/website/docs/user-guide/features/web-dashboard.md index 17eb492b49..e3a8bbff75 100644 --- a/website/docs/user-guide/features/web-dashboard.md +++ b/website/docs/user-guide/features/web-dashboard.md @@ -235,7 +235,7 @@ Config changes take effect on the next agent session or gateway restart. The web Manage the `.env` file where API keys and credentials are stored. Keys are grouped by category: - **LLM Providers** — OpenRouter, Anthropic, OpenAI, DeepSeek, etc. -- **Tool API Keys** — Browserbase, Firecrawl, Keenable, ElevenLabs, etc. +- **Tool API Keys** — Browserbase, Firecrawl, Tavily, Keenable, ElevenLabs, etc. - **Messaging Platforms** — Telegram, Discord, Slack bot tokens, etc. - **Agent Settings** — non-secret env vars like `API_SERVER_ENABLED` diff --git a/website/docs/user-guide/features/web-search.md b/website/docs/user-guide/features/web-search.md index 44911dc5ab..f18345beb8 100644 --- a/website/docs/user-guide/features/web-search.md +++ b/website/docs/user-guide/features/web-search.md @@ -24,10 +24,11 @@ Both are configured through a single backend selection. Providers are chosen via | **DDGS (DuckDuckGo)** | — (no key) | ✔ | — | ✔ Free | | **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with key | | **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key | +| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Opt-in keyless when selected · not in the free-tier ring | | **Keenable** | `KEENABLE_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key | | **xAI (Grok)** | `XAI_API_KEY` or `hermes auth add xai-oauth` | ✔ | — | Paid (SuperGrok or per-token) | -Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below). +Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below). **Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below. @@ -266,13 +267,27 @@ SearXNG handles search; you need a separate provider for `web_extract`. Use the # ~/.hermes/config.yaml web: search_backend: "searxng" - extract_backend: "firecrawl" # or keenable, exa, parallel + extract_backend: "firecrawl" # or tavily, keenable, exa, parallel ``` With this config, Hermes uses SearXNG for all search queries and Firecrawl for URL extraction — combining free search with high-quality extraction. --- +### Tavily + +AI-optimised search and extract. Select Tavily in `hermes tools` (or set `web.backend: tavily`) to use it **keyless** with no account (rate-limited). Tavily is **not** in the zero-config free-tier ring — empty installs rotate across Exa / Parallel / Firecrawl / Keenable. Set an API key when you want higher limits. + +```bash +# optional — skip this for keyless access after selecting Tavily +# ~/.hermes/.env +TAVILY_API_KEY=tvly-your-key-here +``` + +Get a key at [app.tavily.com](https://app.tavily.com/home). See [Tavily keyless](https://docs.tavily.com/documentation/keyless). + +--- + ### Exa Neural search with semantic understanding. Good for research and finding conceptually related content. @@ -338,10 +353,10 @@ web: timeout: 90 # seconds (default) ``` -**Search-only** — pair with Firecrawl / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry. +**Search-only** — pair with Firecrawl / Tavily / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry. :::caution Trust model -Unlike index-backed providers (Brave, Keenable, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input. +Unlike index-backed providers (Brave, Tavily, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input. ::: --- @@ -355,7 +370,7 @@ Set one provider for all web capabilities: ```yaml # ~/.hermes/config.yaml web: - backend: "searxng" # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai + backend: "searxng" # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai ``` ### Per-capability configuration @@ -382,6 +397,7 @@ If no backend has **ever** been selected (no `web.backend` / per-capability key | Credential present | Auto-selected backend | |--------------------|-----------------------| +| `TAVILY_API_KEY` | tavily | | `EXA_API_KEY` | exa | | `PARALLEL_API_KEY` | parallel | | `FIRECRAWL_API_KEY` or `FIRECRAWL_API_URL` (or the Nous Tool Gateway is ready) | firecrawl | @@ -438,7 +454,7 @@ SearXNG cannot extract URL content. Set `web.extract_backend` to a provider that ```yaml web: search_backend: "searxng" - extract_backend: "firecrawl" # or keenable / exa / parallel + extract_backend: "firecrawl" # or tavily / keenable / exa / parallel ``` ### SearXNG returns 0 results