feat(web): add Tavily web search and extract provider

This commit re-introduces the Tavily provider, which supports both search and content extraction capabilities, which was removed in #99199.
This commit is contained in:
Lakshya Agarwal
2026-08-31 15:29:05 -04:00
committed by Teknium
parent f8f4d056f5
commit 428e084dcd
37 changed files with 934 additions and 64 deletions
+2 -2
View File
@@ -59,7 +59,7 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]:
# A function literally named ``web_search`` collides with Grok's native
# server-side tool (incomplete hang or HTTP 400 duplicate names); this alias
# avoids that while still dispatching through Hermes's configured provider
# (Firecrawl / Exa / …). Mapped back to ``web_search`` in normalize_response.
# (Firecrawl / Tavily / …). Mapped back to ``web_search`` in normalize_response.
_XAI_CLIENT_WEB_SEARCH_ALIAS = "hermes_web_search"
# OpenCode's /v1/responses endpoints (Zen and Go, including custom providers
@@ -661,7 +661,7 @@ class ResponsesApiTransport(ProviderTransport):
# fails): drop the client ``web_search`` function and declare
# xAI's built-in instead. 1:1 swap only when client ``web_search``
# was already present — never an additive grant.
# 2. **Client** (Firecrawl / Keenable / Exa / … configured or resolved):
# 2. **Client** (Firecrawl / Tavily / Exa / … configured or resolved):
# keep Hermes dispatch so ``web.backend`` / ``web.search_backend``
# is honored, but rename the wire tool to
# ``hermes_web_search`` so Grok cannot hijack the name. The alias
+3 -3
View File
@@ -13,8 +13,8 @@ Providers live in ``<repo>/plugins/web/<name>/`` (built-in, auto-loaded as
``plugins.enabled``).
This ABC is the SINGLE plugin-facing surface for web providers — every
provider in the tree (brave-free, ddgs, searxng, exa, parallel, keenable,
firecrawl) implements it. The legacy in-tree ``tools.web_providers.base``
provider in the tree (brave-free, ddgs, searxng, exa, parallel, tavily,
keenable, firecrawl) implements it. The legacy in-tree ``tools.web_providers.base``
ABCs were deleted in PR #25182 along with the per-vendor inline helpers
in ``tools/web_tools.py``; the response-shape contract documented below
is preserved bit-for-bit so the tool wrapper does not have to translate.
@@ -93,7 +93,7 @@ class WebSearchProvider(abc.ABC):
:meth:`search` / :meth:`extract`. The :meth:`supports_search` /
:meth:`supports_extract` capability flags let the registry route each
tool call to the right provider, and let multi-capability providers
(Firecrawl, Keenable, Exa, …) advertise multiple capabilities from a
(Firecrawl, Tavily, Exa, …) advertise multiple capabilities from a
single class.
"""
+4 -3
View File
@@ -16,7 +16,7 @@ The active provider is chosen by configuration with this precedence:
2. ``web.backend`` (shared fallback).
3. If exactly one capability-eligible provider is registered AND available,
use it.
4. Legacy preference order — ``firecrawl`` → ``parallel`` →
4. Legacy preference order — ``firecrawl`` → ``parallel`` → ``tavily`` →
``exa`` → ``searxng`` → ``brave-free`` → ``ddgs`` — filtered by
availability. Matches the historic ``tools.web_tools._get_backend()``
candidate order so installs that never set a config key keep landing
@@ -159,6 +159,7 @@ def _read_config_key(*path: str) -> Optional[str]:
_LEGACY_PREFERENCE = (
"firecrawl",
"parallel",
"tavily",
"exa",
"searxng",
"brave-free",
@@ -167,7 +168,7 @@ _LEGACY_PREFERENCE = (
# Keyless free-tier walk — strictly LAST-resort, tried only after the
# availability-filtered legacy walk finds nothing (i.e. the user has zero
# web credentials and no importable ddgs). All five vendors expose public
# web credentials and no importable ddgs). Ring vendors expose public
# anonymous free tiers (see plugins/web/keyless_mcp.py). Unpinned keyless
# traffic round-robins across the ring per request (the ring cursor lives
# in keyless_mcp; an explicit `hermes tools` pick bypasses this walk
@@ -220,7 +221,7 @@ def _resolve(configured: Optional[str], *, capability: str) -> Optional[WebSearc
supports *capability* AND ``is_available()`` reports True, return it.
3. **Legacy preference walk, filtered by availability.** Walk the
:data:`_LEGACY_PREFERENCE` order (firecrawl → parallel →
:data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → tavily →
exa → searxng → brave-free → ddgs) looking for a provider whose
``supports_<capability>()`` is True AND whose ``is_available()`` is
True. Matches the historic ``tools.web_tools._get_backend()``
+1 -1
View File
@@ -63,7 +63,7 @@ with open(os.path.join(hh, "config.yaml"), "w", encoding="utf-8") as f:
os.environ["HERMES_HOME"] = hh
# Strip web-fetch shortcuts: every arm must drive the browser.
os.environ.pop("BROWSER_USE_API_KEY", None)
for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "SERPER_API_KEY"):
for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "TAVILY_API_KEY", "SERPER_API_KEY"):
os.environ.pop(k, None)
os.environ["BU_CDP_URL"] = cdp
os.environ["PATH"] = (
+3 -1
View File
@@ -1111,6 +1111,7 @@ ENV_VARS_BY_VERSION: Dict[int, List[str]] = {
4: ["VOICE_TOOLS_OPENAI_KEY", "ELEVENLABS_API_KEY"],
5: ["WHATSAPP_ENABLED", "WHATSAPP_MODE", "WHATSAPP_ALLOWED_USERS",
"SLACK_BOT_TOKEN", "SLACK_APP_TOKEN", "SLACK_ALLOWED_USERS"],
10: ["TAVILY_API_KEY"],
11: ["TERMINAL_MODAL_MODE"],
}
@@ -1456,7 +1457,7 @@ def _is_env_config_key(key: str) -> bool:
'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY',
'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL',
'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME',
'TOOL_GATEWAY_USER_TOKEN',
'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY',
'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY',
'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN',
'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY',
@@ -5090,6 +5091,7 @@ def show_config():
("EXA_API_KEY", "Exa"),
("PARALLEL_API_KEY", "Parallel"),
("FIRECRAWL_API_KEY", "Firecrawl"),
("TAVILY_API_KEY", "Tavily"),
("BROWSERBASE_API_KEY", "Browserbase"),
("BROWSER_USE_API_KEY", "Browser Use"),
("FAL_KEY", "FAL"),
+13 -4
View File
@@ -555,7 +555,7 @@ DEFAULT_CONFIG = {
"extract_backend": "", # per-capability override for web_extract (e.g. "native")
"extract_char_limit": 15000, # per-page char budget for web_extract; larger pages truncate + store full text in cache/web
# Keyless free-tier ring: with NO web backend configured or keyed,
# web_search/web_extract rotate round-robin across five vendors'
# web_search/web_extract rotate round-robin across four vendors'
# public free tiers (exa, parallel, firecrawl, keenable),
# failing over to the next ring vendor on rate limits. Never
# pre-empts a configured or keyed backend. Set false to disable.
@@ -565,10 +565,11 @@ DEFAULT_CONFIG = {
# free-tier ring — the next call attempts the chosen backend again
# (no sticky failover). Off when keyless_fallback is false.
"keyless_rescue": True,
# Per-provider tier selection for ring vendors with both a keyless
# Per-provider tier selection for vendors with both a keyless
# free endpoint and a keyed paid path (exa, parallel,
# firecrawl, keenable). Set by the `hermes tools` picker's
# "Free (keyless)" / "Paid (API key)" rows.
# firecrawl, keenable on the ring; tavily is opt-in keyless via
# `hermes tools`, not a ring member). Set by the `hermes tools`
# picker's "Free (keyless)" / "Paid (API key)" rows.
# free — always use the anonymous free endpoint (even with a key)
# paid — always use the keyed path (missing key = error; vendor
# is also excluded from the keyless ring)
@@ -4513,6 +4514,14 @@ OPTIONAL_ENV_VARS = {
"category": "tool",
"advanced": True,
},
"TAVILY_API_KEY": {
"description": "Tavily API key for AI-native web search and extract (optional — keyless works when Tavily is selected)",
"prompt": "Tavily API key",
"url": "https://app.tavily.com/home",
"tools": ["web_search", "web_extract"],
"password": True,
"category": "tool",
},
"KEENABLE_API_KEY": {
"description": "Keenable API key for fast independent-index web search and page fetch (optional — keyless free tier works without it)",
"prompt": "Keenable API key",
+1
View File
@@ -388,6 +388,7 @@ def run_dump(args):
("COMMANDCODE_API_KEY", "commandcode"),
("KILOCODE_API_KEY", "kilocode"),
("FIRECRAWL_API_KEY", "firecrawl"),
("TAVILY_API_KEY", "tavily"),
("KEENABLE_API_KEY", "keenable"),
("BROWSERBASE_API_KEY", "browserbase"),
("FAL_KEY", "fal"),
+13
View File
@@ -505,6 +505,10 @@ def get_nous_subscription_features(
direct_exa = bool(get_env_value("EXA_API_KEY"))
direct_firecrawl = bool(get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL"))
direct_parallel = bool(get_env_value("PARALLEL_API_KEY"))
direct_tavily = bool(get_env_value("TAVILY_API_KEY"))
# Keyless Tavily is opt-in: selecting it in `hermes tools` / setup writes
# web.backend (or a per-capability override) without requiring a key.
tavily_selected = "tavily" in {web_backend, web_search_backend, web_extract_backend}
direct_searxng = bool(get_env_value("SEARXNG_URL"))
direct_fal = fal_key_is_configured()
direct_fal_video = direct_fal # same FAL_KEY; separate var so use_gateway is independent
@@ -536,6 +540,8 @@ def get_nous_subscription_features(
direct_firecrawl = False
direct_exa = False
direct_parallel = False
direct_tavily = False
tavily_selected = False
if image_use_gateway:
direct_fal = False
if video_use_gateway:
@@ -624,6 +630,7 @@ def get_nous_subscription_features(
direct_camofox = False
tavily_ready = direct_tavily or tavily_selected
web_managed = web_backend == "firecrawl" and managed_web_available and not direct_firecrawl
web_active = bool(
web_tool_enabled
@@ -632,6 +639,7 @@ def get_nous_subscription_features(
or (web_backend == "exa" and direct_exa)
or (web_backend == "firecrawl" and direct_firecrawl)
or (web_backend == "parallel" and direct_parallel)
or (web_backend == "tavily" and tavily_ready)
or (web_backend == "searxng" and direct_searxng)
# Per-capability overrides: search_backend or extract_backend may be set
# without web.backend (using the new split config from #20061)
@@ -639,6 +647,8 @@ def get_nous_subscription_features(
or (web_search_backend == "exa" and direct_exa)
or (web_search_backend == "firecrawl" and direct_firecrawl)
or (web_search_backend == "parallel" and direct_parallel)
or (web_search_backend == "tavily" and tavily_ready)
or (web_extract_backend == "tavily" and tavily_ready)
)
)
web_available = bool(
@@ -646,6 +656,7 @@ def get_nous_subscription_features(
or direct_exa
or direct_firecrawl
or direct_parallel
or tavily_ready
or direct_searxng
)
@@ -889,6 +900,7 @@ def apply_nous_managed_defaults(
if "web" in selected_toolsets and not features.web.explicit_configured and not (
get_env_value("PARALLEL_API_KEY")
or get_env_value("TAVILY_API_KEY")
or get_env_value("FIRECRAWL_API_KEY")
or get_env_value("FIRECRAWL_API_URL")
):
@@ -986,6 +998,7 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]:
get_env_value("FIRECRAWL_API_KEY")
or get_env_value("FIRECRAWL_API_URL")
or get_env_value("PARALLEL_API_KEY")
or get_env_value("TAVILY_API_KEY")
or get_env_value("EXA_API_KEY")
# Env-configured keyless local backend: a reachable self-hosted
# SearXNG is a working web setup even with no stored selection
+2 -2
View File
@@ -513,7 +513,7 @@ def _print_setup_summary(config: dict, hermes_home):
tool_status.append(("Vision (image analysis)", False, "run 'hermes setup' to configure"))
# Web tools (Exa, Parallel, Firecrawl, or Keenable)
# Web tools (Exa, Parallel, Firecrawl, Tavily, or Keenable)
if subscription_features.web.managed_by_nous:
tool_status.append(("Web Search & Extract (Nous subscription)", True, None))
elif subscription_features.web.available:
@@ -522,7 +522,7 @@ def _print_setup_summary(config: dict, hermes_home):
label = f"Web Search & Extract ({subscription_features.web.current_provider})"
tool_status.append((label, True, None))
else:
tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, KEENABLE_API_KEY, or SEARXNG_URL"))
tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, TAVILY_API_KEY, KEENABLE_API_KEY, or SEARXNG_URL"))
# Browser tools (local Chromium, Camofox, Browserbase, Browser Use, or Firecrawl)
browser_provider = subscription_features.browser.current_provider
+1
View File
@@ -184,6 +184,7 @@ def show_status(args):
"MiniMax-CN": "MINIMAX_CN_API_KEY",
"DeepInfra": "DEEPINFRA_API_KEY",
"Firecrawl": "FIRECRAWL_API_KEY",
"Tavily": "TAVILY_API_KEY",
"Keenable": "KEENABLE_API_KEY",
"Browser Use": "BROWSER_USE_API_KEY", # Optional — local browser works without this
"Browserbase": "BROWSERBASE_API_KEY", # Optional — direct credentials only
+4 -4
View File
@@ -3331,8 +3331,8 @@ def _plugin_video_gen_providers() -> list[dict]:
# Mirror of _plugin_image_gen_providers for web search backends. Surfaces
# every plugin-registered web provider so it appears in the
# "Web Search & Extract" picker. All seven providers (brave-free, ddgs,
# searxng, exa, parallel, firecrawl, keenable) live as plugins after
# "Web Search & Extract" picker. All bundled providers (brave-free, ddgs,
# searxng, exa, parallel, tavily, firecrawl, keenable) live as plugins after
# PR #25182 — this helper is the sole source of truth for the category's
# provider rows. The hardcoded entries that used to drive the category
# were deleted in the same PR; only the two non-provider UX rows
@@ -3348,8 +3348,8 @@ def _plugin_web_search_providers() -> list[dict]:
marker) so the picker behaves identically whether a provider is
hardcoded or plugin-registered.
After PR #25182, all seven web providers (brave-free, ddgs, searxng,
exa, parallel, firecrawl, keenable) are plugins; this helper is the sole
After PR #25182, all bundled web providers (brave-free, ddgs, searxng,
exa, parallel, tavily, firecrawl, keenable) are plugins; this helper is the sole
source of provider rows for the Web Search & Extract category.
"""
try:
+1 -1
View File
@@ -34,7 +34,7 @@ class BraveFreeWebSearchProvider(WebSearchProvider):
"""Search-only Brave provider using the free-tier Data-for-Search API.
Free tier is 2,000 queries/month (1 qps). No content-extraction capability —
users pair this with Firecrawl/Keenable/Exa for ``web_extract``.
users pair this with Firecrawl/Tavily/Exa for ``web_extract``.
"""
@property
+1 -1
View File
@@ -1,7 +1,7 @@
"""SearXNG search plugin — bundled, auto-loaded.
Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``).
Search-only — pair with an extract provider (firecrawl/keenable/exa) for
Search-only — pair with an extract provider (firecrawl/tavily/exa) for
``web_extract`` calls.
"""
+10
View File
@@ -0,0 +1,10 @@
"""Tavily web search + extract plugin — bundled, auto-loaded."""
from __future__ import annotations
from plugins.web.tavily.provider import TavilyWebSearchProvider
def register(ctx) -> None:
"""Register the Tavily provider with the plugin context."""
ctx.register_web_search_provider(TavilyWebSearchProvider())
+7
View File
@@ -0,0 +1,7 @@
name: web-tavily
version: 1.0.0
description: "Tavily web search + extract. Opt-in keyless via hermes tools; set TAVILY_API_KEY for higher limits — https://app.tavily.com/home."
author: NousResearch
kind: backend
provides_web_providers:
- tavily
+313
View File
@@ -0,0 +1,313 @@
"""Tavily web search + content extraction — plugin form.
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two
capabilities advertised:
- ``supports_search()`` -> True (Tavily ``/search``)
- ``supports_extract()`` -> True (Tavily ``/extract``)
Both are sync — the underlying call is ``httpx.post(...)``.
Config keys this provider responds to::
web:
search_backend: "tavily" # explicit per-capability
extract_backend: "tavily" # explicit per-capability
backend: "tavily" # shared fallback for both
Env vars::
TAVILY_API_KEY=... # https://app.tavily.com/home (optional)
TAVILY_BASE_URL=... # optional override of https://api.tavily.com
Auth is header-based. A key uses ``Authorization: Bearer``; without a
key the request is keyless (``X-Tavily-Access-Mode: keyless``). Both
paths send ``X-Client-Name: hermes-agent``.
Tavily is **not** a member of the zero-config keyless ring
(``plugins.web.keyless_mcp._KEYLESS_RING``). Keyless access is opt-in:
select Tavily in ``hermes tools`` (or set ``web.backend: tavily``).
Fresh installs with no web credentials rotate across Exa / Parallel /
Firecrawl / Keenable instead.
"""
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
import httpx
from agent.web_search_provider import WebSearchProvider
logger = logging.getLogger(__name__)
_CLIENT_NAME = "hermes-agent"
_SEARCH_PAYLOAD = {
"include_raw_content": False,
"include_images": False,
}
def _tavily_headers(api_key: str) -> Dict[str, str]:
"""Build Tavily request headers for keyed or keyless access."""
headers = {"X-Client-Name": _CLIENT_NAME}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
else:
headers["X-Tavily-Access-Mode"] = "keyless"
return headers
def _tavily_request(
endpoint: str,
payload: Dict[str, Any],
*,
api_key: Optional[str] = None,
) -> Dict[str, Any]:
"""POST to the Tavily API and return the parsed JSON response.
Keyed when *api_key* (or ``TAVILY_API_KEY``) is set (Bearer auth);
otherwise keyless. Pass ``api_key=""`` to force the keyless header even
when a key is present (``web.provider_tier.tavily: free``). Non-2xx
responses raise ``ValueError`` with the response body so Tavily's
keyless rate-limit / upgrade text reaches the model.
"""
from agent.web_search_provider import get_provider_env
if api_key is None:
api_key = get_provider_env("TAVILY_API_KEY")
base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
url = f"{base_url}/{endpoint.lstrip('/')}"
logger.info("Tavily %s request to %s", endpoint, url)
response = httpx.post(
url,
json=payload,
timeout=60,
headers=_tavily_headers(api_key),
)
if response.status_code >= 400:
body = (response.text or "").strip()
detail = body or f"HTTP {response.status_code}"
raise ValueError(detail)
return response.json()
def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]:
"""Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``."""
web_results = []
for i, result in enumerate(response.get("results", [])):
web_results.append(
{
"title": result.get("title", ""),
"url": result.get("url", ""),
"description": result.get("content", ""),
"position": i + 1,
}
)
return {"success": True, "data": {"web": web_results}}
def _normalize_tavily_documents(
response: Dict[str, Any], fallback_url: str = ""
) -> List[Dict[str, Any]]:
"""Map Tavily ``/extract`` response to standard documents.
Documents follow the legacy LLM post-processing shape::
{"url", "title", "content", "raw_content", "metadata"}
Failures (``failed_results``, ``failed_urls``) become result entries
with an ``error`` field rather than raising.
"""
documents: List[Dict[str, Any]] = []
for result in response.get("results", []):
url = result.get("url", fallback_url)
raw = result.get("raw_content", "") or result.get("content", "")
documents.append(
{
"url": url,
"title": result.get("title", ""),
"content": raw,
"raw_content": raw,
"metadata": {"sourceURL": url, "title": result.get("title", "")},
}
)
for fail in response.get("failed_results", []):
documents.append(
{
"url": fail.get("url", fallback_url),
"title": "",
"content": "",
"raw_content": "",
"error": fail.get("error", "extraction failed"),
"metadata": {"sourceURL": fail.get("url", fallback_url)},
}
)
for fail_url in response.get("failed_urls", []):
url_str = fail_url if isinstance(fail_url, str) else str(fail_url)
documents.append(
{
"url": url_str,
"title": "",
"content": "",
"raw_content": "",
"error": "extraction failed",
"metadata": {"sourceURL": url_str},
}
)
return documents
def _missing_key_error(action: str) -> str:
return (
f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home "
f"or select Tavily in `hermes tools` for opt-in keyless {action}."
)
class TavilyWebSearchProvider(WebSearchProvider):
"""Tavily search + extract provider (keyed, or opt-in keyless)."""
@property
def name(self) -> str:
return "tavily"
@property
def display_name(self) -> str:
return "Tavily"
def is_available(self) -> bool:
"""Return True when ``TAVILY_API_KEY`` is set to a non-empty value."""
from agent.web_search_provider import get_provider_env
return bool(get_provider_env("TAVILY_API_KEY"))
def is_keyless_available(self) -> bool:
"""Tavily serves anonymous keyless requests (X-Tavily-Access-Mode).
Opt-in only — Tavily is not a member of the zero-config keyless
ring. ``is_keyless_available`` is True so an explicit
``web.backend: tavily`` (or ``hermes tools`` pick) works without a
key. False when the user pinned ``web.provider_tier.tavily: paid``.
"""
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
return keyless_enabled() and provider_tier("tavily") != "paid"
def supports_search(self) -> bool:
return True
def supports_extract(self) -> bool:
return True
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
"""Execute a Tavily search (keyed path or opt-in keyless)."""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return {"success": False, "error": "Interrupted"}
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import use_keyless
api_key = get_provider_env("TAVILY_API_KEY")
force_keyless = use_keyless("tavily", api_key)
if not force_keyless and not api_key:
return {"success": False, "error": _missing_key_error("search")}
logger.info(
"Tavily %ssearch: '%s' (limit=%d)",
"keyless " if force_keyless else "",
query,
limit,
)
raw = _tavily_request(
"search",
{
"query": query,
"max_results": min(limit, 20),
**_SEARCH_PAYLOAD,
},
api_key="" if force_keyless else api_key,
)
return _normalize_tavily_search_results(raw)
except ValueError as exc:
return {"success": False, "error": str(exc)}
except Exception as exc: # noqa: BLE001 — including httpx errors
logger.warning("Tavily search error: %s", exc)
return {"success": False, "error": f"Tavily search failed: {exc}"}
def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
"""Extract content from one or more URLs via Tavily.
Sync — the underlying call is httpx.post(...). Returns the legacy
list-of-results shape; per-URL failures become items with ``error``.
Keyless uses Tavily's own endpoint, not the keyless ring.
"""
try:
from tools.interrupt import is_interrupted
if is_interrupted():
return [
{"url": u, "error": "Interrupted", "title": ""} for u in urls
]
from agent.web_search_provider import get_provider_env
from plugins.web.keyless_mcp import use_keyless
api_key = get_provider_env("TAVILY_API_KEY")
force_keyless = use_keyless("tavily", api_key)
if not force_keyless and not api_key:
err = _missing_key_error("extract")
return [
{"url": u, "title": "", "content": "", "error": err}
for u in urls
]
logger.info(
"Tavily %sextract: %d URL(s)",
"keyless " if force_keyless else "",
len(urls),
)
raw = _tavily_request(
"extract",
{
"urls": urls,
"include_images": False,
},
api_key="" if force_keyless else api_key,
)
return _normalize_tavily_documents(
raw, fallback_url=urls[0] if urls else ""
)
except ValueError as exc:
return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
except Exception as exc: # noqa: BLE001
logger.warning("Tavily extract error: %s", exc)
return [
{"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"}
for u in urls
]
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Tavily",
"badge": "free · key optional",
"tag": (
"Search + extract. Opt-in keyless (not in the free-tier ring); "
"set TAVILY_API_KEY for higher limits."
),
"env_vars": [
{
"key": "TAVILY_API_KEY",
"prompt": "Tavily API key (optional — keyless works when Tavily is selected)",
"url": "https://app.tavily.com/home",
},
],
}
+2 -2
View File
@@ -101,12 +101,12 @@ class XAIWebSearchProvider(WebSearchProvider):
back to the Responses API ``citations`` list if Grok ignores the JSON
schema instruction (rare for grok-4.3 but cheap insurance).
No extract capability — pair with Firecrawl / Keenable / Exa for
No extract capability — pair with Firecrawl / Tavily / Exa for
``web_extract`` if you need page content.
Trust model
-----------
Unlike index-backed providers (Brave / Keenable / Exa) which return
Unlike index-backed providers (Brave / Tavily / Exa) which return
verbatim search-engine results, this backend is an LLM in a trench
coat: Grok decides which URLs to surface, generates the titles and
descriptions itself, and is influenced by the *content of the query*.
+1 -1
View File
@@ -186,7 +186,7 @@ _CREDENTIAL_NAMES = frozenset({
"FIRECRAWL_API_KEY",
"PARALLEL_API_KEY",
"EXA_API_KEY",
"TAVILY_API_KEY", # removed backend; still blanked for hermeticity
"TAVILY_API_KEY",
"WANDB_API_KEY",
"ELEVENLABS_API_KEY",
"HONCHO_API_KEY",
+13 -3
View File
@@ -666,13 +666,23 @@ class TestOptionalEnvVarsRegistry:
from hermes_cli.config import OPTIONAL_ENV_VARS
assert OPTIONAL_ENV_VARS["KEENABLE_API_KEY"]["url"] == "https://keenable.ai"
def test_removed_tavily_var_not_in_env_vars_by_version(self):
"""TAVILY_API_KEY was removed with the Tavily backend."""
def test_tavily_api_key_registered(self):
"""TAVILY_API_KEY is listed in OPTIONAL_ENV_VARS."""
from hermes_cli.config import OPTIONAL_ENV_VARS
assert "TAVILY_API_KEY" in OPTIONAL_ENV_VARS
def test_tavily_api_key_has_url(self):
"""TAVILY_API_KEY has a URL."""
from hermes_cli.config import OPTIONAL_ENV_VARS
assert OPTIONAL_ENV_VARS["TAVILY_API_KEY"]["url"] == "https://app.tavily.com/home"
def test_tavily_in_env_vars_by_version(self):
"""TAVILY_API_KEY is listed in ENV_VARS_BY_VERSION."""
from hermes_cli.config import ENV_VARS_BY_VERSION
all_vars = []
for vars_list in ENV_VARS_BY_VERSION.values():
all_vars.extend(vars_list)
assert "TAVILY_API_KEY" not in all_vars
assert "TAVILY_API_KEY" in all_vars
def test_max_iterations_not_offered_as_env_var(self):
"""HERMES_MAX_ITERATIONS must NOT be in OPTIONAL_ENV_VARS (issue #17534).
@@ -47,6 +47,7 @@ def test_dump_leaves_unset_key_untouched(monkeypatch, capsys, tmp_path):
monkeypatch.setattr(dump, "get_project_root", lambda: tmp_path / "noproject")
monkeypatch.delenv("KEENABLE_API_KEY", raising=False)
monkeypatch.delenv("TAVILY_API_KEY", raising=False)
home = get_hermes_home()
home.mkdir(parents=True, exist_ok=True)
@@ -58,6 +58,52 @@ def test_get_nous_subscription_features_recognizes_direct_exa_backend(monkeypatc
assert features.web.current_provider == "exa"
def test_get_nous_subscription_features_recognizes_keyless_tavily_backend(monkeypatch):
"""Selecting Tavily in setup/tools counts as available with no API key.
Mirrors tools.web_tools._is_backend_available('tavily'): keyless is
opt-in via web.backend / search_backend / extract_backend, not a
silent empty-install default. The setup summary previously required
TAVILY_API_KEY and printed a false 'missing' after a skipped key prompt.
"""
monkeypatch.setattr(ns, "get_env_value", lambda name: "")
monkeypatch.setattr(
ns, "get_nous_portal_account_info", lambda: _account(logged_in=False)
)
monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web")
monkeypatch.setattr(ns, "_has_agent_browser", lambda: False)
monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "")
monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False)
features = ns.get_nous_subscription_features({"web": {"backend": "tavily"}})
assert features.web.available is True
assert features.web.active is True
assert features.web.managed_by_nous is False
assert features.web.direct_override is True
assert features.web.current_provider == "tavily"
assert features.web.explicit_configured is True
def test_keyless_tavily_search_backend_without_shared_backend(monkeypatch):
monkeypatch.setattr(ns, "get_env_value", lambda name: "")
monkeypatch.setattr(
ns, "get_nous_portal_account_info", lambda: _account(logged_in=False)
)
monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web")
monkeypatch.setattr(ns, "_has_agent_browser", lambda: False)
monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "")
monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False)
features = ns.get_nous_subscription_features(
{"web": {"search_backend": "tavily"}}
)
assert features.web.available is True
assert features.web.active is True
assert features.web.current_provider == "tavily"
def test_unconfigured_web_without_keys_is_unavailable(monkeypatch):
monkeypatch.setattr(ns, "get_env_value", lambda name: "")
monkeypatch.setattr(
+12
View File
@@ -15,6 +15,18 @@ def test_show_status_all_does_not_print_keenable_key_value(monkeypatch, capsys,
assert sentinel not in output
def test_show_status_all_does_not_print_tavily_key_value(monkeypatch, capsys, tmp_path):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
sentinel = "NONSECRET_SENTINEL_VALUE_DO_NOT_PRINT_TAVILY_123456"
monkeypatch.setenv("TAVILY_API_KEY", sentinel)
show_status(SimpleNamespace(all=True, deep=False))
output = capsys.readouterr().out
assert "Tavily" in output
assert sentinel not in output
def test_show_status_termux_gateway_section_skips_systemctl(monkeypatch, capsys, tmp_path):
from hermes_cli import status as status_mod
import hermes_cli.auth as auth_mod
+1
View File
@@ -245,6 +245,7 @@ def test_first_install_nous_auto_configures_video_gen(monkeypatch):
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"KEENABLE_API_KEY",
"TAVILY_API_KEY",
"PARALLEL_API_KEY",
"BROWSERBASE_API_KEY",
"BROWSERBASE_PROJECT_ID",
@@ -3,7 +3,7 @@
Covers:
- All bundled plugins (brave-free, ddgs, searxng, exa, parallel,
firecrawl, keenable, xai) instantiate and self-report the expected
tavily, firecrawl, keenable, xai) instantiate and self-report the expected
capabilities + ABC-derived defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The web_search_registry resolves an active provider in the documented
@@ -35,6 +35,8 @@ def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
"BRAVE_SEARCH_API_KEY",
"SEARXNG_URL",
"KEENABLE_API_KEY",
"TAVILY_API_KEY",
"TAVILY_BASE_URL",
"EXA_API_KEY",
"PARALLEL_API_KEY",
"PARALLEL_SEARCH_MODE",
@@ -82,6 +84,7 @@ class TestBundledPluginsRegister:
"keenable",
"parallel",
"searxng",
"tavily",
"xai",
]
@@ -94,6 +97,7 @@ class TestBundledPluginsRegister:
("exa", True, True),
("parallel", True, True),
("keenable", True, True),
("tavily", True, True),
("firecrawl", True, True),
# xai: search-only via Grok's agentic web_search tool.
("xai", True, False),
@@ -115,7 +119,7 @@ class TestBundledPluginsRegister:
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "firecrawl", "keenable", "xai"],
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "keenable", "xai"],
)
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
@@ -165,6 +169,16 @@ class TestIsAvailable:
monkeypatch.setenv("KEENABLE_API_KEY", "real")
assert p.is_available() is True
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("TAVILY_API_KEY", "real")
assert p.is_available() is True
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
+2
View File
@@ -83,6 +83,7 @@ def register_all_web_providers():
from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider
from plugins.web.parallel.provider import ParallelWebSearchProvider
from plugins.web.keenable.provider import KeenableWebSearchProvider
from plugins.web.tavily.provider import TavilyWebSearchProvider
from plugins.web.searxng.provider import SearXNGWebSearchProvider
from plugins.web.xai.provider import XAIWebSearchProvider
@@ -94,6 +95,7 @@ def register_all_web_providers():
FirecrawlWebSearchProvider,
ParallelWebSearchProvider,
KeenableWebSearchProvider,
TavilyWebSearchProvider,
SearXNGWebSearchProvider,
XAIWebSearchProvider,
):
+11 -2
View File
@@ -25,7 +25,7 @@ from plugins.web.parallel.provider import ParallelWebSearchProvider
def _no_web_env(monkeypatch):
"""Blank every web credential and neutralize config lookups."""
for var in (
"EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY",
"EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY", "TAVILY_API_KEY",
"FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "BRAVE_SEARCH_API_KEY",
"SEARXNG_URL", "TOOL_GATEWAY_USER_TOKEN",
):
@@ -289,7 +289,7 @@ class TestResolutionOrder:
def test_keyless_ring_rotates_and_covers_all_vendors(self, fresh_registry, monkeypatch):
monkeypatch.setattr(registry, "_read_config_key", lambda *p: None)
# The ring order always contains all five vendors, starting at the
# The ring order always contains all four vendors, starting at the
# current cursor and wrapping.
order = registry._keyless_preference()
assert sorted(order) == sorted(keyless_mcp._KEYLESS_RING)
@@ -303,6 +303,15 @@ class TestResolutionOrder:
assert keyless_mcp._ring_order("keenable")[0] == "keenable"
assert keyless_mcp._ring_order("keenable")[0] == "keenable"
def test_tavily_is_not_a_ring_member(self):
"""Tavily is opt-in keyless; zero-config rotation must not include it."""
from plugins.web import keyless_mcp
assert "tavily" not in keyless_mcp._KEYLESS_RING
assert "tavily" not in keyless_mcp._KEYLESS_SEARCHERS
assert "tavily" not in keyless_mcp._KEYLESS_EXTRACTORS
assert "tavily" not in registry._KEYLESS_PREFERENCE
def test_registry_keyless_disabled_returns_none(self, fresh_registry, monkeypatch):
monkeypatch.setattr(registry, "_read_config_key", lambda *p: None)
monkeypatch.setattr(registry, "_keyless_tier_enabled", lambda: False)
+59 -2
View File
@@ -210,6 +210,7 @@ class TestBackendSelection:
"TOOL_GATEWAY_SCHEME",
"TOOL_GATEWAY_USER_TOKEN",
"KEENABLE_API_KEY",
"TAVILY_API_KEY",
)
def setup_method(self):
@@ -254,7 +255,7 @@ class TestBackendSelection:
assert _get_backend() == "exa"
def test_fallback_exa_takes_priority_over_parallel(self):
"""Direct-credential backends are tried in the order exa > parallel > keenable
"""Direct-credential backends are tried in the order tavily > exa > parallel > keenable
so an explicit Exa key wins when both Exa and Parallel are configured."""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
@@ -275,6 +276,27 @@ class TestBackendSelection:
patch.dict(os.environ, {"EXA_API_KEY": "exa-test", "FIRECRAWL_API_KEY": "fc-test"}):
assert _get_backend() == "exa"
def test_fallback_tavily_only_key(self):
"""Only TAVILY_API_KEY set → 'tavily'."""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}):
assert _get_backend() == "tavily"
def test_fallback_tavily_beats_firecrawl_direct(self):
"""Tavily ranks above firecrawl in the explicit-credential block."""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "FIRECRAWL_API_KEY": "fc-test"}):
assert _get_backend() == "tavily"
def test_fallback_tavily_beats_exa(self):
"""Tavily ranks above Exa in the explicit-credential block."""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "EXA_API_KEY": "exa-test"}):
assert _get_backend() == "tavily"
def test_fallback_parallel_beats_firecrawl_direct(self):
"""Parallel + Firecrawl-direct → parallel (parallel is the higher-priority
@@ -342,6 +364,14 @@ class TestBackendSelection:
patch.dict(os.environ, {"EXA_API_KEY": "exa-test"}):
assert _get_backend() == "exa"
def test_managed_gateway_does_not_preempt_explicit_tavily(self):
"""A Nous OAuth token must not beat an explicit TAVILY_API_KEY."""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}):
assert _get_backend() == "tavily"
def test_managed_gateway_only_falls_through_to_firecrawl(self):
"""When no explicit-credential backend is configured, a Nous-managed
gateway token still selects firecrawl — the convenience path is
@@ -494,6 +524,7 @@ class TestCheckWebApiKey:
"TOOL_GATEWAY_SCHEME",
"TOOL_GATEWAY_USER_TOKEN",
"KEENABLE_API_KEY",
"TAVILY_API_KEY",
)
def setup_method(self):
@@ -597,7 +628,9 @@ class TestCheckWebApiKey:
def test_web_requires_env_includes_exa_key():
from tools.web_tools import _web_requires_env
assert "EXA_API_KEY" in _web_requires_env()
env = _web_requires_env()
assert "EXA_API_KEY" in env
assert "TAVILY_API_KEY" in env
class TestNonBuiltinProviderAvailability:
@@ -625,6 +658,7 @@ class TestNonBuiltinProviderAvailability:
"TOOL_GATEWAY_SCHEME",
"TOOL_GATEWAY_USER_TOKEN",
"KEENABLE_API_KEY",
"TAVILY_API_KEY",
"SEARXNG_URL",
"BRAVE_SEARCH_API_KEY",
"XAI_API_KEY",
@@ -765,6 +799,7 @@ class TestSiblingProvidersEnvResolution:
("plugins.web.exa.provider", "ExaWebSearchProvider", "EXA_API_KEY"),
("plugins.web.parallel.provider", "ParallelWebSearchProvider", "PARALLEL_API_KEY"),
("plugins.web.keenable.provider", "KeenableWebSearchProvider", "KEENABLE_API_KEY"),
("plugins.web.tavily.provider", "TavilyWebSearchProvider", "TAVILY_API_KEY"),
("plugins.web.brave_free.provider", "BraveFreeWebSearchProvider", "BRAVE_SEARCH_API_KEY"),
]
@@ -811,6 +846,28 @@ class TestSiblingProvidersEnvResolution:
assert headers["Authorization"] == "Bearer kn-from-dotenv"
assert headers["X-Keenable-Title"] == "hermes-agent"
def test_tavily_request_reads_key_via_get_env_value(self, monkeypatch):
"""Keyed Tavily must Bearer-auth with a key that lives only in .env."""
monkeypatch.delenv("TAVILY_API_KEY", raising=False)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"results": []}
mock_response.text = "{}"
with patch(
"hermes_cli.config.get_env_value",
side_effect=lambda k: "tvly-from-dotenv" if k == "TAVILY_API_KEY" else None,
), patch(
"plugins.web.tavily.provider.httpx.post", return_value=mock_response
) as mock_post:
from plugins.web.tavily.provider import _tavily_request
_tavily_request("search", {"query": "q"})
headers = mock_post.call_args.kwargs["headers"]
assert headers["Authorization"] == "Bearer tvly-from-dotenv"
assert headers["X-Client-Name"] == "hermes-agent"
assert "X-Tavily-Access-Mode" not in headers
def test_get_provider_env_unset_returns_empty(self, monkeypatch):
monkeypatch.delenv("WSP_TEST_UNSET_KEY", raising=False)
+317
View File
@@ -0,0 +1,317 @@
"""Tests for Tavily web backend integration.
Coverage:
_tavily_request() — keyed Bearer vs keyless header, attribution, error bodies.
_normalize_tavily_search_results() — search response normalization.
_normalize_tavily_documents() — extract response normalization, failed_results.
web_search_tool / web_extract_tool — Tavily dispatch paths.
auto-detect ranking — keyed paid-band; keyless only when Tavily is selected.
"""
import json
import os
import asyncio
import pytest
from unittest.mock import patch, MagicMock
from tests.tools.conftest import register_all_web_providers
def _ok_response(payload=None):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = payload if payload is not None else {"results": []}
mock_response.text = json.dumps(mock_response.json.return_value)
return mock_response
# ─── _tavily_request ─────────────────────────────────────────────────────────
class TestTavilyRequest:
"""Test suite for the _tavily_request helper."""
def test_keyless_when_no_api_key(self):
"""No TAVILY_API_KEY → keyless header, no Authorization, no body key."""
mock_response = _ok_response()
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("TAVILY_API_KEY", None)
with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post:
from plugins.web.tavily.provider import _tavily_request
_tavily_request("search", {"query": "test"})
mock_post.assert_called_once()
headers = mock_post.call_args.kwargs["headers"]
payload = mock_post.call_args.kwargs["json"]
assert headers["X-Client-Name"] == "hermes-agent"
assert headers["X-Tavily-Access-Mode"] == "keyless"
assert "Authorization" not in headers
assert "api_key" not in payload
assert payload["query"] == "test"
assert "api.tavily.com/search" in mock_post.call_args.args[0]
def test_keyed_uses_bearer_not_body(self):
"""TAVILY_API_KEY → Bearer auth, attribution, no body api_key."""
mock_response = _ok_response()
with patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test-key"}):
with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post:
from plugins.web.tavily.provider import _tavily_request
_tavily_request("search", {"query": "hello"})
mock_post.assert_called_once()
headers = mock_post.call_args.kwargs["headers"]
payload = mock_post.call_args.kwargs["json"]
assert headers == {
"X-Client-Name": "hermes-agent",
"Authorization": "Bearer tvly-test-key",
}
assert "X-Tavily-Access-Mode" not in headers
assert "api_key" not in payload
assert payload["query"] == "hello"
assert "api.tavily.com/search" in mock_post.call_args.args[0]
def test_http_error_surfaces_response_body(self):
"""Non-2xx responses raise ValueError with Tavily's response body."""
mock_response = MagicMock()
mock_response.status_code = 429
mock_response.text = "Rate limit hit. Sign up for a free API key at https://app.tavily.com"
mock_response.json.return_value = {}
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("TAVILY_API_KEY", None)
with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response):
from plugins.web.tavily.provider import _tavily_request
with pytest.raises(ValueError, match="Rate limit hit"):
_tavily_request("search", {"query": "test"})
# ─── _normalize_tavily_search_results ─────────────────────────────────────────
class TestNormalizeTavilySearchResults:
"""Test search result normalization."""
def test_basic_normalization(self):
from tools.web_tools import _normalize_tavily_search_results
raw = {
"results": [
{"title": "Python Docs", "url": "https://docs.python.org", "content": "Official docs", "score": 0.9},
{"title": "Tutorial", "url": "https://example.com", "content": "A tutorial", "score": 0.8},
]
}
result = _normalize_tavily_search_results(raw)
assert result["success"] is True
web = result["data"]["web"]
assert len(web) == 2
assert web[0]["title"] == "Python Docs"
assert web[0]["url"] == "https://docs.python.org"
assert web[0]["description"] == "Official docs"
assert web[0]["position"] == 1
assert web[1]["position"] == 2
def test_missing_fields(self):
from tools.web_tools import _normalize_tavily_search_results
result = _normalize_tavily_search_results({"results": [{}]})
web = result["data"]["web"]
assert web[0]["title"] == ""
assert web[0]["url"] == ""
assert web[0]["description"] == ""
# ─── _normalize_tavily_documents ──────────────────────────────────────────────
class TestNormalizeTavilyDocuments:
"""Test extract document normalization."""
def test_basic_document(self):
from tools.web_tools import _normalize_tavily_documents
raw = {
"results": [{
"url": "https://example.com",
"title": "Example",
"raw_content": "Full page content here",
}]
}
docs = _normalize_tavily_documents(raw)
assert len(docs) == 1
assert docs[0]["url"] == "https://example.com"
assert docs[0]["title"] == "Example"
assert docs[0]["content"] == "Full page content here"
assert docs[0]["raw_content"] == "Full page content here"
assert docs[0]["metadata"]["sourceURL"] == "https://example.com"
def test_fallback_url(self):
from tools.web_tools import _normalize_tavily_documents
raw = {"results": [{"content": "data"}]}
docs = _normalize_tavily_documents(raw, fallback_url="https://fallback.com")
assert docs[0]["url"] == "https://fallback.com"
# ─── availability / auto-detect ───────────────────────────────────────────────
class TestTavilyAvailability:
"""Keyed Tavily stays in the paid band; keyless only when selected."""
def test_is_available_without_key(self):
from plugins.web.tavily.provider import TavilyWebSearchProvider
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("TAVILY_API_KEY", None)
assert TavilyWebSearchProvider().is_available() is False
def test_is_backend_available_without_key(self):
from tools.web_tools import _is_backend_available
with patch("tools.web_tools._load_web_config", return_value={}), \
patch.dict(os.environ, {}, clear=False):
os.environ.pop("TAVILY_API_KEY", None)
assert _is_backend_available("tavily") is False
def test_is_backend_available_when_configured_without_key(self):
from tools.web_tools import _is_backend_available
with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \
patch.dict(os.environ, {}, clear=False):
os.environ.pop("TAVILY_API_KEY", None)
assert _is_backend_available("tavily") is True
def test_keyless_does_not_preempt_managed_firecrawl(self):
"""No TAVILY_API_KEY + Nous gateway ready → firecrawl, not keyless tavily."""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \
patch("tools.web_tools._ddgs_package_importable", return_value=False):
os.environ.pop("TAVILY_API_KEY", None)
assert _get_backend() == "firecrawl"
def test_keyless_does_not_preempt_ddgs(self):
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \
patch("tools.web_tools._ddgs_package_importable", return_value=True):
os.environ.pop("TAVILY_API_KEY", None)
assert _get_backend() == "ddgs"
def test_no_keys_defaults_to_firecrawl(self):
"""Keyless tier disabled: zero-credential resolve hits the legacy
firecrawl sentinel. (With the tier on — the default — it resolves
to the Exa/Parallel keyless split; see test_web_keyless_fallback.py.)
"""
from tools.web_tools import _get_backend
with patch("tools.web_tools._load_web_config", return_value={}), \
patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \
patch("tools.web_tools._ddgs_package_importable", return_value=False), \
patch("tools.web_tools._list_registered_web_providers", return_value=[]), \
patch("agent.web_search_registry._keyless_tier_enabled", return_value=False):
os.environ.pop("TAVILY_API_KEY", None)
assert _get_backend() == "firecrawl"
def test_explicit_search_backend_tavily_without_key(self):
"""web.search_backend=tavily sticks even with no TAVILY_API_KEY."""
from tools.web_tools import _get_search_backend
with patch("tools.web_tools._load_web_config",
return_value={"backend": "firecrawl", "search_backend": "tavily"}), \
patch("tools.web_tools._is_tool_gateway_ready", return_value=True):
os.environ.pop("TAVILY_API_KEY", None)
assert _get_search_backend() == "tavily"
def test_check_web_api_key_when_tavily_configured_without_key(self):
from tools.web_tools import check_web_api_key
with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \
patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \
patch("tools.web_tools.check_firecrawl_api_key", return_value=False), \
patch("tools.web_tools._ddgs_package_importable", return_value=False), \
patch("agent.web_search_registry.get_active_search_provider", return_value=None), \
patch("agent.web_search_registry.get_active_extract_provider", return_value=None):
os.environ.pop("TAVILY_API_KEY", None)
assert check_web_api_key() is True
# ─── web_search_tool (Tavily dispatch) ────────────────────────────────────────
class TestWebSearchTavily:
"""Test web_search_tool dispatch to Tavily."""
_register_providers = staticmethod(register_all_web_providers)
@pytest.fixture(autouse=True)
def _populate_web_registry(self):
self._register_providers()
yield
from agent.web_search_registry import _reset_for_tests
_reset_for_tests()
def test_search_dispatches_to_tavily(self):
mock_response = _ok_response({
"results": [{"title": "Result", "url": "https://r.com", "content": "desc", "score": 0.9}]
})
with patch("tools.web_tools._get_backend", return_value="tavily"), \
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \
patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \
patch("tools.interrupt.is_interrupted", return_value=False):
from tools.web_tools import web_search_tool
result = json.loads(web_search_tool("test query", limit=3))
assert result["success"] is True
assert len(result["data"]["web"]) == 1
assert result["data"]["web"][0]["title"] == "Result"
def test_search_keyless_dispatch(self):
"""Opt-in keyless Tavily hits Tavily's own endpoint, not the ring."""
mock_response = _ok_response({
"results": [{"title": "Result", "url": "https://r.com", "content": "desc"}]
})
with patch("tools.web_tools._get_backend", return_value="tavily"), \
patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post, \
patch("tools.interrupt.is_interrupted", return_value=False):
os.environ.pop("TAVILY_API_KEY", None)
from tools.web_tools import web_search_tool
result = json.loads(web_search_tool("test query"))
assert result["success"] is True
headers = mock_post.call_args.kwargs["headers"]
assert headers["X-Tavily-Access-Mode"] == "keyless"
assert headers["X-Client-Name"] == "hermes-agent"
assert "Authorization" not in headers
assert "api.tavily.com/search" in mock_post.call_args.args[0]
def test_tavily_is_not_in_keyless_ring(self):
from plugins.web.keyless_mcp import _KEYLESS_RING, _KEYLESS_SEARCHERS, _KEYLESS_EXTRACTORS
assert "tavily" not in _KEYLESS_RING
assert "tavily" not in _KEYLESS_SEARCHERS
assert "tavily" not in _KEYLESS_EXTRACTORS
# ─── web_extract_tool (Tavily dispatch) ───────────────────────────────────────
class TestWebExtractTavily:
"""Test web_extract_tool dispatch to Tavily."""
_register_providers = staticmethod(register_all_web_providers)
@pytest.fixture(autouse=True)
def _populate_web_registry(self):
self._register_providers()
yield
from agent.web_search_registry import _reset_for_tests
_reset_for_tests()
def test_extract_dispatches_to_tavily(self):
mock_response = _ok_response({
"results": [{"url": "https://example.com", "raw_content": "Extracted content", "title": "Page"}]
})
async def _allow_ssrf(_url: str) -> bool:
return True
with patch("tools.web_tools._get_backend", return_value="tavily"), \
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \
patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \
patch("tools.web_tools.async_is_safe_url", _allow_ssrf):
from tools.web_tools import web_extract_tool
result = json.loads(asyncio.get_event_loop().run_until_complete(
web_extract_tool(["https://example.com"])
))
assert "results" in result
assert len(result["results"]) == 1
assert result["results"][0]["url"] == "https://example.com"
assert "Extracted content" in result["results"][0]["content"]
+1 -1
View File
@@ -21,7 +21,7 @@ Limitations:
connects to the validated IP while preserving Host/SNI semantics.
- Redirect-based bypass is mitigated by httpx event hooks that re-validate
each redirect target in vision_tools, gateway platform adapters, and
media cache helpers. Web tools use third-party SDKs (Firecrawl/Exa)
media cache helpers. Web tools use third-party SDKs (Firecrawl/Tavily)
where redirect handling is on their servers.
"""
+40 -15
View File
@@ -15,6 +15,7 @@ Backend compatibility:
- Exa: https://exa.ai (search, extract)
- Firecrawl: https://docs.firecrawl.dev/introduction (search, extract; direct or derived firecrawl-gateway.<domain> for Nous Subscribers)
- Parallel: https://docs.parallel.ai (search, extract)
- Tavily: https://tavily.com (search, extract; keyed or opt-in keyless, not in the free-tier ring)
LLM Processing:
- Uses OpenRouter API with Gemini 3 Flash Preview for intelligent content extraction
@@ -57,6 +58,13 @@ from plugins.web.firecrawl.provider import (
_is_tool_gateway_ready,
check_firecrawl_api_key,
)
# Tavily helpers re-exported for backward-compat with existing unit tests
# (tests/tools/test_web_tools_tavily.py imports these names directly).
from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names
_normalize_tavily_documents,
_normalize_tavily_search_results,
_tavily_request,
)
# Parallel + Exa clients re-exported for backward-compat with existing
# unit tests (tests/tools/test_web_tools_config.py imports _get_parallel_client
# / _get_async_parallel_client / _get_exa_client directly).
@@ -161,7 +169,7 @@ def _load_web_config() -> dict:
# WebSearchProvider. Keep the two sets aligned by hand: if xai ever ships as
# a registered provider, drop it here so the registry path takes over.
_LEGACY_WEB_BACKENDS = frozenset(
{"parallel", "firecrawl", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"}
{"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"}
)
@@ -244,13 +252,14 @@ def _get_backend() -> str:
return "firecrawl"
# Never-configured install — pick the highest-priority available
# backend. Explicit user credentials (EXA_API_KEY etc.)
# backend. Explicit user credentials (TAVILY_API_KEY etc.)
# beat the managed-tool-gateway probe so a deliberate setup is not
# pre-empted by a Nous OAuth token whose subscription tier may not
# actually grant web-search access (the gateway then fails at runtime
# with "no subscription" and the tool returns an error to the agent
# without falling back). Free-tier backends trail the paid ones.
backend_candidates = (
("tavily", _has_env("TAVILY_API_KEY")),
("exa", _has_env("EXA_API_KEY")),
("parallel", _has_env("PARALLEL_API_KEY")),
("keenable", _has_env("KEENABLE_API_KEY")),
@@ -350,6 +359,13 @@ def _get_capability_backend(capability: str) -> str:
return _get_backend()
def _tavily_explicitly_configured() -> bool:
cfg = _load_web_config()
return any(
(cfg.get(key) or "").lower().strip() == "tavily"
for key in ("backend", "search_backend", "extract_backend")
)
def _is_backend_available(backend: str) -> bool:
"""Return True when the selected backend is currently usable.
@@ -376,6 +392,8 @@ def _is_backend_available(backend: str) -> bool:
return _has_env("KEENABLE_API_KEY")
if backend == "firecrawl":
return check_firecrawl_api_key()
if backend == "tavily":
return _has_env("TAVILY_API_KEY") or _tavily_explicitly_configured()
if backend == "searxng":
return _has_env("SEARXNG_URL")
if backend == "brave-free":
@@ -585,6 +603,7 @@ def _web_requires_env() -> list[str]:
return [
"EXA_API_KEY",
"PARALLEL_API_KEY",
"TAVILY_API_KEY",
"KEENABLE_API_KEY",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
@@ -595,10 +614,11 @@ def _web_requires_env() -> list[str]:
]
# ─── Parallel / Firecrawl helpers — moved into plugins ───────────────────────
# ─── Parallel / Tavily / Firecrawl helpers — moved into plugins ──────────────
# After PR #25182, the per-vendor client construction, request helpers, and
# response normalizers all live in plugins.web.<vendor>.provider:
# - parallel: plugins/web/parallel/provider.py
# - tavily: plugins/web/tavily/provider.py
# - firecrawl: plugins/web/firecrawl/provider.py
# The names from the firecrawl plugin (Firecrawl proxy, _get_firecrawl_client,
# _to_plain_object, _normalize_result_list, _extract_web_search_results,
@@ -790,7 +810,7 @@ def _ensure_web_plugins_loaded() -> None:
"""Idempotently trigger plugin discovery so the web registry is populated.
Every bundled web provider (brave-free, ddgs, searxng, exa, parallel,
firecrawl, keenable) registers itself via ``plugins/web/<vendor>/__init__.py``
tavily, firecrawl, keenable) registers itself via ``plugins/web/<vendor>/__init__.py``
during plugin discovery. Tool dispatch can be reached from contexts that
haven't already triggered discovery — subprocess agent runs, delegate
children, standalone scripts, certain test paths — and without it the
@@ -871,9 +891,9 @@ def web_search_tool(query: str, limit: int = 5) -> str:
if is_interrupted():
return tool_error("Interrupted", success=False)
# Dispatch through the web search registry. All 7 providers
# (brave-free, ddgs, searxng, exa, parallel, firecrawl, keenable)
# now live as plugins; the dispatcher is just a registry lookup +
# Dispatch through the web search registry. All bundled providers
# (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl,
# keenable) now live as plugins; the dispatcher is just a registry lookup +
# delegation. Sync only — every provider's search() is sync.
_ensure_web_plugins_loaded()
from agent.web_search_registry import (
@@ -1034,7 +1054,7 @@ async def web_extract_tool(
Extract content from specific web pages using available extraction API backend.
Returns clean page content (markdown/text) with NO LLM summarization. The
extract backends (Firecrawl, Exa, Parallel, Keenable) already return clean,
extract backends (Firecrawl, Tavily, Exa, Parallel, Keenable) already return clean,
boilerplate-stripped content, so we return it directly and fast. Pages over
``char_limit`` are head+tail truncated with an explicit footer; the full
text is stored under cache/web and the footer tells the model how to
@@ -1142,10 +1162,10 @@ async def web_extract_tool(
else:
backend = _get_extract_backend()
# All seven providers (brave-free, ddgs, searxng, exa, parallel,
# firecrawl, keenable) now live as plugins. The dispatcher is a
# All bundled providers (brave-free, ddgs, searxng, exa, parallel,
# tavily, firecrawl, keenable) now live as plugins. The dispatcher is a
# registry lookup + delegation. Some providers' extract() is
# async (parallel, firecrawl), others sync (exa, keenable) — we
# async (parallel, firecrawl), others sync (exa, tavily, keenable) — we
# detect coroutine functions and await; sync functions run
# inline (the policy gate, SSRF re-check, etc. live inside the
# provider itself for the firecrawl per-URL loop).
@@ -1172,7 +1192,7 @@ async def web_extract_tool(
f"{provider.display_name} is a search-only "
"backend and cannot extract URL content. "
"Set web.extract_backend to firecrawl, "
"keenable, exa, or parallel."
"tavily, keenable, exa, or parallel."
),
},
ensure_ascii=False,
@@ -1235,7 +1255,7 @@ async def web_extract_tool(
"error": (
"No web extract provider configured. "
"Set web.extract_backend to firecrawl, "
"keenable, exa, or parallel."
"tavily, keenable, exa, or parallel."
),
},
ensure_ascii=False,
@@ -1284,7 +1304,7 @@ async def web_extract_tool(
)
# Async-or-sync dispatch: parallel + firecrawl have async
# extract(); exa + keenable are sync.
# extract(); exa + tavily + keenable are sync.
import inspect
_extract_rescued = False
try:
@@ -1568,6 +1588,11 @@ if __name__ == "__main__":
print(" Using Exa API (https://exa.ai)")
elif backend == "parallel":
print(" Using Parallel API (https://parallel.ai)")
elif backend == "tavily":
if _has_env("TAVILY_API_KEY"):
print(" Using Tavily API (https://tavily.com)")
else:
print(" Using Tavily keyless (https://docs.tavily.com/documentation/keyless)")
elif backend == "searxng":
print(f" Using SearXNG (search only): {_env_value('SEARXNG_URL')}")
elif backend == "brave-free":
@@ -1585,7 +1610,7 @@ if __name__ == "__main__":
else:
print("❌ No web search backend configured")
print(
"Set EXA_API_KEY, PARALLEL_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL"
"Set EXA_API_KEY, PARALLEL_API_KEY, TAVILY_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL"
f"{_firecrawl_backend_help_suffix()}"
)
@@ -6,7 +6,7 @@ description: "How to build a web-search/extract/crawl backend plugin for Hermes
# Building a Web Search Provider Plugin
Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web/<name>/`. You can add a new one, or override a bundled one, by dropping a directory next to them.
Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Tavily, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web/<name>/`. You can add a new one, or override a bundled one, by dropping a directory next to them.
:::tip
Web search is one of several **backend plugins** Hermes supports. The others (with their own ABCs) are [Image Generation Provider Plugins](/developer-guide/image-gen-provider-plugin), [Video Generation Provider Plugins](/developer-guide/video-gen-provider-plugin), [Memory Provider Plugins](/developer-guide/memory-provider-plugin), [Context Engine Plugins](/developer-guide/context-engine-plugin), and [Model Provider Plugins](/developer-guide/model-provider-plugin). General tool/hook/CLI plugins live in [Build a Hermes Plugin](/developer-guide/plugins).
@@ -157,7 +157,7 @@ Full contract in `agent/web_search_provider.py`. Methods you may override:
| `search(query, limit)` | conditional | raises | Required when `supports_search()` returns `True` |
| `extract(urls, **kwargs)` | conditional | raises | Required when `supports_extract()` returns `True` |
Providers can advertise multiple capabilities from a single class — Firecrawl, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow.
Providers can advertise multiple capabilities from a single class — Firecrawl, Tavily, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow.
## Response shape
+1 -1
View File
@@ -42,7 +42,7 @@ Quick setup example:
```yaml
web:
backend: firecrawl # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai
backend: firecrawl # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai
```
If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`.
@@ -151,6 +151,8 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe
| `PARALLEL_API_KEY` | AI-native web search ([parallel.ai](https://parallel.ai/)) |
| `FIRECRAWL_API_KEY` | Web scraping and cloud browser ([firecrawl.dev](https://firecrawl.dev/)) |
| `FIRECRAWL_API_URL` | Custom Firecrawl API endpoint for self-hosted instances (optional) |
| `TAVILY_API_KEY` | Optional Tavily API key for higher search/extract limits. After selecting Tavily as the web backend, keyless access works without it ([app.tavily.com](https://app.tavily.com/home), [keyless docs](https://docs.tavily.com/documentation/keyless)) |
| `TAVILY_BASE_URL` | Override the Tavily API endpoint. Useful for corporate proxies and self-hosted Tavily-compatible search backends. Same pattern as `GROQ_BASE_URL`. |
| `SEARXNG_URL` | SearXNG instance URL for free self-hosted web search — no API key required ([searxng.github.io](https://searxng.github.io/searxng/)) |
| `EXA_API_KEY` | Exa API key for AI-native web search and contents ([exa.ai](https://exa.ai/)) |
| `BRAVE_SEARCH_API_KEY` | Brave Search API subscription token for web search (free tier available) ([brave.com/search/api](https://brave.com/search/api/)) |
+2 -2
View File
@@ -320,8 +320,8 @@ The single `video_generate` tool covers both modalities — pass `image_url` to
| Tool | Description | Requires environment |
|------|-------------|----------------------|
| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY |
| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY |
| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY |
| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY |
## `x_search` toolset
+3 -2
View File
@@ -2353,7 +2353,7 @@ The `web_search` and `web_extract` tools support five backend providers. Configu
```yaml
web:
backend: firecrawl # firecrawl | searxng | parallel | keenable | exa
backend: firecrawl # firecrawl | searxng | parallel | tavily | keenable | exa
# Or use per-capability keys to mix providers (e.g. free search + paid extract):
search_backend: "searxng"
@@ -2382,9 +2382,10 @@ web:
| **Firecrawl** (default) | `FIRECRAWL_API_KEY` | ✔ | ✔ |
| **SearXNG** | `SEARXNG_URL` | ✔ | — |
| **Parallel** | `PARALLEL_API_KEY` (optional — keyless free tier) | ✔ | ✔ |
| **Tavily** | `TAVILY_API_KEY` (optional — keyless when selected; not in the free-tier ring) | ✔ | ✔ |
| **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ |
**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Firecrawl or Keenable in `hermes tools` also works without a key.
**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key.
**SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](/user-guide/features/web-search) for Docker setup instructions.
@@ -235,7 +235,7 @@ Config changes take effect on the next agent session or gateway restart. The web
Manage the `.env` file where API keys and credentials are stored. Keys are grouped by category:
- **LLM Providers** — OpenRouter, Anthropic, OpenAI, DeepSeek, etc.
- **Tool API Keys** — Browserbase, Firecrawl, Keenable, ElevenLabs, etc.
- **Tool API Keys** — Browserbase, Firecrawl, Tavily, Keenable, ElevenLabs, etc.
- **Messaging Platforms** — Telegram, Discord, Slack bot tokens, etc.
- **Agent Settings** — non-secret env vars like `API_SERVER_ENABLED`
+22 -6
View File
@@ -24,10 +24,11 @@ Both are configured through a single backend selection. Providers are chosen via
| **DDGS (DuckDuckGo)** | — (no key) | ✔ | — | ✔ Free |
| **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with key |
| **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key |
| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Opt-in keyless when selected · not in the free-tier ring |
| **Keenable** | `KEENABLE_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key |
| **xAI (Grok)** | `XAI_API_KEY` or `hermes auth add xai-oauth` | ✔ | — | Paid (SuperGrok or per-token) |
Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below).
Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below).
**Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below.
@@ -266,13 +267,27 @@ SearXNG handles search; you need a separate provider for `web_extract`. Use the
# ~/.hermes/config.yaml
web:
search_backend: "searxng"
extract_backend: "firecrawl" # or keenable, exa, parallel
extract_backend: "firecrawl" # or tavily, keenable, exa, parallel
```
With this config, Hermes uses SearXNG for all search queries and Firecrawl for URL extraction — combining free search with high-quality extraction.
---
### Tavily
AI-optimised search and extract. Select Tavily in `hermes tools` (or set `web.backend: tavily`) to use it **keyless** with no account (rate-limited). Tavily is **not** in the zero-config free-tier ring — empty installs rotate across Exa / Parallel / Firecrawl / Keenable. Set an API key when you want higher limits.
```bash
# optional — skip this for keyless access after selecting Tavily
# ~/.hermes/.env
TAVILY_API_KEY=tvly-your-key-here
```
Get a key at [app.tavily.com](https://app.tavily.com/home). See [Tavily keyless](https://docs.tavily.com/documentation/keyless).
---
### Exa
Neural search with semantic understanding. Good for research and finding conceptually related content.
@@ -338,10 +353,10 @@ web:
timeout: 90 # seconds (default)
```
**Search-only** — pair with Firecrawl / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry.
**Search-only** — pair with Firecrawl / Tavily / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry.
:::caution Trust model
Unlike index-backed providers (Brave, Keenable, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input.
Unlike index-backed providers (Brave, Tavily, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input.
:::
---
@@ -355,7 +370,7 @@ Set one provider for all web capabilities:
```yaml
# ~/.hermes/config.yaml
web:
backend: "searxng" # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai
backend: "searxng" # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai
```
### Per-capability configuration
@@ -382,6 +397,7 @@ If no backend has **ever** been selected (no `web.backend` / per-capability key
| Credential present | Auto-selected backend |
|--------------------|-----------------------|
| `TAVILY_API_KEY` | tavily |
| `EXA_API_KEY` | exa |
| `PARALLEL_API_KEY` | parallel |
| `FIRECRAWL_API_KEY` or `FIRECRAWL_API_URL` (or the Nous Tool Gateway is ready) | firecrawl |
@@ -438,7 +454,7 @@ SearXNG cannot extract URL content. Set `web.extract_backend` to a provider that
```yaml
web:
search_backend: "searxng"
extract_backend: "firecrawl" # or keenable / exa / parallel
extract_backend: "firecrawl" # or tavily / keenable / exa / parallel
```
### SearXNG returns 0 results