feat(web): add Tavily web search and extract provider
This commit re-introduces the Tavily provider, which supports both search and content extraction capabilities, which was removed in #99199.
This commit is contained in:
@@ -59,7 +59,7 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]:
|
||||
# A function literally named ``web_search`` collides with Grok's native
|
||||
# server-side tool (incomplete hang or HTTP 400 duplicate names); this alias
|
||||
# avoids that while still dispatching through Hermes's configured provider
|
||||
# (Firecrawl / Exa / …). Mapped back to ``web_search`` in normalize_response.
|
||||
# (Firecrawl / Tavily / …). Mapped back to ``web_search`` in normalize_response.
|
||||
_XAI_CLIENT_WEB_SEARCH_ALIAS = "hermes_web_search"
|
||||
|
||||
# OpenCode's /v1/responses endpoints (Zen and Go, including custom providers
|
||||
@@ -661,7 +661,7 @@ class ResponsesApiTransport(ProviderTransport):
|
||||
# fails): drop the client ``web_search`` function and declare
|
||||
# xAI's built-in instead. 1:1 swap only when client ``web_search``
|
||||
# was already present — never an additive grant.
|
||||
# 2. **Client** (Firecrawl / Keenable / Exa / … configured or resolved):
|
||||
# 2. **Client** (Firecrawl / Tavily / Exa / … configured or resolved):
|
||||
# keep Hermes dispatch so ``web.backend`` / ``web.search_backend``
|
||||
# is honored, but rename the wire tool to
|
||||
# ``hermes_web_search`` so Grok cannot hijack the name. The alias
|
||||
|
||||
@@ -13,8 +13,8 @@ Providers live in ``<repo>/plugins/web/<name>/`` (built-in, auto-loaded as
|
||||
``plugins.enabled``).
|
||||
|
||||
This ABC is the SINGLE plugin-facing surface for web providers — every
|
||||
provider in the tree (brave-free, ddgs, searxng, exa, parallel, keenable,
|
||||
firecrawl) implements it. The legacy in-tree ``tools.web_providers.base``
|
||||
provider in the tree (brave-free, ddgs, searxng, exa, parallel, tavily,
|
||||
keenable, firecrawl) implements it. The legacy in-tree ``tools.web_providers.base``
|
||||
ABCs were deleted in PR #25182 along with the per-vendor inline helpers
|
||||
in ``tools/web_tools.py``; the response-shape contract documented below
|
||||
is preserved bit-for-bit so the tool wrapper does not have to translate.
|
||||
@@ -93,7 +93,7 @@ class WebSearchProvider(abc.ABC):
|
||||
:meth:`search` / :meth:`extract`. The :meth:`supports_search` /
|
||||
:meth:`supports_extract` capability flags let the registry route each
|
||||
tool call to the right provider, and let multi-capability providers
|
||||
(Firecrawl, Keenable, Exa, …) advertise multiple capabilities from a
|
||||
(Firecrawl, Tavily, Exa, …) advertise multiple capabilities from a
|
||||
single class.
|
||||
"""
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ The active provider is chosen by configuration with this precedence:
|
||||
2. ``web.backend`` (shared fallback).
|
||||
3. If exactly one capability-eligible provider is registered AND available,
|
||||
use it.
|
||||
4. Legacy preference order — ``firecrawl`` → ``parallel`` →
|
||||
4. Legacy preference order — ``firecrawl`` → ``parallel`` → ``tavily`` →
|
||||
``exa`` → ``searxng`` → ``brave-free`` → ``ddgs`` — filtered by
|
||||
availability. Matches the historic ``tools.web_tools._get_backend()``
|
||||
candidate order so installs that never set a config key keep landing
|
||||
@@ -159,6 +159,7 @@ def _read_config_key(*path: str) -> Optional[str]:
|
||||
_LEGACY_PREFERENCE = (
|
||||
"firecrawl",
|
||||
"parallel",
|
||||
"tavily",
|
||||
"exa",
|
||||
"searxng",
|
||||
"brave-free",
|
||||
@@ -167,7 +168,7 @@ _LEGACY_PREFERENCE = (
|
||||
|
||||
# Keyless free-tier walk — strictly LAST-resort, tried only after the
|
||||
# availability-filtered legacy walk finds nothing (i.e. the user has zero
|
||||
# web credentials and no importable ddgs). All five vendors expose public
|
||||
# web credentials and no importable ddgs). Ring vendors expose public
|
||||
# anonymous free tiers (see plugins/web/keyless_mcp.py). Unpinned keyless
|
||||
# traffic round-robins across the ring per request (the ring cursor lives
|
||||
# in keyless_mcp; an explicit `hermes tools` pick bypasses this walk
|
||||
@@ -220,7 +221,7 @@ def _resolve(configured: Optional[str], *, capability: str) -> Optional[WebSearc
|
||||
supports *capability* AND ``is_available()`` reports True, return it.
|
||||
|
||||
3. **Legacy preference walk, filtered by availability.** Walk the
|
||||
:data:`_LEGACY_PREFERENCE` order (firecrawl → parallel →
|
||||
:data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → tavily →
|
||||
exa → searxng → brave-free → ddgs) looking for a provider whose
|
||||
``supports_<capability>()`` is True AND whose ``is_available()`` is
|
||||
True. Matches the historic ``tools.web_tools._get_backend()``
|
||||
|
||||
@@ -63,7 +63,7 @@ with open(os.path.join(hh, "config.yaml"), "w", encoding="utf-8") as f:
|
||||
os.environ["HERMES_HOME"] = hh
|
||||
# Strip web-fetch shortcuts: every arm must drive the browser.
|
||||
os.environ.pop("BROWSER_USE_API_KEY", None)
|
||||
for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "SERPER_API_KEY"):
|
||||
for k in ("FIRECRAWL_API_KEY", "NOUS_API_KEY", "TAVILY_API_KEY", "SERPER_API_KEY"):
|
||||
os.environ.pop(k, None)
|
||||
os.environ["BU_CDP_URL"] = cdp
|
||||
os.environ["PATH"] = (
|
||||
|
||||
@@ -1111,6 +1111,7 @@ ENV_VARS_BY_VERSION: Dict[int, List[str]] = {
|
||||
4: ["VOICE_TOOLS_OPENAI_KEY", "ELEVENLABS_API_KEY"],
|
||||
5: ["WHATSAPP_ENABLED", "WHATSAPP_MODE", "WHATSAPP_ALLOWED_USERS",
|
||||
"SLACK_BOT_TOKEN", "SLACK_APP_TOKEN", "SLACK_ALLOWED_USERS"],
|
||||
10: ["TAVILY_API_KEY"],
|
||||
11: ["TERMINAL_MODAL_MODE"],
|
||||
}
|
||||
|
||||
@@ -1456,7 +1457,7 @@ def _is_env_config_key(key: str) -> bool:
|
||||
'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY',
|
||||
'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL',
|
||||
'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME',
|
||||
'TOOL_GATEWAY_USER_TOKEN',
|
||||
'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY',
|
||||
'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY',
|
||||
'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN',
|
||||
'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY',
|
||||
@@ -5090,6 +5091,7 @@ def show_config():
|
||||
("EXA_API_KEY", "Exa"),
|
||||
("PARALLEL_API_KEY", "Parallel"),
|
||||
("FIRECRAWL_API_KEY", "Firecrawl"),
|
||||
("TAVILY_API_KEY", "Tavily"),
|
||||
("BROWSERBASE_API_KEY", "Browserbase"),
|
||||
("BROWSER_USE_API_KEY", "Browser Use"),
|
||||
("FAL_KEY", "FAL"),
|
||||
|
||||
@@ -555,7 +555,7 @@ DEFAULT_CONFIG = {
|
||||
"extract_backend": "", # per-capability override for web_extract (e.g. "native")
|
||||
"extract_char_limit": 15000, # per-page char budget for web_extract; larger pages truncate + store full text in cache/web
|
||||
# Keyless free-tier ring: with NO web backend configured or keyed,
|
||||
# web_search/web_extract rotate round-robin across five vendors'
|
||||
# web_search/web_extract rotate round-robin across four vendors'
|
||||
# public free tiers (exa, parallel, firecrawl, keenable),
|
||||
# failing over to the next ring vendor on rate limits. Never
|
||||
# pre-empts a configured or keyed backend. Set false to disable.
|
||||
@@ -565,10 +565,11 @@ DEFAULT_CONFIG = {
|
||||
# free-tier ring — the next call attempts the chosen backend again
|
||||
# (no sticky failover). Off when keyless_fallback is false.
|
||||
"keyless_rescue": True,
|
||||
# Per-provider tier selection for ring vendors with both a keyless
|
||||
# Per-provider tier selection for vendors with both a keyless
|
||||
# free endpoint and a keyed paid path (exa, parallel,
|
||||
# firecrawl, keenable). Set by the `hermes tools` picker's
|
||||
# "Free (keyless)" / "Paid (API key)" rows.
|
||||
# firecrawl, keenable on the ring; tavily is opt-in keyless via
|
||||
# `hermes tools`, not a ring member). Set by the `hermes tools`
|
||||
# picker's "Free (keyless)" / "Paid (API key)" rows.
|
||||
# free — always use the anonymous free endpoint (even with a key)
|
||||
# paid — always use the keyed path (missing key = error; vendor
|
||||
# is also excluded from the keyless ring)
|
||||
@@ -4513,6 +4514,14 @@ OPTIONAL_ENV_VARS = {
|
||||
"category": "tool",
|
||||
"advanced": True,
|
||||
},
|
||||
"TAVILY_API_KEY": {
|
||||
"description": "Tavily API key for AI-native web search and extract (optional — keyless works when Tavily is selected)",
|
||||
"prompt": "Tavily API key",
|
||||
"url": "https://app.tavily.com/home",
|
||||
"tools": ["web_search", "web_extract"],
|
||||
"password": True,
|
||||
"category": "tool",
|
||||
},
|
||||
"KEENABLE_API_KEY": {
|
||||
"description": "Keenable API key for fast independent-index web search and page fetch (optional — keyless free tier works without it)",
|
||||
"prompt": "Keenable API key",
|
||||
|
||||
@@ -388,6 +388,7 @@ def run_dump(args):
|
||||
("COMMANDCODE_API_KEY", "commandcode"),
|
||||
("KILOCODE_API_KEY", "kilocode"),
|
||||
("FIRECRAWL_API_KEY", "firecrawl"),
|
||||
("TAVILY_API_KEY", "tavily"),
|
||||
("KEENABLE_API_KEY", "keenable"),
|
||||
("BROWSERBASE_API_KEY", "browserbase"),
|
||||
("FAL_KEY", "fal"),
|
||||
|
||||
@@ -505,6 +505,10 @@ def get_nous_subscription_features(
|
||||
direct_exa = bool(get_env_value("EXA_API_KEY"))
|
||||
direct_firecrawl = bool(get_env_value("FIRECRAWL_API_KEY") or get_env_value("FIRECRAWL_API_URL"))
|
||||
direct_parallel = bool(get_env_value("PARALLEL_API_KEY"))
|
||||
direct_tavily = bool(get_env_value("TAVILY_API_KEY"))
|
||||
# Keyless Tavily is opt-in: selecting it in `hermes tools` / setup writes
|
||||
# web.backend (or a per-capability override) without requiring a key.
|
||||
tavily_selected = "tavily" in {web_backend, web_search_backend, web_extract_backend}
|
||||
direct_searxng = bool(get_env_value("SEARXNG_URL"))
|
||||
direct_fal = fal_key_is_configured()
|
||||
direct_fal_video = direct_fal # same FAL_KEY; separate var so use_gateway is independent
|
||||
@@ -536,6 +540,8 @@ def get_nous_subscription_features(
|
||||
direct_firecrawl = False
|
||||
direct_exa = False
|
||||
direct_parallel = False
|
||||
direct_tavily = False
|
||||
tavily_selected = False
|
||||
if image_use_gateway:
|
||||
direct_fal = False
|
||||
if video_use_gateway:
|
||||
@@ -624,6 +630,7 @@ def get_nous_subscription_features(
|
||||
direct_camofox = False
|
||||
|
||||
|
||||
tavily_ready = direct_tavily or tavily_selected
|
||||
web_managed = web_backend == "firecrawl" and managed_web_available and not direct_firecrawl
|
||||
web_active = bool(
|
||||
web_tool_enabled
|
||||
@@ -632,6 +639,7 @@ def get_nous_subscription_features(
|
||||
or (web_backend == "exa" and direct_exa)
|
||||
or (web_backend == "firecrawl" and direct_firecrawl)
|
||||
or (web_backend == "parallel" and direct_parallel)
|
||||
or (web_backend == "tavily" and tavily_ready)
|
||||
or (web_backend == "searxng" and direct_searxng)
|
||||
# Per-capability overrides: search_backend or extract_backend may be set
|
||||
# without web.backend (using the new split config from #20061)
|
||||
@@ -639,6 +647,8 @@ def get_nous_subscription_features(
|
||||
or (web_search_backend == "exa" and direct_exa)
|
||||
or (web_search_backend == "firecrawl" and direct_firecrawl)
|
||||
or (web_search_backend == "parallel" and direct_parallel)
|
||||
or (web_search_backend == "tavily" and tavily_ready)
|
||||
or (web_extract_backend == "tavily" and tavily_ready)
|
||||
)
|
||||
)
|
||||
web_available = bool(
|
||||
@@ -646,6 +656,7 @@ def get_nous_subscription_features(
|
||||
or direct_exa
|
||||
or direct_firecrawl
|
||||
or direct_parallel
|
||||
or tavily_ready
|
||||
or direct_searxng
|
||||
)
|
||||
|
||||
@@ -889,6 +900,7 @@ def apply_nous_managed_defaults(
|
||||
|
||||
if "web" in selected_toolsets and not features.web.explicit_configured and not (
|
||||
get_env_value("PARALLEL_API_KEY")
|
||||
or get_env_value("TAVILY_API_KEY")
|
||||
or get_env_value("FIRECRAWL_API_KEY")
|
||||
or get_env_value("FIRECRAWL_API_URL")
|
||||
):
|
||||
@@ -986,6 +998,7 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]:
|
||||
get_env_value("FIRECRAWL_API_KEY")
|
||||
or get_env_value("FIRECRAWL_API_URL")
|
||||
or get_env_value("PARALLEL_API_KEY")
|
||||
or get_env_value("TAVILY_API_KEY")
|
||||
or get_env_value("EXA_API_KEY")
|
||||
# Env-configured keyless local backend: a reachable self-hosted
|
||||
# SearXNG is a working web setup even with no stored selection
|
||||
|
||||
+2
-2
@@ -513,7 +513,7 @@ def _print_setup_summary(config: dict, hermes_home):
|
||||
tool_status.append(("Vision (image analysis)", False, "run 'hermes setup' to configure"))
|
||||
|
||||
|
||||
# Web tools (Exa, Parallel, Firecrawl, or Keenable)
|
||||
# Web tools (Exa, Parallel, Firecrawl, Tavily, or Keenable)
|
||||
if subscription_features.web.managed_by_nous:
|
||||
tool_status.append(("Web Search & Extract (Nous subscription)", True, None))
|
||||
elif subscription_features.web.available:
|
||||
@@ -522,7 +522,7 @@ def _print_setup_summary(config: dict, hermes_home):
|
||||
label = f"Web Search & Extract ({subscription_features.web.current_provider})"
|
||||
tool_status.append((label, True, None))
|
||||
else:
|
||||
tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, KEENABLE_API_KEY, or SEARXNG_URL"))
|
||||
tool_status.append(("Web Search & Extract", False, "EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, TAVILY_API_KEY, KEENABLE_API_KEY, or SEARXNG_URL"))
|
||||
|
||||
# Browser tools (local Chromium, Camofox, Browserbase, Browser Use, or Firecrawl)
|
||||
browser_provider = subscription_features.browser.current_provider
|
||||
|
||||
@@ -184,6 +184,7 @@ def show_status(args):
|
||||
"MiniMax-CN": "MINIMAX_CN_API_KEY",
|
||||
"DeepInfra": "DEEPINFRA_API_KEY",
|
||||
"Firecrawl": "FIRECRAWL_API_KEY",
|
||||
"Tavily": "TAVILY_API_KEY",
|
||||
"Keenable": "KEENABLE_API_KEY",
|
||||
"Browser Use": "BROWSER_USE_API_KEY", # Optional — local browser works without this
|
||||
"Browserbase": "BROWSERBASE_API_KEY", # Optional — direct credentials only
|
||||
|
||||
@@ -3331,8 +3331,8 @@ def _plugin_video_gen_providers() -> list[dict]:
|
||||
|
||||
# Mirror of _plugin_image_gen_providers for web search backends. Surfaces
|
||||
# every plugin-registered web provider so it appears in the
|
||||
# "Web Search & Extract" picker. All seven providers (brave-free, ddgs,
|
||||
# searxng, exa, parallel, firecrawl, keenable) live as plugins after
|
||||
# "Web Search & Extract" picker. All bundled providers (brave-free, ddgs,
|
||||
# searxng, exa, parallel, tavily, firecrawl, keenable) live as plugins after
|
||||
# PR #25182 — this helper is the sole source of truth for the category's
|
||||
# provider rows. The hardcoded entries that used to drive the category
|
||||
# were deleted in the same PR; only the two non-provider UX rows
|
||||
@@ -3348,8 +3348,8 @@ def _plugin_web_search_providers() -> list[dict]:
|
||||
marker) so the picker behaves identically whether a provider is
|
||||
hardcoded or plugin-registered.
|
||||
|
||||
After PR #25182, all seven web providers (brave-free, ddgs, searxng,
|
||||
exa, parallel, firecrawl, keenable) are plugins; this helper is the sole
|
||||
After PR #25182, all bundled web providers (brave-free, ddgs, searxng,
|
||||
exa, parallel, tavily, firecrawl, keenable) are plugins; this helper is the sole
|
||||
source of provider rows for the Web Search & Extract category.
|
||||
"""
|
||||
try:
|
||||
|
||||
@@ -34,7 +34,7 @@ class BraveFreeWebSearchProvider(WebSearchProvider):
|
||||
"""Search-only Brave provider using the free-tier Data-for-Search API.
|
||||
|
||||
Free tier is 2,000 queries/month (1 qps). No content-extraction capability —
|
||||
users pair this with Firecrawl/Keenable/Exa for ``web_extract``.
|
||||
users pair this with Firecrawl/Tavily/Exa for ``web_extract``.
|
||||
"""
|
||||
|
||||
@property
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""SearXNG search plugin — bundled, auto-loaded.
|
||||
|
||||
Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``).
|
||||
Search-only — pair with an extract provider (firecrawl/keenable/exa) for
|
||||
Search-only — pair with an extract provider (firecrawl/tavily/exa) for
|
||||
``web_extract`` calls.
|
||||
"""
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Tavily web search + extract plugin — bundled, auto-loaded."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from plugins.web.tavily.provider import TavilyWebSearchProvider
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register the Tavily provider with the plugin context."""
|
||||
ctx.register_web_search_provider(TavilyWebSearchProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: web-tavily
|
||||
version: 1.0.0
|
||||
description: "Tavily web search + extract. Opt-in keyless via hermes tools; set TAVILY_API_KEY for higher limits — https://app.tavily.com/home."
|
||||
author: NousResearch
|
||||
kind: backend
|
||||
provides_web_providers:
|
||||
- tavily
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Tavily web search + content extraction — plugin form.
|
||||
|
||||
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two
|
||||
capabilities advertised:
|
||||
|
||||
- ``supports_search()`` -> True (Tavily ``/search``)
|
||||
- ``supports_extract()`` -> True (Tavily ``/extract``)
|
||||
|
||||
Both are sync — the underlying call is ``httpx.post(...)``.
|
||||
|
||||
Config keys this provider responds to::
|
||||
|
||||
web:
|
||||
search_backend: "tavily" # explicit per-capability
|
||||
extract_backend: "tavily" # explicit per-capability
|
||||
backend: "tavily" # shared fallback for both
|
||||
|
||||
Env vars::
|
||||
|
||||
TAVILY_API_KEY=... # https://app.tavily.com/home (optional)
|
||||
TAVILY_BASE_URL=... # optional override of https://api.tavily.com
|
||||
|
||||
Auth is header-based. A key uses ``Authorization: Bearer``; without a
|
||||
key the request is keyless (``X-Tavily-Access-Mode: keyless``). Both
|
||||
paths send ``X-Client-Name: hermes-agent``.
|
||||
|
||||
Tavily is **not** a member of the zero-config keyless ring
|
||||
(``plugins.web.keyless_mcp._KEYLESS_RING``). Keyless access is opt-in:
|
||||
select Tavily in ``hermes tools`` (or set ``web.backend: tavily``).
|
||||
Fresh installs with no web credentials rotate across Exa / Parallel /
|
||||
Firecrawl / Keenable instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from agent.web_search_provider import WebSearchProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CLIENT_NAME = "hermes-agent"
|
||||
|
||||
_SEARCH_PAYLOAD = {
|
||||
"include_raw_content": False,
|
||||
"include_images": False,
|
||||
}
|
||||
|
||||
|
||||
def _tavily_headers(api_key: str) -> Dict[str, str]:
|
||||
"""Build Tavily request headers for keyed or keyless access."""
|
||||
headers = {"X-Client-Name": _CLIENT_NAME}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
else:
|
||||
headers["X-Tavily-Access-Mode"] = "keyless"
|
||||
return headers
|
||||
|
||||
|
||||
def _tavily_request(
|
||||
endpoint: str,
|
||||
payload: Dict[str, Any],
|
||||
*,
|
||||
api_key: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""POST to the Tavily API and return the parsed JSON response.
|
||||
|
||||
Keyed when *api_key* (or ``TAVILY_API_KEY``) is set (Bearer auth);
|
||||
otherwise keyless. Pass ``api_key=""`` to force the keyless header even
|
||||
when a key is present (``web.provider_tier.tavily: free``). Non-2xx
|
||||
responses raise ``ValueError`` with the response body so Tavily's
|
||||
keyless rate-limit / upgrade text reaches the model.
|
||||
"""
|
||||
from agent.web_search_provider import get_provider_env
|
||||
|
||||
if api_key is None:
|
||||
api_key = get_provider_env("TAVILY_API_KEY")
|
||||
base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
|
||||
url = f"{base_url}/{endpoint.lstrip('/')}"
|
||||
logger.info("Tavily %s request to %s", endpoint, url)
|
||||
|
||||
response = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
timeout=60,
|
||||
headers=_tavily_headers(api_key),
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
body = (response.text or "").strip()
|
||||
detail = body or f"HTTP {response.status_code}"
|
||||
raise ValueError(detail)
|
||||
return response.json()
|
||||
|
||||
|
||||
def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``."""
|
||||
web_results = []
|
||||
for i, result in enumerate(response.get("results", [])):
|
||||
web_results.append(
|
||||
{
|
||||
"title": result.get("title", ""),
|
||||
"url": result.get("url", ""),
|
||||
"description": result.get("content", ""),
|
||||
"position": i + 1,
|
||||
}
|
||||
)
|
||||
return {"success": True, "data": {"web": web_results}}
|
||||
|
||||
|
||||
def _normalize_tavily_documents(
|
||||
response: Dict[str, Any], fallback_url: str = ""
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Map Tavily ``/extract`` response to standard documents.
|
||||
|
||||
Documents follow the legacy LLM post-processing shape::
|
||||
|
||||
{"url", "title", "content", "raw_content", "metadata"}
|
||||
|
||||
Failures (``failed_results``, ``failed_urls``) become result entries
|
||||
with an ``error`` field rather than raising.
|
||||
"""
|
||||
documents: List[Dict[str, Any]] = []
|
||||
for result in response.get("results", []):
|
||||
url = result.get("url", fallback_url)
|
||||
raw = result.get("raw_content", "") or result.get("content", "")
|
||||
documents.append(
|
||||
{
|
||||
"url": url,
|
||||
"title": result.get("title", ""),
|
||||
"content": raw,
|
||||
"raw_content": raw,
|
||||
"metadata": {"sourceURL": url, "title": result.get("title", "")},
|
||||
}
|
||||
)
|
||||
for fail in response.get("failed_results", []):
|
||||
documents.append(
|
||||
{
|
||||
"url": fail.get("url", fallback_url),
|
||||
"title": "",
|
||||
"content": "",
|
||||
"raw_content": "",
|
||||
"error": fail.get("error", "extraction failed"),
|
||||
"metadata": {"sourceURL": fail.get("url", fallback_url)},
|
||||
}
|
||||
)
|
||||
for fail_url in response.get("failed_urls", []):
|
||||
url_str = fail_url if isinstance(fail_url, str) else str(fail_url)
|
||||
documents.append(
|
||||
{
|
||||
"url": url_str,
|
||||
"title": "",
|
||||
"content": "",
|
||||
"raw_content": "",
|
||||
"error": "extraction failed",
|
||||
"metadata": {"sourceURL": url_str},
|
||||
}
|
||||
)
|
||||
return documents
|
||||
|
||||
|
||||
def _missing_key_error(action: str) -> str:
|
||||
return (
|
||||
f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home "
|
||||
f"or select Tavily in `hermes tools` for opt-in keyless {action}."
|
||||
)
|
||||
|
||||
|
||||
class TavilyWebSearchProvider(WebSearchProvider):
|
||||
"""Tavily search + extract provider (keyed, or opt-in keyless)."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "tavily"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "Tavily"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Return True when ``TAVILY_API_KEY`` is set to a non-empty value."""
|
||||
from agent.web_search_provider import get_provider_env
|
||||
|
||||
return bool(get_provider_env("TAVILY_API_KEY"))
|
||||
|
||||
def is_keyless_available(self) -> bool:
|
||||
"""Tavily serves anonymous keyless requests (X-Tavily-Access-Mode).
|
||||
|
||||
Opt-in only — Tavily is not a member of the zero-config keyless
|
||||
ring. ``is_keyless_available`` is True so an explicit
|
||||
``web.backend: tavily`` (or ``hermes tools`` pick) works without a
|
||||
key. False when the user pinned ``web.provider_tier.tavily: paid``.
|
||||
"""
|
||||
from plugins.web.keyless_mcp import keyless_enabled, provider_tier
|
||||
|
||||
return keyless_enabled() and provider_tier("tavily") != "paid"
|
||||
|
||||
def supports_search(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_extract(self) -> bool:
|
||||
return True
|
||||
|
||||
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
|
||||
"""Execute a Tavily search (keyed path or opt-in keyless)."""
|
||||
try:
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
if is_interrupted():
|
||||
return {"success": False, "error": "Interrupted"}
|
||||
|
||||
from agent.web_search_provider import get_provider_env
|
||||
|
||||
from plugins.web.keyless_mcp import use_keyless
|
||||
|
||||
api_key = get_provider_env("TAVILY_API_KEY")
|
||||
force_keyless = use_keyless("tavily", api_key)
|
||||
if not force_keyless and not api_key:
|
||||
return {"success": False, "error": _missing_key_error("search")}
|
||||
|
||||
logger.info(
|
||||
"Tavily %ssearch: '%s' (limit=%d)",
|
||||
"keyless " if force_keyless else "",
|
||||
query,
|
||||
limit,
|
||||
)
|
||||
raw = _tavily_request(
|
||||
"search",
|
||||
{
|
||||
"query": query,
|
||||
"max_results": min(limit, 20),
|
||||
**_SEARCH_PAYLOAD,
|
||||
},
|
||||
api_key="" if force_keyless else api_key,
|
||||
)
|
||||
return _normalize_tavily_search_results(raw)
|
||||
except ValueError as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
except Exception as exc: # noqa: BLE001 — including httpx errors
|
||||
logger.warning("Tavily search error: %s", exc)
|
||||
return {"success": False, "error": f"Tavily search failed: {exc}"}
|
||||
|
||||
def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
|
||||
"""Extract content from one or more URLs via Tavily.
|
||||
|
||||
Sync — the underlying call is httpx.post(...). Returns the legacy
|
||||
list-of-results shape; per-URL failures become items with ``error``.
|
||||
Keyless uses Tavily's own endpoint, not the keyless ring.
|
||||
"""
|
||||
try:
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
if is_interrupted():
|
||||
return [
|
||||
{"url": u, "error": "Interrupted", "title": ""} for u in urls
|
||||
]
|
||||
|
||||
from agent.web_search_provider import get_provider_env
|
||||
|
||||
from plugins.web.keyless_mcp import use_keyless
|
||||
|
||||
api_key = get_provider_env("TAVILY_API_KEY")
|
||||
force_keyless = use_keyless("tavily", api_key)
|
||||
if not force_keyless and not api_key:
|
||||
err = _missing_key_error("extract")
|
||||
return [
|
||||
{"url": u, "title": "", "content": "", "error": err}
|
||||
for u in urls
|
||||
]
|
||||
|
||||
logger.info(
|
||||
"Tavily %sextract: %d URL(s)",
|
||||
"keyless " if force_keyless else "",
|
||||
len(urls),
|
||||
)
|
||||
raw = _tavily_request(
|
||||
"extract",
|
||||
{
|
||||
"urls": urls,
|
||||
"include_images": False,
|
||||
},
|
||||
api_key="" if force_keyless else api_key,
|
||||
)
|
||||
return _normalize_tavily_documents(
|
||||
raw, fallback_url=urls[0] if urls else ""
|
||||
)
|
||||
except ValueError as exc:
|
||||
return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Tavily extract error: %s", exc)
|
||||
return [
|
||||
{"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"}
|
||||
for u in urls
|
||||
]
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "Tavily",
|
||||
"badge": "free · key optional",
|
||||
"tag": (
|
||||
"Search + extract. Opt-in keyless (not in the free-tier ring); "
|
||||
"set TAVILY_API_KEY for higher limits."
|
||||
),
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "TAVILY_API_KEY",
|
||||
"prompt": "Tavily API key (optional — keyless works when Tavily is selected)",
|
||||
"url": "https://app.tavily.com/home",
|
||||
},
|
||||
],
|
||||
}
|
||||
@@ -101,12 +101,12 @@ class XAIWebSearchProvider(WebSearchProvider):
|
||||
back to the Responses API ``citations`` list if Grok ignores the JSON
|
||||
schema instruction (rare for grok-4.3 but cheap insurance).
|
||||
|
||||
No extract capability — pair with Firecrawl / Keenable / Exa for
|
||||
No extract capability — pair with Firecrawl / Tavily / Exa for
|
||||
``web_extract`` if you need page content.
|
||||
|
||||
Trust model
|
||||
-----------
|
||||
Unlike index-backed providers (Brave / Keenable / Exa) which return
|
||||
Unlike index-backed providers (Brave / Tavily / Exa) which return
|
||||
verbatim search-engine results, this backend is an LLM in a trench
|
||||
coat: Grok decides which URLs to surface, generates the titles and
|
||||
descriptions itself, and is influenced by the *content of the query*.
|
||||
|
||||
+1
-1
@@ -186,7 +186,7 @@ _CREDENTIAL_NAMES = frozenset({
|
||||
"FIRECRAWL_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"EXA_API_KEY",
|
||||
"TAVILY_API_KEY", # removed backend; still blanked for hermeticity
|
||||
"TAVILY_API_KEY",
|
||||
"WANDB_API_KEY",
|
||||
"ELEVENLABS_API_KEY",
|
||||
"HONCHO_API_KEY",
|
||||
|
||||
@@ -666,13 +666,23 @@ class TestOptionalEnvVarsRegistry:
|
||||
from hermes_cli.config import OPTIONAL_ENV_VARS
|
||||
assert OPTIONAL_ENV_VARS["KEENABLE_API_KEY"]["url"] == "https://keenable.ai"
|
||||
|
||||
def test_removed_tavily_var_not_in_env_vars_by_version(self):
|
||||
"""TAVILY_API_KEY was removed with the Tavily backend."""
|
||||
def test_tavily_api_key_registered(self):
|
||||
"""TAVILY_API_KEY is listed in OPTIONAL_ENV_VARS."""
|
||||
from hermes_cli.config import OPTIONAL_ENV_VARS
|
||||
assert "TAVILY_API_KEY" in OPTIONAL_ENV_VARS
|
||||
|
||||
def test_tavily_api_key_has_url(self):
|
||||
"""TAVILY_API_KEY has a URL."""
|
||||
from hermes_cli.config import OPTIONAL_ENV_VARS
|
||||
assert OPTIONAL_ENV_VARS["TAVILY_API_KEY"]["url"] == "https://app.tavily.com/home"
|
||||
|
||||
def test_tavily_in_env_vars_by_version(self):
|
||||
"""TAVILY_API_KEY is listed in ENV_VARS_BY_VERSION."""
|
||||
from hermes_cli.config import ENV_VARS_BY_VERSION
|
||||
all_vars = []
|
||||
for vars_list in ENV_VARS_BY_VERSION.values():
|
||||
all_vars.extend(vars_list)
|
||||
assert "TAVILY_API_KEY" not in all_vars
|
||||
assert "TAVILY_API_KEY" in all_vars
|
||||
|
||||
def test_max_iterations_not_offered_as_env_var(self):
|
||||
"""HERMES_MAX_ITERATIONS must NOT be in OPTIONAL_ENV_VARS (issue #17534).
|
||||
|
||||
@@ -47,6 +47,7 @@ def test_dump_leaves_unset_key_untouched(monkeypatch, capsys, tmp_path):
|
||||
|
||||
monkeypatch.setattr(dump, "get_project_root", lambda: tmp_path / "noproject")
|
||||
monkeypatch.delenv("KEENABLE_API_KEY", raising=False)
|
||||
monkeypatch.delenv("TAVILY_API_KEY", raising=False)
|
||||
|
||||
home = get_hermes_home()
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -58,6 +58,52 @@ def test_get_nous_subscription_features_recognizes_direct_exa_backend(monkeypatc
|
||||
assert features.web.current_provider == "exa"
|
||||
|
||||
|
||||
def test_get_nous_subscription_features_recognizes_keyless_tavily_backend(monkeypatch):
|
||||
"""Selecting Tavily in setup/tools counts as available with no API key.
|
||||
|
||||
Mirrors tools.web_tools._is_backend_available('tavily'): keyless is
|
||||
opt-in via web.backend / search_backend / extract_backend, not a
|
||||
silent empty-install default. The setup summary previously required
|
||||
TAVILY_API_KEY and printed a false 'missing' after a skipped key prompt.
|
||||
"""
|
||||
monkeypatch.setattr(ns, "get_env_value", lambda name: "")
|
||||
monkeypatch.setattr(
|
||||
ns, "get_nous_portal_account_info", lambda: _account(logged_in=False)
|
||||
)
|
||||
monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web")
|
||||
monkeypatch.setattr(ns, "_has_agent_browser", lambda: False)
|
||||
monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "")
|
||||
monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False)
|
||||
|
||||
features = ns.get_nous_subscription_features({"web": {"backend": "tavily"}})
|
||||
|
||||
assert features.web.available is True
|
||||
assert features.web.active is True
|
||||
assert features.web.managed_by_nous is False
|
||||
assert features.web.direct_override is True
|
||||
assert features.web.current_provider == "tavily"
|
||||
assert features.web.explicit_configured is True
|
||||
|
||||
|
||||
def test_keyless_tavily_search_backend_without_shared_backend(monkeypatch):
|
||||
monkeypatch.setattr(ns, "get_env_value", lambda name: "")
|
||||
monkeypatch.setattr(
|
||||
ns, "get_nous_portal_account_info", lambda: _account(logged_in=False)
|
||||
)
|
||||
monkeypatch.setattr(ns, "_toolset_enabled", lambda config, key: key == "web")
|
||||
monkeypatch.setattr(ns, "_has_agent_browser", lambda: False)
|
||||
monkeypatch.setattr(ns, "resolve_openai_audio_api_key", lambda: "")
|
||||
monkeypatch.setattr(ns, "has_direct_modal_credentials", lambda: False)
|
||||
|
||||
features = ns.get_nous_subscription_features(
|
||||
{"web": {"search_backend": "tavily"}}
|
||||
)
|
||||
|
||||
assert features.web.available is True
|
||||
assert features.web.active is True
|
||||
assert features.web.current_provider == "tavily"
|
||||
|
||||
|
||||
def test_unconfigured_web_without_keys_is_unavailable(monkeypatch):
|
||||
monkeypatch.setattr(ns, "get_env_value", lambda name: "")
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -15,6 +15,18 @@ def test_show_status_all_does_not_print_keenable_key_value(monkeypatch, capsys,
|
||||
assert sentinel not in output
|
||||
|
||||
|
||||
def test_show_status_all_does_not_print_tavily_key_value(monkeypatch, capsys, tmp_path):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
sentinel = "NONSECRET_SENTINEL_VALUE_DO_NOT_PRINT_TAVILY_123456"
|
||||
monkeypatch.setenv("TAVILY_API_KEY", sentinel)
|
||||
|
||||
show_status(SimpleNamespace(all=True, deep=False))
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert "Tavily" in output
|
||||
assert sentinel not in output
|
||||
|
||||
|
||||
def test_show_status_termux_gateway_section_skips_systemctl(monkeypatch, capsys, tmp_path):
|
||||
from hermes_cli import status as status_mod
|
||||
import hermes_cli.auth as auth_mod
|
||||
|
||||
@@ -245,6 +245,7 @@ def test_first_install_nous_auto_configures_video_gen(monkeypatch):
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
"KEENABLE_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"BROWSERBASE_API_KEY",
|
||||
"BROWSERBASE_PROJECT_ID",
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Covers:
|
||||
|
||||
- All bundled plugins (brave-free, ddgs, searxng, exa, parallel,
|
||||
firecrawl, keenable, xai) instantiate and self-report the expected
|
||||
tavily, firecrawl, keenable, xai) instantiate and self-report the expected
|
||||
capabilities + ABC-derived defaults.
|
||||
- Each plugin's ``is_available()`` correctly reflects env-var presence.
|
||||
- The web_search_registry resolves an active provider in the documented
|
||||
@@ -35,6 +35,8 @@ def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"BRAVE_SEARCH_API_KEY",
|
||||
"SEARXNG_URL",
|
||||
"KEENABLE_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
"TAVILY_BASE_URL",
|
||||
"EXA_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"PARALLEL_SEARCH_MODE",
|
||||
@@ -82,6 +84,7 @@ class TestBundledPluginsRegister:
|
||||
"keenable",
|
||||
"parallel",
|
||||
"searxng",
|
||||
"tavily",
|
||||
"xai",
|
||||
]
|
||||
|
||||
@@ -94,6 +97,7 @@ class TestBundledPluginsRegister:
|
||||
("exa", True, True),
|
||||
("parallel", True, True),
|
||||
("keenable", True, True),
|
||||
("tavily", True, True),
|
||||
("firecrawl", True, True),
|
||||
# xai: search-only via Grok's agentic web_search tool.
|
||||
("xai", True, False),
|
||||
@@ -115,7 +119,7 @@ class TestBundledPluginsRegister:
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "firecrawl", "keenable", "xai"],
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "keenable", "xai"],
|
||||
)
|
||||
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
@@ -165,6 +169,16 @@ class TestIsAvailable:
|
||||
monkeypatch.setenv("KEENABLE_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("TAVILY_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
@@ -83,6 +83,7 @@ def register_all_web_providers():
|
||||
from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider
|
||||
from plugins.web.parallel.provider import ParallelWebSearchProvider
|
||||
from plugins.web.keenable.provider import KeenableWebSearchProvider
|
||||
from plugins.web.tavily.provider import TavilyWebSearchProvider
|
||||
from plugins.web.searxng.provider import SearXNGWebSearchProvider
|
||||
from plugins.web.xai.provider import XAIWebSearchProvider
|
||||
|
||||
@@ -94,6 +95,7 @@ def register_all_web_providers():
|
||||
FirecrawlWebSearchProvider,
|
||||
ParallelWebSearchProvider,
|
||||
KeenableWebSearchProvider,
|
||||
TavilyWebSearchProvider,
|
||||
SearXNGWebSearchProvider,
|
||||
XAIWebSearchProvider,
|
||||
):
|
||||
|
||||
@@ -25,7 +25,7 @@ from plugins.web.parallel.provider import ParallelWebSearchProvider
|
||||
def _no_web_env(monkeypatch):
|
||||
"""Blank every web credential and neutralize config lookups."""
|
||||
for var in (
|
||||
"EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY",
|
||||
"EXA_API_KEY", "PARALLEL_API_KEY", "KEENABLE_API_KEY", "TAVILY_API_KEY",
|
||||
"FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "BRAVE_SEARCH_API_KEY",
|
||||
"SEARXNG_URL", "TOOL_GATEWAY_USER_TOKEN",
|
||||
):
|
||||
@@ -289,7 +289,7 @@ class TestResolutionOrder:
|
||||
|
||||
def test_keyless_ring_rotates_and_covers_all_vendors(self, fresh_registry, monkeypatch):
|
||||
monkeypatch.setattr(registry, "_read_config_key", lambda *p: None)
|
||||
# The ring order always contains all five vendors, starting at the
|
||||
# The ring order always contains all four vendors, starting at the
|
||||
# current cursor and wrapping.
|
||||
order = registry._keyless_preference()
|
||||
assert sorted(order) == sorted(keyless_mcp._KEYLESS_RING)
|
||||
@@ -303,6 +303,15 @@ class TestResolutionOrder:
|
||||
assert keyless_mcp._ring_order("keenable")[0] == "keenable"
|
||||
assert keyless_mcp._ring_order("keenable")[0] == "keenable"
|
||||
|
||||
def test_tavily_is_not_a_ring_member(self):
|
||||
"""Tavily is opt-in keyless; zero-config rotation must not include it."""
|
||||
from plugins.web import keyless_mcp
|
||||
|
||||
assert "tavily" not in keyless_mcp._KEYLESS_RING
|
||||
assert "tavily" not in keyless_mcp._KEYLESS_SEARCHERS
|
||||
assert "tavily" not in keyless_mcp._KEYLESS_EXTRACTORS
|
||||
assert "tavily" not in registry._KEYLESS_PREFERENCE
|
||||
|
||||
def test_registry_keyless_disabled_returns_none(self, fresh_registry, monkeypatch):
|
||||
monkeypatch.setattr(registry, "_read_config_key", lambda *p: None)
|
||||
monkeypatch.setattr(registry, "_keyless_tier_enabled", lambda: False)
|
||||
|
||||
@@ -210,6 +210,7 @@ class TestBackendSelection:
|
||||
"TOOL_GATEWAY_SCHEME",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
"KEENABLE_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
)
|
||||
|
||||
def setup_method(self):
|
||||
@@ -254,7 +255,7 @@ class TestBackendSelection:
|
||||
assert _get_backend() == "exa"
|
||||
|
||||
def test_fallback_exa_takes_priority_over_parallel(self):
|
||||
"""Direct-credential backends are tried in the order exa > parallel > keenable
|
||||
"""Direct-credential backends are tried in the order tavily > exa > parallel > keenable
|
||||
so an explicit Exa key wins when both Exa and Parallel are configured."""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
@@ -275,6 +276,27 @@ class TestBackendSelection:
|
||||
patch.dict(os.environ, {"EXA_API_KEY": "exa-test", "FIRECRAWL_API_KEY": "fc-test"}):
|
||||
assert _get_backend() == "exa"
|
||||
|
||||
def test_fallback_tavily_only_key(self):
|
||||
"""Only TAVILY_API_KEY set → 'tavily'."""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}):
|
||||
assert _get_backend() == "tavily"
|
||||
|
||||
def test_fallback_tavily_beats_firecrawl_direct(self):
|
||||
"""Tavily ranks above firecrawl in the explicit-credential block."""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "FIRECRAWL_API_KEY": "fc-test"}):
|
||||
assert _get_backend() == "tavily"
|
||||
|
||||
def test_fallback_tavily_beats_exa(self):
|
||||
"""Tavily ranks above Exa in the explicit-credential block."""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test", "EXA_API_KEY": "exa-test"}):
|
||||
assert _get_backend() == "tavily"
|
||||
|
||||
|
||||
def test_fallback_parallel_beats_firecrawl_direct(self):
|
||||
"""Parallel + Firecrawl-direct → parallel (parallel is the higher-priority
|
||||
@@ -342,6 +364,14 @@ class TestBackendSelection:
|
||||
patch.dict(os.environ, {"EXA_API_KEY": "exa-test"}):
|
||||
assert _get_backend() == "exa"
|
||||
|
||||
def test_managed_gateway_does_not_preempt_explicit_tavily(self):
|
||||
"""A Nous OAuth token must not beat an explicit TAVILY_API_KEY."""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \
|
||||
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}):
|
||||
assert _get_backend() == "tavily"
|
||||
|
||||
def test_managed_gateway_only_falls_through_to_firecrawl(self):
|
||||
"""When no explicit-credential backend is configured, a Nous-managed
|
||||
gateway token still selects firecrawl — the convenience path is
|
||||
@@ -494,6 +524,7 @@ class TestCheckWebApiKey:
|
||||
"TOOL_GATEWAY_SCHEME",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
"KEENABLE_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
)
|
||||
|
||||
def setup_method(self):
|
||||
@@ -597,7 +628,9 @@ class TestCheckWebApiKey:
|
||||
def test_web_requires_env_includes_exa_key():
|
||||
from tools.web_tools import _web_requires_env
|
||||
|
||||
assert "EXA_API_KEY" in _web_requires_env()
|
||||
env = _web_requires_env()
|
||||
assert "EXA_API_KEY" in env
|
||||
assert "TAVILY_API_KEY" in env
|
||||
|
||||
|
||||
class TestNonBuiltinProviderAvailability:
|
||||
@@ -625,6 +658,7 @@ class TestNonBuiltinProviderAvailability:
|
||||
"TOOL_GATEWAY_SCHEME",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
"KEENABLE_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
"SEARXNG_URL",
|
||||
"BRAVE_SEARCH_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
@@ -765,6 +799,7 @@ class TestSiblingProvidersEnvResolution:
|
||||
("plugins.web.exa.provider", "ExaWebSearchProvider", "EXA_API_KEY"),
|
||||
("plugins.web.parallel.provider", "ParallelWebSearchProvider", "PARALLEL_API_KEY"),
|
||||
("plugins.web.keenable.provider", "KeenableWebSearchProvider", "KEENABLE_API_KEY"),
|
||||
("plugins.web.tavily.provider", "TavilyWebSearchProvider", "TAVILY_API_KEY"),
|
||||
("plugins.web.brave_free.provider", "BraveFreeWebSearchProvider", "BRAVE_SEARCH_API_KEY"),
|
||||
]
|
||||
|
||||
@@ -811,6 +846,28 @@ class TestSiblingProvidersEnvResolution:
|
||||
assert headers["Authorization"] == "Bearer kn-from-dotenv"
|
||||
assert headers["X-Keenable-Title"] == "hermes-agent"
|
||||
|
||||
def test_tavily_request_reads_key_via_get_env_value(self, monkeypatch):
|
||||
"""Keyed Tavily must Bearer-auth with a key that lives only in .env."""
|
||||
monkeypatch.delenv("TAVILY_API_KEY", raising=False)
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"results": []}
|
||||
mock_response.text = "{}"
|
||||
|
||||
with patch(
|
||||
"hermes_cli.config.get_env_value",
|
||||
side_effect=lambda k: "tvly-from-dotenv" if k == "TAVILY_API_KEY" else None,
|
||||
), patch(
|
||||
"plugins.web.tavily.provider.httpx.post", return_value=mock_response
|
||||
) as mock_post:
|
||||
from plugins.web.tavily.provider import _tavily_request
|
||||
|
||||
_tavily_request("search", {"query": "q"})
|
||||
headers = mock_post.call_args.kwargs["headers"]
|
||||
assert headers["Authorization"] == "Bearer tvly-from-dotenv"
|
||||
assert headers["X-Client-Name"] == "hermes-agent"
|
||||
assert "X-Tavily-Access-Mode" not in headers
|
||||
|
||||
|
||||
def test_get_provider_env_unset_returns_empty(self, monkeypatch):
|
||||
monkeypatch.delenv("WSP_TEST_UNSET_KEY", raising=False)
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Tests for Tavily web backend integration.
|
||||
|
||||
Coverage:
|
||||
_tavily_request() — keyed Bearer vs keyless header, attribution, error bodies.
|
||||
_normalize_tavily_search_results() — search response normalization.
|
||||
_normalize_tavily_documents() — extract response normalization, failed_results.
|
||||
web_search_tool / web_extract_tool — Tavily dispatch paths.
|
||||
auto-detect ranking — keyed paid-band; keyless only when Tavily is selected.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import asyncio
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from tests.tools.conftest import register_all_web_providers
|
||||
|
||||
|
||||
def _ok_response(payload=None):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = payload if payload is not None else {"results": []}
|
||||
mock_response.text = json.dumps(mock_response.json.return_value)
|
||||
return mock_response
|
||||
|
||||
|
||||
# ─── _tavily_request ─────────────────────────────────────────────────────────
|
||||
|
||||
class TestTavilyRequest:
|
||||
"""Test suite for the _tavily_request helper."""
|
||||
|
||||
def test_keyless_when_no_api_key(self):
|
||||
"""No TAVILY_API_KEY → keyless header, no Authorization, no body key."""
|
||||
mock_response = _ok_response()
|
||||
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post:
|
||||
from plugins.web.tavily.provider import _tavily_request
|
||||
_tavily_request("search", {"query": "test"})
|
||||
|
||||
mock_post.assert_called_once()
|
||||
headers = mock_post.call_args.kwargs["headers"]
|
||||
payload = mock_post.call_args.kwargs["json"]
|
||||
assert headers["X-Client-Name"] == "hermes-agent"
|
||||
assert headers["X-Tavily-Access-Mode"] == "keyless"
|
||||
assert "Authorization" not in headers
|
||||
assert "api_key" not in payload
|
||||
assert payload["query"] == "test"
|
||||
assert "api.tavily.com/search" in mock_post.call_args.args[0]
|
||||
|
||||
def test_keyed_uses_bearer_not_body(self):
|
||||
"""TAVILY_API_KEY → Bearer auth, attribution, no body api_key."""
|
||||
mock_response = _ok_response()
|
||||
|
||||
with patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test-key"}):
|
||||
with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post:
|
||||
from plugins.web.tavily.provider import _tavily_request
|
||||
_tavily_request("search", {"query": "hello"})
|
||||
|
||||
mock_post.assert_called_once()
|
||||
headers = mock_post.call_args.kwargs["headers"]
|
||||
payload = mock_post.call_args.kwargs["json"]
|
||||
assert headers == {
|
||||
"X-Client-Name": "hermes-agent",
|
||||
"Authorization": "Bearer tvly-test-key",
|
||||
}
|
||||
assert "X-Tavily-Access-Mode" not in headers
|
||||
assert "api_key" not in payload
|
||||
assert payload["query"] == "hello"
|
||||
assert "api.tavily.com/search" in mock_post.call_args.args[0]
|
||||
|
||||
def test_http_error_surfaces_response_body(self):
|
||||
"""Non-2xx responses raise ValueError with Tavily's response body."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 429
|
||||
mock_response.text = "Rate limit hit. Sign up for a free API key at https://app.tavily.com"
|
||||
mock_response.json.return_value = {}
|
||||
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
with patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response):
|
||||
from plugins.web.tavily.provider import _tavily_request
|
||||
with pytest.raises(ValueError, match="Rate limit hit"):
|
||||
_tavily_request("search", {"query": "test"})
|
||||
|
||||
|
||||
# ─── _normalize_tavily_search_results ─────────────────────────────────────────
|
||||
|
||||
class TestNormalizeTavilySearchResults:
|
||||
"""Test search result normalization."""
|
||||
|
||||
def test_basic_normalization(self):
|
||||
from tools.web_tools import _normalize_tavily_search_results
|
||||
raw = {
|
||||
"results": [
|
||||
{"title": "Python Docs", "url": "https://docs.python.org", "content": "Official docs", "score": 0.9},
|
||||
{"title": "Tutorial", "url": "https://example.com", "content": "A tutorial", "score": 0.8},
|
||||
]
|
||||
}
|
||||
result = _normalize_tavily_search_results(raw)
|
||||
assert result["success"] is True
|
||||
web = result["data"]["web"]
|
||||
assert len(web) == 2
|
||||
assert web[0]["title"] == "Python Docs"
|
||||
assert web[0]["url"] == "https://docs.python.org"
|
||||
assert web[0]["description"] == "Official docs"
|
||||
assert web[0]["position"] == 1
|
||||
assert web[1]["position"] == 2
|
||||
|
||||
|
||||
def test_missing_fields(self):
|
||||
from tools.web_tools import _normalize_tavily_search_results
|
||||
result = _normalize_tavily_search_results({"results": [{}]})
|
||||
web = result["data"]["web"]
|
||||
assert web[0]["title"] == ""
|
||||
assert web[0]["url"] == ""
|
||||
assert web[0]["description"] == ""
|
||||
|
||||
|
||||
# ─── _normalize_tavily_documents ──────────────────────────────────────────────
|
||||
|
||||
class TestNormalizeTavilyDocuments:
|
||||
"""Test extract document normalization."""
|
||||
|
||||
def test_basic_document(self):
|
||||
from tools.web_tools import _normalize_tavily_documents
|
||||
raw = {
|
||||
"results": [{
|
||||
"url": "https://example.com",
|
||||
"title": "Example",
|
||||
"raw_content": "Full page content here",
|
||||
}]
|
||||
}
|
||||
docs = _normalize_tavily_documents(raw)
|
||||
assert len(docs) == 1
|
||||
assert docs[0]["url"] == "https://example.com"
|
||||
assert docs[0]["title"] == "Example"
|
||||
assert docs[0]["content"] == "Full page content here"
|
||||
assert docs[0]["raw_content"] == "Full page content here"
|
||||
assert docs[0]["metadata"]["sourceURL"] == "https://example.com"
|
||||
|
||||
|
||||
def test_fallback_url(self):
|
||||
from tools.web_tools import _normalize_tavily_documents
|
||||
raw = {"results": [{"content": "data"}]}
|
||||
docs = _normalize_tavily_documents(raw, fallback_url="https://fallback.com")
|
||||
assert docs[0]["url"] == "https://fallback.com"
|
||||
|
||||
|
||||
# ─── availability / auto-detect ───────────────────────────────────────────────
|
||||
|
||||
class TestTavilyAvailability:
|
||||
"""Keyed Tavily stays in the paid band; keyless only when selected."""
|
||||
|
||||
def test_is_available_without_key(self):
|
||||
from plugins.web.tavily.provider import TavilyWebSearchProvider
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert TavilyWebSearchProvider().is_available() is False
|
||||
|
||||
def test_is_backend_available_without_key(self):
|
||||
from tools.web_tools import _is_backend_available
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert _is_backend_available("tavily") is False
|
||||
|
||||
def test_is_backend_available_when_configured_without_key(self):
|
||||
from tools.web_tools import _is_backend_available
|
||||
with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \
|
||||
patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert _is_backend_available("tavily") is True
|
||||
|
||||
def test_keyless_does_not_preempt_managed_firecrawl(self):
|
||||
"""No TAVILY_API_KEY + Nous gateway ready → firecrawl, not keyless tavily."""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch("tools.web_tools._is_tool_gateway_ready", return_value=True), \
|
||||
patch("tools.web_tools._ddgs_package_importable", return_value=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert _get_backend() == "firecrawl"
|
||||
|
||||
def test_keyless_does_not_preempt_ddgs(self):
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \
|
||||
patch("tools.web_tools._ddgs_package_importable", return_value=True):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert _get_backend() == "ddgs"
|
||||
|
||||
def test_no_keys_defaults_to_firecrawl(self):
|
||||
"""Keyless tier disabled: zero-credential resolve hits the legacy
|
||||
firecrawl sentinel. (With the tier on — the default — it resolves
|
||||
to the Exa/Parallel keyless split; see test_web_keyless_fallback.py.)
|
||||
"""
|
||||
from tools.web_tools import _get_backend
|
||||
with patch("tools.web_tools._load_web_config", return_value={}), \
|
||||
patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \
|
||||
patch("tools.web_tools._ddgs_package_importable", return_value=False), \
|
||||
patch("tools.web_tools._list_registered_web_providers", return_value=[]), \
|
||||
patch("agent.web_search_registry._keyless_tier_enabled", return_value=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert _get_backend() == "firecrawl"
|
||||
|
||||
def test_explicit_search_backend_tavily_without_key(self):
|
||||
"""web.search_backend=tavily sticks even with no TAVILY_API_KEY."""
|
||||
from tools.web_tools import _get_search_backend
|
||||
with patch("tools.web_tools._load_web_config",
|
||||
return_value={"backend": "firecrawl", "search_backend": "tavily"}), \
|
||||
patch("tools.web_tools._is_tool_gateway_ready", return_value=True):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert _get_search_backend() == "tavily"
|
||||
|
||||
def test_check_web_api_key_when_tavily_configured_without_key(self):
|
||||
from tools.web_tools import check_web_api_key
|
||||
with patch("tools.web_tools._load_web_config", return_value={"backend": "tavily"}), \
|
||||
patch("tools.web_tools._is_tool_gateway_ready", return_value=False), \
|
||||
patch("tools.web_tools.check_firecrawl_api_key", return_value=False), \
|
||||
patch("tools.web_tools._ddgs_package_importable", return_value=False), \
|
||||
patch("agent.web_search_registry.get_active_search_provider", return_value=None), \
|
||||
patch("agent.web_search_registry.get_active_extract_provider", return_value=None):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
assert check_web_api_key() is True
|
||||
|
||||
|
||||
# ─── web_search_tool (Tavily dispatch) ────────────────────────────────────────
|
||||
|
||||
class TestWebSearchTavily:
|
||||
"""Test web_search_tool dispatch to Tavily."""
|
||||
|
||||
_register_providers = staticmethod(register_all_web_providers)
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _populate_web_registry(self):
|
||||
self._register_providers()
|
||||
yield
|
||||
from agent.web_search_registry import _reset_for_tests
|
||||
_reset_for_tests()
|
||||
|
||||
def test_search_dispatches_to_tavily(self):
|
||||
mock_response = _ok_response({
|
||||
"results": [{"title": "Result", "url": "https://r.com", "content": "desc", "score": 0.9}]
|
||||
})
|
||||
|
||||
with patch("tools.web_tools._get_backend", return_value="tavily"), \
|
||||
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \
|
||||
patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \
|
||||
patch("tools.interrupt.is_interrupted", return_value=False):
|
||||
from tools.web_tools import web_search_tool
|
||||
result = json.loads(web_search_tool("test query", limit=3))
|
||||
assert result["success"] is True
|
||||
assert len(result["data"]["web"]) == 1
|
||||
assert result["data"]["web"][0]["title"] == "Result"
|
||||
|
||||
def test_search_keyless_dispatch(self):
|
||||
"""Opt-in keyless Tavily hits Tavily's own endpoint, not the ring."""
|
||||
mock_response = _ok_response({
|
||||
"results": [{"title": "Result", "url": "https://r.com", "content": "desc"}]
|
||||
})
|
||||
|
||||
with patch("tools.web_tools._get_backend", return_value="tavily"), \
|
||||
patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response) as mock_post, \
|
||||
patch("tools.interrupt.is_interrupted", return_value=False):
|
||||
os.environ.pop("TAVILY_API_KEY", None)
|
||||
from tools.web_tools import web_search_tool
|
||||
result = json.loads(web_search_tool("test query"))
|
||||
assert result["success"] is True
|
||||
headers = mock_post.call_args.kwargs["headers"]
|
||||
assert headers["X-Tavily-Access-Mode"] == "keyless"
|
||||
assert headers["X-Client-Name"] == "hermes-agent"
|
||||
assert "Authorization" not in headers
|
||||
assert "api.tavily.com/search" in mock_post.call_args.args[0]
|
||||
|
||||
def test_tavily_is_not_in_keyless_ring(self):
|
||||
from plugins.web.keyless_mcp import _KEYLESS_RING, _KEYLESS_SEARCHERS, _KEYLESS_EXTRACTORS
|
||||
assert "tavily" not in _KEYLESS_RING
|
||||
assert "tavily" not in _KEYLESS_SEARCHERS
|
||||
assert "tavily" not in _KEYLESS_EXTRACTORS
|
||||
|
||||
|
||||
# ─── web_extract_tool (Tavily dispatch) ───────────────────────────────────────
|
||||
|
||||
class TestWebExtractTavily:
|
||||
"""Test web_extract_tool dispatch to Tavily."""
|
||||
|
||||
_register_providers = staticmethod(register_all_web_providers)
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _populate_web_registry(self):
|
||||
self._register_providers()
|
||||
yield
|
||||
from agent.web_search_registry import _reset_for_tests
|
||||
_reset_for_tests()
|
||||
|
||||
def test_extract_dispatches_to_tavily(self):
|
||||
mock_response = _ok_response({
|
||||
"results": [{"url": "https://example.com", "raw_content": "Extracted content", "title": "Page"}]
|
||||
})
|
||||
|
||||
async def _allow_ssrf(_url: str) -> bool:
|
||||
return True
|
||||
|
||||
with patch("tools.web_tools._get_backend", return_value="tavily"), \
|
||||
patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}), \
|
||||
patch("plugins.web.tavily.provider.httpx.post", return_value=mock_response), \
|
||||
patch("tools.web_tools.async_is_safe_url", _allow_ssrf):
|
||||
from tools.web_tools import web_extract_tool
|
||||
result = json.loads(asyncio.get_event_loop().run_until_complete(
|
||||
web_extract_tool(["https://example.com"])
|
||||
))
|
||||
assert "results" in result
|
||||
assert len(result["results"]) == 1
|
||||
assert result["results"][0]["url"] == "https://example.com"
|
||||
assert "Extracted content" in result["results"][0]["content"]
|
||||
+1
-1
@@ -21,7 +21,7 @@ Limitations:
|
||||
connects to the validated IP while preserving Host/SNI semantics.
|
||||
- Redirect-based bypass is mitigated by httpx event hooks that re-validate
|
||||
each redirect target in vision_tools, gateway platform adapters, and
|
||||
media cache helpers. Web tools use third-party SDKs (Firecrawl/Exa)
|
||||
media cache helpers. Web tools use third-party SDKs (Firecrawl/Tavily)
|
||||
where redirect handling is on their servers.
|
||||
"""
|
||||
|
||||
|
||||
+40
-15
@@ -15,6 +15,7 @@ Backend compatibility:
|
||||
- Exa: https://exa.ai (search, extract)
|
||||
- Firecrawl: https://docs.firecrawl.dev/introduction (search, extract; direct or derived firecrawl-gateway.<domain> for Nous Subscribers)
|
||||
- Parallel: https://docs.parallel.ai (search, extract)
|
||||
- Tavily: https://tavily.com (search, extract; keyed or opt-in keyless, not in the free-tier ring)
|
||||
|
||||
LLM Processing:
|
||||
- Uses OpenRouter API with Gemini 3 Flash Preview for intelligent content extraction
|
||||
@@ -57,6 +58,13 @@ from plugins.web.firecrawl.provider import (
|
||||
_is_tool_gateway_ready,
|
||||
check_firecrawl_api_key,
|
||||
)
|
||||
# Tavily helpers re-exported for backward-compat with existing unit tests
|
||||
# (tests/tools/test_web_tools_tavily.py imports these names directly).
|
||||
from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names
|
||||
_normalize_tavily_documents,
|
||||
_normalize_tavily_search_results,
|
||||
_tavily_request,
|
||||
)
|
||||
# Parallel + Exa clients re-exported for backward-compat with existing
|
||||
# unit tests (tests/tools/test_web_tools_config.py imports _get_parallel_client
|
||||
# / _get_async_parallel_client / _get_exa_client directly).
|
||||
@@ -161,7 +169,7 @@ def _load_web_config() -> dict:
|
||||
# WebSearchProvider. Keep the two sets aligned by hand: if xai ever ships as
|
||||
# a registered provider, drop it here so the registry path takes over.
|
||||
_LEGACY_WEB_BACKENDS = frozenset(
|
||||
{"parallel", "firecrawl", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"}
|
||||
{"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"}
|
||||
)
|
||||
|
||||
|
||||
@@ -244,13 +252,14 @@ def _get_backend() -> str:
|
||||
return "firecrawl"
|
||||
|
||||
# Never-configured install — pick the highest-priority available
|
||||
# backend. Explicit user credentials (EXA_API_KEY etc.)
|
||||
# backend. Explicit user credentials (TAVILY_API_KEY etc.)
|
||||
# beat the managed-tool-gateway probe so a deliberate setup is not
|
||||
# pre-empted by a Nous OAuth token whose subscription tier may not
|
||||
# actually grant web-search access (the gateway then fails at runtime
|
||||
# with "no subscription" and the tool returns an error to the agent
|
||||
# without falling back). Free-tier backends trail the paid ones.
|
||||
backend_candidates = (
|
||||
("tavily", _has_env("TAVILY_API_KEY")),
|
||||
("exa", _has_env("EXA_API_KEY")),
|
||||
("parallel", _has_env("PARALLEL_API_KEY")),
|
||||
("keenable", _has_env("KEENABLE_API_KEY")),
|
||||
@@ -350,6 +359,13 @@ def _get_capability_backend(capability: str) -> str:
|
||||
return _get_backend()
|
||||
|
||||
|
||||
def _tavily_explicitly_configured() -> bool:
|
||||
cfg = _load_web_config()
|
||||
return any(
|
||||
(cfg.get(key) or "").lower().strip() == "tavily"
|
||||
for key in ("backend", "search_backend", "extract_backend")
|
||||
)
|
||||
|
||||
|
||||
def _is_backend_available(backend: str) -> bool:
|
||||
"""Return True when the selected backend is currently usable.
|
||||
@@ -376,6 +392,8 @@ def _is_backend_available(backend: str) -> bool:
|
||||
return _has_env("KEENABLE_API_KEY")
|
||||
if backend == "firecrawl":
|
||||
return check_firecrawl_api_key()
|
||||
if backend == "tavily":
|
||||
return _has_env("TAVILY_API_KEY") or _tavily_explicitly_configured()
|
||||
if backend == "searxng":
|
||||
return _has_env("SEARXNG_URL")
|
||||
if backend == "brave-free":
|
||||
@@ -585,6 +603,7 @@ def _web_requires_env() -> list[str]:
|
||||
return [
|
||||
"EXA_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
"KEENABLE_API_KEY",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
@@ -595,10 +614,11 @@ def _web_requires_env() -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
# ─── Parallel / Firecrawl helpers — moved into plugins ───────────────────────
|
||||
# ─── Parallel / Tavily / Firecrawl helpers — moved into plugins ──────────────
|
||||
# After PR #25182, the per-vendor client construction, request helpers, and
|
||||
# response normalizers all live in plugins.web.<vendor>.provider:
|
||||
# - parallel: plugins/web/parallel/provider.py
|
||||
# - tavily: plugins/web/tavily/provider.py
|
||||
# - firecrawl: plugins/web/firecrawl/provider.py
|
||||
# The names from the firecrawl plugin (Firecrawl proxy, _get_firecrawl_client,
|
||||
# _to_plain_object, _normalize_result_list, _extract_web_search_results,
|
||||
@@ -790,7 +810,7 @@ def _ensure_web_plugins_loaded() -> None:
|
||||
"""Idempotently trigger plugin discovery so the web registry is populated.
|
||||
|
||||
Every bundled web provider (brave-free, ddgs, searxng, exa, parallel,
|
||||
firecrawl, keenable) registers itself via ``plugins/web/<vendor>/__init__.py``
|
||||
tavily, firecrawl, keenable) registers itself via ``plugins/web/<vendor>/__init__.py``
|
||||
during plugin discovery. Tool dispatch can be reached from contexts that
|
||||
haven't already triggered discovery — subprocess agent runs, delegate
|
||||
children, standalone scripts, certain test paths — and without it the
|
||||
@@ -871,9 +891,9 @@ def web_search_tool(query: str, limit: int = 5) -> str:
|
||||
if is_interrupted():
|
||||
return tool_error("Interrupted", success=False)
|
||||
|
||||
# Dispatch through the web search registry. All 7 providers
|
||||
# (brave-free, ddgs, searxng, exa, parallel, firecrawl, keenable)
|
||||
# now live as plugins; the dispatcher is just a registry lookup +
|
||||
# Dispatch through the web search registry. All bundled providers
|
||||
# (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl,
|
||||
# keenable) now live as plugins; the dispatcher is just a registry lookup +
|
||||
# delegation. Sync only — every provider's search() is sync.
|
||||
_ensure_web_plugins_loaded()
|
||||
from agent.web_search_registry import (
|
||||
@@ -1034,7 +1054,7 @@ async def web_extract_tool(
|
||||
Extract content from specific web pages using available extraction API backend.
|
||||
|
||||
Returns clean page content (markdown/text) with NO LLM summarization. The
|
||||
extract backends (Firecrawl, Exa, Parallel, Keenable) already return clean,
|
||||
extract backends (Firecrawl, Tavily, Exa, Parallel, Keenable) already return clean,
|
||||
boilerplate-stripped content, so we return it directly and fast. Pages over
|
||||
``char_limit`` are head+tail truncated with an explicit footer; the full
|
||||
text is stored under cache/web and the footer tells the model how to
|
||||
@@ -1142,10 +1162,10 @@ async def web_extract_tool(
|
||||
else:
|
||||
backend = _get_extract_backend()
|
||||
|
||||
# All seven providers (brave-free, ddgs, searxng, exa, parallel,
|
||||
# firecrawl, keenable) now live as plugins. The dispatcher is a
|
||||
# All bundled providers (brave-free, ddgs, searxng, exa, parallel,
|
||||
# tavily, firecrawl, keenable) now live as plugins. The dispatcher is a
|
||||
# registry lookup + delegation. Some providers' extract() is
|
||||
# async (parallel, firecrawl), others sync (exa, keenable) — we
|
||||
# async (parallel, firecrawl), others sync (exa, tavily, keenable) — we
|
||||
# detect coroutine functions and await; sync functions run
|
||||
# inline (the policy gate, SSRF re-check, etc. live inside the
|
||||
# provider itself for the firecrawl per-URL loop).
|
||||
@@ -1172,7 +1192,7 @@ async def web_extract_tool(
|
||||
f"{provider.display_name} is a search-only "
|
||||
"backend and cannot extract URL content. "
|
||||
"Set web.extract_backend to firecrawl, "
|
||||
"keenable, exa, or parallel."
|
||||
"tavily, keenable, exa, or parallel."
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
@@ -1235,7 +1255,7 @@ async def web_extract_tool(
|
||||
"error": (
|
||||
"No web extract provider configured. "
|
||||
"Set web.extract_backend to firecrawl, "
|
||||
"keenable, exa, or parallel."
|
||||
"tavily, keenable, exa, or parallel."
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
@@ -1284,7 +1304,7 @@ async def web_extract_tool(
|
||||
)
|
||||
|
||||
# Async-or-sync dispatch: parallel + firecrawl have async
|
||||
# extract(); exa + keenable are sync.
|
||||
# extract(); exa + tavily + keenable are sync.
|
||||
import inspect
|
||||
_extract_rescued = False
|
||||
try:
|
||||
@@ -1568,6 +1588,11 @@ if __name__ == "__main__":
|
||||
print(" Using Exa API (https://exa.ai)")
|
||||
elif backend == "parallel":
|
||||
print(" Using Parallel API (https://parallel.ai)")
|
||||
elif backend == "tavily":
|
||||
if _has_env("TAVILY_API_KEY"):
|
||||
print(" Using Tavily API (https://tavily.com)")
|
||||
else:
|
||||
print(" Using Tavily keyless (https://docs.tavily.com/documentation/keyless)")
|
||||
elif backend == "searxng":
|
||||
print(f" Using SearXNG (search only): {_env_value('SEARXNG_URL')}")
|
||||
elif backend == "brave-free":
|
||||
@@ -1585,7 +1610,7 @@ if __name__ == "__main__":
|
||||
else:
|
||||
print("❌ No web search backend configured")
|
||||
print(
|
||||
"Set EXA_API_KEY, PARALLEL_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL"
|
||||
"Set EXA_API_KEY, PARALLEL_API_KEY, TAVILY_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL"
|
||||
f"{_firecrawl_backend_help_suffix()}"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ description: "How to build a web-search/extract/crawl backend plugin for Hermes
|
||||
|
||||
# Building a Web Search Provider Plugin
|
||||
|
||||
Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web/<name>/`. You can add a new one, or override a bundled one, by dropping a directory next to them.
|
||||
Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Tavily, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web/<name>/`. You can add a new one, or override a bundled one, by dropping a directory next to them.
|
||||
|
||||
:::tip
|
||||
Web search is one of several **backend plugins** Hermes supports. The others (with their own ABCs) are [Image Generation Provider Plugins](/developer-guide/image-gen-provider-plugin), [Video Generation Provider Plugins](/developer-guide/video-gen-provider-plugin), [Memory Provider Plugins](/developer-guide/memory-provider-plugin), [Context Engine Plugins](/developer-guide/context-engine-plugin), and [Model Provider Plugins](/developer-guide/model-provider-plugin). General tool/hook/CLI plugins live in [Build a Hermes Plugin](/developer-guide/plugins).
|
||||
@@ -157,7 +157,7 @@ Full contract in `agent/web_search_provider.py`. Methods you may override:
|
||||
| `search(query, limit)` | conditional | raises | Required when `supports_search()` returns `True` |
|
||||
| `extract(urls, **kwargs)` | conditional | raises | Required when `supports_extract()` returns `True` |
|
||||
|
||||
Providers can advertise multiple capabilities from a single class — Firecrawl, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow.
|
||||
Providers can advertise multiple capabilities from a single class — Firecrawl, Tavily, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow.
|
||||
|
||||
## Response shape
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ Quick setup example:
|
||||
|
||||
```yaml
|
||||
web:
|
||||
backend: firecrawl # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai
|
||||
backend: firecrawl # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai
|
||||
```
|
||||
|
||||
If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`.
|
||||
|
||||
@@ -151,6 +151,8 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe
|
||||
| `PARALLEL_API_KEY` | AI-native web search ([parallel.ai](https://parallel.ai/)) |
|
||||
| `FIRECRAWL_API_KEY` | Web scraping and cloud browser ([firecrawl.dev](https://firecrawl.dev/)) |
|
||||
| `FIRECRAWL_API_URL` | Custom Firecrawl API endpoint for self-hosted instances (optional) |
|
||||
| `TAVILY_API_KEY` | Optional Tavily API key for higher search/extract limits. After selecting Tavily as the web backend, keyless access works without it ([app.tavily.com](https://app.tavily.com/home), [keyless docs](https://docs.tavily.com/documentation/keyless)) |
|
||||
| `TAVILY_BASE_URL` | Override the Tavily API endpoint. Useful for corporate proxies and self-hosted Tavily-compatible search backends. Same pattern as `GROQ_BASE_URL`. |
|
||||
| `SEARXNG_URL` | SearXNG instance URL for free self-hosted web search — no API key required ([searxng.github.io](https://searxng.github.io/searxng/)) |
|
||||
| `EXA_API_KEY` | Exa API key for AI-native web search and contents ([exa.ai](https://exa.ai/)) |
|
||||
| `BRAVE_SEARCH_API_KEY` | Brave Search API subscription token for web search (free tier available) ([brave.com/search/api](https://brave.com/search/api/)) |
|
||||
|
||||
@@ -320,8 +320,8 @@ The single `video_generate` tool covers both modalities — pass `image_url` to
|
||||
|
||||
| Tool | Description | Requires environment |
|
||||
|------|-------------|----------------------|
|
||||
| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY |
|
||||
| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or KEENABLE_API_KEY |
|
||||
| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY |
|
||||
| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY |
|
||||
|
||||
## `x_search` toolset
|
||||
|
||||
|
||||
@@ -2353,7 +2353,7 @@ The `web_search` and `web_extract` tools support five backend providers. Configu
|
||||
|
||||
```yaml
|
||||
web:
|
||||
backend: firecrawl # firecrawl | searxng | parallel | keenable | exa
|
||||
backend: firecrawl # firecrawl | searxng | parallel | tavily | keenable | exa
|
||||
|
||||
# Or use per-capability keys to mix providers (e.g. free search + paid extract):
|
||||
search_backend: "searxng"
|
||||
@@ -2382,9 +2382,10 @@ web:
|
||||
| **Firecrawl** (default) | `FIRECRAWL_API_KEY` | ✔ | ✔ |
|
||||
| **SearXNG** | `SEARXNG_URL` | ✔ | — |
|
||||
| **Parallel** | `PARALLEL_API_KEY` (optional — keyless free tier) | ✔ | ✔ |
|
||||
| **Tavily** | `TAVILY_API_KEY` (optional — keyless when selected; not in the free-tier ring) | ✔ | ✔ |
|
||||
| **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ |
|
||||
|
||||
**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Firecrawl or Keenable in `hermes tools` also works without a key.
|
||||
**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key.
|
||||
|
||||
**SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](/user-guide/features/web-search) for Docker setup instructions.
|
||||
|
||||
|
||||
@@ -235,7 +235,7 @@ Config changes take effect on the next agent session or gateway restart. The web
|
||||
Manage the `.env` file where API keys and credentials are stored. Keys are grouped by category:
|
||||
|
||||
- **LLM Providers** — OpenRouter, Anthropic, OpenAI, DeepSeek, etc.
|
||||
- **Tool API Keys** — Browserbase, Firecrawl, Keenable, ElevenLabs, etc.
|
||||
- **Tool API Keys** — Browserbase, Firecrawl, Tavily, Keenable, ElevenLabs, etc.
|
||||
- **Messaging Platforms** — Telegram, Discord, Slack bot tokens, etc.
|
||||
- **Agent Settings** — non-secret env vars like `API_SERVER_ENABLED`
|
||||
|
||||
|
||||
@@ -24,10 +24,11 @@ Both are configured through a single backend selection. Providers are chosen via
|
||||
| **DDGS (DuckDuckGo)** | — (no key) | ✔ | — | ✔ Free |
|
||||
| **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with key |
|
||||
| **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key |
|
||||
| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Opt-in keyless when selected · not in the free-tier ring |
|
||||
| **Keenable** | `KEENABLE_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key |
|
||||
| **xAI (Grok)** | `XAI_API_KEY` or `hermes auth add xai-oauth` | ✔ | — | Paid (SuperGrok or per-token) |
|
||||
|
||||
Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below).
|
||||
Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below).
|
||||
|
||||
**Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below.
|
||||
|
||||
@@ -266,13 +267,27 @@ SearXNG handles search; you need a separate provider for `web_extract`. Use the
|
||||
# ~/.hermes/config.yaml
|
||||
web:
|
||||
search_backend: "searxng"
|
||||
extract_backend: "firecrawl" # or keenable, exa, parallel
|
||||
extract_backend: "firecrawl" # or tavily, keenable, exa, parallel
|
||||
```
|
||||
|
||||
With this config, Hermes uses SearXNG for all search queries and Firecrawl for URL extraction — combining free search with high-quality extraction.
|
||||
|
||||
---
|
||||
|
||||
### Tavily
|
||||
|
||||
AI-optimised search and extract. Select Tavily in `hermes tools` (or set `web.backend: tavily`) to use it **keyless** with no account (rate-limited). Tavily is **not** in the zero-config free-tier ring — empty installs rotate across Exa / Parallel / Firecrawl / Keenable. Set an API key when you want higher limits.
|
||||
|
||||
```bash
|
||||
# optional — skip this for keyless access after selecting Tavily
|
||||
# ~/.hermes/.env
|
||||
TAVILY_API_KEY=tvly-your-key-here
|
||||
```
|
||||
|
||||
Get a key at [app.tavily.com](https://app.tavily.com/home). See [Tavily keyless](https://docs.tavily.com/documentation/keyless).
|
||||
|
||||
---
|
||||
|
||||
### Exa
|
||||
|
||||
Neural search with semantic understanding. Good for research and finding conceptually related content.
|
||||
@@ -338,10 +353,10 @@ web:
|
||||
timeout: 90 # seconds (default)
|
||||
```
|
||||
|
||||
**Search-only** — pair with Firecrawl / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry.
|
||||
**Search-only** — pair with Firecrawl / Tavily / Keenable / Exa / Parallel if you also need `web_extract`. On 401 the provider performs a single forced OAuth-token refresh and retries (covers mid-window revocation and opaque tokens the proactive expiry check can't decode); env-var credentials skip the retry.
|
||||
|
||||
:::caution Trust model
|
||||
Unlike index-backed providers (Brave, Keenable, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input.
|
||||
Unlike index-backed providers (Brave, Tavily, Exa) which return verbatim search-engine results, xAI is an LLM choosing which URLs to surface and writing the titles and descriptions itself. The *content* of the query influences the output, so a maliciously crafted query (e.g. injected via untrusted upstream input the agent picked up) can in principle steer Grok into emitting attacker-chosen URLs. Treat returned URLs the same way you'd treat any model-generated link — validate before fetching, especially if the query came from untrusted input.
|
||||
:::
|
||||
|
||||
---
|
||||
@@ -355,7 +370,7 @@ Set one provider for all web capabilities:
|
||||
```yaml
|
||||
# ~/.hermes/config.yaml
|
||||
web:
|
||||
backend: "searxng" # firecrawl | searxng | brave-free | ddgs | keenable | exa | parallel | xai
|
||||
backend: "searxng" # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai
|
||||
```
|
||||
|
||||
### Per-capability configuration
|
||||
@@ -382,6 +397,7 @@ If no backend has **ever** been selected (no `web.backend` / per-capability key
|
||||
|
||||
| Credential present | Auto-selected backend |
|
||||
|--------------------|-----------------------|
|
||||
| `TAVILY_API_KEY` | tavily |
|
||||
| `EXA_API_KEY` | exa |
|
||||
| `PARALLEL_API_KEY` | parallel |
|
||||
| `FIRECRAWL_API_KEY` or `FIRECRAWL_API_URL` (or the Nous Tool Gateway is ready) | firecrawl |
|
||||
@@ -438,7 +454,7 @@ SearXNG cannot extract URL content. Set `web.extract_backend` to a provider that
|
||||
```yaml
|
||||
web:
|
||||
search_backend: "searxng"
|
||||
extract_backend: "firecrawl" # or keenable / exa / parallel
|
||||
extract_backend: "firecrawl" # or tavily / keenable / exa / parallel
|
||||
```
|
||||
|
||||
### SearXNG returns 0 results
|
||||
|
||||
Reference in New Issue
Block a user