feat(web): Perplexity Search API as a web_search + web_extract backend
Adds plugins/web/perplexity — a keyed-only WebSearchProvider over httpx: - search: POST https://api.perplexity.ai/search (documented Search API), search_context_size=low so `snippet` stays description-sized; results[].snippet -> description, max_results capped at the API's 20. - extract: POST /sdk/content/snippets — the query-relevant page-excerpt route behind `pplx content snippets` (the CLI's `content fetch` is deprecated upstream). web_extract has no query, so the URLs' path words serve as the relevance query; per-URL `error` entries survive a 200. - Wired into the same touchpoints as the other keyed vendors: legacy backend set + credential ladder + availability probe (web_tools), registry preference walk, OPTIONAL_ENV_VARS, `hermes config`/status/ dump key lists, nous_subscription direct-credential detection, setup summary, test conftests, docs. Not a keyless-ring member (Perplexity has no anonymous tier). Related closed PRs #9192 / #23981 / #45225 predate the plugin ABC.
This commit is contained in:
Teknium
@@ -57,7 +57,7 @@ def _configured_backend(capability: str) -> Optional[str]:
|
||||
|
||||
# Paid providers first so existing paid setups don't get downgraded to a free
|
||||
# tier on upgrade; filtered by ``is_available()`` at walk time.
|
||||
_LEGACY_PREFERENCE = ("firecrawl", "parallel", "tavily", "exa", "searxng", "brave-free", "ddgs")
|
||||
_LEGACY_PREFERENCE = ("firecrawl", "parallel", "tavily", "perplexity", "exa", "searxng", "brave-free", "ddgs")
|
||||
|
||||
# Anonymous public free tiers (see plugins/web/keyless_mcp.py); strictly last
|
||||
# resort, i.e. zero web credentials and no importable ddgs. Unpinned keyless
|
||||
|
||||
@@ -945,7 +945,7 @@ _ENV_CONFIG_KEYS = frozenset({
|
||||
'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY',
|
||||
'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL',
|
||||
'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME',
|
||||
'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', 'API_SERVER_KEY',
|
||||
'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', 'PERPLEXITY_API_KEY', 'API_SERVER_KEY',
|
||||
'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY',
|
||||
'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN',
|
||||
'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY',
|
||||
@@ -2800,6 +2800,7 @@ _SHOW_CONFIG_API_KEYS = (
|
||||
("PARALLEL_API_KEY", "Parallel"),
|
||||
("FIRECRAWL_API_KEY", "Firecrawl"),
|
||||
("TAVILY_API_KEY", "Tavily"),
|
||||
("PERPLEXITY_API_KEY", "Perplexity"),
|
||||
("BROWSERBASE_API_KEY", "Browserbase"),
|
||||
("BROWSER_USE_API_KEY", "Browser Use"),
|
||||
("FAL_KEY", "FAL"))
|
||||
|
||||
@@ -2490,6 +2490,10 @@ OPTIONAL_ENV_VARS = {
|
||||
"Tavily API key for AI-native web search and extract (optional — keyless works when "
|
||||
"Tavily is selected)", "Tavily API key", "https://app.tavily.com/home",
|
||||
tools=["web_search", "web_extract"]),
|
||||
"PERPLEXITY_API_KEY": _tool(
|
||||
"Perplexity API key for the Search API web backend (ranked results + query-relevant page "
|
||||
"snippets)", "Perplexity API key", "https://www.perplexity.ai/account/api",
|
||||
tools=["web_search", "web_extract"]),
|
||||
"KEENABLE_API_KEY": _tool(
|
||||
"Keenable API key for fast independent-index web search and page fetch (optional — "
|
||||
"keyless free tier works without it)", "Keenable API key", "https://keenable.ai",
|
||||
|
||||
+1
-1
@@ -167,7 +167,7 @@ _API_KEYS = [
|
||||
("DASHSCOPE_API_KEY", "dashscope"), ("HF_TOKEN", "huggingface"), ("NVIDIA_API_KEY", "nvidia"),
|
||||
("AI_GATEWAY_API_KEY", "ai_gateway"), ("OPENCODE_ZEN_API_KEY", "opencode_zen"),
|
||||
("OPENCODE_GO_API_KEY", "opencode_go"), ("COMMANDCODE_API_KEY", "commandcode"),
|
||||
("KILOCODE_API_KEY", "kilocode"), ("FIRECRAWL_API_KEY", "firecrawl"), ("TAVILY_API_KEY", "tavily"),
|
||||
("KILOCODE_API_KEY", "kilocode"), ("FIRECRAWL_API_KEY", "firecrawl"), ("TAVILY_API_KEY", "tavily"), ("PERPLEXITY_API_KEY", "perplexity"),
|
||||
("KEENABLE_API_KEY", "keenable"), ("BROWSERBASE_API_KEY", "browserbase"), ("FAL_KEY", "fal"),
|
||||
("ELEVENLABS_API_KEY", "elevenlabs"), ("GITHUB_TOKEN", "github"),
|
||||
]
|
||||
|
||||
@@ -43,8 +43,8 @@ class _FeatureSpec:
|
||||
_FEATURES: Dict[str, _FeatureSpec] = {
|
||||
"web": _FeatureSpec(
|
||||
"Web tools", True, "firecrawl", "firecrawl", ("web", "backend"),
|
||||
"Web search & extract (Firecrawl)", "Firecrawl/Exa/Parallel/Keenable key or SearXNG",
|
||||
("PARALLEL_API_KEY", "TAVILY_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL"),
|
||||
"Web search & extract (Firecrawl)", "Firecrawl/Exa/Parallel/Tavily/Perplexity/Keenable key or SearXNG",
|
||||
("PARALLEL_API_KEY", "TAVILY_API_KEY", "PERPLEXITY_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL"),
|
||||
),
|
||||
"image_gen": _FeatureSpec(
|
||||
"Image generation", True, "fal", "fal-queue", ("image_gen", "provider"), "Image generation (FAL)", "FAL key",
|
||||
@@ -296,10 +296,11 @@ def _web_feature(web_cfg: Dict[str, object], tool_enabled: bool, managed: bool,
|
||||
"firecrawl": direct_firecrawl,
|
||||
"parallel": _any_env("PARALLEL_API_KEY") and not web_gw,
|
||||
"tavily": (_any_env("TAVILY_API_KEY") or "tavily" in {backend, search_backend, extract_backend}) and not web_gw,
|
||||
"perplexity": _any_env("PERPLEXITY_API_KEY") and not web_gw,
|
||||
"searxng": _any_env("SEARXNG_URL"),
|
||||
}
|
||||
web_managed = backend == "firecrawl" and managed and not direct_firecrawl
|
||||
active = web_managed or direct.get(backend) or direct.get(search_backend) or (extract_backend == "tavily" and direct["tavily"])
|
||||
active = web_managed or direct.get(backend) or direct.get(search_backend) or (extract_backend in ("tavily", "perplexity") and direct[extract_backend])
|
||||
return _state(
|
||||
"web", available=bool(managed or any(direct.values())), active=bool(tool_enabled and active),
|
||||
managed_by_nous=web_managed, toolset_enabled=tool_enabled,
|
||||
@@ -527,7 +528,7 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]:
|
||||
fal_direct = fal_key_is_configured()
|
||||
audio_direct = bool(resolve_openai_audio_api_key())
|
||||
return {
|
||||
"web": _any_env("FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "SEARXNG_URL"),
|
||||
"web": _any_env("FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY", "TAVILY_API_KEY", "PERPLEXITY_API_KEY", "EXA_API_KEY", "SEARXNG_URL"),
|
||||
# Env-configured keyless local backend: a reachable self-hosted SearXNG is a working web setup even
|
||||
# with no stored selection (the autodetect cascade in tools/web_tools.py picks it up), so it must
|
||||
# not be classified "unconfigured" and pre-checked (#92647).
|
||||
|
||||
@@ -31,7 +31,7 @@ _BROWSER_MISSING_HINTS = {
|
||||
"Local browser": "npm install -g agent-browser && agent-browser install --with-deps"}
|
||||
_BROWSER_MISSING_DEFAULT = "npm install -g agent-browser, set CAMOFOX_URL, or configure Browser Use or Browserbase"
|
||||
_WEB_MISSING = ("EXA_API_KEY, PARALLEL_API_KEY, FIRECRAWL_API_KEY/FIRECRAWL_API_URL, TAVILY_API_KEY, "
|
||||
"KEENABLE_API_KEY, or SEARXNG_URL")
|
||||
"PERPLEXITY_API_KEY, KEENABLE_API_KEY, or SEARXNG_URL")
|
||||
|
||||
_DONE_BANNER = (
|
||||
"┌─────────────────────────────────────────────────────────┐",
|
||||
|
||||
@@ -51,7 +51,7 @@ _API_KEYS: dict[str, str | tuple[str, ...]] = {
|
||||
"xAI / Grok": "XAI_API_KEY", "NVIDIA NIM": "NVIDIA_API_KEY", "Z.AI / GLM": "GLM_API_KEY",
|
||||
"Kimi": "KIMI_API_KEY", "StepFun Step Plan": "STEPFUN_API_KEY", "MiniMax": "MINIMAX_API_KEY",
|
||||
"MiniMax-CN": "MINIMAX_CN_API_KEY", "DeepInfra": "DEEPINFRA_API_KEY", "Firecrawl": "FIRECRAWL_API_KEY",
|
||||
"Tavily": "TAVILY_API_KEY", "Keenable": "KEENABLE_API_KEY",
|
||||
"Tavily": "TAVILY_API_KEY", "Perplexity": "PERPLEXITY_API_KEY", "Keenable": "KEENABLE_API_KEY",
|
||||
"Browser Use": "BROWSER_USE_API_KEY", # Optional — local browser works without this
|
||||
"Browserbase": "BROWSERBASE_API_KEY", # Optional — direct credentials only
|
||||
"FAL": "FAL_KEY", "ElevenLabs": "ELEVENLABS_API_KEY", "GitHub": "GITHUB_TOKEN"}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Perplexity web search + snippets plugin — bundled, auto-loaded.
|
||||
|
||||
Backed by the Perplexity Search API over httpx. Both search and extract are
|
||||
sync; the dispatcher in :mod:`tools.web_tools` handles the wrap when the
|
||||
caller is async.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from plugins.web.perplexity.provider import PerplexityWebSearchProvider
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register the Perplexity provider with the plugin context."""
|
||||
ctx.register_web_search_provider(PerplexityWebSearchProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: web-perplexity
|
||||
version: 1.0.0
|
||||
description: "Perplexity Search API web search and query-relevant page snippets. Requires PERPLEXITY_API_KEY — get one at https://www.perplexity.ai/account/api."
|
||||
author: NousResearch
|
||||
kind: backend
|
||||
provides_web_providers:
|
||||
- perplexity
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Perplexity web search + page snippets — plugin form.
|
||||
|
||||
Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two
|
||||
capabilities advertised:
|
||||
|
||||
- ``supports_search()`` -> True (Perplexity Search API ``POST /search``)
|
||||
- ``supports_extract()`` -> True (``POST /sdk/content/snippets`` — the
|
||||
query-relevant page-excerpt route behind ``pplx content snippets``)
|
||||
|
||||
Both are sync — the underlying call is ``httpx.post(...)``.
|
||||
|
||||
Config keys this provider responds to::
|
||||
|
||||
web:
|
||||
search_backend: "perplexity" # explicit per-capability
|
||||
extract_backend: "perplexity" # explicit per-capability
|
||||
backend: "perplexity" # shared fallback for both
|
||||
|
||||
Env vars::
|
||||
|
||||
PERPLEXITY_API_KEY=... # https://www.perplexity.ai/account/api (required)
|
||||
PERPLEXITY_BASE_URL=... # optional override of https://api.perplexity.ai
|
||||
|
||||
Keyed only — Perplexity has no anonymous tier, so this provider is not a
|
||||
member of the zero-config keyless ring and never resolves without a key.
|
||||
|
||||
Extract caveat: Perplexity's only supported page-content route returns the
|
||||
passages of a page relevant to a *query* (elisions marked ``…``), not the
|
||||
whole page. ``web_extract`` has no query, so the URL's own path words are
|
||||
used as the relevance query, which approximates "what is this page about".
|
||||
Use Firecrawl / Exa / Parallel when a verbatim full-page dump is required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from agent.web_search_provider import WebSearchProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_BASE_URL = "https://api.perplexity.ai"
|
||||
_KEY_URL = "https://www.perplexity.ai/account/api"
|
||||
|
||||
# Search API hard cap for search_type=web.
|
||||
_MAX_SEARCH_RESULTS = 20
|
||||
# Snippet budgets (backend limits: max_tokens 1-16384, per page 1-4096).
|
||||
_MAX_TOKENS = 16384
|
||||
_MAX_TOKENS_PER_PAGE = 4096
|
||||
|
||||
|
||||
def _missing_key_error() -> str:
|
||||
return f"PERPLEXITY_API_KEY is not set. Get a key at {_KEY_URL}"
|
||||
|
||||
|
||||
def _perplexity_request(endpoint: str, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""POST to the Perplexity API and return the parsed JSON response.
|
||||
|
||||
Raises ``ValueError`` when the key is missing or on any non-2xx status,
|
||||
carrying the response body so Perplexity's own error text (invalid key,
|
||||
BAD_REQUEST, rate limit) reaches the model verbatim.
|
||||
"""
|
||||
from agent.web_search_provider import get_provider_env
|
||||
|
||||
api_key = get_provider_env("PERPLEXITY_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError(_missing_key_error())
|
||||
base_url = (get_provider_env("PERPLEXITY_BASE_URL") or _DEFAULT_BASE_URL).rstrip("/")
|
||||
url = f"{base_url}/{endpoint.lstrip('/')}"
|
||||
logger.info("Perplexity %s request to %s", endpoint, url)
|
||||
|
||||
response = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
timeout=60,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
body = (response.text or "").strip()
|
||||
raise ValueError(body or f"HTTP {response.status_code}")
|
||||
return response.json()
|
||||
|
||||
|
||||
def _normalize_search_results(response: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Map Search API ``{results: [{title,url,snippet,...}]}`` to the tool shape."""
|
||||
web_results = []
|
||||
for i, result in enumerate(response.get("results") or []):
|
||||
web_results.append(
|
||||
{
|
||||
"title": result.get("title", "") or "",
|
||||
"url": result.get("url", "") or "",
|
||||
"description": result.get("snippet", "") or "",
|
||||
"position": i + 1,
|
||||
}
|
||||
)
|
||||
return {"success": True, "data": {"web": web_results}}
|
||||
|
||||
|
||||
def _normalize_snippets(response: Dict[str, Any], urls: List[str]) -> List[Dict[str, Any]]:
|
||||
"""Map ``{results: [{url,text?,tokens_count?,error?}]}`` to extract documents.
|
||||
|
||||
One document per requested URL, in request order. A URL the backend
|
||||
omitted or flagged with ``error`` becomes a document carrying ``error``
|
||||
rather than raising — a 200 does not mean every page succeeded.
|
||||
"""
|
||||
by_url = {r.get("url", ""): r for r in (response.get("results") or []) if isinstance(r, dict)}
|
||||
documents: List[Dict[str, Any]] = []
|
||||
for url in urls:
|
||||
result = by_url.get(url, {})
|
||||
text = result.get("text") or ""
|
||||
doc: Dict[str, Any] = {
|
||||
"url": url,
|
||||
"title": "",
|
||||
"content": text,
|
||||
"raw_content": text,
|
||||
"metadata": {"sourceURL": url},
|
||||
}
|
||||
error = result.get("error")
|
||||
if error or not text:
|
||||
doc["error"] = str(error) if error else "no content returned"
|
||||
documents.append(doc)
|
||||
return documents
|
||||
|
||||
|
||||
def _query_for_urls(urls: List[str]) -> str:
|
||||
"""Derive a relevance query from URL path words (``/bloom-filter`` -> ``bloom filter``)."""
|
||||
words: List[str] = []
|
||||
for url in urls:
|
||||
parsed = urlparse(url)
|
||||
for token in parsed.path.replace("-", " ").replace("_", " ").replace("/", " ").split():
|
||||
if token.lower() not in words and not token.isdigit():
|
||||
words.append(token.lower())
|
||||
if not parsed.path.strip("/"):
|
||||
words.append(parsed.netloc)
|
||||
return " ".join(words)[:500] or " ".join(urls)[:500]
|
||||
|
||||
|
||||
class PerplexityWebSearchProvider(WebSearchProvider):
|
||||
"""Perplexity Search API (search) + content snippets (extract), keyed only."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "perplexity"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "Perplexity"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Return True when ``PERPLEXITY_API_KEY`` is set to a non-empty value."""
|
||||
from agent.web_search_provider import get_provider_env
|
||||
|
||||
return bool(get_provider_env("PERPLEXITY_API_KEY"))
|
||||
|
||||
def supports_search(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_extract(self) -> bool:
|
||||
return True
|
||||
|
||||
def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
|
||||
"""Execute a Perplexity Search API query.
|
||||
|
||||
``search_context_size: low`` keeps ``snippet`` at description length;
|
||||
the default (``high``) returns multi-KB page excerpts per hit, which
|
||||
belongs in ``web_extract`` rather than a results list.
|
||||
"""
|
||||
try:
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
if is_interrupted():
|
||||
return {"success": False, "error": "Interrupted"}
|
||||
|
||||
logger.info("Perplexity search: '%s' (limit=%d)", query, limit)
|
||||
raw = _perplexity_request(
|
||||
"search",
|
||||
{
|
||||
"query": query,
|
||||
"max_results": max(1, min(limit, _MAX_SEARCH_RESULTS)),
|
||||
"search_context_size": "low",
|
||||
},
|
||||
)
|
||||
return _normalize_search_results(raw)
|
||||
except ValueError as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
except Exception as exc: # noqa: BLE001 — including httpx errors
|
||||
logger.warning("Perplexity search error: %s", exc)
|
||||
return {"success": False, "error": f"Perplexity search failed: {exc}"}
|
||||
|
||||
def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
|
||||
"""Return query-relevant snippets for one or more URLs.
|
||||
|
||||
Sync — the underlying call is httpx.post(...). Per-URL failures
|
||||
become items with ``error``; a missing key errors every URL.
|
||||
"""
|
||||
try:
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
if is_interrupted():
|
||||
return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
|
||||
|
||||
logger.info("Perplexity snippets: %d URL(s)", len(urls))
|
||||
raw = _perplexity_request(
|
||||
"sdk/content/snippets",
|
||||
{
|
||||
"query": _query_for_urls(urls),
|
||||
"urls": list(urls),
|
||||
"max_tokens": _MAX_TOKENS,
|
||||
"max_tokens_per_page": _MAX_TOKENS_PER_PAGE,
|
||||
},
|
||||
)
|
||||
return _normalize_snippets(raw, list(urls))
|
||||
except ValueError as exc:
|
||||
return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Perplexity extract error: %s", exc)
|
||||
return [
|
||||
{"url": u, "title": "", "content": "", "error": f"Perplexity extract failed: {exc}"}
|
||||
for u in urls
|
||||
]
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "Perplexity",
|
||||
"badge": "paid",
|
||||
"tag": (
|
||||
"Perplexity Search API — ranked, date-stamped web results plus "
|
||||
"query-relevant page snippets for extract."
|
||||
),
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "PERPLEXITY_API_KEY",
|
||||
"prompt": "Perplexity API key",
|
||||
"url": _KEY_URL,
|
||||
},
|
||||
],
|
||||
"web_tier": "paid",
|
||||
}
|
||||
@@ -187,6 +187,7 @@ _CREDENTIAL_NAMES = frozenset({
|
||||
"PARALLEL_API_KEY",
|
||||
"EXA_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
"PERPLEXITY_API_KEY",
|
||||
"WANDB_API_KEY",
|
||||
"ELEVENLABS_API_KEY",
|
||||
"HONCHO_API_KEY",
|
||||
|
||||
@@ -83,6 +83,7 @@ class TestBundledPluginsRegister:
|
||||
"firecrawl",
|
||||
"keenable",
|
||||
"parallel",
|
||||
"perplexity",
|
||||
"searxng",
|
||||
"tavily",
|
||||
"xai",
|
||||
@@ -98,6 +99,7 @@ class TestBundledPluginsRegister:
|
||||
("parallel", True, True),
|
||||
("keenable", True, True),
|
||||
("tavily", True, True),
|
||||
("perplexity", True, True),
|
||||
("firecrawl", True, True),
|
||||
# xai: search-only via Grok's agentic web_search tool.
|
||||
("xai", True, False),
|
||||
@@ -119,7 +121,7 @@ class TestBundledPluginsRegister:
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "keenable", "xai"],
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "perplexity", "firecrawl", "keenable", "xai"],
|
||||
)
|
||||
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
|
||||
@@ -84,6 +84,7 @@ def register_all_web_providers():
|
||||
from plugins.web.parallel.provider import ParallelWebSearchProvider
|
||||
from plugins.web.keenable.provider import KeenableWebSearchProvider
|
||||
from plugins.web.tavily.provider import TavilyWebSearchProvider
|
||||
from plugins.web.perplexity.provider import PerplexityWebSearchProvider
|
||||
from plugins.web.searxng.provider import SearXNGWebSearchProvider
|
||||
from plugins.web.xai.provider import XAIWebSearchProvider
|
||||
|
||||
@@ -96,6 +97,7 @@ def register_all_web_providers():
|
||||
ParallelWebSearchProvider,
|
||||
KeenableWebSearchProvider,
|
||||
TavilyWebSearchProvider,
|
||||
PerplexityWebSearchProvider,
|
||||
SearXNGWebSearchProvider,
|
||||
XAIWebSearchProvider,
|
||||
):
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Perplexity web backend — search + snippets dispatch through the real tools."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from tests.tools.conftest import register_all_web_providers
|
||||
|
||||
|
||||
def _ok(payload):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = payload
|
||||
resp.text = json.dumps(payload)
|
||||
return resp
|
||||
|
||||
|
||||
def test_search_dispatch_maps_search_api_shape():
|
||||
"""web_search on backend=perplexity hits /search with Bearer auth and maps snippet→description."""
|
||||
import tools.web_tools as wt
|
||||
|
||||
register_all_web_providers()
|
||||
payload = {
|
||||
"results": [
|
||||
{"title": "Bloom filter", "url": "https://en.wikipedia.org/wiki/Bloom_filter",
|
||||
"snippet": "space-efficient probabilistic structure", "date": "2004-04-17"},
|
||||
],
|
||||
"id": "abc",
|
||||
}
|
||||
with patch.dict(os.environ, {"PERPLEXITY_API_KEY": "pplx-test"}), \
|
||||
patch.object(wt, "_get_search_backend", return_value="perplexity"), \
|
||||
patch("plugins.web.perplexity.provider.httpx.post", return_value=_ok(payload)) as post:
|
||||
out = json.loads(wt.web_search_tool("bloom filter", limit=3))
|
||||
|
||||
assert post.call_args.args[0] == "https://api.perplexity.ai/search"
|
||||
assert post.call_args.kwargs["headers"]["Authorization"] == "Bearer pplx-test"
|
||||
body = post.call_args.kwargs["json"]
|
||||
assert body["query"] == "bloom filter"
|
||||
assert 1 <= body["max_results"] <= 20 # dispatcher bucket-rounds the fetch limit
|
||||
assert body["search_context_size"] == "low"
|
||||
assert out["success"] is True
|
||||
assert out["data"]["web"][0] == {
|
||||
"title": "Bloom filter",
|
||||
"url": "https://en.wikipedia.org/wiki/Bloom_filter",
|
||||
"description": "space-efficient probabilistic structure",
|
||||
"position": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_dispatch_snippets_per_url_and_missing_key():
|
||||
"""web_extract on backend=perplexity posts every URL to /sdk/content/snippets;
|
||||
a URL the backend failed carries ``error`` instead of content; no key → error, no HTTP."""
|
||||
import tools.web_tools as wt
|
||||
|
||||
register_all_web_providers()
|
||||
urls = ["https://tokio.rs/tokio/tutorial", "https://docs.rs/smol"]
|
||||
payload = {"results": [
|
||||
{"url": urls[0], "text": "Tokio is an asynchronous runtime … for Rust.", "tokens_count": 12},
|
||||
{"url": urls[1], "error": "Page not found or unavailable."},
|
||||
]}
|
||||
with patch.dict(os.environ, {"PERPLEXITY_API_KEY": "pplx-test"}), \
|
||||
patch.object(wt, "_get_extract_backend", return_value="perplexity"), \
|
||||
patch("plugins.web.perplexity.provider.httpx.post", return_value=_ok(payload)) as post:
|
||||
out = json.loads(asyncio.run(wt.web_extract_tool(urls)))
|
||||
|
||||
assert post.call_args.args[0] == "https://api.perplexity.ai/sdk/content/snippets"
|
||||
body = post.call_args.kwargs["json"]
|
||||
assert body["urls"] == urls
|
||||
assert body["query"] == "tokio tutorial smol"
|
||||
assert body["max_tokens_per_page"] <= body["max_tokens"]
|
||||
by_url = {r["url"]: r for r in out["results"]}
|
||||
assert "Tokio is an asynchronous runtime" in by_url[urls[0]]["content"]
|
||||
assert by_url[urls[1]]["error"] == "Page not found or unavailable."
|
||||
|
||||
with patch.dict(os.environ, {}, clear=False), \
|
||||
patch("plugins.web.perplexity.provider.httpx.post") as post:
|
||||
os.environ.pop("PERPLEXITY_API_KEY", None)
|
||||
from plugins.web.perplexity.provider import PerplexityWebSearchProvider
|
||||
p = PerplexityWebSearchProvider()
|
||||
assert p.is_available() is False
|
||||
res = p.search("x")
|
||||
assert res["success"] is False and "PERPLEXITY_API_KEY" in res["error"]
|
||||
docs = p.extract(["https://example.com"])
|
||||
assert "PERPLEXITY_API_KEY" in docs[0]["error"]
|
||||
post.assert_not_called()
|
||||
+4
-2
@@ -113,7 +113,8 @@ def _get_backend() -> str:
|
||||
# token's tier may not grant web access; the gateway then fails at runtime with no fallback).
|
||||
# Free tiers trail paid.
|
||||
backend_candidates = (
|
||||
("tavily", _has_env("TAVILY_API_KEY")), ("exa", _has_env("EXA_API_KEY")),
|
||||
("tavily", _has_env("TAVILY_API_KEY")), ("perplexity", _has_env("PERPLEXITY_API_KEY")),
|
||||
("exa", _has_env("EXA_API_KEY")),
|
||||
("parallel", _has_env("PARALLEL_API_KEY")), ("keenable", _has_env("KEENABLE_API_KEY")),
|
||||
("firecrawl", _has_env("FIRECRAWL_API_KEY") or _has_env("FIRECRAWL_API_URL")),
|
||||
("firecrawl", _is_tool_gateway_ready()), ("searxng", _has_env("SEARXNG_URL")),
|
||||
@@ -183,6 +184,7 @@ _BUILTIN_AVAILABILITY = {
|
||||
"firecrawl": lambda: check_firecrawl_api_key(),
|
||||
"tavily": lambda: _has_env("TAVILY_API_KEY")
|
||||
or any(_configured_backend(k) == "tavily" for k in ("backend", "search_backend", "extract_backend")),
|
||||
"perplexity": lambda: _has_env("PERPLEXITY_API_KEY"),
|
||||
"searxng": lambda: _has_env("SEARXNG_URL"),
|
||||
"brave-free": lambda: _has_env("BRAVE_SEARCH_API_KEY"),
|
||||
"ddgs": lambda: _ddgs_package_importable(),
|
||||
@@ -217,7 +219,7 @@ def _web_requires_env() -> list[str]:
|
||||
on ``managed_nous_tools_enabled()`` cost a synchronous portal HTTP refresh at every CLI startup.
|
||||
Contract: set var -> tool sees it; extras are harmless for the not-logged-in."""
|
||||
return [
|
||||
"EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY",
|
||||
"EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "PERPLEXITY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
]
|
||||
|
||||
@@ -6,7 +6,7 @@ description: "How to build a web-search/extract/crawl backend plugin for Hermes
|
||||
|
||||
# Building a Web Search Provider Plugin
|
||||
|
||||
Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Tavily, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web/<name>/`. You can add a new one, or override a bundled one, by dropping a directory next to them.
|
||||
Web-search provider plugins register a backend that services `web_search`, `web_extract`, and (optionally) deep-crawl tool calls. Built-in providers — Firecrawl, SearXNG, Tavily, Perplexity, Exa, Parallel, Keenable, Brave Search (free tier), xAI, and DDGS — all ship as plugins under `plugins/web/<name>/`. You can add a new one, or override a bundled one, by dropping a directory next to them.
|
||||
|
||||
:::tip
|
||||
Web search is one of several **backend plugins** Hermes supports. The others (with their own ABCs) are [Image Generation Provider Plugins](/developer-guide/image-gen-provider-plugin), [Video Generation Provider Plugins](/developer-guide/video-gen-provider-plugin), [Memory Provider Plugins](/developer-guide/memory-provider-plugin), [Context Engine Plugins](/developer-guide/context-engine-plugin), and [Model Provider Plugins](/developer-guide/model-provider-plugin). General tool/hook/CLI plugins live in [Build a Hermes Plugin](/developer-guide/plugins).
|
||||
@@ -157,7 +157,7 @@ Full contract in `agent/web_search_provider.py`. Methods you may override:
|
||||
| `search(query, limit)` | conditional | raises | Required when `supports_search()` returns `True` |
|
||||
| `extract(urls, **kwargs)` | conditional | raises | Required when `supports_extract()` returns `True` |
|
||||
|
||||
Providers can advertise multiple capabilities from a single class — Firecrawl, Tavily, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow.
|
||||
Providers can advertise multiple capabilities from a single class — Firecrawl, Tavily, Perplexity, Keenable, Exa, and Parallel all implement both search and extract. Brave Search and DDGS are search-only; SearXNG is search-only with a documented "pair me with an extract provider" workflow.
|
||||
|
||||
## Response shape
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ Quick setup example:
|
||||
|
||||
```yaml
|
||||
web:
|
||||
backend: firecrawl # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai
|
||||
backend: firecrawl # firecrawl | searxng | brave-free | ddgs | tavily | perplexity | keenable | exa | parallel | xai
|
||||
```
|
||||
|
||||
If `web.backend` is not set, the backend is auto-detected from whichever API key is available. Self-hosted Firecrawl is also supported via `FIRECRAWL_API_URL`.
|
||||
|
||||
@@ -155,6 +155,8 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe
|
||||
| `FIRECRAWL_API_URL` | Custom Firecrawl API endpoint for self-hosted instances (optional) |
|
||||
| `TAVILY_API_KEY` | Optional Tavily API key for higher search/extract limits. After selecting Tavily as the web backend, keyless access works without it ([app.tavily.com](https://app.tavily.com/home), [keyless docs](https://docs.tavily.com/documentation/keyless)) |
|
||||
| `TAVILY_BASE_URL` | Override the Tavily API endpoint. Useful for corporate proxies and self-hosted Tavily-compatible search backends. Same pattern as `GROQ_BASE_URL`. |
|
||||
| `PERPLEXITY_API_KEY` | Perplexity Search API key for the `perplexity` web backend — ranked search results plus query-relevant page snippets for extract ([perplexity.ai/account/api](https://www.perplexity.ai/account/api)) |
|
||||
| `PERPLEXITY_BASE_URL` | Override the Perplexity API endpoint (default `https://api.perplexity.ai`) for proxies (optional) |
|
||||
| `SEARXNG_URL` | SearXNG instance URL for free self-hosted web search — no API key required ([searxng.github.io](https://searxng.github.io/searxng/)) |
|
||||
| `EXA_API_KEY` | Exa API key for AI-native web search and contents ([exa.ai](https://exa.ai/)) |
|
||||
| `BRAVE_SEARCH_API_KEY` | Brave Search API subscription token for web search (free tier available) ([brave.com/search/api](https://brave.com/search/api/)) |
|
||||
|
||||
@@ -320,8 +320,8 @@ The single `video_generate` tool covers both modalities — pass `image_url` to
|
||||
|
||||
| Tool | Description | Requires environment |
|
||||
|------|-------------|----------------------|
|
||||
| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY |
|
||||
| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or KEENABLE_API_KEY |
|
||||
| `web_search` | Search the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. Accepts an optional `limit` (1-100, default 5). The query is passed through to the configured backend, so operators such as `site:domain`, `filetype:pdf`, `intitle:word`, `-term`, and `"exact phrase"` may work when the backend supports them. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or PERPLEXITY_API_KEY or KEENABLE_API_KEY |
|
||||
| `web_extract` | Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer pointing at the full text saved on disk. Max 5 URLs per call. | EXA_API_KEY or PARALLEL_API_KEY or FIRECRAWL_API_KEY or TAVILY_API_KEY or PERPLEXITY_API_KEY or KEENABLE_API_KEY |
|
||||
|
||||
## `x_search` toolset
|
||||
|
||||
|
||||
@@ -2407,7 +2407,7 @@ The `web_search` and `web_extract` tools support five backend providers. Configu
|
||||
|
||||
```yaml
|
||||
web:
|
||||
backend: firecrawl # firecrawl | searxng | parallel | tavily | keenable | exa
|
||||
backend: firecrawl # firecrawl | searxng | parallel | tavily | perplexity | keenable | exa
|
||||
|
||||
# Or use per-capability keys to mix providers (e.g. free search + paid extract):
|
||||
search_backend: "searxng"
|
||||
@@ -2437,9 +2437,10 @@ web:
|
||||
| **SearXNG** | `SEARXNG_URL` | ✔ | — |
|
||||
| **Parallel** | `PARALLEL_API_KEY` (optional — keyless free tier) | ✔ | ✔ |
|
||||
| **Tavily** | `TAVILY_API_KEY` (optional — keyless when selected) | ✔ | ✔ |
|
||||
| **Perplexity** | `PERPLEXITY_API_KEY` | ✔ | ✔ (query-relevant snippets) |
|
||||
| **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ |
|
||||
|
||||
**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key.
|
||||
**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PERPLEXITY_API_KEY` is set, Perplexity; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](/user-guide/features/web-search) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key.
|
||||
|
||||
**SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](/user-guide/features/web-search) for Docker setup instructions.
|
||||
|
||||
|
||||
@@ -25,10 +25,11 @@ Both are configured through a single backend selection. Providers are chosen via
|
||||
| **Exa** | `EXA_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · 1 000 searches/mo with key |
|
||||
| **Parallel** | `PARALLEL_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key |
|
||||
| **Tavily** | `TAVILY_API_KEY` (optional) | ✔ | ✔ | ✔ Opt-in keyless when selected |
|
||||
| **Perplexity** | `PERPLEXITY_API_KEY` | ✔ | ✔ (query-relevant snippets) | Paid (per-request Search API pricing) |
|
||||
| **Keenable** | `KEENABLE_API_KEY` (optional) | ✔ | ✔ | ✔ Keyless ring member · paid with key |
|
||||
| **xAI (Grok)** | `XAI_API_KEY` or `hermes auth add xai-oauth` | ✔ | — | Paid (SuperGrok or per-token) |
|
||||
|
||||
Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below).
|
||||
Brave Search, DDGS, and xAI are **search-only** — pair any of them with Firecrawl/Tavily/Perplexity/Keenable/Exa/Parallel when you also need `web_extract`. DDGS uses the [`ddgs` Python package](https://pypi.org/project/ddgs/) under the hood; if it isn't already installed, run `pip install ddgs` (or let Hermes lazy-install it on first use). xAI runs Grok's server-side `web_search` tool on the Responses API — results are LLM-generated rather than index-backed, so titles, descriptions, and URL choice are all model output (see the [trust-model caveat](#xai-grok) below).
|
||||
|
||||
**Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below.
|
||||
|
||||
@@ -267,7 +268,7 @@ SearXNG handles search; you need a separate provider for `web_extract`. Use the
|
||||
# ~/.hermes/config.yaml
|
||||
web:
|
||||
search_backend: "searxng"
|
||||
extract_backend: "firecrawl" # or tavily, keenable, exa, parallel
|
||||
extract_backend: "firecrawl" # or tavily, perplexity, keenable, exa, parallel
|
||||
```
|
||||
|
||||
With this config, Hermes uses SearXNG for all search queries and Firecrawl for URL extraction — combining free search with high-quality extraction.
|
||||
@@ -288,6 +289,19 @@ Get a key at [app.tavily.com](https://app.tavily.com/home). See [Tavily keyless]
|
||||
|
||||
---
|
||||
|
||||
### Perplexity
|
||||
|
||||
[Perplexity's Search API](https://docs.perplexity.ai/docs/search/quickstart) returns ranked, date-stamped results from Perplexity's own index (`web_search`). For `web_extract` it uses the same query-relevant *snippets* route as the official `pplx` CLI: you get the passages of each page that matter, with elisions marked `…`, rather than a verbatim full-page dump — pick Firecrawl / Exa / Parallel as `web.extract_backend` when you need the whole page. Keyed only; there is no anonymous tier.
|
||||
|
||||
```bash
|
||||
# ~/.hermes/.env
|
||||
PERPLEXITY_API_KEY=pplx-your-key-here
|
||||
```
|
||||
|
||||
Get a key at [perplexity.ai/account/api](https://www.perplexity.ai/account/api). Set `PERPLEXITY_BASE_URL` to route through a proxy.
|
||||
|
||||
---
|
||||
|
||||
### Exa
|
||||
|
||||
Neural search with semantic understanding. Good for research and finding conceptually related content.
|
||||
@@ -370,7 +384,7 @@ Set one provider for all web capabilities:
|
||||
```yaml
|
||||
# ~/.hermes/config.yaml
|
||||
web:
|
||||
backend: "searxng" # firecrawl | searxng | brave-free | ddgs | tavily | keenable | exa | parallel | xai
|
||||
backend: "searxng" # firecrawl | searxng | brave-free | ddgs | tavily | perplexity | keenable | exa | parallel | xai
|
||||
```
|
||||
|
||||
### Per-capability configuration
|
||||
@@ -398,6 +412,7 @@ If no backend has **ever** been selected (no `web.backend` / per-capability key
|
||||
| Credential present | Auto-selected backend |
|
||||
|--------------------|-----------------------|
|
||||
| `TAVILY_API_KEY` | tavily |
|
||||
| `PERPLEXITY_API_KEY` | perplexity |
|
||||
| `EXA_API_KEY` | exa |
|
||||
| `PARALLEL_API_KEY` | parallel |
|
||||
| `FIRECRAWL_API_KEY` or `FIRECRAWL_API_URL` (or the Nous Tool Gateway is ready) | firecrawl |
|
||||
@@ -454,7 +469,7 @@ SearXNG cannot extract URL content. Set `web.extract_backend` to a provider that
|
||||
```yaml
|
||||
web:
|
||||
search_backend: "searxng"
|
||||
extract_backend: "firecrawl" # or tavily / keenable / exa / parallel
|
||||
extract_backend: "firecrawl" # or tavily / perplexity / keenable / exa / parallel
|
||||
```
|
||||
|
||||
### SearXNG returns 0 results
|
||||
|
||||
Reference in New Issue
Block a user