ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 9 Packages Projects Releases Wiki Activity

refactor(tools): derive _LEGACY_WEB_BACKENDS from probe table, top-level selection helpers, fold single-use policy/cache helpers

Browse Source
This commit is contained in:
Teknium
2026-09-02 22:57:35 -07:00
parent ab5f70073e
commit 3106197c40
6 changed files with 172 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/web_result_cache.py
+22 -32
View File
@@ -4,8 +4,7 @@
paid request. Limits bucket to 10/20/50/100 so near-identical requests share an entry.
* **Extract cache** — disk-backed under ``cache/web`` (cross-process) with a JSON sidecar index:
URL digest → (file, fetched_at, title). Hits re-run the normal truncate pipeline.
Lives here, not in generic tool dispatch, so hits sit *after* every safety check (secret-in-URL, SSRF,
policy) and skip only the vendor call.
Lives here, not in tool dispatch, so hits sit *after* every safety check and skip only the vendor call.
"""
import hashlib
@@ -102,12 +101,10 @@ class SearchMemo:
key = self._key(provider, query, limit)
with self._store_lock:
hit = self._store.get(key)
if hit is None:
return None
expires, response = hit
if time.monotonic() >= expires:
del self._store[key]
if hit is None or time.monotonic() >= hit[0]:
self._store.pop(key, None)
return None
response = hit[1]
logger.info("web_search cache hit: %r via %s", query, provider)
return _deep_copy(response)
@@ -176,32 +173,25 @@ def _cache_dir() -> Optional[Path]:
return None
def _index_path() -> Optional[Path]:
return (d / _INDEX_FILENAME) if (d := _cache_dir()) else None
def _load_index() -> dict:
path = _index_path()
if path is None or not path.exists():
return {}
path = (d / _INDEX_FILENAME) if (d := _cache_dir()) else None
try:
data = json.loads(path.read_text(encoding="utf-8"))
data = json.loads(path.read_text(encoding="utf-8")) if path else {}
return data if isinstance(data, dict) else {}
except Exception: # noqa: BLE001 — corrupt index == empty cache
except Exception: # noqa: BLE001 — missing/corrupt index == empty cache
return {}
def _save_index(index: dict) -> None:
path = _index_path()
if path is None:
if (d := _cache_dir()) is None:
return
path = d / _INDEX_FILENAME
try:
if len(index) > _INDEX_MAX_ENTRIES:
newest = sorted(index.items(), key=lambda kv: kv[1].get("fetched_at", 0), reverse=True)
index = dict(newest[:_INDEX_MAX_ENTRIES])
# Per-process tmp name: CLI, gateway, cron, and subagents all write this index; a shared tmp
# name would let concurrent writers truncate each other. os.replace is atomic, so the worst
# outcome is a lost insert.
# Per-process tmp name: CLI, gateway, cron, and subagents all write this index; a shared tmp name
# would let concurrent writers truncate each other. os.replace is atomic: worst case is a lost insert.
tmp = path.with_suffix(f".tmp.{os.getpid()}")
tmp.write_text(json.dumps(index), encoding="utf-8")
tmp.replace(path)
@@ -241,24 +231,22 @@ def _host_matches_pattern(host: str, pattern: str) -> bool:
def _is_cache_exempt_host(url: str) -> bool:
"""True when the host matches ``web.cache_exempt_hosts`` — sites the user is
developing over public DNS (staging, tunnels, previews) that must fetch live."""
"""True when the host matches ``web.cache_exempt_hosts`` — sites the user develops over public DNS
(staging, tunnels, previews) that must fetch live."""
try:
patterns = _web_config().get("cache_exempt_hosts") or []
if not isinstance(patterns, (list, tuple)) or not patterns:
return False
host = _url_host(url)
return bool(host) and any(_host_matches_pattern(host, str(p)) for p in patterns)
if not isinstance(patterns, (list, tuple)) or not host:
return False
return any(_host_matches_pattern(host, str(p)) for p in patterns)
except Exception: # noqa: BLE001 — config problems never break tools
return False
def _is_local_dev_url(url: str) -> bool:
"""True for loopback/private/LAN URLs — never cached: they are the user's own fast-changing dev servers.
Hostname heuristics only, no DNS: this is a freshness decision, not a security boundary (SSRF
enforcement lives in tools/url_safety.py, which blocks these by default anyway).
"""
Hostname heuristics only, no DNS: this is a freshness decision, not a security boundary (SSRF enforcement
lives in tools/url_safety.py, which blocks these by default anyway)."""
try:
host = _url_host(url).lower()
if not host:
@@ -272,7 +260,7 @@ def _is_local_dev_url(url: str) -> bool:
ip = ipaddress.ip_address(host)
except ValueError:
return False # public DNS name
return bool(ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified)
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified
except Exception: # noqa: BLE001 — on doubt, don't cache
return True
@@ -302,7 +290,9 @@ def extract_cache_get(url: str, format: Optional[str] = None, provider: str = ""
return {"url": url, "title": entry.get("title", ""), "content": content, "error": None, "cached": True}
def extract_cache_put(url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = "") -> None:
def extract_cache_put(
url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = ""
) -> None:
"""Store one successful extraction's full clean text for TTL reuse; pages over the truncate-store
ceiling are not cached (serving a capped copy back as if whole would silently lose the tail)."""
if not content or not _cacheable(url):
tools/web_tools.py
+64 -83
View File
@@ -20,7 +20,9 @@ from plugins.web.firecrawl.provider import ( # noqa: F401 — backward-compat n
Firecrawl, _firecrawl_backend_help_suffix, _get_firecrawl_client, _get_firecrawl_gateway_url,
_is_tool_gateway_ready, check_firecrawl_api_key,
)
from plugins.web.tavily.provider import _normalize_tavily_documents, _normalize_tavily_search_results, _tavily_request # noqa: F401
from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names
_normalize_tavily_documents, _normalize_tavily_search_results, _tavily_request,
)
from plugins.web.parallel.provider import _get_async_parallel_client, _get_parallel_client # noqa: F401
from plugins.web.exa.provider import _get_exa_client # noqa: F401
@@ -36,14 +38,15 @@ from tools.managed_tool_gateway import ( # noqa: F401 — backward-compat names
build_vendor_gateway_url, resolve_managed_tool_gateway,
peek_nous_access_token as _peek_nous_access_token, read_nous_access_token as _read_nous_access_token,
)
from tools.tool_backend_helpers import ( # noqa: F401
from tools.tool_backend_helpers import ( # noqa: F401 — first three are backward-compat re-exports
managed_nous_tools_enabled, nous_tool_gateway_unavailable_message, prefers_gateway,
NOUS_MANAGED_PROVIDER, selection_exists,
)
from tools.url_safety import async_is_safe_url
from tools.web_tools_rescue import ( # noqa: F401 — re-exported (tests patch tools.web_tools.<name>)
_keyless_rescue_enabled, _policy_blocked_result, _rescue_eligible, _rescue_extract, _rescue_search,
)
from tools.web_tools_truncate import ( # noqa: F401 — re-exported (tests + web_result_cache import via tools.web_tools)
from tools.web_tools_truncate import ( # noqa: F401 — re-exported (tests + web_result_cache import these)
DEFAULT_EXTRACT_CHAR_LIMIT, MAX_STORED_TEXT_CHARS, _clamp_char_limit, _effective_char_limit,
_get_extract_char_limit, _store_full_text, _trim_results, _truncate_results, _truncate_with_footer,
convert_base64_images_to_links,
@@ -60,7 +63,7 @@ logger = logging.getLogger(__name__)
# ─── Backend Selection ────────────────────────────────────────────────────────
def _env_value(name: str) -> str:
"""Resolve ``name`` via the Hermes config-aware env layer (``hermes config set`` values), then process env."""
"""Resolve ``name`` via the config-aware env layer (``hermes config set`` values), then process env."""
try:
from hermes_cli.config import get_env_value
val = get_env_value(name)
@@ -87,16 +90,8 @@ def _configured_backend(key: str = "backend") -> str:
return (_load_web_config().get(key) or "").lower().strip()
# Built-in backends probed by _BUILTIN_AVAILABILITY; any other name is a plugin-registered provider resolved
# via the registry's ``is_available()``. Includes ``xai`` (probed via has_xai_credentials(), not a registered
# provider) though the registry's _LEGACY_PREFERENCE omits it — drop it here if xai ever registers.
_LEGACY_WEB_BACKENDS = frozenset(
{"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"}
)
def _registry_call(func_name: str, default, *args):
"""``agent.web_search_registry.<func_name>(*args)``, or *default* if it raised (registry is optional, never fatal)."""
"""``agent.web_search_registry.<func_name>(*args)``, or *default* if it raised (registry never fatal)."""
try:
import agent.web_search_registry as registry_mod
return getattr(registry_mod, func_name)(*args)
@@ -111,14 +106,13 @@ def _registered_web_provider(backend: str):
def _probe(provider, method: str, context: str = "") -> Optional[bool]:
"""``bool(provider.<method>())``, or ``None`` if it raised (logged; a broken provider is unavailable).
``context`` is appended to the debug log line (e.g. " during readiness check").
"""
"""``bool(provider.<method>())``, or ``None`` if it raised (a broken provider is unavailable; *context* is
appended to the debug log line, e.g. " during readiness check")."""
try:
return bool(getattr(provider, method)())
except Exception as exc: # noqa: BLE001 — a broken provider is "unavailable"
logger.debug("web provider %r.%s() raised%s: %s", getattr(provider, "name", provider), method, context, exc)
name = getattr(provider, "name", provider)
logger.debug("web provider %r.%s() raised%s: %s", name, method, context, exc)
return None
@@ -128,20 +122,14 @@ def _list_registered_web_providers():
def _get_backend() -> str:
"""Shared web backend name.
A stored ``web.backend`` is returned as-is — no availability probe, no fallback — so a broken selection
surfaces the vendor's honest error rather than silently rerouting. Autodetect runs ONLY when no web
selection has ever been stored.
"""
"""Shared web backend name. A stored ``web.backend`` is returned as-is — no availability probe, no
fallback — so a broken selection surfaces the vendor's honest error rather than silently rerouting.
Autodetect runs ONLY when no web selection has ever been stored."""
configured = _configured_backend()
if configured:
# "nous" (managed subscription) is serviced by the firecrawl provider, whose client
# resolver routes it through the managed Tool Gateway.
from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER
return "firecrawl" if configured == NOUS_MANAGED_PROVIDER else configured
from tools.tool_backend_helpers import selection_exists
if selection_exists("web"):
# Selection exists (use_gateway / per-capability keys) but no shared name: keep the
# firecrawl default rather than credential-laddering.
@@ -196,12 +184,8 @@ def _get_extract_backend() -> str:
return _configured_backend("extract_backend") or _get_backend()
def _tavily_explicitly_configured() -> bool:
return any(_configured_backend(key) == "tavily" for key in ("backend", "search_backend", "extract_backend"))
def _xai_available() -> bool:
# Cheap probe only (env var OR auth.json OAuth): resolve_xai_http_credentials() may refresh over the network.
# Cheap probe only (env var OR auth.json OAuth): resolve_xai_http_credentials() may hit the network.
try:
from tools.xai_http import has_xai_credentials
return has_xai_credentials()
@@ -218,27 +202,29 @@ def _ddgs_package_importable() -> bool:
return False
# Built-in availability probes (see _LEGACY_WEB_BACKENDS). Lambdas so test patches of module-level helpers
# (e.g. _ddgs_package_importable, check_firecrawl_api_key) are honored at call time.
# Built-in backends and their cheap availability probes; any other name is a plugin-registered provider
# resolved via the registry's ``is_available()``. Lambdas so test patches of module-level helpers (e.g.
# _ddgs_package_importable, check_firecrawl_api_key) are honored at call time. Includes ``xai`` (probed via
# has_xai_credentials(), not a registered provider) though the registry's _LEGACY_PREFERENCE omits it —
# drop it here if xai ever registers.
_BUILTIN_AVAILABILITY = {
"exa": lambda: _has_env("EXA_API_KEY"),
"parallel": lambda: _has_env("PARALLEL_API_KEY"),
"keenable": lambda: _has_env("KEENABLE_API_KEY"),
"firecrawl": lambda: check_firecrawl_api_key(),
"tavily": lambda: _has_env("TAVILY_API_KEY") or _tavily_explicitly_configured(),
"tavily": lambda: _has_env("TAVILY_API_KEY")
or any(_configured_backend(k) == "tavily" for k in ("backend", "search_backend", "extract_backend")),
"searxng": lambda: _has_env("SEARXNG_URL"),
"brave-free": lambda: _has_env("BRAVE_SEARCH_API_KEY"),
"ddgs": lambda: _ddgs_package_importable(),
"xai": _xai_available,
}
_LEGACY_WEB_BACKENDS = frozenset(_BUILTIN_AVAILABILITY)
def _is_backend_available(backend: str) -> bool:
"""True when *backend* is usable — the single availability chokepoint.
Non-legacy names delegate to the registered provider's ``is_available()`` (unregistered names
fall through); built-ins use the cheap hardcoded probes.
"""
"""True when *backend* is usable — the single availability chokepoint. Non-legacy names delegate to the
registered provider's ``is_available()`` (unregistered names fall through); built-ins use cheap probes."""
backend = (backend or "").lower().strip()
if backend not in _LEGACY_WEB_BACKENDS:
provider = _registered_web_provider(backend)
@@ -249,11 +235,9 @@ def _is_backend_available(backend: str) -> bool:
def _web_requires_env() -> list[str]:
"""Tool-registry metadata env vars for the web backends.
Gateway vars are always listed: gating them on ``managed_nous_tools_enabled()`` cost a synchronous portal
HTTP refresh at every CLI startup. Contract: set var -> tool sees it; extras are harmless for the not-logged-in.
"""
"""Tool-registry metadata env vars for the web backends. Gateway vars are always listed: gating them
on ``managed_nous_tools_enabled()`` cost a synchronous portal HTTP refresh at every CLI startup.
Contract: set var -> tool sees it; extras are harmless for the not-logged-in."""
return [
"EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME",
@@ -267,11 +251,9 @@ _debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
# ─── Dispatch ─────────────────────────────────────────────────────────────────
def _ensure_web_plugins_loaded() -> None:
"""Idempotently run plugin discovery so the web registry is populated.
Dispatch is reachable from contexts that never triggered discovery (subprocess agent runs, delegate
children, scripts); without it a configured backend yields a misleading "No web ... provider" error.
"""
"""Idempotently run plugin discovery so the web registry is populated. Dispatch is reachable from contexts
that never triggered discovery (subprocess agent runs, delegate children, scripts); without it a
configured backend yields a misleading "No web ... provider" error."""
try:
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
@@ -312,14 +294,12 @@ def web_search_tool(query: str, limit: int = 5) -> str:
from tools.interrupt import is_interrupted
if is_interrupted():
return tool_error("Interrupted", success=False)
# Sync only — every provider's search() is sync.
_ensure_web_plugins_loaded()
from agent.web_search_registry import get_active_search_provider, get_provider as _wsp_get_provider
backend = _get_search_backend()
provider = _wsp_get_provider(backend) if backend else None
if provider is None or not provider.supports_search():
from tools.tool_backend_helpers import selection_exists
if provider is None and backend and selection_exists("web"):
error_text = debug_call_data["error"] = _strict_selection_error("search", backend)
_finish_debug("web_search_tool", debug_call_data)
@@ -328,8 +308,12 @@ def web_search_tool(query: str, limit: int = 5) -> str:
provider = get_active_search_provider()
if provider is None:
error_text = _no_provider_error("search", "No web search provider configured. Run `hermes tools` to set one up.")
response_data = {"success": False, "error": error_text}
response_data = {
"success": False,
"error": _no_provider_error(
"search", "No web search provider configured. Run `hermes tools` to set one up."
),
}
else:
logger.info("Web search via %s: '%s' (limit: %d)", provider.name, query, limit)
response_data = _memoized_search(provider, query, limit)
@@ -339,18 +323,15 @@ def web_search_tool(query: str, limit: int = 5) -> str:
debug_call_data["final_response_size"] = len(result_json)
_finish_debug("web_search_tool", debug_call_data)
return result_json
except Exception as e:
return _debug_error("web_search_tool", debug_call_data, f"Error searching web: {str(e)}")
def _memoized_search(provider, query: str, limit: int) -> dict:
"""TTL memo + single-flight around the paid vendor call (tools/web_result_cache.py).
Sits after every safety/config check. The provider is asked for the BUCKETED count so near-identical
limits share an entry; the caller's count is sliced out. Only successful, non-rescued responses are
cached — caching a rescue would make the one-shot ring fallback sticky for a whole TTL.
"""
"""TTL memo + single-flight around the paid vendor call (tools/web_result_cache.py); sits after every
safety/config check. The provider is asked for the BUCKETED count so near-identical limits share an entry;
the caller's count is sliced out. Only successful, non-rescued responses are cached — caching a rescue
would make the one-shot ring fallback sticky for a whole TTL."""
from tools.web_result_cache import bucket_limit, search_memo, slice_search_response
def _paid_search() -> tuple[dict, bool]:
fetch_limit = bucket_limit(limit)
@@ -402,21 +383,23 @@ async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Opti
safe_urls.append(url)
safe_indices.append(index)
else:
ssrf_blocked[index] = _result_entry(url, "Blocked: URL targets a private or internal network address")
ssrf_blocked[index] = _result_entry(
url, "Blocked: URL targets a private or internal network address"
)
if not safe_urls:
results = []
else:
results = []
if safe_urls:
backend = _get_extract_backend()
_ensure_web_plugins_loaded()
provider, error_json = _resolve_extract_provider(backend)
if error_json is not None:
return error_json
results = await _extract_safe_urls(provider, safe_urls, format)
# Reconstruct input order across invalid, blocked, and provider entries (providers preserve safe-list order).
# Reconstruct input order across invalid, blocked, and provider entries (providers preserve
# the order of the safe URL list they receive).
if invalid_urls or ssrf_blocked:
results = _merge_in_order(len(urls), {**ssrf_blocked, **invalid_urls}, safe_indices, safe_urls, results)
fixed = {**ssrf_blocked, **invalid_urls}
results = _merge_in_order(len(urls), fixed, safe_indices, safe_urls, results)
logger.info("Extracted content from %d pages", len(results))
debug_call_data["pages_extracted"] = len(results)
@@ -424,17 +407,15 @@ async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Opti
debug_call_data["processing_applied"].append("truncate_and_store")
_truncate_results(results, _effective_char_limit(char_limit), debug_call_data)
trimmed = _trim_results(results)
if not trimmed:
result_json = tool_error("Content was inaccessible or not found")
else:
result_json = json.dumps({"results": trimmed}, indent=2, ensure_ascii=False)
# Belt-and-suspenders sweep over the serialized JSON in case a provider tucked a base64 blob in metadata.
result_json = json.dumps({"results": trimmed}, indent=2, ensure_ascii=False) if trimmed else tool_error(
"Content was inaccessible or not found"
)
# Belt-and-suspenders sweep of the serialized JSON: a provider may tuck a base64 blob in metadata.
cleaned_result = convert_base64_images_to_links(result_json)
debug_call_data["final_response_size"] = len(cleaned_result)
debug_call_data["processing_applied"].append("base64_image_conversion")
_finish_debug("web_extract_tool", debug_call_data)
return cleaned_result
except Exception as e:
return _debug_error("web_extract_tool", debug_call_data, f"Error extracting content: {str(e)}")
@@ -442,9 +423,9 @@ async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Opti
def _provider_is_ready(provider) -> bool:
"""True when *provider* is keyed-available OR keyless-capable, without raising.
``get_active_*_provider()`` returns an explicitly configured backend even when ``is_available()`` is False
(so dispatch can emit a precise error), so readiness gates (tool check_fn, ``hermes doctor``) must probe
for real. Keyless mode (Exa/Parallel free tier) is a working state, not a misconfig.
``get_active_*_provider()`` returns an explicitly configured backend even when ``is_available()`` is
False (so dispatch can emit a precise error), so readiness gates (tool check_fn, ``hermes doctor``)
must probe for real. Keyless mode (Exa/Parallel free tier) is a working state, not a misconfig.
"""
if provider is None:
return False
@@ -460,8 +441,8 @@ def _provider_is_ready(provider) -> bool:
def check_web_api_key() -> bool:
"""``check_fn`` gate for web_search / web_extract: is any web backend available?
A plugin-registered provider reporting ``is_available()`` must light the tools up even with no built-in
credentials; resolution funnels through :func:`_is_backend_available`.
A plugin-registered provider reporting ``is_available()`` must light the tools up even with no
built-in credentials; resolution funnels through :func:`_is_backend_available`.
"""
configured = _configured_backend()
if configured and _is_backend_available(configured):
@@ -473,15 +454,15 @@ def check_web_api_key() -> bool:
try:
_ensure_web_plugins_loaded()
from agent.web_search_registry import get_active_search_provider, get_active_extract_provider
return _provider_is_ready(get_active_search_provider()) or _provider_is_ready(get_active_extract_provider())
return _provider_is_ready(get_active_search_provider()) or _provider_is_ready(
get_active_extract_provider()
)
except Exception as exc: # noqa: BLE001 — registry optional; never fatal
logger.debug("web provider registry availability check failed: %s", exc)
return False
# ---------------------------------------------------------------------------
# Registry
# ---------------------------------------------------------------------------
# ─── Registry ─────────────────────────────────────────────────────────────────
from tools.registry import registry, tool_error
WEB_SEARCH_SCHEMA = {
tools/web_tools_extract.py
+27 -30
View File
@@ -12,6 +12,7 @@ import json
import logging
from typing import Any, Dict, List, Optional
from tools.tool_backend_helpers import selection_error, selection_exists
from tools.url_safety import normalize_url_for_request, sensitive_query_param_name
from tools.web_tools_rescue import _rescue_eligible, _rescue_extract
@@ -38,26 +39,12 @@ def _disabled_plugin_error(capability: str, disabled_key: str) -> str:
"""Error text when the configured backend's bundled plugin is disabled in config."""
vendor = disabled_key.split("/", 1)[-1]
return (
f"web.{capability}_backend is set to '{vendor}', but its "
f"plugin ('{disabled_key}') is disabled in config. "
f"Re-enable it with `hermes plugins enable {disabled_key}` "
f"web.{capability}_backend is set to '{vendor}', but its plugin ('{disabled_key}') is disabled "
f"in config. Re-enable it with `hermes plugins enable {disabled_key}` "
"(or remove it from plugins.disabled)."
)
def _strict_selection_error(capability: str, backend: str) -> str:
"""Error for a stored-but-unregistered backend: name the disabled plugin, else the bad selection.
Strict selection never silently switches to whatever the availability walk finds.
"""
from agent.web_search_registry import _disabled_web_plugin_for
from tools.tool_backend_helpers import selection_error
disabled_key = _disabled_web_plugin_for(capability=capability)
if disabled_key:
return _disabled_plugin_error(capability, disabled_key)
return selection_error("web", f"'{backend}'", f"no registered web {capability} provider has that name")
def _no_provider_error(capability: str, fallback: str) -> str:
"""Error when no provider resolved: point at a disabled bundled plugin if that is the real cause."""
from agent.web_search_registry import _disabled_web_plugin_for
@@ -65,6 +52,14 @@ def _no_provider_error(capability: str, fallback: str) -> str:
return _disabled_plugin_error(capability, disabled_key) if disabled_key else fallback
def _strict_selection_error(capability: str, backend: str) -> str:
"""Error for a stored-but-unregistered backend: name the disabled plugin, else the bad selection.
Strict selection never silently switches to whatever the availability walk finds."""
failure = f"no registered web {capability} provider has that name"
fallback = selection_error("web", f"'{backend}'", failure)
return _no_provider_error(capability, fallback)
def _result_entry(url: str, error: Optional[str]) -> Dict[str, Any]:
return {"url": url, "title": "", "content": "", "error": error}
@@ -85,7 +80,8 @@ def _merge_in_order(
*fetch_positions* (a short provider list yields ``_NO_RESULT_ERROR`` entries for the rest)."""
merged = dict(fixed)
for pos, position in enumerate(fetch_positions):
merged[position] = results[pos] if pos < len(results) else _result_entry(fetch_urls[pos], _NO_RESULT_ERROR)
missing = _result_entry(fetch_urls[pos], _NO_RESULT_ERROR)
merged[position] = results[pos] if pos < len(results) else missing
return [merged[i] for i in range(total)]
@@ -105,13 +101,16 @@ def _validate_extract_urls(urls: List[Any]):
_url = _web_extract_url(item)
if _url is None:
invalid_urls[index] = _result_entry(
"", f"Invalid URL item at index {index}: expected a URL string or an object with a string 'url' or 'href' field"
"",
f"Invalid URL item at index {index}: expected a URL string "
"or an object with a string 'url' or 'href' field",
)
continue
normalized_url = normalize_url_for_request(_url)
if any(_PREFIX_RE.search(c) for c in (_url, unquote(_url), normalized_url, unquote(normalized_url))):
return _refuse_all(
"Blocked: URL contains what appears to be an API key or token. Secrets must not be sent in URLs."
"Blocked: URL contains what appears to be an API key or token. "
"Secrets must not be sent in URLs."
)
sensitive_query_key = sensitive_query_param_name(normalized_url)
if sensitive_query_key:
@@ -129,8 +128,8 @@ def _validate_extract_urls(urls: List[Any]):
def _resolve_extract_provider(backend: str):
"""Resolve the extract provider for *backend*; returns ``(provider, error_json)``.
A registered search-only backend is a typed error (never a silent switch). An unregistered name with a
stored web selection is a strict-selection error; with no selection, fall through to the availability walk.
A registered search-only backend is a typed error (never a silent switch). An unregistered name with
a stored web selection is a strict-selection error; with no selection, fall through to the walk.
"""
from agent.web_search_registry import get_active_extract_provider, get_provider as _wsp_get_provider
provider = _wsp_get_provider(backend) if backend else None
@@ -138,17 +137,16 @@ def _resolve_extract_provider(backend: str):
return provider, None
if provider is not None:
return None, _extract_error_json(
f"{provider.display_name} is a search-only "
"backend and cannot extract URL content. "
f"{provider.display_name} is a search-only backend and cannot extract URL content. "
"Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT
)
from tools.tool_backend_helpers import selection_exists
if backend and selection_exists("web"):
return None, _extract_error_json(_strict_selection_error("extract", backend))
provider = get_active_extract_provider()
if provider is None:
return None, _extract_error_json(_no_provider_error(
"extract", "No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT,
"extract",
"No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT,
))
return provider, None
@@ -180,7 +178,7 @@ async def _dispatch_extract(provider, fetch_urls: List[str], format: Optional[st
continue
_content = fetched.get("raw_content", "") or fetched.get("content", "")
if _content:
extract_cache_put(url, _content, title=fetched.get("title", ""), format=format, provider=provider.name)
extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
return results
@@ -188,9 +186,9 @@ async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[st
"""Serve cache hits, fetch the rest, and merge back in ``safe_urls`` order.
The disk cache (tools/web_result_cache.py) sits AFTER the secret-URL gate, SSRF gate, and provider
resolution, and is gated per-URL on the website policy — a hit skips only the vendor call, never a control.
Policy-blocked URLs are cache misses so dispatch handles them exactly as without a cache. Keys include
provider and format, so switching either within the TTL never serves the other's content.
resolution, and is gated per-URL on the website policy — a hit skips only the vendor call, never a
control. Policy-blocked URLs are cache misses so dispatch handles them exactly as without a cache.
Keys include provider and format, so switching either within the TTL never serves the other's content.
"""
from tools.web_result_cache import extract_cache_get
from tools.website_policy import check_website_access as _check_site
@@ -211,7 +209,6 @@ async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[st
if not fetch_urls:
return [cached_results[i] for i in range(len(safe_urls))]
logger.info("Web extract via %s: %d URL(s)", provider.name, len(fetch_urls))
results = await _dispatch_extract(provider, fetch_urls, format)
if not cached_results:
tools/web_tools_rescue.py
+26 -21
View File
@@ -1,9 +1,8 @@
"""One-shot keyless-ring rescue for failed keyed/configured web calls.
Stateless by design: a rescue routes THIS call through the free-tier ring
(plugins/web/keyless_mcp.py); the next web_search/web_extract call attempts
the chosen backend again. Callers must never cache a rescue-served response,
or the one-shot rescue becomes sticky for a whole TTL. Names are re-imported by
Stateless by design: a rescue routes THIS call through the free-tier ring (plugins/web/keyless_mcp.py);
the next web_search/web_extract call attempts the chosen backend again. Callers must never cache a
rescue-served response, or the one-shot rescue becomes sticky for a whole TTL. Names are re-imported by
tools/web_tools.py (``tools.web_tools._rescue_eligible``); logs under the origin logger.
"""
@@ -13,7 +12,8 @@ logger = logging.getLogger("tools.web_tools")
# Ring vendor -> env var holding its paid key (keyed mode ⇒ eligible for rescue).
_RING_KEY_VARS = {
"exa": "EXA_API_KEY", "parallel": "PARALLEL_API_KEY", "firecrawl": "FIRECRAWL_API_KEY", "keenable": "KEENABLE_API_KEY",
"exa": "EXA_API_KEY", "parallel": "PARALLEL_API_KEY",
"firecrawl": "FIRECRAWL_API_KEY", "keenable": "KEENABLE_API_KEY",
}
@@ -54,35 +54,41 @@ def _rescue_eligible(provider) -> bool:
def _rescue_search(provider_name: str, original_error: str, query: str, limit: int) -> dict:
"""Rescue a failed search via the ring; annotate the result with the original failure."""
from plugins.web.keyless_mcp import search_with_failover
logger.warning("web_search backend '%s' failed (%s); one-shot keyless rescue", provider_name, (original_error or "")[:200])
logger.warning(
"web_search backend '%s' failed (%s); one-shot keyless rescue",
provider_name, (original_error or "")[:200],
)
rescued = search_with_failover(provider_name, query, limit)
if rescued.get("success"):
data = rescued.setdefault("data", {})
data["rescued_from"] = provider_name
data["backend_error"] = (
f"Configured backend '{provider_name}' failed this call ({(original_error or 'unknown error')[:300]}); "
f"result served by the keyless free tier. The next call will use '{provider_name}' again."
f"Configured backend '{provider_name}' failed this call "
f"({(original_error or 'unknown error')[:300]}); result served by the keyless free tier. "
f"The next call will use '{provider_name}' again."
)
return rescued
# Ring also failed: the ORIGINAL error names the user's setup, so lead with it.
return {
"success": False,
"error": f"{original_error or 'search failed'} (keyless rescue also failed: {rescued.get('error', 'unknown')})",
"error": (
f"{original_error or 'search failed'} "
f"(keyless rescue also failed: {rescued.get('error', 'unknown')})"
),
}
def _policy_blocked_result(result: dict) -> bool:
"""True for a website-policy refusal — intentional, never rescued (it would fetch blocked content)."""
return bool(result.get("blocked_by_policy")) or "blocked by website policy" in str(result.get("error") or "").lower()
if result.get("blocked_by_policy"):
return True
return "blocked by website policy" in str(result.get("error") or "").lower()
def _rescue_extract(provider_name: str, urls: list, results: list) -> list:
"""Rescue a whole-batch extract failure via the ring.
Only genuine failures are re-fetched; policy-blocked entries are preserved verbatim. If the
provider broke url/result order parity, every entry is treated as rescueable and the ring's
list replaces the batch wholesale.
"""
"""Rescue a whole-batch extract failure via the ring. Only genuine failures are re-fetched; policy-blocked
entries are preserved verbatim. If the provider broke url/result order parity, every entry is treated as
rescueable and the ring's list replaces the batch wholesale."""
from plugins.web.keyless_mcp import extract_with_failover
parity = len(results) == len(urls)
rescue_idx = [i for i, r in enumerate(results) if not parity or not _policy_blocked_result(r)]
@@ -90,7 +96,8 @@ def _rescue_extract(provider_name: str, urls: list, results: list) -> list:
return results # every failure is an intentional policy block
rescue_urls = [urls[i] for i in rescue_idx] if parity else list(urls)
original_error = next((results[i].get("error") for i in rescue_idx if results[i].get("error")), "extract failed")
errors = (results[i].get("error") for i in rescue_idx if results[i].get("error"))
original_error = next(errors, "extract failed")
logger.warning(
"web_extract backend '%s' failed all %d URL(s) (%s); one-shot keyless rescue",
provider_name, len(rescue_urls), (original_error or "")[:200],
@@ -104,8 +111,6 @@ def _rescue_extract(provider_name: str, urls: list, results: list) -> list:
meta["rescued_from"] = provider_name
meta["backend_error"] = (original_error or "")[:300]
if parity and len(rescued) == len(rescue_idx):
merged = list(results)
for pos, i in enumerate(rescue_idx):
merged[i] = rescued[pos]
return merged
replacements = dict(zip(rescue_idx, rescued))
return [replacements.get(i, r) for i, r in enumerate(results)]
return rescued
tools/web_tools_truncate.py
+25 -32
View File
@@ -1,11 +1,9 @@
"""Truncate-and-store pipeline for web_extract (no LLM).
Pages at or under the char budget are returned whole; larger pages become a
head+tail window plus a footer that says how much is shown, where the full text
is stored (cache/web) and the exact read_file call that pages the omitted middle.
Inline base64 images are replaced with ``[IMAGE: alt]`` placeholders. Names are
re-imported by tools/web_tools.py (``tools.web_tools.MAX_STORED_TEXT_CHARS``); logs
under the origin logger name for parity.
Pages at or under the char budget are returned whole; larger pages become a head+tail window plus a
footer that says how much is shown, where the full text is stored (cache/web) and the exact read_file
call that pages the omitted middle. Inline base64 images become ``[IMAGE: alt]`` placeholders. Names are
re-imported by tools/web_tools.py (``tools.web_tools.MAX_STORED_TEXT_CHARS``); logs under the origin logger.
"""
import logging
@@ -25,10 +23,8 @@ _CHAR_LIMIT_FLOOR, _CHAR_LIMIT_CEILING = 2000, 500_000
def _clamp_char_limit(value: Any) -> int:
"""Clamp to [2k, 500k]; raises TypeError/ValueError for non-numeric input.
Floor: below 2k the truncation footer dominates. Ceiling: a config typo must not blow up context.
"""
"""Clamp to [2k, 500k] (below 2k the truncation footer dominates; a config typo must not blow up context);
raises TypeError/ValueError for non-numeric input."""
return max(_CHAR_LIMIT_FLOOR, min(int(value), _CHAR_LIMIT_CEILING))
@@ -45,11 +41,9 @@ def _get_extract_char_limit() -> int:
def convert_base64_images_to_links(text: str) -> str:
"""Replace inline base64 image blobs (token bombs) with ``[IMAGE: alt]`` placeholders.
Handles markdown images (alt text kept), parenthesised blobs, and bare ``data:image/...;base64,`` payloads.
Real http(s) markdown image links are left untouched so the agent can ``web_extract`` / ``vision_analyze`` them.
"""
"""Replace inline base64 image blobs (token bombs) with ``[IMAGE: alt]`` placeholders: markdown images
(alt kept), parenthesised blobs, and bare ``data:image/...;base64,`` payloads. Real http(s) markdown
image links are left untouched so the agent can ``web_extract`` / ``vision_analyze`` them."""
def _md_repl(m: "re.Match[str]") -> str:
alt = (m.group("alt") or "").strip()
return f"[IMAGE: {alt}]" if alt else "[IMAGE]"
@@ -61,12 +55,9 @@ def convert_base64_images_to_links(text: str) -> str:
def _store_full_text(url: str, content: str) -> Optional[str]:
"""Write the full page to cache/web; absolute path or None.
cache/web is mounted read-only into remote backends (credential_files _CACHE_DIRS) so read_file
can page the complete text on any backend. Best-effort: on failure the truncated content is
still returned to the model.
"""
"""Write the full page to cache/web; absolute path or None. cache/web is mounted read-only into remote
backends (credential_files _CACHE_DIRS) so read_file can page the complete text on any backend.
Best-effort: on failure the truncated content is still returned to the model."""
try:
import hashlib
from hermes_constants import get_hermes_dir
@@ -91,12 +82,9 @@ def _store_full_text(url: str, content: str) -> Optional[str]:
def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str, bool]:
"""Return (model_text, was_truncated).
Pages over ``char_limit`` become a ~75% head / ~25% tail window cut on line boundaries, plus a
footer saying how much is shown, where the full text is stored, and the read_file call that
pages the omitted middle. Deterministic.
"""
"""Return (model_text, was_truncated). Pages over ``char_limit`` become a ~75% head / ~25% tail window cut
on line boundaries, plus a footer saying how much is shown, where the full text is stored, and the
read_file call that pages the omitted middle. Deterministic."""
if len(content) <= char_limit:
return content, False
head_budget = int(char_limit * 0.75)
@@ -114,14 +102,17 @@ def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str,
footer_lines = [
"",
"─" * 8 + " [TRUNCATED] " + "─" * 8,
f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) of {len(content):,} total clean characters.",
f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) "
f"of {len(content):,} total clean characters.",
]
if stored_path:
# read_file is 1-indexed; +2 lands on the first line after the shown head.
middle_start_line = head.count("\n") + 2
footer_lines += [
f"Full text saved to: {stored_path}",
f'To read the omitted middle: read_file path="{stored_path}" offset={head.count(chr(10)) + 2} limit=200 '
"(the file is the complete page; raise/lower offset to page through it).",
f'To read the omitted middle: read_file path="{stored_path}" '
f"offset={middle_start_line} limit=200 (the file is the complete page; "
f"raise/lower offset to page through it).",
]
else:
footer_lines.append(
@@ -129,7 +120,8 @@ def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str,
"specific URL or use browser_navigate for the complete page."
)
footer_lines.append("─" * 29)
return head + "\n\n[... middle omitted — see footer ...]\n\n" + tail + "\n" + "\n".join(footer_lines), True
model_text = head + "\n\n[... middle omitted — see footer ...]\n\n" + tail
return model_text + "\n" + "\n".join(footer_lines), True
def _effective_char_limit(char_limit: Optional[int]) -> int:
@@ -168,7 +160,8 @@ def _trim_results(results: List[dict]) -> List[dict]:
"""Keep only url/title/content/error per entry (+ blocked_by_policy when present)."""
return [
{
"url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""), "error": r.get("error"),
"url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""),
"error": r.get("error"),
**({"blocked_by_policy": r["blocked_by_policy"]} if "blocked_by_policy" in r else {}),
}
for r in results
tools/website_policy.py
+8 -10
View File
@@ -51,8 +51,8 @@ def _normalize_rule(rule: Any) -> Optional[str]:
def _iter_blocklist_file_rules(path: Path) -> List[str]:
"""Rules from a shared blocklist file; missing/unreadable files warn and yield nothing
rather than raising — a bad file path must not disable all web tools."""
"""Rules from a shared blocklist file; missing/unreadable files warn and yield nothing rather than
raising — a bad file path must not disable all web tools."""
try:
raw = path.read_text(encoding="utf-8")
except FileNotFoundError:
@@ -66,11 +66,9 @@ def _iter_blocklist_file_rules(path: Path) -> List[str]:
def _require_mapping(value: Any, label: str) -> Dict[str, Any]:
"""``None`` (empty YAML section) counts as an empty mapping; other non-dicts are errors."""
if value is None:
return {}
if not isinstance(value, dict):
if value is not None and not isinstance(value, dict):
raise WebsitePolicyError(f"{label} must be a mapping")
return value
return value or {}
def _load_policy_config(config_path: Path) -> Dict[str, Any]:
@@ -90,7 +88,6 @@ def _load_policy_config(config_path: Path) -> Dict[str, Any]:
raise WebsitePolicyError(f"Failed to read config file {config_path}: {exc}") from exc
if not isinstance(config, dict):
raise WebsitePolicyError("config root must be a mapping")
security = _require_mapping(config.get("security", {}), "security")
website_blocklist = _require_mapping(security.get("website_blocklist", {}), "security.website_blocklist")
return {**_DEFAULT_WEBSITE_BLOCKLIST, **website_blocklist}
@@ -102,13 +99,14 @@ def _require_type(policy: Dict[str, Any], key: str, kind: type, default: Any) ->
if kind is list:
value = value or []
if not isinstance(value, kind):
raise WebsitePolicyError(f"security.website_blocklist.{key} must be a {'boolean' if kind is bool else 'list'}")
kind_name = "boolean" if kind is bool else "list"
raise WebsitePolicyError(f"security.website_blocklist.{key} must be a {kind_name}")
return value
def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]:
"""Parsed website blocklist policy (``{"enabled", "rules"}``); cached for ``_CACHE_TTL_SECONDS`` for the
default config path only — an explicit ``config_path`` (tests) always bypasses and never populates the cache."""
"""Parsed website blocklist policy (``{"enabled", "rules"}``); cached for ``_CACHE_TTL_SECONDS`` for
the default config path only — an explicit ``config_path`` (tests) bypasses and never populates it."""
global _cached_policy, _cached_policy_path, _cached_policy_time
default_path = get_hermes_home() / "config.yaml"
resolved_path = str(config_path or default_path)