diff --git a/tools/web_result_cache.py b/tools/web_result_cache.py index 8bcc23cbf8..e6a99c8567 100644 --- a/tools/web_result_cache.py +++ b/tools/web_result_cache.py @@ -4,8 +4,7 @@ paid request. Limits bucket to 10/20/50/100 so near-identical requests share an entry. * **Extract cache** — disk-backed under ``cache/web`` (cross-process) with a JSON sidecar index: URL digest → (file, fetched_at, title). Hits re-run the normal truncate pipeline. -Lives here, not in generic tool dispatch, so hits sit *after* every safety check (secret-in-URL, SSRF, -policy) and skip only the vendor call. +Lives here, not in tool dispatch, so hits sit *after* every safety check and skip only the vendor call. """ import hashlib @@ -102,12 +101,10 @@ class SearchMemo: key = self._key(provider, query, limit) with self._store_lock: hit = self._store.get(key) - if hit is None: - return None - expires, response = hit - if time.monotonic() >= expires: - del self._store[key] + if hit is None or time.monotonic() >= hit[0]: + self._store.pop(key, None) return None + response = hit[1] logger.info("web_search cache hit: %r via %s", query, provider) return _deep_copy(response) @@ -176,32 +173,25 @@ def _cache_dir() -> Optional[Path]: return None -def _index_path() -> Optional[Path]: - return (d / _INDEX_FILENAME) if (d := _cache_dir()) else None - - def _load_index() -> dict: - path = _index_path() - if path is None or not path.exists(): - return {} + path = (d / _INDEX_FILENAME) if (d := _cache_dir()) else None try: - data = json.loads(path.read_text(encoding="utf-8")) + data = json.loads(path.read_text(encoding="utf-8")) if path else {} return data if isinstance(data, dict) else {} - except Exception: # noqa: BLE001 — corrupt index == empty cache + except Exception: # noqa: BLE001 — missing/corrupt index == empty cache return {} def _save_index(index: dict) -> None: - path = _index_path() - if path is None: + if (d := _cache_dir()) is None: return + path = d / _INDEX_FILENAME try: if len(index) > _INDEX_MAX_ENTRIES: newest = sorted(index.items(), key=lambda kv: kv[1].get("fetched_at", 0), reverse=True) index = dict(newest[:_INDEX_MAX_ENTRIES]) - # Per-process tmp name: CLI, gateway, cron, and subagents all write this index; a shared tmp - # name would let concurrent writers truncate each other. os.replace is atomic, so the worst - # outcome is a lost insert. + # Per-process tmp name: CLI, gateway, cron, and subagents all write this index; a shared tmp name + # would let concurrent writers truncate each other. os.replace is atomic: worst case is a lost insert. tmp = path.with_suffix(f".tmp.{os.getpid()}") tmp.write_text(json.dumps(index), encoding="utf-8") tmp.replace(path) @@ -241,24 +231,22 @@ def _host_matches_pattern(host: str, pattern: str) -> bool: def _is_cache_exempt_host(url: str) -> bool: - """True when the host matches ``web.cache_exempt_hosts`` — sites the user is - developing over public DNS (staging, tunnels, previews) that must fetch live.""" + """True when the host matches ``web.cache_exempt_hosts`` — sites the user develops over public DNS + (staging, tunnels, previews) that must fetch live.""" try: patterns = _web_config().get("cache_exempt_hosts") or [] - if not isinstance(patterns, (list, tuple)) or not patterns: - return False host = _url_host(url) - return bool(host) and any(_host_matches_pattern(host, str(p)) for p in patterns) + if not isinstance(patterns, (list, tuple)) or not host: + return False + return any(_host_matches_pattern(host, str(p)) for p in patterns) except Exception: # noqa: BLE001 — config problems never break tools return False def _is_local_dev_url(url: str) -> bool: """True for loopback/private/LAN URLs — never cached: they are the user's own fast-changing dev servers. - - Hostname heuristics only, no DNS: this is a freshness decision, not a security boundary (SSRF - enforcement lives in tools/url_safety.py, which blocks these by default anyway). - """ + Hostname heuristics only, no DNS: this is a freshness decision, not a security boundary (SSRF enforcement + lives in tools/url_safety.py, which blocks these by default anyway).""" try: host = _url_host(url).lower() if not host: @@ -272,7 +260,7 @@ def _is_local_dev_url(url: str) -> bool: ip = ipaddress.ip_address(host) except ValueError: return False # public DNS name - return bool(ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified) + return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified except Exception: # noqa: BLE001 — on doubt, don't cache return True @@ -302,7 +290,9 @@ def extract_cache_get(url: str, format: Optional[str] = None, provider: str = "" return {"url": url, "title": entry.get("title", ""), "content": content, "error": None, "cached": True} -def extract_cache_put(url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = "") -> None: +def extract_cache_put( + url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = "" +) -> None: """Store one successful extraction's full clean text for TTL reuse; pages over the truncate-store ceiling are not cached (serving a capped copy back as if whole would silently lose the tail).""" if not content or not _cacheable(url): diff --git a/tools/web_tools.py b/tools/web_tools.py index 44b87f019e..14730963a4 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -20,7 +20,9 @@ from plugins.web.firecrawl.provider import ( # noqa: F401 — backward-compat n Firecrawl, _firecrawl_backend_help_suffix, _get_firecrawl_client, _get_firecrawl_gateway_url, _is_tool_gateway_ready, check_firecrawl_api_key, ) -from plugins.web.tavily.provider import _normalize_tavily_documents, _normalize_tavily_search_results, _tavily_request # noqa: F401 +from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names + _normalize_tavily_documents, _normalize_tavily_search_results, _tavily_request, +) from plugins.web.parallel.provider import _get_async_parallel_client, _get_parallel_client # noqa: F401 from plugins.web.exa.provider import _get_exa_client # noqa: F401 @@ -36,14 +38,15 @@ from tools.managed_tool_gateway import ( # noqa: F401 — backward-compat names build_vendor_gateway_url, resolve_managed_tool_gateway, peek_nous_access_token as _peek_nous_access_token, read_nous_access_token as _read_nous_access_token, ) -from tools.tool_backend_helpers import ( # noqa: F401 +from tools.tool_backend_helpers import ( # noqa: F401 — first three are backward-compat re-exports managed_nous_tools_enabled, nous_tool_gateway_unavailable_message, prefers_gateway, + NOUS_MANAGED_PROVIDER, selection_exists, ) from tools.url_safety import async_is_safe_url from tools.web_tools_rescue import ( # noqa: F401 — re-exported (tests patch tools.web_tools.) _keyless_rescue_enabled, _policy_blocked_result, _rescue_eligible, _rescue_extract, _rescue_search, ) -from tools.web_tools_truncate import ( # noqa: F401 — re-exported (tests + web_result_cache import via tools.web_tools) +from tools.web_tools_truncate import ( # noqa: F401 — re-exported (tests + web_result_cache import these) DEFAULT_EXTRACT_CHAR_LIMIT, MAX_STORED_TEXT_CHARS, _clamp_char_limit, _effective_char_limit, _get_extract_char_limit, _store_full_text, _trim_results, _truncate_results, _truncate_with_footer, convert_base64_images_to_links, @@ -60,7 +63,7 @@ logger = logging.getLogger(__name__) # ─── Backend Selection ──────────────────────────────────────────────────────── def _env_value(name: str) -> str: - """Resolve ``name`` via the Hermes config-aware env layer (``hermes config set`` values), then process env.""" + """Resolve ``name`` via the config-aware env layer (``hermes config set`` values), then process env.""" try: from hermes_cli.config import get_env_value val = get_env_value(name) @@ -87,16 +90,8 @@ def _configured_backend(key: str = "backend") -> str: return (_load_web_config().get(key) or "").lower().strip() -# Built-in backends probed by _BUILTIN_AVAILABILITY; any other name is a plugin-registered provider resolved -# via the registry's ``is_available()``. Includes ``xai`` (probed via has_xai_credentials(), not a registered -# provider) though the registry's _LEGACY_PREFERENCE omits it — drop it here if xai ever registers. -_LEGACY_WEB_BACKENDS = frozenset( - {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"} -) - - def _registry_call(func_name: str, default, *args): - """``agent.web_search_registry.(*args)``, or *default* if it raised (registry is optional, never fatal).""" + """``agent.web_search_registry.(*args)``, or *default* if it raised (registry never fatal).""" try: import agent.web_search_registry as registry_mod return getattr(registry_mod, func_name)(*args) @@ -111,14 +106,13 @@ def _registered_web_provider(backend: str): def _probe(provider, method: str, context: str = "") -> Optional[bool]: - """``bool(provider.())``, or ``None`` if it raised (logged; a broken provider is unavailable). - - ``context`` is appended to the debug log line (e.g. " during readiness check"). - """ + """``bool(provider.())``, or ``None`` if it raised (a broken provider is unavailable; *context* is + appended to the debug log line, e.g. " during readiness check").""" try: return bool(getattr(provider, method)()) except Exception as exc: # noqa: BLE001 — a broken provider is "unavailable" - logger.debug("web provider %r.%s() raised%s: %s", getattr(provider, "name", provider), method, context, exc) + name = getattr(provider, "name", provider) + logger.debug("web provider %r.%s() raised%s: %s", name, method, context, exc) return None @@ -128,20 +122,14 @@ def _list_registered_web_providers(): def _get_backend() -> str: - """Shared web backend name. - - A stored ``web.backend`` is returned as-is — no availability probe, no fallback — so a broken selection - surfaces the vendor's honest error rather than silently rerouting. Autodetect runs ONLY when no web - selection has ever been stored. - """ + """Shared web backend name. A stored ``web.backend`` is returned as-is — no availability probe, no + fallback — so a broken selection surfaces the vendor's honest error rather than silently rerouting. + Autodetect runs ONLY when no web selection has ever been stored.""" configured = _configured_backend() if configured: # "nous" (managed subscription) is serviced by the firecrawl provider, whose client # resolver routes it through the managed Tool Gateway. - from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER return "firecrawl" if configured == NOUS_MANAGED_PROVIDER else configured - - from tools.tool_backend_helpers import selection_exists if selection_exists("web"): # Selection exists (use_gateway / per-capability keys) but no shared name: keep the # firecrawl default rather than credential-laddering. @@ -196,12 +184,8 @@ def _get_extract_backend() -> str: return _configured_backend("extract_backend") or _get_backend() -def _tavily_explicitly_configured() -> bool: - return any(_configured_backend(key) == "tavily" for key in ("backend", "search_backend", "extract_backend")) - - def _xai_available() -> bool: - # Cheap probe only (env var OR auth.json OAuth): resolve_xai_http_credentials() may refresh over the network. + # Cheap probe only (env var OR auth.json OAuth): resolve_xai_http_credentials() may hit the network. try: from tools.xai_http import has_xai_credentials return has_xai_credentials() @@ -218,27 +202,29 @@ def _ddgs_package_importable() -> bool: return False -# Built-in availability probes (see _LEGACY_WEB_BACKENDS). Lambdas so test patches of module-level helpers -# (e.g. _ddgs_package_importable, check_firecrawl_api_key) are honored at call time. +# Built-in backends and their cheap availability probes; any other name is a plugin-registered provider +# resolved via the registry's ``is_available()``. Lambdas so test patches of module-level helpers (e.g. +# _ddgs_package_importable, check_firecrawl_api_key) are honored at call time. Includes ``xai`` (probed via +# has_xai_credentials(), not a registered provider) though the registry's _LEGACY_PREFERENCE omits it — +# drop it here if xai ever registers. _BUILTIN_AVAILABILITY = { "exa": lambda: _has_env("EXA_API_KEY"), "parallel": lambda: _has_env("PARALLEL_API_KEY"), "keenable": lambda: _has_env("KEENABLE_API_KEY"), "firecrawl": lambda: check_firecrawl_api_key(), - "tavily": lambda: _has_env("TAVILY_API_KEY") or _tavily_explicitly_configured(), + "tavily": lambda: _has_env("TAVILY_API_KEY") + or any(_configured_backend(k) == "tavily" for k in ("backend", "search_backend", "extract_backend")), "searxng": lambda: _has_env("SEARXNG_URL"), "brave-free": lambda: _has_env("BRAVE_SEARCH_API_KEY"), "ddgs": lambda: _ddgs_package_importable(), "xai": _xai_available, } +_LEGACY_WEB_BACKENDS = frozenset(_BUILTIN_AVAILABILITY) def _is_backend_available(backend: str) -> bool: - """True when *backend* is usable — the single availability chokepoint. - - Non-legacy names delegate to the registered provider's ``is_available()`` (unregistered names - fall through); built-ins use the cheap hardcoded probes. - """ + """True when *backend* is usable — the single availability chokepoint. Non-legacy names delegate to the + registered provider's ``is_available()`` (unregistered names fall through); built-ins use cheap probes.""" backend = (backend or "").lower().strip() if backend not in _LEGACY_WEB_BACKENDS: provider = _registered_web_provider(backend) @@ -249,11 +235,9 @@ def _is_backend_available(backend: str) -> bool: def _web_requires_env() -> list[str]: - """Tool-registry metadata env vars for the web backends. - - Gateway vars are always listed: gating them on ``managed_nous_tools_enabled()`` cost a synchronous portal - HTTP refresh at every CLI startup. Contract: set var -> tool sees it; extras are harmless for the not-logged-in. - """ + """Tool-registry metadata env vars for the web backends. Gateway vars are always listed: gating them + on ``managed_nous_tools_enabled()`` cost a synchronous portal HTTP refresh at every CLI startup. + Contract: set var -> tool sees it; extras are harmless for the not-logged-in.""" return [ "EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME", @@ -267,11 +251,9 @@ _debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG") # ─── Dispatch ───────────────────────────────────────────────────────────────── def _ensure_web_plugins_loaded() -> None: - """Idempotently run plugin discovery so the web registry is populated. - - Dispatch is reachable from contexts that never triggered discovery (subprocess agent runs, delegate - children, scripts); without it a configured backend yields a misleading "No web ... provider" error. - """ + """Idempotently run plugin discovery so the web registry is populated. Dispatch is reachable from contexts + that never triggered discovery (subprocess agent runs, delegate children, scripts); without it a + configured backend yields a misleading "No web ... provider" error.""" try: from hermes_cli.plugins import _ensure_plugins_discovered _ensure_plugins_discovered() @@ -312,14 +294,12 @@ def web_search_tool(query: str, limit: int = 5) -> str: from tools.interrupt import is_interrupted if is_interrupted(): return tool_error("Interrupted", success=False) - # Sync only — every provider's search() is sync. _ensure_web_plugins_loaded() from agent.web_search_registry import get_active_search_provider, get_provider as _wsp_get_provider backend = _get_search_backend() provider = _wsp_get_provider(backend) if backend else None if provider is None or not provider.supports_search(): - from tools.tool_backend_helpers import selection_exists if provider is None and backend and selection_exists("web"): error_text = debug_call_data["error"] = _strict_selection_error("search", backend) _finish_debug("web_search_tool", debug_call_data) @@ -328,8 +308,12 @@ def web_search_tool(query: str, limit: int = 5) -> str: provider = get_active_search_provider() if provider is None: - error_text = _no_provider_error("search", "No web search provider configured. Run `hermes tools` to set one up.") - response_data = {"success": False, "error": error_text} + response_data = { + "success": False, + "error": _no_provider_error( + "search", "No web search provider configured. Run `hermes tools` to set one up." + ), + } else: logger.info("Web search via %s: '%s' (limit: %d)", provider.name, query, limit) response_data = _memoized_search(provider, query, limit) @@ -339,18 +323,15 @@ def web_search_tool(query: str, limit: int = 5) -> str: debug_call_data["final_response_size"] = len(result_json) _finish_debug("web_search_tool", debug_call_data) return result_json - except Exception as e: return _debug_error("web_search_tool", debug_call_data, f"Error searching web: {str(e)}") def _memoized_search(provider, query: str, limit: int) -> dict: - """TTL memo + single-flight around the paid vendor call (tools/web_result_cache.py). - - Sits after every safety/config check. The provider is asked for the BUCKETED count so near-identical - limits share an entry; the caller's count is sliced out. Only successful, non-rescued responses are - cached — caching a rescue would make the one-shot ring fallback sticky for a whole TTL. - """ + """TTL memo + single-flight around the paid vendor call (tools/web_result_cache.py); sits after every + safety/config check. The provider is asked for the BUCKETED count so near-identical limits share an entry; + the caller's count is sliced out. Only successful, non-rescued responses are cached — caching a rescue + would make the one-shot ring fallback sticky for a whole TTL.""" from tools.web_result_cache import bucket_limit, search_memo, slice_search_response def _paid_search() -> tuple[dict, bool]: fetch_limit = bucket_limit(limit) @@ -402,21 +383,23 @@ async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Opti safe_urls.append(url) safe_indices.append(index) else: - ssrf_blocked[index] = _result_entry(url, "Blocked: URL targets a private or internal network address") + ssrf_blocked[index] = _result_entry( + url, "Blocked: URL targets a private or internal network address" + ) - if not safe_urls: - results = [] - else: + results = [] + if safe_urls: backend = _get_extract_backend() _ensure_web_plugins_loaded() provider, error_json = _resolve_extract_provider(backend) if error_json is not None: return error_json results = await _extract_safe_urls(provider, safe_urls, format) - - # Reconstruct input order across invalid, blocked, and provider entries (providers preserve safe-list order). + # Reconstruct input order across invalid, blocked, and provider entries (providers preserve + # the order of the safe URL list they receive). if invalid_urls or ssrf_blocked: - results = _merge_in_order(len(urls), {**ssrf_blocked, **invalid_urls}, safe_indices, safe_urls, results) + fixed = {**ssrf_blocked, **invalid_urls} + results = _merge_in_order(len(urls), fixed, safe_indices, safe_urls, results) logger.info("Extracted content from %d pages", len(results)) debug_call_data["pages_extracted"] = len(results) @@ -424,17 +407,15 @@ async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Opti debug_call_data["processing_applied"].append("truncate_and_store") _truncate_results(results, _effective_char_limit(char_limit), debug_call_data) trimmed = _trim_results(results) - if not trimmed: - result_json = tool_error("Content was inaccessible or not found") - else: - result_json = json.dumps({"results": trimmed}, indent=2, ensure_ascii=False) - # Belt-and-suspenders sweep over the serialized JSON in case a provider tucked a base64 blob in metadata. + result_json = json.dumps({"results": trimmed}, indent=2, ensure_ascii=False) if trimmed else tool_error( + "Content was inaccessible or not found" + ) + # Belt-and-suspenders sweep of the serialized JSON: a provider may tuck a base64 blob in metadata. cleaned_result = convert_base64_images_to_links(result_json) debug_call_data["final_response_size"] = len(cleaned_result) debug_call_data["processing_applied"].append("base64_image_conversion") _finish_debug("web_extract_tool", debug_call_data) return cleaned_result - except Exception as e: return _debug_error("web_extract_tool", debug_call_data, f"Error extracting content: {str(e)}") @@ -442,9 +423,9 @@ async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Opti def _provider_is_ready(provider) -> bool: """True when *provider* is keyed-available OR keyless-capable, without raising. - ``get_active_*_provider()`` returns an explicitly configured backend even when ``is_available()`` is False - (so dispatch can emit a precise error), so readiness gates (tool check_fn, ``hermes doctor``) must probe - for real. Keyless mode (Exa/Parallel free tier) is a working state, not a misconfig. + ``get_active_*_provider()`` returns an explicitly configured backend even when ``is_available()`` is + False (so dispatch can emit a precise error), so readiness gates (tool check_fn, ``hermes doctor``) + must probe for real. Keyless mode (Exa/Parallel free tier) is a working state, not a misconfig. """ if provider is None: return False @@ -460,8 +441,8 @@ def _provider_is_ready(provider) -> bool: def check_web_api_key() -> bool: """``check_fn`` gate for web_search / web_extract: is any web backend available? - A plugin-registered provider reporting ``is_available()`` must light the tools up even with no built-in - credentials; resolution funnels through :func:`_is_backend_available`. + A plugin-registered provider reporting ``is_available()`` must light the tools up even with no + built-in credentials; resolution funnels through :func:`_is_backend_available`. """ configured = _configured_backend() if configured and _is_backend_available(configured): @@ -473,15 +454,15 @@ def check_web_api_key() -> bool: try: _ensure_web_plugins_loaded() from agent.web_search_registry import get_active_search_provider, get_active_extract_provider - return _provider_is_ready(get_active_search_provider()) or _provider_is_ready(get_active_extract_provider()) + return _provider_is_ready(get_active_search_provider()) or _provider_is_ready( + get_active_extract_provider() + ) except Exception as exc: # noqa: BLE001 — registry optional; never fatal logger.debug("web provider registry availability check failed: %s", exc) return False -# --------------------------------------------------------------------------- -# Registry -# --------------------------------------------------------------------------- +# ─── Registry ───────────────────────────────────────────────────────────────── from tools.registry import registry, tool_error WEB_SEARCH_SCHEMA = { diff --git a/tools/web_tools_extract.py b/tools/web_tools_extract.py index 249ec09821..fecb4f1ffc 100644 --- a/tools/web_tools_extract.py +++ b/tools/web_tools_extract.py @@ -12,6 +12,7 @@ import json import logging from typing import Any, Dict, List, Optional +from tools.tool_backend_helpers import selection_error, selection_exists from tools.url_safety import normalize_url_for_request, sensitive_query_param_name from tools.web_tools_rescue import _rescue_eligible, _rescue_extract @@ -38,26 +39,12 @@ def _disabled_plugin_error(capability: str, disabled_key: str) -> str: """Error text when the configured backend's bundled plugin is disabled in config.""" vendor = disabled_key.split("/", 1)[-1] return ( - f"web.{capability}_backend is set to '{vendor}', but its " - f"plugin ('{disabled_key}') is disabled in config. " - f"Re-enable it with `hermes plugins enable {disabled_key}` " + f"web.{capability}_backend is set to '{vendor}', but its plugin ('{disabled_key}') is disabled " + f"in config. Re-enable it with `hermes plugins enable {disabled_key}` " "(or remove it from plugins.disabled)." ) -def _strict_selection_error(capability: str, backend: str) -> str: - """Error for a stored-but-unregistered backend: name the disabled plugin, else the bad selection. - - Strict selection never silently switches to whatever the availability walk finds. - """ - from agent.web_search_registry import _disabled_web_plugin_for - from tools.tool_backend_helpers import selection_error - disabled_key = _disabled_web_plugin_for(capability=capability) - if disabled_key: - return _disabled_plugin_error(capability, disabled_key) - return selection_error("web", f"'{backend}'", f"no registered web {capability} provider has that name") - - def _no_provider_error(capability: str, fallback: str) -> str: """Error when no provider resolved: point at a disabled bundled plugin if that is the real cause.""" from agent.web_search_registry import _disabled_web_plugin_for @@ -65,6 +52,14 @@ def _no_provider_error(capability: str, fallback: str) -> str: return _disabled_plugin_error(capability, disabled_key) if disabled_key else fallback +def _strict_selection_error(capability: str, backend: str) -> str: + """Error for a stored-but-unregistered backend: name the disabled plugin, else the bad selection. + Strict selection never silently switches to whatever the availability walk finds.""" + failure = f"no registered web {capability} provider has that name" + fallback = selection_error("web", f"'{backend}'", failure) + return _no_provider_error(capability, fallback) + + def _result_entry(url: str, error: Optional[str]) -> Dict[str, Any]: return {"url": url, "title": "", "content": "", "error": error} @@ -85,7 +80,8 @@ def _merge_in_order( *fetch_positions* (a short provider list yields ``_NO_RESULT_ERROR`` entries for the rest).""" merged = dict(fixed) for pos, position in enumerate(fetch_positions): - merged[position] = results[pos] if pos < len(results) else _result_entry(fetch_urls[pos], _NO_RESULT_ERROR) + missing = _result_entry(fetch_urls[pos], _NO_RESULT_ERROR) + merged[position] = results[pos] if pos < len(results) else missing return [merged[i] for i in range(total)] @@ -105,13 +101,16 @@ def _validate_extract_urls(urls: List[Any]): _url = _web_extract_url(item) if _url is None: invalid_urls[index] = _result_entry( - "", f"Invalid URL item at index {index}: expected a URL string or an object with a string 'url' or 'href' field" + "", + f"Invalid URL item at index {index}: expected a URL string " + "or an object with a string 'url' or 'href' field", ) continue normalized_url = normalize_url_for_request(_url) if any(_PREFIX_RE.search(c) for c in (_url, unquote(_url), normalized_url, unquote(normalized_url))): return _refuse_all( - "Blocked: URL contains what appears to be an API key or token. Secrets must not be sent in URLs." + "Blocked: URL contains what appears to be an API key or token. " + "Secrets must not be sent in URLs." ) sensitive_query_key = sensitive_query_param_name(normalized_url) if sensitive_query_key: @@ -129,8 +128,8 @@ def _validate_extract_urls(urls: List[Any]): def _resolve_extract_provider(backend: str): """Resolve the extract provider for *backend*; returns ``(provider, error_json)``. - A registered search-only backend is a typed error (never a silent switch). An unregistered name with a - stored web selection is a strict-selection error; with no selection, fall through to the availability walk. + A registered search-only backend is a typed error (never a silent switch). An unregistered name with + a stored web selection is a strict-selection error; with no selection, fall through to the walk. """ from agent.web_search_registry import get_active_extract_provider, get_provider as _wsp_get_provider provider = _wsp_get_provider(backend) if backend else None @@ -138,17 +137,16 @@ def _resolve_extract_provider(backend: str): return provider, None if provider is not None: return None, _extract_error_json( - f"{provider.display_name} is a search-only " - "backend and cannot extract URL content. " + f"{provider.display_name} is a search-only backend and cannot extract URL content. " "Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT ) - from tools.tool_backend_helpers import selection_exists if backend and selection_exists("web"): return None, _extract_error_json(_strict_selection_error("extract", backend)) provider = get_active_extract_provider() if provider is None: return None, _extract_error_json(_no_provider_error( - "extract", "No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT, + "extract", + "No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT, )) return provider, None @@ -180,7 +178,7 @@ async def _dispatch_extract(provider, fetch_urls: List[str], format: Optional[st continue _content = fetched.get("raw_content", "") or fetched.get("content", "") if _content: - extract_cache_put(url, _content, title=fetched.get("title", ""), format=format, provider=provider.name) + extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name) return results @@ -188,9 +186,9 @@ async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[st """Serve cache hits, fetch the rest, and merge back in ``safe_urls`` order. The disk cache (tools/web_result_cache.py) sits AFTER the secret-URL gate, SSRF gate, and provider - resolution, and is gated per-URL on the website policy — a hit skips only the vendor call, never a control. - Policy-blocked URLs are cache misses so dispatch handles them exactly as without a cache. Keys include - provider and format, so switching either within the TTL never serves the other's content. + resolution, and is gated per-URL on the website policy — a hit skips only the vendor call, never a + control. Policy-blocked URLs are cache misses so dispatch handles them exactly as without a cache. + Keys include provider and format, so switching either within the TTL never serves the other's content. """ from tools.web_result_cache import extract_cache_get from tools.website_policy import check_website_access as _check_site @@ -211,7 +209,6 @@ async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[st if not fetch_urls: return [cached_results[i] for i in range(len(safe_urls))] - logger.info("Web extract via %s: %d URL(s)", provider.name, len(fetch_urls)) results = await _dispatch_extract(provider, fetch_urls, format) if not cached_results: diff --git a/tools/web_tools_rescue.py b/tools/web_tools_rescue.py index 5c9dde021e..5f548ebc5b 100644 --- a/tools/web_tools_rescue.py +++ b/tools/web_tools_rescue.py @@ -1,9 +1,8 @@ """One-shot keyless-ring rescue for failed keyed/configured web calls. -Stateless by design: a rescue routes THIS call through the free-tier ring -(plugins/web/keyless_mcp.py); the next web_search/web_extract call attempts -the chosen backend again. Callers must never cache a rescue-served response, -or the one-shot rescue becomes sticky for a whole TTL. Names are re-imported by +Stateless by design: a rescue routes THIS call through the free-tier ring (plugins/web/keyless_mcp.py); +the next web_search/web_extract call attempts the chosen backend again. Callers must never cache a +rescue-served response, or the one-shot rescue becomes sticky for a whole TTL. Names are re-imported by tools/web_tools.py (``tools.web_tools._rescue_eligible``); logs under the origin logger. """ @@ -13,7 +12,8 @@ logger = logging.getLogger("tools.web_tools") # Ring vendor -> env var holding its paid key (keyed mode ⇒ eligible for rescue). _RING_KEY_VARS = { - "exa": "EXA_API_KEY", "parallel": "PARALLEL_API_KEY", "firecrawl": "FIRECRAWL_API_KEY", "keenable": "KEENABLE_API_KEY", + "exa": "EXA_API_KEY", "parallel": "PARALLEL_API_KEY", + "firecrawl": "FIRECRAWL_API_KEY", "keenable": "KEENABLE_API_KEY", } @@ -54,35 +54,41 @@ def _rescue_eligible(provider) -> bool: def _rescue_search(provider_name: str, original_error: str, query: str, limit: int) -> dict: """Rescue a failed search via the ring; annotate the result with the original failure.""" from plugins.web.keyless_mcp import search_with_failover - logger.warning("web_search backend '%s' failed (%s); one-shot keyless rescue", provider_name, (original_error or "")[:200]) + logger.warning( + "web_search backend '%s' failed (%s); one-shot keyless rescue", + provider_name, (original_error or "")[:200], + ) rescued = search_with_failover(provider_name, query, limit) if rescued.get("success"): data = rescued.setdefault("data", {}) data["rescued_from"] = provider_name data["backend_error"] = ( - f"Configured backend '{provider_name}' failed this call ({(original_error or 'unknown error')[:300]}); " - f"result served by the keyless free tier. The next call will use '{provider_name}' again." + f"Configured backend '{provider_name}' failed this call " + f"({(original_error or 'unknown error')[:300]}); result served by the keyless free tier. " + f"The next call will use '{provider_name}' again." ) return rescued # Ring also failed: the ORIGINAL error names the user's setup, so lead with it. return { "success": False, - "error": f"{original_error or 'search failed'} (keyless rescue also failed: {rescued.get('error', 'unknown')})", + "error": ( + f"{original_error or 'search failed'} " + f"(keyless rescue also failed: {rescued.get('error', 'unknown')})" + ), } def _policy_blocked_result(result: dict) -> bool: """True for a website-policy refusal — intentional, never rescued (it would fetch blocked content).""" - return bool(result.get("blocked_by_policy")) or "blocked by website policy" in str(result.get("error") or "").lower() + if result.get("blocked_by_policy"): + return True + return "blocked by website policy" in str(result.get("error") or "").lower() def _rescue_extract(provider_name: str, urls: list, results: list) -> list: - """Rescue a whole-batch extract failure via the ring. - - Only genuine failures are re-fetched; policy-blocked entries are preserved verbatim. If the - provider broke url/result order parity, every entry is treated as rescueable and the ring's - list replaces the batch wholesale. - """ + """Rescue a whole-batch extract failure via the ring. Only genuine failures are re-fetched; policy-blocked + entries are preserved verbatim. If the provider broke url/result order parity, every entry is treated as + rescueable and the ring's list replaces the batch wholesale.""" from plugins.web.keyless_mcp import extract_with_failover parity = len(results) == len(urls) rescue_idx = [i for i, r in enumerate(results) if not parity or not _policy_blocked_result(r)] @@ -90,7 +96,8 @@ def _rescue_extract(provider_name: str, urls: list, results: list) -> list: return results # every failure is an intentional policy block rescue_urls = [urls[i] for i in rescue_idx] if parity else list(urls) - original_error = next((results[i].get("error") for i in rescue_idx if results[i].get("error")), "extract failed") + errors = (results[i].get("error") for i in rescue_idx if results[i].get("error")) + original_error = next(errors, "extract failed") logger.warning( "web_extract backend '%s' failed all %d URL(s) (%s); one-shot keyless rescue", provider_name, len(rescue_urls), (original_error or "")[:200], @@ -104,8 +111,6 @@ def _rescue_extract(provider_name: str, urls: list, results: list) -> list: meta["rescued_from"] = provider_name meta["backend_error"] = (original_error or "")[:300] if parity and len(rescued) == len(rescue_idx): - merged = list(results) - for pos, i in enumerate(rescue_idx): - merged[i] = rescued[pos] - return merged + replacements = dict(zip(rescue_idx, rescued)) + return [replacements.get(i, r) for i, r in enumerate(results)] return rescued diff --git a/tools/web_tools_truncate.py b/tools/web_tools_truncate.py index 47d09e0543..c930b89ec3 100644 --- a/tools/web_tools_truncate.py +++ b/tools/web_tools_truncate.py @@ -1,11 +1,9 @@ """Truncate-and-store pipeline for web_extract (no LLM). -Pages at or under the char budget are returned whole; larger pages become a -head+tail window plus a footer that says how much is shown, where the full text -is stored (cache/web) and the exact read_file call that pages the omitted middle. -Inline base64 images are replaced with ``[IMAGE: alt]`` placeholders. Names are -re-imported by tools/web_tools.py (``tools.web_tools.MAX_STORED_TEXT_CHARS``); logs -under the origin logger name for parity. +Pages at or under the char budget are returned whole; larger pages become a head+tail window plus a +footer that says how much is shown, where the full text is stored (cache/web) and the exact read_file +call that pages the omitted middle. Inline base64 images become ``[IMAGE: alt]`` placeholders. Names are +re-imported by tools/web_tools.py (``tools.web_tools.MAX_STORED_TEXT_CHARS``); logs under the origin logger. """ import logging @@ -25,10 +23,8 @@ _CHAR_LIMIT_FLOOR, _CHAR_LIMIT_CEILING = 2000, 500_000 def _clamp_char_limit(value: Any) -> int: - """Clamp to [2k, 500k]; raises TypeError/ValueError for non-numeric input. - - Floor: below 2k the truncation footer dominates. Ceiling: a config typo must not blow up context. - """ + """Clamp to [2k, 500k] (below 2k the truncation footer dominates; a config typo must not blow up context); + raises TypeError/ValueError for non-numeric input.""" return max(_CHAR_LIMIT_FLOOR, min(int(value), _CHAR_LIMIT_CEILING)) @@ -45,11 +41,9 @@ def _get_extract_char_limit() -> int: def convert_base64_images_to_links(text: str) -> str: - """Replace inline base64 image blobs (token bombs) with ``[IMAGE: alt]`` placeholders. - - Handles markdown images (alt text kept), parenthesised blobs, and bare ``data:image/...;base64,`` payloads. - Real http(s) markdown image links are left untouched so the agent can ``web_extract`` / ``vision_analyze`` them. - """ + """Replace inline base64 image blobs (token bombs) with ``[IMAGE: alt]`` placeholders: markdown images + (alt kept), parenthesised blobs, and bare ``data:image/...;base64,`` payloads. Real http(s) markdown + image links are left untouched so the agent can ``web_extract`` / ``vision_analyze`` them.""" def _md_repl(m: "re.Match[str]") -> str: alt = (m.group("alt") or "").strip() return f"[IMAGE: {alt}]" if alt else "[IMAGE]" @@ -61,12 +55,9 @@ def convert_base64_images_to_links(text: str) -> str: def _store_full_text(url: str, content: str) -> Optional[str]: - """Write the full page to cache/web; absolute path or None. - - cache/web is mounted read-only into remote backends (credential_files _CACHE_DIRS) so read_file - can page the complete text on any backend. Best-effort: on failure the truncated content is - still returned to the model. - """ + """Write the full page to cache/web; absolute path or None. cache/web is mounted read-only into remote + backends (credential_files _CACHE_DIRS) so read_file can page the complete text on any backend. + Best-effort: on failure the truncated content is still returned to the model.""" try: import hashlib from hermes_constants import get_hermes_dir @@ -91,12 +82,9 @@ def _store_full_text(url: str, content: str) -> Optional[str]: def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str, bool]: - """Return (model_text, was_truncated). - - Pages over ``char_limit`` become a ~75% head / ~25% tail window cut on line boundaries, plus a - footer saying how much is shown, where the full text is stored, and the read_file call that - pages the omitted middle. Deterministic. - """ + """Return (model_text, was_truncated). Pages over ``char_limit`` become a ~75% head / ~25% tail window cut + on line boundaries, plus a footer saying how much is shown, where the full text is stored, and the + read_file call that pages the omitted middle. Deterministic.""" if len(content) <= char_limit: return content, False head_budget = int(char_limit * 0.75) @@ -114,14 +102,17 @@ def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str, footer_lines = [ "", "─" * 8 + " [TRUNCATED] " + "─" * 8, - f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) of {len(content):,} total clean characters.", + f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) " + f"of {len(content):,} total clean characters.", ] if stored_path: # read_file is 1-indexed; +2 lands on the first line after the shown head. + middle_start_line = head.count("\n") + 2 footer_lines += [ f"Full text saved to: {stored_path}", - f'To read the omitted middle: read_file path="{stored_path}" offset={head.count(chr(10)) + 2} limit=200 ' - "(the file is the complete page; raise/lower offset to page through it).", + f'To read the omitted middle: read_file path="{stored_path}" ' + f"offset={middle_start_line} limit=200 (the file is the complete page; " + f"raise/lower offset to page through it).", ] else: footer_lines.append( @@ -129,7 +120,8 @@ def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str, "specific URL or use browser_navigate for the complete page." ) footer_lines.append("─" * 29) - return head + "\n\n[... middle omitted — see footer ...]\n\n" + tail + "\n" + "\n".join(footer_lines), True + model_text = head + "\n\n[... middle omitted — see footer ...]\n\n" + tail + return model_text + "\n" + "\n".join(footer_lines), True def _effective_char_limit(char_limit: Optional[int]) -> int: @@ -168,7 +160,8 @@ def _trim_results(results: List[dict]) -> List[dict]: """Keep only url/title/content/error per entry (+ blocked_by_policy when present).""" return [ { - "url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""), "error": r.get("error"), + "url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""), + "error": r.get("error"), **({"blocked_by_policy": r["blocked_by_policy"]} if "blocked_by_policy" in r else {}), } for r in results diff --git a/tools/website_policy.py b/tools/website_policy.py index 49da31f37a..7abbb3d042 100644 --- a/tools/website_policy.py +++ b/tools/website_policy.py @@ -51,8 +51,8 @@ def _normalize_rule(rule: Any) -> Optional[str]: def _iter_blocklist_file_rules(path: Path) -> List[str]: - """Rules from a shared blocklist file; missing/unreadable files warn and yield nothing - rather than raising — a bad file path must not disable all web tools.""" + """Rules from a shared blocklist file; missing/unreadable files warn and yield nothing rather than + raising — a bad file path must not disable all web tools.""" try: raw = path.read_text(encoding="utf-8") except FileNotFoundError: @@ -66,11 +66,9 @@ def _iter_blocklist_file_rules(path: Path) -> List[str]: def _require_mapping(value: Any, label: str) -> Dict[str, Any]: """``None`` (empty YAML section) counts as an empty mapping; other non-dicts are errors.""" - if value is None: - return {} - if not isinstance(value, dict): + if value is not None and not isinstance(value, dict): raise WebsitePolicyError(f"{label} must be a mapping") - return value + return value or {} def _load_policy_config(config_path: Path) -> Dict[str, Any]: @@ -90,7 +88,6 @@ def _load_policy_config(config_path: Path) -> Dict[str, Any]: raise WebsitePolicyError(f"Failed to read config file {config_path}: {exc}") from exc if not isinstance(config, dict): raise WebsitePolicyError("config root must be a mapping") - security = _require_mapping(config.get("security", {}), "security") website_blocklist = _require_mapping(security.get("website_blocklist", {}), "security.website_blocklist") return {**_DEFAULT_WEBSITE_BLOCKLIST, **website_blocklist} @@ -102,13 +99,14 @@ def _require_type(policy: Dict[str, Any], key: str, kind: type, default: Any) -> if kind is list: value = value or [] if not isinstance(value, kind): - raise WebsitePolicyError(f"security.website_blocklist.{key} must be a {'boolean' if kind is bool else 'list'}") + kind_name = "boolean" if kind is bool else "list" + raise WebsitePolicyError(f"security.website_blocklist.{key} must be a {kind_name}") return value def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]: - """Parsed website blocklist policy (``{"enabled", "rules"}``); cached for ``_CACHE_TTL_SECONDS`` for the - default config path only — an explicit ``config_path`` (tests) always bypasses and never populates the cache.""" + """Parsed website blocklist policy (``{"enabled", "rules"}``); cached for ``_CACHE_TTL_SECONDS`` for + the default config path only — an explicit ``config_path`` (tests) bypasses and never populates it.""" global _cached_policy, _cached_policy_path, _cached_policy_time default_path = get_hermes_home() / "config.yaml" resolved_path = str(config_path or default_path)