diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 8aaecf3989..63c3a80991 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -648,6 +648,7 @@ def classify_api_error( """Classify an API error into a structured recovery recommendation. Priority-ordered pipeline: + 0. Plugin ``classify_api_error`` hooks (first valid result wins) 1. Special-case provider-specific patterns (thinking sigs, tier gates) 2. HTTP status code + message-aware refinement 3. Error code classification (from body) @@ -731,6 +732,41 @@ def classify_api_error( defaults.update(overrides) return ClassifiedError(**defaults) + # ── 0. Plugin classifiers (first valid result wins) ───────────── + # + # Consulted BEFORE the built-in pipeline so a provider plugin can both + # add classifications the core patterns miss and correct ones they get + # wrong for its provider (see the ``classify_api_error`` entry in + # hermes_cli.plugins.VALID_HOOKS for the callback contract). Callback + # exceptions are isolated inside invoke_hook and malformed returns are + # dropped by the helper, so a broken plugin can never break + # classification — the guard here only covers import/dispatch failure. + try: + from hermes_cli.plugins import get_plugin_error_classification + plugin_classification = get_plugin_error_classification( + provider=provider, + model=model, + status_code=status_code, + error_type=error_type, + error_code=error_code, + error_message=error_msg, + error_body=body, + error=error, + approx_tokens=approx_tokens, + context_length=context_length, + num_messages=num_messages, + ) + except Exception as exc: + logger.debug("Plugin error classification unavailable: %s", exc) + plugin_classification = None + if plugin_classification is not None: + reason = plugin_classification.pop("reason") + logger.info( + "API error classified by plugin hook: %s (provider=%s, status=%s)", + reason.value, provider, status_code, + ) + return _result(reason, **plugin_classification) + # ── 1. Provider-specific patterns (highest priority) ──────────── # Provider content-policy / safety-filter block. The provider has made a diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index 7f5070ee2a..82165083c5 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -184,6 +184,21 @@ VALID_HOOKS: Set[str] = { "pre_api_request", "post_api_request", "api_request_error", + # API-error classification override. Fired once per failed API call at + # the top of ``agent/error_classifier.classify_api_error()``, BEFORE the + # built-in pipeline, so provider plugins can own their provider's error + # quirks without core patches. Callbacks receive the parsed error context + # (provider, model, status_code, error_type, error_code, error_message, + # error_body, error, approx_tokens, context_length, num_messages) and + # should self-scope on ``provider``. Return None to pass, or a dict:: + # {"reason": "", # required + # "retryable": bool, "should_compress": bool, + # "should_rotate_credential": bool, "should_fallback": bool, + # "message": str, "error_context": dict} # all optional + # First valid result (registration order) wins. Invalid dicts and + # unknown reasons are skipped; exceptions are isolated — a broken + # plugin can never break error classification. + "classify_api_error", "on_session_start", "on_session_end", "on_session_finalize", @@ -5768,6 +5783,87 @@ def get_pre_verify_continue_message( return None +def get_plugin_error_classification( + *, + provider: str = "", + model: str = "", + status_code: Optional[int] = None, + error_type: str = "", + error_code: str = "", + error_message: str = "", + error_body: Optional[Dict[str, Any]] = None, + error: Optional[BaseException] = None, + approx_tokens: int = 0, + context_length: int = 0, + num_messages: int = 0, +) -> Optional[Dict[str, Any]]: + """Check ``classify_api_error`` hooks for a classification directive. + + Consulted by :func:`agent.error_classifier.classify_api_error` BEFORE + its built-in pipeline, so a provider plugin can both add classifications + the core patterns miss and correct ones they get wrong for its provider. + + A callback returns ``None`` to pass, or a dict with a required + ``"reason"`` (a :class:`agent.error_classifier.FailoverReason` member or + its string name) plus optional recovery-hint overrides. The first result + carrying a valid reason wins — mirroring + :func:`get_pre_tool_call_block_message`, invalid or irrelevant returns + are silently ignored so a misbehaving plugin degrades to a no-op. + + Returns a sanitized dict (``reason`` coerced to ``FailoverReason``, hint + fields coerced to ``bool``) or ``None`` when no plugin claimed the error. + """ + from agent.error_classifier import FailoverReason + + hook_results = invoke_hook( + "classify_api_error", + provider=provider, + model=model, + status_code=status_code, + error_type=error_type, + error_code=error_code, + error_message=error_message, + error_body=error_body if isinstance(error_body, dict) else {}, + error=error, + approx_tokens=approx_tokens, + context_length=context_length, + num_messages=num_messages, + ) + + for result in hook_results: + if not isinstance(result, dict): + continue + reason_raw = result.get("reason") + if isinstance(reason_raw, FailoverReason): + reason = reason_raw + elif isinstance(reason_raw, str): + try: + reason = FailoverReason(reason_raw.strip().lower()) + except ValueError: + continue + else: + continue + + out: Dict[str, Any] = {"reason": reason} + for key in ( + "retryable", + "should_compress", + "should_rotate_credential", + "should_fallback", + ): + if key in result: + out[key] = bool(result[key]) + message = result.get("message") + if isinstance(message, str) and message.strip(): + out["message"] = message.strip()[:500] + error_context = result.get("error_context") + if isinstance(error_context, dict): + out["error_context"] = error_context + return out + + return None + + def _ensure_plugins_discovered(force: bool = False) -> PluginManager: """Return the global manager after ensuring plugin discovery has run. diff --git a/plugins/openrouter-tool-use-404/README.md b/plugins/openrouter-tool-use-404/README.md new file mode 100644 index 0000000000..0af3e98925 --- /dev/null +++ b/plugins/openrouter-tool-use-404/README.md @@ -0,0 +1,24 @@ +# openrouter-tool-use-404 + +Classifies OpenRouter's `No endpoints found that support tool use` 404 as +`model_not_found` (`retryable=False`, `should_fallback=True`) so the agent +fails over to a configured fallback model immediately instead of retrying a +deterministic rejection 3–5 times. + +Also the reference implementation for the `classify_api_error` plugin hook: +provider plugins can own their provider's error quirks without patching +`agent/error_classifier.py`. The callback contract is documented on the +`classify_api_error` entry in `hermes_cli.plugins.VALID_HOOKS`. + +## Enable + +Like all bundled standalone plugins, this ships opt-in: + +```bash +hermes plugins enable openrouter-tool-use-404 +``` + +## Origin + +Classification logic from PR #58451 by @webtecnica, re-implemented as a +plugin to demonstrate the hook. diff --git a/plugins/openrouter-tool-use-404/__init__.py b/plugins/openrouter-tool-use-404/__init__.py new file mode 100644 index 0000000000..7101982710 --- /dev/null +++ b/plugins/openrouter-tool-use-404/__init__.py @@ -0,0 +1,57 @@ +"""openrouter-tool-use-404 — provider error classification as a plugin. + +When OpenRouter routes a tool-calling request to a model with no +tool-capable endpoint, it returns HTTP 404 with a body like:: + + No endpoints found that support tool use. Try disabling "browser_back". + To learn more about provider routing, visit: + https://openrouter.ai/docs/guides/routing/provider-selection + +The body carries none of the core ``_MODEL_NOT_FOUND_PATTERNS`` signals, +so the built-in classifier files it under ``unknown`` (retryable) and the +retry loop burns several attempts on a deterministic rejection before +surfacing a generic error. + +This plugin classifies it as ``model_not_found`` (``retryable=False``, +``should_fallback=True``) so the client-error fast-fallback path in +``conversation_loop.py`` switches to a configured fallback model before +the user ever sees the error. + +It is also the reference implementation for the ``classify_api_error`` +hook: a provider plugin self-scopes on the ``provider`` kwarg, matches +its provider's quirk in the pre-parsed error context, and returns a +classification dict — no core patches required. See the hook contract in +``hermes_cli.plugins.VALID_HOOKS``. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +_PATTERN = "no endpoints found that support tool use" + + +def classify( + provider: str = "", + status_code: Optional[int] = None, + error_message: str = "", + **_kwargs: Any, +) -> Optional[Dict[str, Any]]: + """Claim OpenRouter's tool-use-404, pass on everything else.""" + if (provider or "").strip().lower() != "openrouter": + return None + # OpenRouter surfaces this as 404, but some SDK wrappers drop the + # status; the phrase alone is unambiguous, so accept a missing code. + if status_code not in (None, 404): + return None + if _PATTERN not in (error_message or "").lower(): + return None + return { + "reason": "model_not_found", + "retryable": False, + "should_fallback": True, + } + + +def register(ctx) -> None: + ctx.register_hook("classify_api_error", classify) diff --git a/plugins/openrouter-tool-use-404/plugin.yaml b/plugins/openrouter-tool-use-404/plugin.yaml new file mode 100644 index 0000000000..c1c706ecb9 --- /dev/null +++ b/plugins/openrouter-tool-use-404/plugin.yaml @@ -0,0 +1,6 @@ +name: openrouter-tool-use-404 +version: 1.0.0 +description: "Classify OpenRouter's 'No endpoints found that support tool use' 404 as model_not_found so the agent fails over to a configured fallback model instead of retrying a deterministic rejection. Reference implementation for the classify_api_error plugin hook." +author: "NousResearch (hook demo; classification from PR #58451 by @webtecnica)" +provides_hooks: + - classify_api_error diff --git a/tests/test_classify_api_error_hook.py b/tests/test_classify_api_error_hook.py new file mode 100644 index 0000000000..6396c21de5 --- /dev/null +++ b/tests/test_classify_api_error_hook.py @@ -0,0 +1,278 @@ +"""Tests for the ``classify_api_error`` plugin hook. + +Covers the seam in ``agent.error_classifier.classify_api_error`` (step 0, +consulted before the built-in pipeline), the sanitization contract of +``hermes_cli.plugins.get_plugin_error_classification``, and the bundled +``openrouter-tool-use-404`` reference plugin. + +Mirrors the ``transform_tool_result`` hook tests: patch the symbol the +call site actually imports (``hermes_cli.plugins.*``) rather than the +consuming module, because the import happens at call time. +""" + +import importlib.util +from pathlib import Path + +import hermes_cli.plugins as plugins_mod +from agent.error_classifier import FailoverReason, classify_api_error + + +class _FakeAPIError(Exception): + def __init__(self, message, status_code=None, body=None): + super().__init__(message) + if status_code is not None: + self.status_code = status_code + self.body = body or {} + + +_TOOL_USE_404 = 'No endpoints found that support tool use. Try disabling "browser_back".' + + +def _classify_tool_use_404(**kwargs): + return classify_api_error( + _FakeAPIError(_TOOL_USE_404, status_code=404), + provider="openrouter", + model="deepseek/deepseek-chat", + **kwargs, + ) + + +# ── Baseline: no plugins ──────────────────────────────────────────────── + + +def test_no_hook_falls_through_to_builtin(monkeypatch): + # Fresh manager so no stale plugin hooks pollute state. + monkeypatch.setattr(plugins_mod, "_plugin_manager", plugins_mod.PluginManager()) + + result = _classify_tool_use_404() + # Documents the gap the demo plugin closes: a generic 404 with no + # model-not-found signal classifies as unknown/retryable today. + assert result.reason == FailoverReason.unknown + assert result.retryable is True + + +# ── Plugin classification wins over built-ins ─────────────────────────── + + +def test_plugin_classification_wins(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [ + {"reason": "model_not_found", "retryable": False, "should_fallback": True} + ], + ) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.model_not_found + assert result.retryable is False + assert result.should_fallback is True + # Extracted context is preserved on the ClassifiedError. + assert result.provider == "openrouter" + assert result.status_code == 404 + + +def test_plugin_overrides_builtin_classification(monkeypatch): + # A 429 classifies as rate_limit built-in; a plugin can reclassify it. + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [{"reason": "overloaded"}], + ) + + result = classify_api_error( + _FakeAPIError("too many requests", status_code=429), + provider="zai", + ) + assert result.reason == FailoverReason.overloaded + + +def test_enum_reason_accepted(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [{"reason": FailoverReason.billing}], + ) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.billing + + +def test_reason_only_dict_uses_dataclass_defaults(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [{"reason": "server_error"}], + ) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.server_error + assert result.retryable is True + assert result.should_compress is False + assert result.should_rotate_credential is False + assert result.should_fallback is False + + +# ── Invalid returns are ignored, first valid wins ─────────────────────── + + +def test_invalid_reason_falls_through_to_builtin(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [{"reason": "not_a_real_reason"}], + ) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.unknown + + +def test_non_dict_results_ignored(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: ["model_not_found", 123, ["nope"], None], + ) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.unknown + + +def test_first_valid_result_wins(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [ + {"reason": "bogus"}, + {"reason": "billing"}, + {"reason": "rate_limit"}, + ], + ) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.billing + + +def test_helper_exception_never_breaks_classification(monkeypatch): + def _boom(**kwargs): + raise RuntimeError("plugin infrastructure exploded") + + monkeypatch.setattr(plugins_mod, "get_plugin_error_classification", _boom) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.unknown + assert result.retryable is True + + +# ── Hook kwargs contract ──────────────────────────────────────────────── + + +def test_hook_receives_parsed_error_context(monkeypatch): + seen = {} + + def _capture(name, **kw): + seen.update(kw, hook_name=name) + return [] + + monkeypatch.setattr(plugins_mod, "invoke_hook", _capture) + + _classify_tool_use_404(approx_tokens=1234, num_messages=7) + + assert seen["hook_name"] == "classify_api_error" + assert seen["provider"] == "openrouter" + assert seen["model"] == "deepseek/deepseek-chat" + assert seen["status_code"] == 404 + assert seen["error_type"] == "_FakeAPIError" + assert "no endpoints found that support tool use" in seen["error_message"] + assert seen["approx_tokens"] == 1234 + assert seen["num_messages"] == 7 + assert isinstance(seen["error_body"], dict) + assert isinstance(seen["error"], _FakeAPIError) + + +def test_message_override_and_error_context_sanitized(monkeypatch): + monkeypatch.setattr( + plugins_mod, "invoke_hook", + lambda name, **kw: [{ + "reason": "model_not_found", + "message": " custom guidance ", + "error_context": {"upstream_provider": "DeepSeek"}, + }], + ) + + result = _classify_tool_use_404() + assert result.message == "custom guidance" + assert result.error_context == {"upstream_provider": "DeepSeek"} + + +# ── Bundled reference plugin ──────────────────────────────────────────── + + +def _load_demo_plugin(): + plugin_init = ( + Path(__file__).resolve().parent.parent + / "plugins" / "openrouter-tool-use-404" / "__init__.py" + ) + spec = importlib.util.spec_from_file_location( + "openrouter_tool_use_404_demo", plugin_init, + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_demo_plugin_claims_openrouter_tool_use_404(): + demo = _load_demo_plugin() + result = demo.classify( + provider="openrouter", + status_code=404, + error_message=_TOOL_USE_404.lower(), + ) + assert result == { + "reason": "model_not_found", + "retryable": False, + "should_fallback": True, + } + + +def test_demo_plugin_self_scopes(): + demo = _load_demo_plugin() + # Different provider: pass. + assert demo.classify( + provider="anthropic", status_code=404, + error_message=_TOOL_USE_404.lower(), + ) is None + # Different status: pass. + assert demo.classify( + provider="openrouter", status_code=400, + error_message=_TOOL_USE_404.lower(), + ) is None + # Different message: pass. + assert demo.classify( + provider="openrouter", status_code=404, + error_message="model not found", + ) is None + # Missing status but unambiguous phrase: claim. + assert demo.classify( + provider="openrouter", status_code=None, + error_message=_TOOL_USE_404.lower(), + ) is not None + + +def test_demo_plugin_end_to_end(monkeypatch): + """register() + real invoke_hook + classify_api_error, no mocks.""" + demo = _load_demo_plugin() + manager = plugins_mod.PluginManager() + monkeypatch.setattr(plugins_mod, "_plugin_manager", manager) + + class _Ctx: + def register_hook(self, name, cb): + manager._hooks.setdefault(name, []).append(cb) + + demo.register(_Ctx()) + + result = _classify_tool_use_404() + assert result.reason == FailoverReason.model_not_found + assert result.retryable is False + assert result.should_fallback is True + + # And the built-in pipeline is untouched for everything the plugin + # doesn't claim. + other = classify_api_error( + _FakeAPIError("rate limit exceeded", status_code=429), + provider="openrouter", + ) + assert other.reason == FailoverReason.rate_limit