diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index 868b58db14..965bbcd3d0 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -369,7 +369,7 @@ def _parse_hooks_block(hooks_cfg: Any) -> List[ShellHookSpec]: Malformed entries warn-and-skip — we never raise from config parsing because a broken hook must not crash the agent. """ - from hermes_cli.plugins import VALID_HOOKS + from hermes_cli.plugins import SHELL_UNSUPPORTED_HOOKS, VALID_HOOKS if not isinstance(hooks_cfg, dict): return [] @@ -383,6 +383,17 @@ def _parse_hooks_block(hooks_cfg: Any) -> List[ShellHookSpec]: # the latter parsed by agent/outbound_webhooks.py). if event_name in ("output_spill", "outbound"): continue + if event_name in SHELL_UNSUPPORTED_HOOKS: + # Registering would "succeed" while the hook's return value is + # silently dropped (_parse_response has no channel for these + # events' directives) — refuse loudly instead. + logger.warning( + "hook event %r is Python-plugin-only: shell hooks cannot " + "return its directive, so this registration is refused " + "rather than silently ignored", + event_name, + ) + continue if event_name not in VALID_HOOKS: suggestion = difflib.get_close_matches( str(event_name), VALID_HOOKS, n=1, cutoff=0.6, diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index 82165083c5..6ee2ddd92e 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -371,6 +371,16 @@ VALID_HOOKS: Set[str] = { "pre_command", } +# Hooks whose return value carries a directive that the shell-hook response +# parser (``agent/shell_hooks._parse_response``) has no channel for. +# ``VALID_HOOKS`` doubles as the shell-hook config allow-list, so without +# this exclusion a shell hook could register for one of these events and +# have its output silently ignored — registration is refused loudly instead. +# Support for a shell response shape can lift an event out of this set. +SHELL_UNSUPPORTED_HOOKS: Set[str] = { + "classify_api_error", +} + ENTRY_POINTS_GROUP = "hermes_agent.plugins" # System-prompt sections are deliberately more constrained than lifecycle diff --git a/plugins/openrouter-tool-use-404/README.md b/plugins/openrouter-tool-use-404/README.md deleted file mode 100644 index 0af3e98925..0000000000 --- a/plugins/openrouter-tool-use-404/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# openrouter-tool-use-404 - -Classifies OpenRouter's `No endpoints found that support tool use` 404 as -`model_not_found` (`retryable=False`, `should_fallback=True`) so the agent -fails over to a configured fallback model immediately instead of retrying a -deterministic rejection 3–5 times. - -Also the reference implementation for the `classify_api_error` plugin hook: -provider plugins can own their provider's error quirks without patching -`agent/error_classifier.py`. The callback contract is documented on the -`classify_api_error` entry in `hermes_cli.plugins.VALID_HOOKS`. - -## Enable - -Like all bundled standalone plugins, this ships opt-in: - -```bash -hermes plugins enable openrouter-tool-use-404 -``` - -## Origin - -Classification logic from PR #58451 by @webtecnica, re-implemented as a -plugin to demonstrate the hook. diff --git a/plugins/openrouter-tool-use-404/__init__.py b/plugins/openrouter-tool-use-404/__init__.py deleted file mode 100644 index 7101982710..0000000000 --- a/plugins/openrouter-tool-use-404/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -"""openrouter-tool-use-404 — provider error classification as a plugin. - -When OpenRouter routes a tool-calling request to a model with no -tool-capable endpoint, it returns HTTP 404 with a body like:: - - No endpoints found that support tool use. Try disabling "browser_back". - To learn more about provider routing, visit: - https://openrouter.ai/docs/guides/routing/provider-selection - -The body carries none of the core ``_MODEL_NOT_FOUND_PATTERNS`` signals, -so the built-in classifier files it under ``unknown`` (retryable) and the -retry loop burns several attempts on a deterministic rejection before -surfacing a generic error. - -This plugin classifies it as ``model_not_found`` (``retryable=False``, -``should_fallback=True``) so the client-error fast-fallback path in -``conversation_loop.py`` switches to a configured fallback model before -the user ever sees the error. - -It is also the reference implementation for the ``classify_api_error`` -hook: a provider plugin self-scopes on the ``provider`` kwarg, matches -its provider's quirk in the pre-parsed error context, and returns a -classification dict — no core patches required. See the hook contract in -``hermes_cli.plugins.VALID_HOOKS``. -""" - -from __future__ import annotations - -from typing import Any, Dict, Optional - -_PATTERN = "no endpoints found that support tool use" - - -def classify( - provider: str = "", - status_code: Optional[int] = None, - error_message: str = "", - **_kwargs: Any, -) -> Optional[Dict[str, Any]]: - """Claim OpenRouter's tool-use-404, pass on everything else.""" - if (provider or "").strip().lower() != "openrouter": - return None - # OpenRouter surfaces this as 404, but some SDK wrappers drop the - # status; the phrase alone is unambiguous, so accept a missing code. - if status_code not in (None, 404): - return None - if _PATTERN not in (error_message or "").lower(): - return None - return { - "reason": "model_not_found", - "retryable": False, - "should_fallback": True, - } - - -def register(ctx) -> None: - ctx.register_hook("classify_api_error", classify) diff --git a/plugins/openrouter-tool-use-404/plugin.yaml b/plugins/openrouter-tool-use-404/plugin.yaml deleted file mode 100644 index c1c706ecb9..0000000000 --- a/plugins/openrouter-tool-use-404/plugin.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: openrouter-tool-use-404 -version: 1.0.0 -description: "Classify OpenRouter's 'No endpoints found that support tool use' 404 as model_not_found so the agent fails over to a configured fallback model instead of retrying a deterministic rejection. Reference implementation for the classify_api_error plugin hook." -author: "NousResearch (hook demo; classification from PR #58451 by @webtecnica)" -provides_hooks: - - classify_api_error diff --git a/tests/agent/test_shell_hooks.py b/tests/agent/test_shell_hooks.py index 6fb8f78460..9eb1a4fac9 100644 --- a/tests/agent/test_shell_hooks.py +++ b/tests/agent/test_shell_hooks.py @@ -270,6 +270,25 @@ class TestParseHooksBlock: + def test_python_only_event_refused(self, caplog): + # classify_api_error returns a classification directive that + # _parse_response has no channel for — a shell registration would + # be silently ignored, so it must be refused with a warning. + specs = shell_hooks._parse_hooks_block({ + "classify_api_error": [ + {"command": "/tmp/hook.sh"}, + ], + }) + assert specs == [] + assert any("Python-plugin-only" in r.message for r in caplog.records) + + def test_timeout_clamped_to_max(self): + specs = shell_hooks._parse_hooks_block({ + "post_tool_call": [ + {"command": "/tmp/slow.sh", "timeout": 9999}, + ], + }) + assert specs[0].timeout == shell_hooks.MAX_TIMEOUT_SECONDS diff --git a/tests/test_classify_api_error_hook.py b/tests/test_classify_api_error_hook.py index 6396c21de5..3c410f1b00 100644 --- a/tests/test_classify_api_error_hook.py +++ b/tests/test_classify_api_error_hook.py @@ -1,9 +1,13 @@ """Tests for the ``classify_api_error`` plugin hook. Covers the seam in ``agent.error_classifier.classify_api_error`` (step 0, -consulted before the built-in pipeline), the sanitization contract of -``hermes_cli.plugins.get_plugin_error_classification``, and the bundled -``openrouter-tool-use-404`` reference plugin. +consulted before the built-in pipeline) and the sanitization contract of +``hermes_cli.plugins.get_plugin_error_classification``. + +The fixture error is deliberately synthetic (fake provider, made-up +message, no status code) so no present or future built-in rule can claim +it — the earlier OpenRouter tool-use-404 fixture went stale the moment +core learned that exact phrase. Mirrors the ``transform_tool_result`` hook tests: patch the symbol the call site actually imports (``hermes_cli.plugins.*``) rather than the @@ -11,7 +15,6 @@ consuming module, because the import happens at call time. """ import importlib.util -from pathlib import Path import hermes_cli.plugins as plugins_mod from agent.error_classifier import FailoverReason, classify_api_error @@ -25,14 +28,18 @@ class _FakeAPIError(Exception): self.body = body or {} -_TOOL_USE_404 = 'No endpoints found that support tool use. Try disabling "browser_back".' +_UNCLAIMED_MESSAGE = "flux capacitor drift detected in shard seven" -def _classify_tool_use_404(**kwargs): +def _classify_unclaimed_error(**kwargs): return classify_api_error( - _FakeAPIError(_TOOL_USE_404, status_code=404), - provider="openrouter", - model="deepseek/deepseek-chat", + # No status code on purpose: the built-in pipeline claims whole + # status classes (4xx -> format_error, 429 -> rate_limit, ...), so + # a status-less neutral message is the only shape guaranteed to + # reach the unknown/retryable fall-through. + _FakeAPIError(_UNCLAIMED_MESSAGE), + provider="acmecloud", + model="acme/large-1", **kwargs, ) @@ -44,9 +51,10 @@ def test_no_hook_falls_through_to_builtin(monkeypatch): # Fresh manager so no stale plugin hooks pollute state. monkeypatch.setattr(plugins_mod, "_plugin_manager", plugins_mod.PluginManager()) - result = _classify_tool_use_404() - # Documents the gap the demo plugin closes: a generic 404 with no - # model-not-found signal classifies as unknown/retryable today. + result = _classify_unclaimed_error() + # The synthetic error matches no built-in rule: unknown/retryable is + # the pipeline's fall-through, which is exactly the class of error + # this hook lets provider plugins claim. assert result.reason == FailoverReason.unknown assert result.retryable is True @@ -62,13 +70,13 @@ def test_plugin_classification_wins(monkeypatch): ], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.model_not_found assert result.retryable is False assert result.should_fallback is True # Extracted context is preserved on the ClassifiedError. - assert result.provider == "openrouter" - assert result.status_code == 404 + assert result.provider == "acmecloud" + assert result.status_code is None def test_plugin_overrides_builtin_classification(monkeypatch): @@ -91,7 +99,7 @@ def test_enum_reason_accepted(monkeypatch): lambda name, **kw: [{"reason": FailoverReason.billing}], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.billing @@ -101,7 +109,7 @@ def test_reason_only_dict_uses_dataclass_defaults(monkeypatch): lambda name, **kw: [{"reason": "server_error"}], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.server_error assert result.retryable is True assert result.should_compress is False @@ -118,7 +126,7 @@ def test_invalid_reason_falls_through_to_builtin(monkeypatch): lambda name, **kw: [{"reason": "not_a_real_reason"}], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.unknown @@ -128,7 +136,7 @@ def test_non_dict_results_ignored(monkeypatch): lambda name, **kw: ["model_not_found", 123, ["nope"], None], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.unknown @@ -142,7 +150,7 @@ def test_first_valid_result_wins(monkeypatch): ], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.billing @@ -152,7 +160,7 @@ def test_helper_exception_never_breaks_classification(monkeypatch): monkeypatch.setattr(plugins_mod, "get_plugin_error_classification", _boom) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.reason == FailoverReason.unknown assert result.retryable is True @@ -169,14 +177,14 @@ def test_hook_receives_parsed_error_context(monkeypatch): monkeypatch.setattr(plugins_mod, "invoke_hook", _capture) - _classify_tool_use_404(approx_tokens=1234, num_messages=7) + _classify_unclaimed_error(approx_tokens=1234, num_messages=7) assert seen["hook_name"] == "classify_api_error" - assert seen["provider"] == "openrouter" - assert seen["model"] == "deepseek/deepseek-chat" - assert seen["status_code"] == 404 + assert seen["provider"] == "acmecloud" + assert seen["model"] == "acme/large-1" + assert seen["status_code"] is None assert seen["error_type"] == "_FakeAPIError" - assert "no endpoints found that support tool use" in seen["error_message"] + assert "flux capacitor drift" in seen["error_message"] assert seen["approx_tokens"] == 1234 assert seen["num_messages"] == 7 assert isinstance(seen["error_body"], dict) @@ -189,72 +197,60 @@ def test_message_override_and_error_context_sanitized(monkeypatch): lambda name, **kw: [{ "reason": "model_not_found", "message": " custom guidance ", - "error_context": {"upstream_provider": "DeepSeek"}, + "error_context": {"upstream_provider": "AcmeCloud"}, }], ) - result = _classify_tool_use_404() + result = _classify_unclaimed_error() assert result.message == "custom guidance" - assert result.error_context == {"upstream_provider": "DeepSeek"} + assert result.error_context == {"upstream_provider": "AcmeCloud"} -# ── Bundled reference plugin ──────────────────────────────────────────── +# ── Plugin register() end-to-end (synthetic, written at test time) ────── + +_SYNTHETIC_PLUGIN = ''' +def classify(provider=None, error_message=None, **kwargs): + """Self-scoped classifier for acmecloud's flux-drift errors.""" + if provider != "acmecloud": + return None + if "flux capacitor drift" not in (error_message or ""): + return None + return {"reason": "overloaded", "retryable": True, "should_fallback": True} -def _load_demo_plugin(): - plugin_init = ( - Path(__file__).resolve().parent.parent - / "plugins" / "openrouter-tool-use-404" / "__init__.py" - ) - spec = importlib.util.spec_from_file_location( - "openrouter_tool_use_404_demo", plugin_init, - ) +def register(ctx): + ctx.register_hook("classify_api_error", classify) +''' + + +def _load_synthetic_plugin(tmp_path): + plugin_init = tmp_path / "acmecloud_classifier.py" + plugin_init.write_text(_SYNTHETIC_PLUGIN, encoding="utf-8") + spec = importlib.util.spec_from_file_location("acmecloud_classifier", plugin_init) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module -def test_demo_plugin_claims_openrouter_tool_use_404(): - demo = _load_demo_plugin() - result = demo.classify( - provider="openrouter", - status_code=404, - error_message=_TOOL_USE_404.lower(), - ) - assert result == { - "reason": "model_not_found", - "retryable": False, - "should_fallback": True, - } - - -def test_demo_plugin_self_scopes(): - demo = _load_demo_plugin() +def test_synthetic_plugin_self_scopes(tmp_path): + demo = _load_synthetic_plugin(tmp_path) # Different provider: pass. assert demo.classify( - provider="anthropic", status_code=404, - error_message=_TOOL_USE_404.lower(), - ) is None - # Different status: pass. - assert demo.classify( - provider="openrouter", status_code=400, - error_message=_TOOL_USE_404.lower(), + provider="anthropic", error_message=_UNCLAIMED_MESSAGE, ) is None # Different message: pass. assert demo.classify( - provider="openrouter", status_code=404, - error_message="model not found", + provider="acmecloud", error_message="model not found", ) is None - # Missing status but unambiguous phrase: claim. + # Provider and unambiguous phrase: claim. assert demo.classify( - provider="openrouter", status_code=None, - error_message=_TOOL_USE_404.lower(), + provider="acmecloud", error_message=_UNCLAIMED_MESSAGE, ) is not None -def test_demo_plugin_end_to_end(monkeypatch): +def test_synthetic_plugin_end_to_end(tmp_path, monkeypatch): """register() + real invoke_hook + classify_api_error, no mocks.""" - demo = _load_demo_plugin() + demo = _load_synthetic_plugin(tmp_path) manager = plugins_mod.PluginManager() monkeypatch.setattr(plugins_mod, "_plugin_manager", manager) @@ -264,15 +260,15 @@ def test_demo_plugin_end_to_end(monkeypatch): demo.register(_Ctx()) - result = _classify_tool_use_404() - assert result.reason == FailoverReason.model_not_found - assert result.retryable is False + result = _classify_unclaimed_error() + assert result.reason == FailoverReason.overloaded + assert result.retryable is True assert result.should_fallback is True # And the built-in pipeline is untouched for everything the plugin # doesn't claim. other = classify_api_error( _FakeAPIError("rate limit exceeded", status_code=429), - provider="openrouter", + provider="acmecloud", ) assert other.reason == FailoverReason.rate_limit diff --git a/website/docs/user-guide/features/hooks.md b/website/docs/user-guide/features/hooks.md index 54c8fdc7ac..c72a241dfe 100644 --- a/website/docs/user-guide/features/hooks.md +++ b/website/docs/user-guide/features/hooks.md @@ -453,6 +453,7 @@ Payload fields below are the exact event-specific fields supplied by each call s | `on_stream_delta` | Observer | Dispatched per normalized streaming text delta via the bounded observer queue; a stalled callback drops only its own oldest events; return ignored. | `delta`, `kind` (`text` or `reasoning`), `turn_id`, `iteration`, `session_id`, `model`, `provider`, `surface` | Delta text is raw model output; reasoning deltas require the `plugins.stream_reasoning_deltas` opt-in. | | `on_stream_end` | Observer | Dispatched when a streaming response finishes or errors, after the stream closes; return ignored. | `final_text`, `finished`, `error`, `turn_id`, `iteration`, `session_id`, `model`, `provider`, `surface` | Full assembled response text; error text may include provider data. | | `on_interim_message` | Observer | Dispatched when a mid-loop assistant message is surfaced before the final answer (streaming or non-streaming); return ignored. | `text`, `already_streamed`, `turn_id`, `iteration`, `session_id`, `model`, `provider`, `surface` | Full interim assistant text. | +| `classify_api_error` | Directive/control | On each failed provider attempt, at the top of the built-in classifier; all callbacks run, then the first dict with a valid `reason` wins (run-all-then-pick-first). Python plugins only. | `provider`, `model`, `status_code`, `error_type`, `error_code`, `error_message`, `error_body`, `error`, `approx_tokens`, `context_length`, `num_messages` | `error_message` and `error_body` may contain raw provider/user data. | | `on_session_start` | Observer | First turn of a new session; return ignored. | `session_id`, `model`, `platform` | Identifiers and routing metadata only. | | `on_session_end` | Observer | Canonically at each turn finalization; CLI/TUI exits have additional reduced legacy shapes. Return ignored. | Canonical: `session_id`, `task_id`, `turn_id`, `completed`, `failed`, `interrupted`, `turn_exit_reason`, `model`, `platform`; exit paths may add `reason`/`api_request_id` and omit fields. | IDs, model/platform, and outcome; canonical payload has no message body. | | `on_session_finalize` | Observer | CLI/TUI/gateway teardown through `finalize_session`; gateway shutdown or expiry may finalize without a reset. Return ignored. | Surface-dependent `session_id`, `platform`, optionally `reason`, `old_session_id`, `new_session_id` | Session and routing identifiers. | @@ -846,6 +847,54 @@ For standing guidance that should shape the built-in missing-evidence nudge, use --- +### `classify_api_error` + +Fires **once per failed API call**, at the top of `agent/error_classifier.classify_api_error()` — BEFORE the built-in classification pipeline. Provider plugins use it to own their provider's error quirks (a vendor-specific 404 that should fast-fallback, a misleading status code) without core patches. + +This hook is **behavior-changing**: the returned classification drives retry, compression, credential-rotation, and fallback routing for the failed call. + +**Callback signature:** + +```python +def my_callback(provider: str, model: str, status_code, error_type: str, + error_code, error_message: str, error_body, error, + approx_tokens, context_length, num_messages, **kwargs): +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `provider` | `str` | The provider whose call failed — **self-scope on this** | +| `model` | `str` | The model identifier for the failed call | +| `status_code` | `int \| None` | HTTP status, when the error carried one | +| `error_type` | `str` | The exception class name | +| `error_code` | `str \| None` | Provider error code, when present | +| `error_message` | `str` | Lower-cased error message text | +| `error_body` | `dict` | Parsed provider error body, when present | +| `error` | `Exception` | The original exception object | +| `approx_tokens` | `int \| None` | Approximate prompt tokens of the failed request | +| `context_length` | `int \| None` | Model context length, when known | +| `num_messages` | `int \| None` | Message count of the failed request | + +**Return value — claim the error:** + +```python +return { + "reason": "model_not_found", # required: a FailoverReason name + "retryable": False, # optional recovery-hint overrides + "should_fallback": True, + "should_compress": False, + "should_rotate_credential": False, + "message": "...", # optional user-facing guidance + "error_context": {...}, # optional extra context +} +``` + +Return `None` (or nothing) to pass the error to the built-in pipeline. The first valid result in registration order wins; invalid dicts and unknown reasons are skipped; exceptions are isolated so a broken plugin can never break error classification. + +**Python plugins only.** Shell hooks cannot register for this event: the shell response parser has no channel for the classification directive, so a shell registration is refused at config parse with a warning rather than being silently ignored. + +--- + ### `on_session_start` Fires **once** when a brand-new session is created. Does **not** fire on session continuation (when the user sends a second message in an existing session).