From 7e0b5cd235b341e8d382864a811c808308885757 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:03:40 -0700 Subject: [PATCH] fix(auxiliary): auto never bills a provider the user did not select MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With a main provider selected, an unusable main route (expired xAI/Codex OAuth token, 401/402/429 mid-session) fell through the built-in discovery chain (OpenRouter -> Nous -> custom -> api-key) and quietly ran every compression, title and memory-flush call on whichever OTHER account was still logged in. Reported as "using Grok on my Premium+ sub, my Nous Portal balance kept draining" — the chat visibly stayed on Grok while the side tasks were billed elsewhere, and re-logging into X did not help because the aux side never consulted the selected provider. The discovery chain is now reserved for installs with no selected main provider (`model.provider: auto` / unset). Otherwise the ladder is main -> auxiliary..fallback_chain -> fallback_providers -> refuse with a warning naming the dead provider and the fix. Both entry points gate on the same predicate: the resolve-time route and the mid-request payment/auth hop (_try_payment_fallback). Existing chain tests that asserted the hop now pin `provider=auto`, the one case where discovery is still the contract. --- agent/auxiliary_client.py | 19 +++++++++ ...t_auxiliary_auto_never_guesses_provider.py | 41 +++++++++++++++++++ tests/agent/test_auxiliary_client.py | 8 ++-- .../test_models_detect_vendor_ownership.py | 35 ++++++++++++++++ website/docs/user-guide/configuring-models.md | 4 +- .../user-guide/features/fallback-providers.md | 8 ++-- 6 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 tests/agent/test_auxiliary_auto_never_guesses_provider.py create mode 100644 tests/hermes_cli/test_models_detect_vendor_ownership.py diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 121fe0f352..ac5c1633df 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -3827,6 +3827,8 @@ def _try_payment_fallback( provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "").""" skip = failed_provider.lower().strip() main_provider = _read_main_provider() + if not _discovery_chain_allowed(main_provider, task): + return None, None, "" skip_labels = {skip} if main_provider and main_provider.lower() in skip: skip_labels.add(main_provider.lower()) @@ -4200,6 +4202,21 @@ def _try_main_provider_route( return client, resolved or main_model, resolved_provider +def _discovery_chain_allowed(main_provider: str, task: Optional[str] = None) -> bool: + """The built-in discovery chain is a convenience for installs with NO selected main provider. + Once the user picked one, every auxiliary route must be a provider they configured (main, + ``auxiliary.``, ``fallback_providers``); guessing "whatever else is logged in" bills an + account they never pointed this session at (xAI OAuth session with a dead token → every + compression silently charged to a Nous Portal balance).""" + if (main_provider or "").strip().lower() in {"", "auto"}: + return True + logger.warning( + "Auxiliary %s: main provider %s is unavailable and no fallback_chain / fallback_providers is " + "configured — refusing to guess another logged-in provider. Re-authenticate (`hermes model`) " + "or declare a fallback.", task or "call", main_provider) + return False + + def _try_discovery_chain() -> Tuple[Optional[OpenAI], Optional[str], str]: """Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers.""" tried = [] @@ -4249,6 +4266,8 @@ def _resolve_auto_route( task, main_provider or "auto", reason="main provider unavailable") if fb_client is not None: return fb_client, fb_model, fb_label + if not _discovery_chain_allowed(main_provider, task): + return None, None, "" return _try_discovery_chain() diff --git a/tests/agent/test_auxiliary_auto_never_guesses_provider.py b/tests/agent/test_auxiliary_auto_never_guesses_provider.py new file mode 100644 index 0000000000..04b3821e7b --- /dev/null +++ b/tests/agent/test_auxiliary_auto_never_guesses_provider.py @@ -0,0 +1,41 @@ +"""``auxiliary.*.provider: auto`` never bills a provider the user did not select. + +Regression for the Grok/Nous incident: main provider ``xai-oauth`` with an expired token, Nous +Portal still logged in from an earlier setup, no ``fallback_providers``. Every compression, title +and memory-flush call for the Grok conversation fell through to the built-in discovery chain and +was charged to the Portal balance while the chat visibly stayed on Grok. The discovery chain is +only for installs that have no selected main provider at all. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from agent import auxiliary_client as aux + + +@pytest.fixture +def nous_is_the_only_working_provider(): + """Main provider unusable; Nous would win the discovery chain; no configured fallback policy.""" + aux._aux_unhealthy_until.clear() + with patch.object(aux, "_try_main_provider_route", return_value=None), \ + patch.object(aux, "_try_configured_fallback_chain", return_value=(None, None, "")), \ + patch.object(aux, "_try_main_fallback_chain", return_value=(None, None, "")), \ + patch.object(aux, "_try_openrouter", return_value=(None, None)), \ + patch.object(aux, "_try_nous", return_value=(MagicMock(name="nous"), "nous-model")): + yield + + +def test_selected_main_provider_down_refuses_to_guess_another_account(nous_is_the_only_working_provider): + runtime = {"provider": "xai-oauth", "model": "grok-4.6", "base_url": "https://api.x.ai/v1", "api_key": "dead"} + with patch.object(aux, "_read_main_provider", return_value="xai-oauth"): + assert aux._resolve_auto_route(main_runtime=runtime, task="compression") == (None, None, "") + assert aux._try_payment_fallback("xai-oauth", task="compression") == (None, None, "") + + +def test_no_selected_main_provider_still_discovers(nous_is_the_only_working_provider): + with patch.object(aux, "_read_main_provider", return_value="auto"): + client, model, label = aux._resolve_auto_route(main_runtime={"provider": "auto"}, task="compression") + assert client is not None and (model, label) == ("nous-model", "nous") diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index c1d5398e9b..ecd99c70a3 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -1596,10 +1596,12 @@ class TestTryPaymentFallback: _aux_unhealthy_logged_at.clear() def test_skips_failed_provider(self): + """Discovery only walks with no selected main provider (``auto``); a selected provider that + fails never hops to another logged-in account (test_auxiliary_auto_never_guesses_provider).""" mock_client = MagicMock() with patch("agent.auxiliary_client._try_openrouter", return_value=(None, None)), \ patch("agent.auxiliary_client._try_nous", return_value=(mock_client, "nous-model")), \ - patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"): + patch("agent.auxiliary_client._read_main_provider", return_value="auto"): client, model, label = _try_payment_fallback("openrouter", task="compression") assert client is mock_client assert model == "nous-model" @@ -1617,7 +1619,7 @@ class TestTryPaymentFallback: patch("agent.auxiliary_client._try_nous", return_value=(None, None)), \ patch("agent.auxiliary_client._try_custom_endpoint", return_value=(None, None)), \ patch("agent.auxiliary_client._resolve_api_key_provider", return_value=(None, None)), \ - patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"): + patch("agent.auxiliary_client._read_main_provider", return_value="auto"): client, model, label = _try_payment_fallback("openrouter") assert client is None assert model is None @@ -4168,7 +4170,7 @@ class TestAuxUnhealthyCache: # Mark BOTH the failed provider (openrouter) and a sibling (custom) # unhealthy. The chain should still find nous. _mark_provider_unhealthy("local/custom") - with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + with patch("agent.auxiliary_client._read_main_provider", return_value="auto"), \ patch("agent.auxiliary_client._try_openrouter") as or_try, \ patch("agent.auxiliary_client._try_nous", return_value=(nous_client, "n-model")), \ patch("agent.auxiliary_client._try_custom_endpoint") as custom_try, \ diff --git a/tests/hermes_cli/test_models_detect_vendor_ownership.py b/tests/hermes_cli/test_models_detect_vendor_ownership.py new file mode 100644 index 0000000000..18dc5f45a4 --- /dev/null +++ b/tests/hermes_cli/test_models_detect_vendor_ownership.py @@ -0,0 +1,35 @@ +"""A first-party session never re-routes its own vendor's model when the live catalog cannot vouch. + +Second half of the Astra incident (#97487): the live-catalog guard only helps when the fetch +succeeds. A transient Codex outage, or a static fallback that lags an early-access rollout, left +``/model gpt-6-astra`` on ``openai-codex`` walking the ladder to OpenRouter (which relists every +vendor) and silently rebuilding the session on a metered aggregator. +""" + +from __future__ import annotations + +import pytest + +from hermes_cli import models, models_detect + + +@pytest.fixture +def ladder_would_hijack(monkeypatch): + """Live catalog unavailable; OpenRouter lists everything; the user holds an OpenRouter key.""" + monkeypatch.setattr(models, "cached_provider_model_ids", lambda provider, **_: []) + monkeypatch.setattr(models, "_find_openrouter_slug", lambda name: f"vendor/{name}") + monkeypatch.setattr(models_detect, "provider_has_credentials", lambda p: p == "openrouter") + + +@pytest.mark.parametrize("provider,model", [ + ("openai-codex", "gpt-6-astra"), + ("xai-oauth", "grok-5-preview"), + ("anthropic", "claude-opus-5-early"), +]) +def test_own_vendor_id_stays_when_live_catalog_is_empty(ladder_would_hijack, provider, model): + assert models.detect_provider_for_model(model, provider) is None + + +def test_other_vendor_id_still_remaps_to_keyed_aggregator(ladder_would_hijack): + assert models.detect_provider_for_model("claude-opus-4.7", "openai-codex") == ( + "openrouter", "vendor/claude-opus-4.7") diff --git a/website/docs/user-guide/configuring-models.md b/website/docs/user-guide/configuring-models.md index 2378ea231a..c200d46b81 100644 --- a/website/docs/user-guide/configuring-models.md +++ b/website/docs/user-guide/configuring-models.md @@ -73,7 +73,7 @@ Click **Show auxiliary** to reveal the 11 task slots: ![Auxiliary panel expanded](/img/docs/dashboard-models/auxiliary-expanded.png) -Every auxiliary task defaults to `auto` — meaning Hermes tries your main model for that job too. If that route is unavailable or hits a capacity-style failure, `auto` follows any task-specific `auxiliary..fallback_chain`, then the main `fallback_providers` / `fallback_model` chain, then Hermes' built-in auxiliary discovery chain. Override a specific task when you want a cheaper or faster model for a side-job. +Every auxiliary task defaults to `auto` — meaning Hermes tries your main model for that job too. If that route is unavailable or hits a capacity-style failure, `auto` follows any task-specific `auxiliary..fallback_chain`, then the main `fallback_providers` / `fallback_model` chain. It never guesses a provider you did not configure: with a main provider selected and no fallback declared, the side task is skipped with a warning rather than billed to another account you happen to be logged into. (Hermes' built-in discovery chain only runs when no main provider is selected at all.) Override a specific task when you want a cheaper or faster model for a side-job. ### Common override patterns @@ -163,7 +163,7 @@ auxiliary: model: inclusionai/ring-2.6-1t:free ``` -When `fallback_chain` is absent, `auto` uses the top-level `fallback_providers` chain before the built-in auxiliary discovery chain. +When `fallback_chain` is absent, `auto` uses the top-level `fallback_providers` chain. If that is also absent and the main provider cannot serve the call, the task is skipped with a warning — Hermes does not fall through to other logged-in providers. ## Per-provider request options diff --git a/website/docs/user-guide/features/fallback-providers.md b/website/docs/user-guide/features/fallback-providers.md index 82dc096324..50d465f655 100644 --- a/website/docs/user-guide/features/fallback-providers.md +++ b/website/docs/user-guide/features/fallback-providers.md @@ -214,11 +214,11 @@ Hermes uses separate lightweight models for side tasks. Each task has its own pr ### Auto-Detection Chain -When a task's provider is set to `"auto"` (the default), Hermes first tries the main provider + main model for that auxiliary task. If that route is unavailable or later fails with a capacity-style error, Hermes now honors user-configured fallback policy before using the built-in discovery chain: +When a task's provider is set to `"auto"` (the default), Hermes first tries the main provider + main model for that auxiliary task. If that route is unavailable or later fails with a capacity-style error, Hermes follows your configured fallback policy and then stops: ```text Main provider + main model → auxiliary..fallback_chain → -fallback_providers / fallback_model → built-in auxiliary discovery chain +fallback_providers / fallback_model → skip the task (warn) ``` A billing or quota failure quarantines only the failed custom endpoint for the auxiliary health cooldown, not every route registered as `custom`. A healthy local endpoint with a different base URL remains eligible for fallback and subsequent auto routing. Aliases for the same custom endpoint share its health state. Built-in providers retain their shared-account health checks. @@ -239,7 +239,7 @@ Main provider (if vision-capable) → OpenRouter → Nous Portal → Codex OAuth → Anthropic → Custom endpoint → give up ``` -Those built-in chains are a convenience fallback for users who have not declared a task-specific or main fallback policy. +Those built-in chains run **only when no main provider is selected** (`model.provider: auto` or unset). Once you have picked a main provider, an unavailable main route with no `fallback_chain` / `fallback_providers` skips the auxiliary task with a warning instead of guessing another provider you happen to be logged into — an expired xAI or Codex session must never bill your Nous Portal or OpenRouter balance behind your back. Declare a fallback if you want one. ### Configuring Auxiliary Providers @@ -269,7 +269,7 @@ auxiliary: model: "" ``` -Every task above follows the same **provider / model / base_url** pattern. Each task can also declare its own `fallback_chain`; if omitted, `provider: auto` uses the top-level `fallback_providers` chain before Hermes' built-in auxiliary discovery chain. +Every task above follows the same **provider / model / base_url** pattern. Each task can also declare its own `fallback_chain`; if omitted, `provider: auto` uses the top-level `fallback_providers` chain (the built-in discovery chain applies only when no main provider is selected). Context compression is configured under `auxiliary.compression`: