From 4caeb02735cdbbe302d0f28bf1a0ef14aa1af3e8 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:12:58 -0300 Subject: [PATCH 01/21] fix(models): key the pricing cache on auth state, not just the base URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fetch_models_with_pricing` checked its cache above the point where the Authorization header is built, and keyed that cache on the base URL alone. Whichever read of a given base URL landed first in a process therefore answered every later read, whatever key it passed — a non-empty result is held for the life of the process. That is wrong for any endpoint whose answer depends on who is asking. The Nous inference gateway filters `GET /v1/models` by the caller's org model policy, so an anonymous read landing first makes a later authenticated read return the full, unfiltered catalog without a request going out. Separate the URL root from the cache key and fold auth state into the latter. Only whether a key was supplied participates, never its value, so no secret reaches the key. `credits_tracker` peeked into the private `_pricing_cache` and duplicated the key shape to do it; it now calls `peek_cached_pricing`, which owns both the /v1-suffix normalization and the preference for the authenticated catalog. Co-Authored-By: Claude Opus 5 (1M context) --- agent/credits_tracker.py | 14 +- hermes_cli/models.py | 41 +++++- .../hermes_cli/test_pricing_cache_auth_key.py | 127 ++++++++++++++++++ 3 files changed, 171 insertions(+), 11 deletions(-) create mode 100644 tests/hermes_cli/test_pricing_cache_auth_key.py diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 39c74ea58b..2d0873c563 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -252,15 +252,13 @@ def is_free_tier_model(model: str, base_url: str = "") -> bool: if not base_url: return False try: - from hermes_cli.models import _is_model_free, _pricing_cache + from hermes_cli.models import _is_model_free, peek_cached_pricing - # Mirror get_pricing_for_provider's key normalization: the agent's - # Nous base_url is /v1-suffixed (https://inference-api.nousresearch.com/v1) - # but the picker keys _pricing_cache on the pre-/v1 root. - key = base_url.rstrip("/") - if key.endswith("/v1"): - key = key[:-3].rstrip("/") - pricing = _pricing_cache.get(key) + # The agent's Nous base_url is /v1-suffixed + # (https://inference-api.nousresearch.com/v1) but the catalog fetchers + # key on the pre-/v1 root, and on auth state besides; peek_cached_pricing + # owns both details. + pricing = peek_cached_pricing(base_url) if not pricing: return False return _is_model_free(model, pricing) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 389cbaed72..cc2ba908b8 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2255,6 +2255,39 @@ def _cache_catalog( return result +# A governed endpoint answers an authenticated read with a policy-filtered +# catalog and an anonymous read with the full one, so auth state is part of the +# cache identity. NUL cannot appear in a URL, so the suffix cannot collide with +# a base URL that happens to end this way. +_PRICING_AUTH_KEY_SUFFIX = "\x00auth" + + +def _pricing_cache_key(url_root: str, api_key: str | None) -> str: + """The ``_pricing_cache`` key for a read of *url_root*. + + Only *whether* a key was supplied participates — never its value, so no + secret reaches the cache key. + """ + return url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root + + +def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: + """Pricing already cached for *base_url*, or ``{}``. Never fetches. + + Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the + catalog fetchers key on. Prefers the authenticated catalog, which is the + one scoped to the caller's org. + """ + root = (base_url or "").rstrip("/") + if root.endswith("/v1"): + root = root[:-3].rstrip("/") + for key in (root + _PRICING_AUTH_KEY_SUFFIX, root): + cached = _pricing_cache.get(key) + if cached: + return cached + return {} + + def _format_price_per_mtok(per_token_str: str) -> str: """Convert a per-token price string to a human-friendly $/Mtok string. @@ -2391,7 +2424,8 @@ def fetch_models_with_pricing( ) -> dict[str, dict[str, Any]]: """Fetch ``/v1/models`` and return ``{model_id: {prompt, completion, ...}}``. - Results are cached per *base_url* so repeated calls are free. + Results are cached per *base_url* and per auth state, so repeated calls + are free and an authenticated read never answers an anonymous one. Works with any OpenRouter-compatible endpoint (OpenRouter, Nous Portal). When *include_sale_original* is true (Nous Portal only) and the gateway @@ -2402,13 +2436,14 @@ def fetch_models_with_pricing( ``{prompt, completion}`` shape even if a response happens to nest ``original``. """ - cache_key = (base_url or "").rstrip("/") + url_root = (base_url or "").rstrip("/") + cache_key = _pricing_cache_key(url_root, api_key) if not force_refresh: cached = _cached_catalog(cache_key) if cached is not None: return cached - url = cache_key + "/v1/models" + url = url_root + "/v1/models" headers: dict[str, str] = { "Accept": "application/json", "User-Agent": _HERMES_USER_AGENT, diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py new file mode 100644 index 0000000000..a1ae120365 --- /dev/null +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -0,0 +1,127 @@ +"""``_pricing_cache`` keys on auth state, not just the base URL. + +A governed endpoint (Nous ``/v1/models`` filtered by an org's model policy) +answers an authenticated read with a narrower catalog than an anonymous one. +Keyed on the base URL alone, whichever read landed first in a process answered +every later one — so an authenticated caller could be handed the full, +unfiltered catalog without a request going out. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest + +import hermes_cli.models as models_mod +from hermes_cli.models import fetch_models_with_pricing, peek_cached_pricing + +BASE = "https://inference-api.example.com" + +# What the endpoint serves anonymously vs. to a policy-restricted caller. +_FULL = ["vendor/allowed", "vendor/blocked"] +_FILTERED = ["vendor/allowed"] + + +@pytest.fixture(autouse=True) +def _clear_pricing_cache(): + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + yield + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + + +@pytest.fixture +def catalog(monkeypatch): + """Serve the filtered catalog to an authenticated read, the full one to an + anonymous read, and record every request.""" + requests: list[str | None] = [] + + def _fake_urlopen(req, timeout=8.0): + auth = req.get_header("Authorization") + requests.append(auth) + ids = _FILTERED if auth else _FULL + payload = { + "data": [ + {"id": mid, "pricing": {"prompt": "0.000002", "completion": "0.00001"}} + for mid in ids + ] + } + resp = MagicMock() + resp.read.return_value = json.dumps(payload).encode() + resp.__enter__ = lambda self: self + resp.__exit__ = lambda *a: False + return resp + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen) + return requests + + +def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): + """The bug: an anonymous read landing first must not answer the next + authenticated read out of cache.""" + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + + assert sorted(anon) == sorted(_FULL) + assert sorted(authed) == sorted(_FILTERED) + assert len(catalog) == 2, "the authenticated read must reach the network" + assert catalog[0] is None and catalog[1] == "Bearer sk-test" + + +def test_anonymous_read_is_not_answered_by_an_authenticated_one(catalog): + """And the reverse direction, so neither entry can shadow the other.""" + authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + + assert sorted(authed) == sorted(_FILTERED) + assert sorted(anon) == sorted(_FULL) + assert len(catalog) == 2 + + +@pytest.mark.parametrize("api_key", ["sk-test", ""]) +def test_repeated_read_still_hits_the_cache(catalog, api_key): + """Widening the key must not cost the caching it was there for.""" + first = fetch_models_with_pricing(api_key=api_key, base_url=BASE) + second = fetch_models_with_pricing(api_key=api_key, base_url=BASE) + + assert first == second + assert len(catalog) == 1, "second read should be served from cache" + + +def test_force_refresh_replaces_only_its_own_entry(catalog): + """A forced authenticated re-read must leave the anonymous entry intact.""" + fetch_models_with_pricing(api_key="", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE, force_refresh=True) + + assert len(catalog) == 3 + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + assert sorted(anon) == sorted(_FULL) + assert len(catalog) == 3, "the anonymous entry should have survived" + + +class TestPeekCachedPricing: + def test_returns_empty_when_nothing_cached(self): + assert peek_cached_pricing(BASE) == {} + + def test_accepts_a_v1_suffixed_url(self, catalog): + """The agent holds a /v1-suffixed base URL; the fetchers key on the root.""" + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert sorted(peek_cached_pricing(BASE + "/v1")) == sorted(_FILTERED) + + def test_prefers_the_authenticated_catalog(self, catalog): + """It is the one scoped to the caller's org.""" + fetch_models_with_pricing(api_key="", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert sorted(peek_cached_pricing(BASE)) == sorted(_FILTERED) + + def test_falls_back_to_the_anonymous_catalog(self, catalog): + fetch_models_with_pricing(api_key="", base_url=BASE) + assert sorted(peek_cached_pricing(BASE)) == sorted(_FULL) + + def test_never_fetches(self, catalog): + peek_cached_pricing(BASE) + assert catalog == [] From c248d5356c38b8bef1e4c03c241b8832bb1f47b8 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:13:13 -0300 Subject: [PATCH 02/21] feat(nous): read the org model policy and expose it as a list filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Nous team admin can restrict which models and which serving providers their org may use. The inference gateway applies that policy to `GET /v1/models`, omitting blocked rows with no marker field, so the keys of an authenticated catalog read are the reachable set. Add the two pieces the pickers need: `nous_policy_present()` reads the `policy_present` claim off the OAuth access token, which costs no request. `/api/oauth/account` does not carry the claim, so this reads the token rather than going through `get_nous_portal_account_info`. The claim is tri-state — absent means an older mint, which is not the same as "no policy" and must not be reported as one. `nous_policy_allowed_ids()` turns the authenticated pricing response into that set, reusing the cache entry a caller asking for pricing already populates rather than issuing a second round trip. It returns None — "leave the list alone" — for an org with no policy, for an anonymous read whose catalog is unfiltered, and for an empty read, each of which would otherwise narrow a list on evidence that cannot support it. `restrict_to_nous_policy()` applies the set while preserving the caller's order, and keeps a `:free` sibling whose base model is reachable. The gateway admits a row when any of its requestable ids passes and treats anything unknown as a keep, on the grounds that over-listing costs a 403 from the authoritative gate while hiding a row the gate would serve is unrecoverable from the client. This mirrors that. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/models.py | 68 +++++++++ hermes_cli/nous_account.py | 32 ++++ tests/hermes_cli/test_nous_policy_filter.py | 159 ++++++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 tests/hermes_cli/test_nous_policy_filter.py diff --git a/hermes_cli/models.py b/hermes_cli/models.py index cc2ba908b8..c1f7d580f1 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2604,6 +2604,74 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]: return (api_key, base_url) +def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]: + """The Nous model ids the caller's org may reach, or ``None`` to not filter. + + The gateway filters ``GET /v1/models`` by the org's model policy for an + authenticated read, omitting blocked rows with no marker field, so the keys + of the authenticated pricing response are the reachable set. This reuses + that response rather than issuing a second round trip. + + Returns ``None`` — meaning "leave the caller's list alone" — in three cases, + each of which would otherwise narrow a list on evidence that cannot support + it: + + * the org carries no policy, or the token is too old to say (see + :func:`~hermes_cli.nous_account.nous_policy_present`). Filtering an + unrestricted org's list buys nothing and risks dropping a model the + Portal recommends before the gateway catalog lists it. + * credential resolution failed, so the read is anonymous and therefore + unfiltered. A full catalog must not be mistaken for a policy-filtered one. + * the read came back empty, which is a fetch failure rather than an org + that may reach nothing. + """ + try: + from hermes_cli.nous_account import nous_policy_present + + if nous_policy_present() is not True: + return None + except Exception: + return None + + api_key, base_url = _resolve_nous_pricing_credentials() + if not api_key or not base_url: + return None + + # Same arguments as get_pricing_for_provider's nous branch, so a caller + # that also asks for pricing shares this cache entry instead of paying for + # a second request. + pricing = fetch_models_with_pricing( + api_key=api_key, + base_url=base_url, + force_refresh=force_refresh, + include_sale_original=True, + ) + return set(pricing) or None + + +def restrict_to_nous_policy( + model_ids: list[str], allowed: Optional[set[str]] +) -> list[str]: + """*model_ids* narrowed to *allowed*, preserving the caller's order. + + A ``None`` or empty *allowed* leaves the list untouched — see + :func:`nous_policy_allowed_ids` for when that happens. + + A ``:free`` sibling is kept when its base model is reachable. The gateway + admits a row when any of its requestable ids passes, and treats anything + unknown as a keep on the grounds that over-listing costs a 403 from the + authoritative gate while hiding a row the gate would serve is unrecoverable + from the client. This mirrors that. + """ + if not allowed: + return list(model_ids) + return [ + mid + for mid in model_ids + if mid in allowed or mid.split(":", 1)[0] in allowed + ] + + def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita).""" normalized = normalize_provider(provider) diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index 654487e684..4c3bf0a51f 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -99,6 +99,7 @@ class NousPortalAccountInfo: subscription: Optional[NousPortalSubscriptionInfo] = None paid_service_access: Optional[bool] = None paid_service_access_info: Optional[NousPaidServiceAccessInfo] = None + policy_present: Optional[bool] = None tool_access: Optional[NousToolAccessInfo] = None raw_claims: Optional[dict[str, Any]] = None raw_account: Optional[dict[str, Any]] = None @@ -396,6 +397,36 @@ def get_nous_portal_account_info( ) +def nous_policy_present() -> Optional[bool]: + """Whether the caller's org carries a restrictive model/provider policy. + + Read from the ``policy_present`` claim on the Nous OAuth access token, so + this costs no request. ``/api/oauth/account`` does not carry the claim, + which is why this reads the token directly rather than going through + :func:`get_nous_portal_account_info`. + + ``None`` means unknown — an older mint, an unreadable token, or a + non-boolean claim. Unknown is NOT "no policy": callers must not report the + absence of the claim as the absence of a restriction. + + The claim is stamped at mint time, so it goes stale until the next token + refresh. + """ + try: + from hermes_cli.auth import get_provider_auth_state, _decode_jwt_claims + + state = get_provider_auth_state("nous") or {} + access_token = state.get("access_token") + if not isinstance(access_token, str) or not access_token.strip(): + return None + claims = _decode_jwt_claims(access_token) + if not claims: + return None + return _coerce_bool(claims.get("policy_present")) + except Exception: + return None + + def _fresh_account_info( *, state: dict[str, Any], @@ -642,6 +673,7 @@ def _info_from_valid_jwt( expires_at=datetime.fromtimestamp(exp, tz=timezone.utc), paid_service_access=paid_access, paid_service_access_info=access_info, + policy_present=_coerce_bool(claims.get("policy_present")), tool_access=_tool_access_from_value(claims.get("tool_access")), raw_claims=dict(claims), ) diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py new file mode 100644 index 0000000000..a225d560dc --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -0,0 +1,159 @@ +"""Narrowing the Nous model lists to an org's policy. + +The inference gateway omits policy-blocked rows from an authenticated +``GET /v1/models`` with no marker field, so the keys of the authenticated +catalog read are the reachable set. These helpers turn that into a filter the +pickers can apply without a second round trip, and — just as importantly — +decline to filter when the evidence cannot support it. +""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +import hermes_cli.models as models_mod +import hermes_cli.nous_account as account_mod +from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy +from hermes_cli.nous_account import nous_policy_present + + +def _jwt(claims: dict) -> str: + def seg(obj): + raw = json.dumps(obj).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + return f"{seg({'alg': 'RS256'})}.{seg(claims)}.sig" + + +class TestRestrictToNousPolicy: + def test_none_leaves_the_list_untouched(self): + ids = ["a/one", "b/two"] + assert restrict_to_nous_policy(ids, None) == ids + + def test_empty_set_leaves_the_list_untouched(self): + """Empty is a failed read, not an org that may reach nothing.""" + ids = ["a/one", "b/two"] + assert restrict_to_nous_policy(ids, set()) == ids + + def test_drops_ids_outside_the_policy(self): + assert restrict_to_nous_policy( + ["a/one", "b/two", "c/three"], {"a/one", "c/three"} + ) == ["a/one", "c/three"] + + def test_preserves_curated_order(self): + """The pickers show a curated order deliberately; filtering must not + reorder it into the catalog's alphabetical order.""" + curated = ["z/last", "a/first", "m/middle"] + allowed = {"a/first", "m/middle", "z/last"} + assert restrict_to_nous_policy(curated, allowed) == curated + + def test_keeps_a_free_sibling_when_its_base_is_reachable(self): + """Portal free recommendations are ``:free`` ids; the gateway admits a + row when any of its requestable ids passes.""" + assert restrict_to_nous_policy(["vendor/model:free"], {"vendor/model"}) == [ + "vendor/model:free" + ] + + def test_keeps_a_free_id_listed_in_its_own_right(self): + assert restrict_to_nous_policy( + ["vendor/model:free"], {"vendor/model:free"} + ) == ["vendor/model:free"] + + def test_drops_a_free_sibling_whose_base_is_blocked(self): + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] + + +class TestNousPolicyAllowedIds: + @pytest.fixture(autouse=True) + def _clear_cache(self): + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + yield + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + + def _patch(self, monkeypatch, *, policy_present, api_key="sk-test", pricing=None): + calls = [] + monkeypatch.setattr( + account_mod, "nous_policy_present", lambda: policy_present + ) + monkeypatch.setattr( + models_mod, + "_resolve_nous_pricing_credentials", + lambda: (api_key, "https://inference.example.com"), + ) + + def _fake_fetch(**kwargs): + calls.append(kwargs) + return pricing if pricing is not None else {} + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + return calls + + def test_returns_the_authenticated_catalog_keys(self, monkeypatch): + calls = self._patch( + monkeypatch, + policy_present=True, + pricing={"a/one": {}, "b/two": {}}, + ) + assert nous_policy_allowed_ids() == {"a/one", "b/two"} + assert len(calls) == 1 + assert calls[0]["api_key"] == "sk-test" + + def test_declines_to_filter_an_unrestricted_org(self, monkeypatch): + calls = self._patch(monkeypatch, policy_present=False, pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + assert calls == [], "an unrestricted org should not pay for the read" + + def test_declines_to_filter_when_the_claim_is_unknown(self, monkeypatch): + """Absent is an older mint, not an unrestricted org.""" + calls = self._patch(monkeypatch, policy_present=None, pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + assert calls == [] + + def test_declines_to_filter_on_an_anonymous_read(self, monkeypatch): + """An anonymous read returns the full catalog; treating it as the + policy-filtered set would silently widen the list to everything.""" + self._patch(monkeypatch, policy_present=True, api_key="", pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + + def test_declines_to_filter_on_an_empty_read(self, monkeypatch): + """A failed fetch must not read as an org that may reach nothing.""" + self._patch(monkeypatch, policy_present=True, pricing={}) + assert nous_policy_allowed_ids() is None + + +class TestNousPolicyPresent: + def _patch_token(self, monkeypatch, token): + import hermes_cli.auth as auth_mod + + monkeypatch.setattr( + auth_mod, + "get_provider_auth_state", + lambda _p: {"access_token": token} if token is not None else {}, + ) + + @pytest.mark.parametrize("claim,expected", [(True, True), (False, False)]) + def test_reads_the_claim(self, monkeypatch, claim, expected): + self._patch_token(monkeypatch, _jwt({"policy_present": claim})) + assert nous_policy_present() is expected + + def test_absent_claim_is_unknown_not_false(self, monkeypatch): + self._patch_token(monkeypatch, _jwt({"org_id": "org_1"})) + assert nous_policy_present() is None + + def test_non_boolean_claim_is_unknown(self, monkeypatch): + """The gateway refuses to read a corrupt claim as "no policy".""" + self._patch_token(monkeypatch, _jwt({"policy_present": "yes"})) + assert nous_policy_present() is None + + def test_no_token_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, None) + assert nous_policy_present() is None + + def test_undecodable_token_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, "not-a-jwt") + assert nous_policy_present() is None From b1ea9196f7ed87dc4d919b528c3b006bdfd174c3 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:13:53 -0300 Subject: [PATCH 03/21] fix(nous): narrow every model list to the org's policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four surfaces list Nous models, and none of them was filtered. All four seed from the docs-hosted curated manifest and union the Portal's `recommended-models` endpoint; neither source is authenticated, so org policy had no effect on the model a user picks — which is the model they then use. The Portal endpoint compounds it, serving one globally CDN-cached payload for the whole platform, invalidated only by admin pricing edits and never by a policy change, so it can put a hidden model straight back into a list. Narrow all four against the authenticated catalog: - `_login_nous`, which chooses the model the session starts on - `_model_flow_nous`, the `hermes model` picker - `list_authenticated_providers`, the `/model` picker - `/api/model/recommended-default`, dashboard onboarding The list stays curated and curated-ordered — the policy set only ever subtracts. Replacing a list with the catalog's keys would swap a curated agentic list for a large alphabetical dump of vendor-prefixed models, which is the regression the picker's nous branch already exists to avoid. The `/model` picker's filter sits outside the try that wraps the Portal union, so a Portal outage still yields a policy-filtered curated list. `_login_nous` and `_model_flow_nous` also narrow their unavailable lists, so a policy-hidden model is not offered as a free-tier upsell either. For an org with no policy — the common case — the filter is a no-op and every list is what it was. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/auth.py | 9 + hermes_cli/model_setup_flows.py | 9 + hermes_cli/model_switch.py | 13 ++ hermes_cli/web_server.py | 7 + tests/hermes_cli/test_nous_policy_surfaces.py | 162 ++++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100644 tests/hermes_cli/test_nous_policy_surfaces.py diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 8d3a3d13ae..f97f91bca5 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9377,6 +9377,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: from hermes_cli.models import ( get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, partition_nous_models_by_tier, + nous_policy_allowed_ids, restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -9427,6 +9428,14 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) + # The curated list and the Portal's recommendations are both + # unauthenticated, so neither knows what the org may reach. + # Narrow both lists to the policy before they are shown. + _policy_allowed = nous_policy_allowed_ids() + model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + unavailable_models = restrict_to_nous_policy( + unavailable_models, _policy_allowed, + ) _portal = auth_state.get("portal_base_url", "") if model_ids: print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index fbb6f35c94..90075912a1 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -559,6 +559,15 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing, _nous_portal_url, ) + # The curated list and the Portal's recommendations are both + # unauthenticated, so neither knows what the org may reach. Narrow both + # lists to the policy before they are shown. + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + _policy_allowed = nous_policy_allowed_ids() + model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed) + if not model_ids and not unavailable_models: print("No models available for Nous Portal after filtering.") return diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index d4689df874..cb7f9cf6df 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -3096,6 +3096,19 @@ def list_authenticated_providers( # curated list alone (still correct, just may lag newly # launched models, exactly like an offline CLI run). pass + # Both the curated list and the Portal's recommendations are + # unauthenticated, so neither knows what the org may reach. Narrow + # to the policy outside the try, so a failed recommendation fetch + # still yields a filtered curated list. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids as _nous_policy, + restrict_to_nous_policy as _nous_restrict, + ) + + model_ids = _nous_restrict(model_ids, _nous_policy()) + except Exception: + pass else: # Unified pathway — see Section 1 rationale. Fall back to the # curated dict (with models.dev merge for preferred providers) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 73b2014e8e..41577df7d4 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7485,8 +7485,10 @@ def get_recommended_default_model(provider: str = ""): get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, + nous_policy_allowed_ids, partition_nous_models_by_tier, pick_silent_default_model, + restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -7515,6 +7517,11 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing, portal_url ) + # Neither the curated list nor the Portal's recommendations know + # what the org may reach, and this endpoint picks the model a user + # lands on without choosing it. + model_ids = restrict_to_nous_policy(model_ids, nous_policy_allowed_ids()) + model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} except Exception: diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py new file mode 100644 index 0000000000..c8f4cc9d18 --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -0,0 +1,162 @@ +"""Every Nous model list is narrowed to the org's policy before it is shown. + +Four surfaces build a Nous list from the curated manifest unioned with the +Portal's ``recommended-models`` endpoint. Neither source is authenticated, so +without this filter an org's hidden model is offered to the user and then +refused at request time with ``model_blocked_by_org_policy``. +""" + +from __future__ import annotations + +import argparse + +import pytest + +import hermes_cli.models as models_mod + +CURATED = ["vendor/allowed", "vendor/blocked"] +ALLOWED = {"vendor/allowed"} + + +@pytest.fixture +def policy(monkeypatch): + """An org whose policy admits only ``vendor/allowed``.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: ALLOWED) + return ALLOWED + + +@pytest.fixture +def no_policy(monkeypatch): + """An unrestricted org — lists must come through untouched.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: None) + + +class TestLoginNous: + """``_login_nous`` — the model picked at login is the model then used.""" + + def _run(self, monkeypatch, tmp_path): + import hermes_cli.auth as auth_mod + import hermes_cli.nous_subscription as ns + + seen: dict = {} + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr( + auth_mod, + "_nous_device_code_login", + lambda **_k: { + "access_token": "tok", + "agent_key": "key", + "inference_base_url": "https://inference.example.com", + "portal_base_url": "https://portal.example.com", + "refresh_token": "r", + "token_expires_at": 9999999999, + }, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(ns, "prompt_enable_tool_gateway", lambda _c: None) + + def _capture(model_ids, **kwargs): + seen["model_ids"] = list(model_ids) + return None + + monkeypatch.setattr(auth_mod, "_prompt_model_selection", _capture) + + args = argparse.Namespace( + portal_url=None, inference_url=None, client_id=None, scope=None, + no_browser=True, timeout=15.0, ca_bundle=None, insecure=False, + ) + auth_mod._login_nous(args, auth_mod.PROVIDER_REGISTRY["nous"]) + return seen + + def test_hidden_model_is_not_offered(self, monkeypatch, tmp_path, policy): + assert self._run(monkeypatch, tmp_path).get("model_ids") == ["vendor/allowed"] + + def test_unrestricted_org_sees_the_full_curated_list( + self, monkeypatch, tmp_path, no_policy + ): + assert self._run(monkeypatch, tmp_path).get("model_ids") == CURATED + + +class TestModelSwitchPicker: + """The ``/model`` picker's nous branch (``list_authenticated_providers``).""" + + def _rows(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, + "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + rows = ms.list_authenticated_providers(max_models=10) + return next((r for r in rows if r["slug"] == "nous"), None) + + def test_hidden_model_is_filtered(self, monkeypatch, policy): + row = self._rows(monkeypatch) + assert row is not None, "nous row should be listed" + assert "vendor/blocked" not in row["models"] + assert "vendor/allowed" in row["models"] + + def test_unrestricted_org_keeps_both(self, monkeypatch, no_policy): + row = self._rows(monkeypatch) + assert row is not None + assert set(CURATED) <= set(row["models"]) + + def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy): + """The filter sits outside the try that wraps the Portal union, so a + Portal outage still yields a policy-filtered curated list.""" + + def _boom(_p): + raise RuntimeError("portal down") + + monkeypatch.setattr(models_mod, "get_pricing_for_provider", _boom) + row = self._rows(monkeypatch) + assert row is not None + assert "vendor/blocked" not in row["models"] + + +class TestRecommendedDefaultEndpoint: + """``GET /api/model/recommended-default`` picks a model the user never sees + chosen, so an unreachable one there is worse than in a picker.""" + + def _call(self, monkeypatch): + import hermes_cli.auth as auth_mod + from hermes_cli.web_server import get_recommended_default_model + + # Blocked first, so an unfiltered list would make it the silent + # default — otherwise this passes whether or not the filter runs. + monkeypatch.setattr( + models_mod, "get_curated_nous_model_ids", + lambda: ["vendor/blocked", "vendor/allowed"], + ) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(auth_mod, "get_provider_auth_state", lambda _p: {}) + return get_recommended_default_model(provider="nous") + + def test_hidden_model_is_never_the_silent_default(self, monkeypatch, policy): + assert self._call(monkeypatch)["model"] == "vendor/allowed" + + def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy): + assert self._call(monkeypatch)["model"] == "vendor/blocked" From 35e0d158615ff5aae4d2e5a8b358ebcb69478870 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:14:22 -0300 Subject: [PATCH 04/21] fix(aux): read the Nous fast-model catalog with credentials, and filter it `_fast_model_from_catalog` treats the catalog's keys as a source of ids, scanning them for a cheap model to use for side tasks like titling. Two problems for Nous. The credential lookup goes through `resolve_api_key_provider_credentials`, which raises for Nous because it is OAuth. The read then went out anonymous and came back with the full catalog rather than the one the org may reach, so a policy-hidden model could be selected and then refused at request time with `model_blocked_by_org_policy`. Fall back to the Nous credential resolver when the api-key path raises, and narrow the resulting ids by the org policy the same way the pickers' lists are narrowed. Co-Authored-By: Claude Opus 5 (1M context) --- agent/auxiliary_client.py | 24 +++++++++++ tests/hermes_cli/test_nous_policy_surfaces.py | 41 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index e371b0f676..534b42184a 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -884,6 +884,18 @@ def _fast_model_from_catalog(provider_id: str) -> str: # fetch below still works for the catalogs that allow it. logger.debug("No credentials for %s catalog", provider_id, exc_info=True) + if not api_key and provider_id.strip().lower() == "nous": + # Nous is OAuth, so the api-key resolver above raises for it. An + # anonymous read returns the full catalog rather than the one the + # org may reach, and a model picked from it is refused at request + # time with model_blocked_by_org_policy. + try: + from hermes_cli.models import _resolve_nous_pricing_credentials + + api_key, base_url = _resolve_nous_pricing_credentials() + except Exception: + logger.debug("No Nous credentials for catalog", exc_info=True) + if not base_url: base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "") base_url = base_url.rstrip("/") @@ -900,6 +912,18 @@ def _fast_model_from_catalog(provider_id: str) -> str: return "" ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True) + if provider_id.strip().lower() == "nous": + # The catalog's keys are a source of ids here, so the policy has to + # narrow them the same way it narrows the pickers' lists. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids, + restrict_to_nous_policy, + ) + + ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids()) + except Exception: + logger.debug("Nous policy filter unavailable", exc_info=True) for family in _FAST_MODEL_FAMILIES: for model_id in ids: lowered = model_id.lower() diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index c8f4cc9d18..d3103c9619 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -160,3 +160,44 @@ class TestRecommendedDefaultEndpoint: def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy): assert self._call(monkeypatch)["model"] == "vendor/blocked" + + +class TestAuxiliaryFastModel: + """``_fast_model_from_catalog`` treats the catalog's keys as a source of + ids, so an anonymous read there can select a model the gateway refuses.""" + + def _pick(self, monkeypatch, *, catalog): + import agent.auxiliary_client as aux + + seen: dict = {} + + def _fake_fetch(*, api_key=None, base_url="", timeout=8.0, **_k): + seen["api_key"] = api_key + return {mid: {} for mid in catalog} + + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("sk-nous", "https://inference.example.com"), + ) + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + picked = aux._fast_model_from_catalog("nous") + return picked, seen + + def test_reads_the_catalog_with_nous_oauth_credentials(self, monkeypatch, no_policy): + """The api-key resolver raises for OAuth providers; without a fallback + the read goes out anonymous and returns the unfiltered catalog.""" + _, seen = self._pick(monkeypatch, catalog=["vendor/haiku-fast"]) + assert seen["api_key"] == "sk-nous" + + def test_hidden_model_is_not_selected(self, monkeypatch, policy): + import agent.auxiliary_client as aux + + monkeypatch.setattr( + models_mod, "nous_policy_allowed_ids", lambda **_k: {"vendor/allowed"} + ) + monkeypatch.setattr(aux, "_FAST_MODEL_FAMILIES", ("vendor/",)) + monkeypatch.setattr(aux, "_FAST_MODEL_EXCLUDE", ()) + picked, _ = self._pick( + monkeypatch, catalog=["vendor/blocked", "vendor/allowed"] + ) + assert picked == "vendor/allowed" From 9fc43919cc58772569a4bd9254ed180d55c0f900 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:14:38 -0300 Subject: [PATCH 05/21] perf(nous): stop prefetching a catalog nothing reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `/model` picker warms `provider_models_cache.json` in parallel before its serial build loop, and Nous was collected into that prefetch because the credential scan treats any auth.json providers entry as credentials regardless of auth type. Nothing reads the result. The picker's nous branch builds from the curated list rather than `cached_provider_model_ids`, and Nous cannot reach the api_key-only unified pathway that would call it. Because the prefetch forces a refresh it also skips the cache read, so the entry is written and never read — a live authenticated /v1/models round trip per picker open for nothing. Exclude it. Also add the plan this and the preceding commits implement. Co-Authored-By: Claude Opus 5 (1M context) --- docs/nous-org-model-policy.md | 269 ++++++++++++++++++ hermes_cli/model_switch.py | 7 +- tests/hermes_cli/test_nous_policy_surfaces.py | 16 ++ 3 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 docs/nous-org-model-policy.md diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md new file mode 100644 index 0000000000..174f65fec7 --- /dev/null +++ b/docs/nous-org-model-policy.md @@ -0,0 +1,269 @@ +# Honouring the Nous org model policy in the pickers + +> **Audience:** Contributors touching Nous model selection +> **Source files:** `hermes_cli/auth.py` (`_login_nous`, `fetch_nous_models`, +> `_prompt_model_selection`), `hermes_cli/models.py` (`fetch_models_with_pricing`, +> `get_pricing_for_provider`, `union_with_portal_*`, `partition_nous_models_by_tier`), +> `hermes_cli/model_setup_flows.py` (`_model_flow_nous`), +> `hermes_cli/model_switch.py` (`list_authenticated_providers`), +> `hermes_cli/web_server.py` (`/api/model/recommended-default`), +> `hermes_cli/nous_account.py` (`_info_from_valid_jwt`) +> **Related:** Inference gateway PR #164 (filters `GET /v1/models` by org policy), +> NAS #941 (team admins restrict providers), NAS `openrouter-provider-map` +> (publishes the model→providers map the gateway filter needs) + +## What changed upstream + +A Nous team admin can restrict which models and which serving providers their +org may use. The inference gateway applies that policy to `GET /v1/models`, so +an authenticated catalog read returns only what the caller may actually reach. +Blocked models are **omitted** — the row is skipped, no marker field is added +(`api/src/handlers/models.ts:99-138`). An anonymous read is still allowed and +still returns the full catalog (`api/src/app.ts:309-313` — no auth middleware +on the route). + +Two things bound how urgent this is. + +**The gateway is authoritative and this is cosmetic.** Asking for a hidden +model is refused at request time with `403 model_blocked_by_org_policy` +(`api/src/middleware/model_entitlement_gate.ts:337-356`). The listing fails +open; the request gate fails closed. Nothing here is a security boundary — the +cost of a wrong list is a predictable 403, and the gateway PR states that +tradeoff deliberately. This document is only about the client showing the +right list. + +**It is inert today.** PR #164 is merged, but is switched off until NAS +publishes the policy fields and the provider map, and the admin surface sits +behind the `org-model-policy` Vercel flag. The `openrouter-provider-map` branch +is the publisher half (a daily cron writing `openrouter_model_providers` to the +entitlement Redis). Until that lands, every caller — anonymous and +authenticated — gets the same unfiltered list. **No change here is verifiable +end to end yet; every test mocks the filtered response.** + +## Where we stand + +Four surfaces list Nous models. **None of them is filtered.** + +| surface | builds its list from | filtered | +| --- | --- | --- | +| Login (`_login_nous`, `auth.py:9383`) | `get_curated_nous_model_ids()` ∪ Portal recommendations | no | +| `hermes model` (`_model_flow_nous`, `model_setup_flows.py:399`) | same | no | +| `/model` picker (`list_authenticated_providers`, `model_switch.py:3062`) | same | no | +| Dashboard onboarding (`web_server.py:7486`) | same | no | + +All four seed from the docs-hosted manifest and union the Portal's +`recommended-models` endpoint. Neither source is authenticated, so org policy +has no effect on any list a user picks from. + +`cached_provider_model_ids("nous")` — which *does* reach the authenticated +`fetch_nous_models` — is not consulted by any of them. The `/model` picker +handles nous in its own branch that deliberately bypasses it, and nous cannot +reach the generic pathway at `model_switch.py:2898` because line 2861 skips +every non-`api_key` provider. Its only caller for nous is the background +prefetch (`model_switch.py:2390`), which writes an entry nothing reads. + +Two things that are already fine, and should stay that way: + +- `nous` is **not** in `_MODELS_DEV_PREFERRED`, so no models.dev entries are + merged on top of the live list. +- The nous fallback ladder in `provider_model_ids` is a *chain* (live → + manifest → in-repo snapshot), not a merge, so a successful live fetch is + used exclusively. + +--- + +## Fix 0 — put auth state in the pricing cache key + +**This is a prerequisite for fix 1, and worth landing on its own merits.** + +**Problem.** `fetch_models_with_pricing` caches on the base URL alone, and the +cache check happens *above* the point where the `Authorization` header is built +(`models.py:2404`): + +```python +cache_key = (base_url or "").rstrip("/") +if not force_refresh: + cached = _cached_catalog(cache_key) + if cached is not None: + return cached +... +if api_key: + headers["Authorization"] = f"Bearer {api_key}" +``` + +`_pricing_cache` is process-lifetime with no expiry for a non-empty result +(`models.py:2231-2253`). So whichever read of a given base URL lands first — +authenticated or anonymous — answers every later read in that process, +whatever key it passes. An anonymous read landing first (the auxiliary-model +path in fix 2 is one) makes a later authenticated read return an unfiltered +list without touching the network. A fix built on this cache looks like it +works and does not. + +**Do.** Fold auth state into the cache key. Distinguishing authenticated from +anonymous is enough — the token value need not be in the key, and keeping it +out avoids hashing a secret. + +**Do** update `agent/credits_tracker.py:257`, which reaches into the private +`_pricing_cache` dict assuming one entry per base URL. + +**Test.** An anonymous read followed by an authenticated read of the same base +URL issues two requests and returns two different lists. Independently +testable today, unlike everything below. + +## Fix 1 — narrow each list to the org's policy + +**Problem.** All four surfaces build their list from +`get_curated_nous_model_ids()` unioned with the Portal's `recommended-models` +endpoint. Neither is authenticated, so org policy has no effect on the model a +user picks — which is the model they then use. The Portal endpoint compounds +it: it takes no auth and no parameters, returns one globally CDN-cached payload +for the whole platform, and is invalidated only by admin pricing edits — never +by a policy change. It can put a hidden model straight back into a list. There +is no policy-aware variant of it and no parameter that would make one. + +Each surface, however, already fetches `/v1/models`. +`get_pricing_for_provider("nous")` calls `fetch_models_with_pricing`, which +reads that endpoint and returns `{model_id: {...}}`, and already resolves +credentials (`_resolve_nous_pricing_credentials`), so it is already the +authenticated read. Its keys are the reachable set. + +**Do.** Use that set to *narrow* each list, keeping the curated order. +`nous_policy_allowed_ids()` obtains the set; `restrict_to_nous_policy()` +applies it. Both live in `models.py`, and the fetch reuses the pricing cache +entry the surface already populates, so no surface makes an extra request. + +**Do not** replace a list with the response's keys. Every surface shows the +curated agentic list in curated order deliberately — the live catalog is a +large alphabetical dump of vendor-prefixed models, and swapping it in is the +regression `model_switch.py:3070` records. Recommendations should be able to +*reveal* a newly launched model; the policy set should only ever subtract. + +**Do not** narrow a list on evidence that cannot support it. +`nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — +in three cases, and each matters: + +- **The org has no policy, or the token is too old to say.** Gated on the + `policy_present` claim (fix 4). For an unrestricted org — the common case — + filtering buys nothing and risks dropping a Portal recommendation the + gateway catalog has not caught up on yet. This keeps the change a no-op for + everyone the policy does not apply to. +- **Credential resolution failed**, so the read was anonymous and therefore + unfiltered. A full catalog must not be mistaken for a filtered one. A stated + degradation, not a silent one. +- **The read came back empty**, which is a fetch failure, not an org that may + reach nothing. + +A `:free` sibling is kept when its base model is reachable, mirroring the +gateway, which admits a row when any of its requestable ids passes and treats +anything unknown as a keep — "over-listing costs a 403 from the authoritative +gate, while hiding a row the gate would serve is unrecoverable from the client" +(`api/src/libs/catalog_policy.ts:74-78`). Prefer over-listing here too. + +**Test.** With a policy hiding model X: X is absent from each of the four +lists, and no surface makes more Nous requests than it does today. With no +policy, with credentials broken, or with an empty read, every list is byte-for- +byte what it is today. A model the Portal flags as free but the org hides stays +out; curated ordering survives filtering. + +## Fix 2 — audit the other readers of the pricing map + +**Problem.** `fetch_models_with_pricing` is shared, so any caller that treats +its keys as "the models that exist" inherits whatever authentication the first +caller happened to have. Fix 0 stops the *authentication* from leaking between +callers; this fix is about which callers may treat the map as a source of ids +at all. + +**Do.** Make the map a lookup *for* ids already in the list, never a source of +ids. Two consumers are already correct and should stay that way: +`partition_nous_models_by_tier` only looks up ids it was given, and the +`union_with_portal_*` pair only ever writes into the map — their id-widening +comes from the Portal endpoint (fix 2), not from the map. + +The one that is wrong is `agent/auxiliary_client.py:869-908` +(`_fast_model_from_catalog`), which iterates the map's keys directly as its +candidate list off an anonymous read. Reachable for nous on the titling path, +where it can select a policy-hidden model that then 403s at request time. + +**Test.** With credentials broken so the read falls back to anonymous, no +list grows. + +## Fix 3 — stop prefetching the nous catalog + +**Problem.** The background prefetch calls +`cached_provider_model_ids("nous", force_refresh=True)` +(`model_switch.py:2390`); nous is collected into it because +`_collect_authed_provider_slugs` treats any `auth.json` providers entry as +credentials regardless of `auth_type` (`model_switch.py:2519-2526`). Because +`force_refresh=True` skips the cache read and no nous surface reads the entry, +this is a live authenticated `/v1/models` round trip per picker open written to +a location nothing consults. + +**Do.** Exclude nous from the prefetch and delete the write-only entry. + +This replaces what an earlier draft proposed here — folding `org_id` into +`_credential_fingerprint` and shortening `_PROVIDER_MODELS_STALE_SERVE_MAX` +for nous (a single global constant, `models.py:4204`, with no per-provider +branching today). Both would have hardened a cache that, after fix 1, has no +nous readers to protect. If a future surface routes nous through +`cached_provider_model_ids` again, revisit the fingerprint then: it hashes +env-var values and `auth.json` mtime and carries no org signal +(`models.py:4277`), so two orgs on one machine can serve each other's list. + +**Test.** Opening the `/model` picker makes no Nous `/v1/models` request beyond +the one the displayed list is built from. + +## Fix 4 — the `policy_present` claim + +**Problem.** Under omission a blocked model simply vanishes, which reads as +"Hermes does not support this" rather than "your org disallows it". + +**Do.** Read the `policy_present` claim off the Nous OAuth access token and, +when it is `true`, show a single line stating that the org restricts which +models are available. No enumeration, no per-model marking. + +The claim rides the same JWT as `org_id` +(`access-token-issuer.ts:552,595`, `token_use: "access"`) — the token the +client already decodes — and `_info_from_valid_jwt` already retains every +claim in `raw_claims` (`nous_account.py:600-647`), so surfacing it is one +typed field on `NousPortalAccountInfo` and no new request. + +It is already widened to cover provider-only restrictions, not just model +allowlists (`nous-account-service/src/server/entitlement-snapshot.ts:478-480`). +Two NAS docs still describe it as allowlist-only and list the widening as +pending — they are stale; trust that expression. + +**Do not** enumerate the blocked set. Model policy is allowlist-only — +`denyModels` is a dead column (`nous-account-service/src/server/model-policy.ts:230`) +— so an org that allows five models blocks the entire rest of the catalog. +Graying hundreds of rows is a worse UI than omitting them. An earlier draft +proposed deriving the blocked set by diffing the anonymous and authenticated +reads and feeding it to `_prompt_model_selection`'s `unavailable_models`; that +is the wrong shape twice over, because that picker carries one +`unavailable_message` for the whole list and cannot say "free-tier-gated" and +"policy-hidden" at once. + +**Do not** report the absence of the claim as the absence of a policy. It is +tri-state: `true`, `false`, and absent, where absent means unknown — an older +mint, not an unrestricted org. The gateway rejects a corrupt (non-boolean) +claim outright rather than reading it as "no policy" +(`api/src/middleware/nas_jwt_auth.ts:179`). Show the line only on `true`. + +**Known bound:** the claim is stamped at mint time, so it goes stale until the +next token refresh — the line can lag a policy change by up to the access +token's lifetime. Acceptable, and worth stating rather than rediscovering. + +**Test.** With `policy_present` true the line shows; with it false or absent it +does not. + +--- + +## Order + +Fix 0 first: fix 1 is silently wrong without it, and it is the only piece +testable before NAS switches the feature on. Fix 4's claim gates fix 1, so the +two land together. Fix 1 is the correctness work — without it the policy is +bypassed on every surface a user picks from. Fix 2 keeps the pricing map from +becoming another way to widen a list. Fix 3 is a deletion that fix 1 makes +safe. + +Run tests with `scripts/run_tests.sh` — not bare `pytest`. diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index cb7f9cf6df..3516cdfe24 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2565,7 +2565,12 @@ def _collect_authed_provider_slugs( slugs.append(_cp.slug) seen.add(_cp.slug.lower()) - return slugs + # Nous is deliberately excluded. Its picker branch builds from the curated + # list rather than cached_provider_model_ids, and nous cannot reach the + # api_key-only unified pathway, so a prefetched entry is written and never + # read — a live authenticated /v1/models round trip per picker open for + # nothing. + return [s for s in slugs if s != "nous"] def list_authenticated_providers( diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index d3103c9619..e279162b0c 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -201,3 +201,19 @@ class TestAuxiliaryFastModel: monkeypatch, catalog=["vendor/blocked", "vendor/allowed"] ) assert picked == "vendor/allowed" + + +class TestNousPrefetch: + """The nous disk-cache entry is write-only: its picker branch builds from + the curated list, so prefetching it is a round trip for nothing.""" + + def test_nous_is_not_collected_for_prefetch(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + slugs = ms._collect_authed_provider_slugs({}, {"nous": list(CURATED)}, []) + assert "nous" not in slugs From c38d62aefe9f0a394fb417c598e9458fe2c193d9 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:17:56 -0300 Subject: [PATCH 06/21] feat(nous): tell a governed org its model choice is restricted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway omits a policy-blocked model from `/v1/models` rather than marking it, so after the preceding commits a restricted model is simply absent from the pickers. That reads as "Hermes does not support this" instead of "your organization disallows it". Show one line when the org is governed, in the two flows where a user picks a model. It enumerates nothing: model policy is an allowlist, so an org admitting a handful of models blocks the whole rest of the catalog, and graying hundreds of rows would be a worse UI than omitting them. Driven by the `policy_present` claim, which is tri-state — the line shows only when it is explicitly true, because an absent claim means an older mint rather than an unrestricted org. The claim is stamped at mint time, so the line can lag a policy change by up to the access token's lifetime. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/auth.py | 5 ++++ hermes_cli/model_setup_flows.py | 5 ++++ hermes_cli/nous_account.py | 21 +++++++++++++++ tests/hermes_cli/test_nous_policy_filter.py | 27 +++++++++++++++++++ tests/hermes_cli/test_nous_policy_surfaces.py | 20 ++++++++++++++ 5 files changed, 78 insertions(+) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index f97f91bca5..eebbc5609c 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9438,6 +9438,11 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: ) _portal = auth_state.get("portal_base_url", "") if model_ids: + from hermes_cli.nous_account import nous_policy_notice + + _policy_notice = nous_policy_notice() + if _policy_notice: + print(_policy_notice) print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") selected_model = _prompt_model_selection( model_ids, pricing=pricing, diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 90075912a1..9489692159 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -581,6 +581,11 @@ def _model_flow_nous(config, current_model="", args=None): print(unavailable_message or f"Upgrade at {_url} to access paid models.") return + from hermes_cli.nous_account import nous_policy_notice + + _policy_notice = nous_policy_notice() + if _policy_notice: + print(_policy_notice) print( f'Showing {len(model_ids)} curated models — use "Enter custom model name" for others.' ) diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index 4c3bf0a51f..c63d090fc4 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -427,6 +427,27 @@ def nous_policy_present() -> Optional[bool]: return None +def nous_policy_notice() -> str: + """A one-line notice for an org that restricts model choice, else ``""``. + + Under the gateway's policy filter a blocked model is omitted rather than + marked, which reads as "Hermes does not support this" instead of "your org + disallows it". This says which it is without enumerating anything: model + policy is an allowlist, so an org that admits a handful of models blocks + the whole rest of the catalog, and listing those would be a worse UI than + omitting them. + + Silent unless the claim is explicitly true — absent means an older mint, + not an unrestricted org. + """ + if nous_policy_present() is not True: + return "" + return ( + "Your organization restricts which models are available — " + "models outside its policy are not listed." + ) + + def _fresh_account_info( *, state: dict[str, Any], diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index a225d560dc..edb7e6dc20 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -157,3 +157,30 @@ class TestNousPolicyPresent: def test_undecodable_token_is_unknown(self, monkeypatch): self._patch_token(monkeypatch, "not-a-jwt") assert nous_policy_present() is None + + +class TestNousPolicyNotice: + """A governed org is told its choice is restricted, rather than left to + read an omitted model as one Hermes does not support.""" + + def _patch(self, monkeypatch, present): + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: present) + + def test_shows_a_line_for_a_governed_org(self, monkeypatch): + self._patch(monkeypatch, True) + assert "restricts which models" in account_mod.nous_policy_notice() + + @pytest.mark.parametrize("present", [False, None]) + def test_silent_otherwise(self, monkeypatch, present): + """Absent is an older mint, not an unrestricted org — either way there + is nothing truthful to say.""" + self._patch(monkeypatch, present) + assert account_mod.nous_policy_notice() == "" + + def test_names_no_models(self, monkeypatch): + """Policy is an allowlist, so the blocked set is most of the catalog; + the notice must not try to enumerate it.""" + self._patch(monkeypatch, True) + notice = account_mod.nous_policy_notice() + assert "/" not in notice, f"looks like it names a model: {notice}" + assert len(notice.splitlines()) == 1 diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index e279162b0c..e59974c511 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -217,3 +217,23 @@ class TestNousPrefetch: ) slugs = ms._collect_authed_provider_slugs({}, {"nous": list(CURATED)}, []) assert "nous" not in slugs + + +class TestPolicyNoticeIsShown: + """The notice reaches the two flows where a user picks a model.""" + + def test_login_prints_it(self, monkeypatch, tmp_path, policy, capsys): + import hermes_cli.nous_account as account_mod + + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: True) + TestLoginNous()._run(monkeypatch, tmp_path) + assert "restricts which models" in capsys.readouterr().out + + def test_login_silent_for_an_ungoverned_org( + self, monkeypatch, tmp_path, no_policy, capsys + ): + import hermes_cli.nous_account as account_mod + + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: False) + TestLoginNous()._run(monkeypatch, tmp_path) + assert "restricts which models" not in capsys.readouterr().out From 117e7fef88e18cbc9fe120eeff5f0e3456370f69 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 12:24:55 -0300 Subject: [PATCH 07/21] fix(nous): surface allowed models the curated list does not carry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An org allowlist can name a model the docs-hosted curated manifest has never heard of. Intersecting the curated list against the reachable set then produced an empty picker — "No models available for Nous Portal after filtering" — which is strictly worse than showing an unfiltered list, because the one model the org may actually use is the one that got dropped. When the reachable set is small enough to be a human-authored allowlist, append whatever it admits that the curated list is missing, after the curated entries so their order survives. Bounded by size, which is what separates the two kinds of policy: an allowlist is small, while a provider-only policy leaves the whole catalog reachable and appending it would bury the curated order. Past the cap the intersection stands alone and the picker's custom-model entry remains the way to reach anything omitted. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/models.py | 30 +++++++++++++++- tests/hermes_cli/test_nous_policy_filter.py | 38 ++++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index c1f7d580f1..4e5f4507a6 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2649,6 +2649,13 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] return set(pricing) or None +# Above this many reachable models, an allowed set is treated as catalog-wide +# rather than as an allowlist worth enumerating in a picker. NAS caps an +# allowlist at 512, but a set this large is indistinguishable from the full +# catalog for display purposes. +_NOUS_POLICY_APPEND_MAX = 64 + + def restrict_to_nous_policy( model_ids: list[str], allowed: Optional[set[str]] ) -> list[str]: @@ -2665,12 +2672,33 @@ def restrict_to_nous_policy( """ if not allowed: return list(model_ids) - return [ + kept = [ mid for mid in model_ids if mid in allowed or mid.split(":", 1)[0] in allowed ] + # An allowlist can admit models the curated manifest has never heard of, and + # intersecting alone would then leave the user with nothing to pick at all — + # strictly worse than the unfiltered list. When the reachable set is no + # larger than what would have been shown anyway, it IS the list: append + # whatever it admits that the curated list is missing. + # + # Bounded by size, which is what separates the two kinds of policy: a model + # allowlist is human-authored and small, while a provider-only policy leaves + # the whole catalog reachable. Appending several hundred alphabetical + # vendor-prefixed ids would bury the curated order — the regression the + # pickers' curated branch exists to avoid. Past the cap the intersection + # stands on its own, and the picker's custom-model entry remains the way to + # reach anything it omits. + if len(allowed) <= _NOUS_POLICY_APPEND_MAX: + covered: set[str] = set() + for mid in kept: + covered.add(mid) + covered.add(mid.split(":", 1)[0]) + kept.extend(sorted(a for a in allowed if a not in covered)) + return kept + def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita).""" diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index edb7e6dc20..7da078c77e 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -63,7 +63,11 @@ class TestRestrictToNousPolicy: ) == ["vendor/model:free"] def test_drops_a_free_sibling_whose_base_is_blocked(self): - assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] + """The blocked sibling goes; the model the org may actually use takes + its place rather than leaving the picker empty.""" + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [ + "other/model" + ] class TestNousPolicyAllowedIds: @@ -184,3 +188,35 @@ class TestNousPolicyNotice: notice = account_mod.nous_policy_notice() assert "/" not in notice, f"looks like it names a model: {notice}" assert len(notice.splitlines()) == 1 + + +class TestAllowlistOutsideTheCuratedList: + """An allowlist can name a model the curated manifest has never heard of. + + Intersecting alone leaves the picker empty in that case — strictly worse + than showing an unfiltered list, because the one model the org may use is + the one that got dropped. + """ + + def test_surfaces_an_allowed_model_the_curated_list_lacks(self): + assert restrict_to_nous_policy( + ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"} + ) == ["amazon/nova-2-lite-v1"] + + def test_keeps_curated_order_then_appends_the_rest(self): + kept = restrict_to_nous_policy( + ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"} + ) + assert kept == ["z/curated", "a/curated", "new/model"] + + def test_does_not_append_a_free_sibling_already_covered(self): + assert restrict_to_nous_policy(["vendor/m:free"], {"vendor/m"}) == [ + "vendor/m:free" + ] + + def test_a_provider_only_policy_does_not_bury_the_curated_order(self): + """Such a policy leaves the whole catalog reachable; appending it would + drop hundreds of alphabetical ids into the picker.""" + curated = ["vendor/one", "vendor/two"] + catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) + assert restrict_to_nous_policy(curated, catalog) == curated From bafaac5e61aaa1a30a0db67db638972eb54cba5f Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 12:26:30 -0300 Subject: [PATCH 08/21] docs(nous): correct the subtract-only claim in the policy plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan stated the policy set should only ever subtract from a list. That is wrong when an allowlist names a model the curated manifest lacks, which empties the picker instead of narrowing it — the behaviour fixed in 117e7fef88. Co-Authored-By: Claude Opus 5 (1M context) --- docs/nous-org-model-policy.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md index 174f65fec7..942c798507 100644 --- a/docs/nous-org-model-policy.md +++ b/docs/nous-org-model-policy.md @@ -135,8 +135,16 @@ entry the surface already populates, so no surface makes an extra request. **Do not** replace a list with the response's keys. Every surface shows the curated agentic list in curated order deliberately — the live catalog is a large alphabetical dump of vendor-prefixed models, and swapping it in is the -regression `model_switch.py:3070` records. Recommendations should be able to -*reveal* a newly launched model; the policy set should only ever subtract. +regression `model_switch.py:3070` records. + +**Do not** treat the policy set as subtract-only either. An allowlist can name +a model the curated manifest has never heard of, and intersecting alone then +empties the picker — strictly worse than an unfiltered list, because the one +model the org may use is the one dropped. When the reachable set is small +enough to be a human-authored allowlist, append what it admits that the +curated list lacks, after the curated entries so their order survives. Bound +it by size: a provider-only policy leaves the whole catalog reachable, and +appending that would bury the curated order. **Do not** narrow a list on evidence that cannot support it. `nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — From 04647f15c88a7dfdaf0c9b060b85b519d0835d50 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 12:39:29 -0300 Subject: [PATCH 09/21] fix(nous): only fall back to the reachable set when the overlap is empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfacing allowed models the curated list lacks was gated on the size of the reachable set alone. A jurisdiction or provider policy leaves few enough models to pass that cap, so it appended the remainder — pushing non-curated alphabetical ids into a picker that shows a curated order on purpose, and making the list long enough that the non-curses fallback's input prompt scrolled off screen and read as a hang. Gate on the intersection instead. The fallback exists for an allowlist that names nothing curated, which is the empty-overlap case; a policy that merely narrows the catalog keeps the curated overlap and needs no help. The size cap stays as a guard on that one path. Co-Authored-By: Claude Opus 5 (1M context) --- docs/nous-org-model-policy.md | 18 ++++++----- hermes_cli/models.py | 35 +++++++++------------ tests/hermes_cli/test_nous_policy_filter.py | 17 ++++++++-- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md index 942c798507..7b4027ba72 100644 --- a/docs/nous-org-model-policy.md +++ b/docs/nous-org-model-policy.md @@ -138,13 +138,17 @@ large alphabetical dump of vendor-prefixed models, and swapping it in is the regression `model_switch.py:3070` records. **Do not** treat the policy set as subtract-only either. An allowlist can name -a model the curated manifest has never heard of, and intersecting alone then -empties the picker — strictly worse than an unfiltered list, because the one -model the org may use is the one dropped. When the reachable set is small -enough to be a human-authored allowlist, append what it admits that the -curated list lacks, after the curated entries so their order survives. Bound -it by size: a provider-only policy leaves the whole catalog reachable, and -appending that would bury the curated order. +only models the curated manifest has never heard of, and intersecting alone +then empties the picker — strictly worse than an unfiltered list, because the +models the org may use are the ones dropped. Fall back to the reachable set +itself in exactly that case. + +Only when the intersection is empty, and only when the set is small enough to +be an allowlist rather than a whole catalog. A jurisdiction or provider policy +narrows the catalog without emptying the curated overlap; appending its +remainder pushes non-curated alphabetical ids into a picker that shows a +curated order on purpose. A size cap alone does not catch this — a region +filter can leave few enough models to pass it. **Do not** narrow a list on evidence that cannot support it. `nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 4e5f4507a6..e7754fa196 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2650,9 +2650,9 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] # Above this many reachable models, an allowed set is treated as catalog-wide -# rather than as an allowlist worth enumerating in a picker. NAS caps an -# allowlist at 512, but a set this large is indistinguishable from the full -# catalog for display purposes. +# rather than as an allowlist worth showing in place of an empty picker. NAS +# caps an allowlist at 512, but a set this large is indistinguishable from the +# full catalog for display purposes. _NOUS_POLICY_APPEND_MAX = 64 @@ -2678,25 +2678,18 @@ def restrict_to_nous_policy( if mid in allowed or mid.split(":", 1)[0] in allowed ] - # An allowlist can admit models the curated manifest has never heard of, and - # intersecting alone would then leave the user with nothing to pick at all — - # strictly worse than the unfiltered list. When the reachable set is no - # larger than what would have been shown anyway, it IS the list: append - # whatever it admits that the curated list is missing. + # An allowlist can admit only models the curated manifest has never heard + # of, leaving nothing to intersect and an empty picker — strictly worse than + # the unfiltered list, because the models the org may actually use are the + # ones dropped. Fall back to the reachable set itself in exactly that case. # - # Bounded by size, which is what separates the two kinds of policy: a model - # allowlist is human-authored and small, while a provider-only policy leaves - # the whole catalog reachable. Appending several hundred alphabetical - # vendor-prefixed ids would bury the curated order — the regression the - # pickers' curated branch exists to avoid. Past the cap the intersection - # stands on its own, and the picker's custom-model entry remains the way to - # reach anything it omits. - if len(allowed) <= _NOUS_POLICY_APPEND_MAX: - covered: set[str] = set() - for mid in kept: - covered.add(mid) - covered.add(mid.split(":", 1)[0]) - kept.extend(sorted(a for a in allowed if a not in covered)) + # Only when the intersection is empty. A jurisdiction or provider policy + # narrows the catalog without emptying the curated overlap, and appending + # its remainder would push non-curated alphabetical ids into a picker that + # shows a curated order on purpose. Anything omitted is still reachable + # through the picker's custom-model entry. + if not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: + return sorted(allowed) return kept diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index 7da078c77e..b91fe10148 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -203,11 +203,24 @@ class TestAllowlistOutsideTheCuratedList: ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"} ) == ["amazon/nova-2-lite-v1"] - def test_keeps_curated_order_then_appends_the_rest(self): + def test_does_not_append_when_the_curated_overlap_is_non_empty(self): + """A jurisdiction or provider policy narrows the catalog without + emptying the curated overlap. Appending its remainder would push + non-curated alphabetical ids into a deliberately curated order.""" kept = restrict_to_nous_policy( ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"} ) - assert kept == ["z/curated", "a/curated", "new/model"] + assert kept == ["z/curated", "a/curated"] + + def test_jurisdiction_policy_never_grows_the_list(self): + """Regression: a region filter leaves few enough models to slip under + the size cap, so a size-only guard let it append.""" + curated = ["vendor/one", "vendor/two", "vendor/three"] + reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} + assert restrict_to_nous_policy(curated, reachable) == [ + "vendor/one", + "vendor/two", + ] def test_does_not_append_a_free_sibling_already_covered(self): assert restrict_to_nous_policy(["vendor/m:free"], {"vendor/m"}) == [ From da3c2435e2ad850469d8b2031a74585b850d2f94 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 13:34:27 -0300 Subject: [PATCH 10/21] fix(nous): only rescue an empty list where emptiness means "filtered out" The fallback also ran on unavailable_models, which is legitimately empty on a paid tier, filling the picker with the whole reachable set. Make it opt-in. --- docs/nous-org-model-policy.md | 17 ++++--- hermes_cli/auth.py | 4 +- hermes_cli/model_setup_flows.py | 4 +- hermes_cli/model_switch.py | 4 +- hermes_cli/models.py | 22 +++++---- hermes_cli/web_server.py | 4 +- tests/hermes_cli/test_nous_policy_filter.py | 49 ++++++++++++++++----- 7 files changed, 74 insertions(+), 30 deletions(-) diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md index 7b4027ba72..8b1cbc4573 100644 --- a/docs/nous-org-model-policy.md +++ b/docs/nous-org-model-policy.md @@ -143,12 +143,17 @@ then empties the picker — strictly worse than an unfiltered list, because the models the org may use are the ones dropped. Fall back to the reachable set itself in exactly that case. -Only when the intersection is empty, and only when the set is small enough to -be an allowlist rather than a whole catalog. A jurisdiction or provider policy -narrows the catalog without emptying the curated overlap; appending its -remainder pushes non-curated alphabetical ids into a picker that shows a -curated order on purpose. A size cap alone does not catch this — a region -filter can leave few enough models to pass it. +Only when the intersection is empty, only when the set is small enough to be +an allowlist rather than a whole catalog, and **only for the list a user picks +from**. Callers opt in per list. Any list whose emptiness carries meaning must +not get the rescue: a paid-tier user's unavailable list is legitimately empty, +and rescuing it reads that as "nothing survived" and fills the picker's +unavailable block with the entire reachable set. + +A size cap alone does not make this safe — a jurisdiction filter can leave few +enough models to pass it — and neither does gating on an empty intersection, +because an intentionally empty input is indistinguishable from a fully filtered +one. The opt-in is what separates them. **Do not** narrow a list on evidence that cannot support it. `nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index eebbc5609c..c6a7330e29 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9432,7 +9432,9 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: # unauthenticated, so neither knows what the org may reach. # Narrow both lists to the policy before they are shown. _policy_allowed = nous_policy_allowed_ids() - model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) unavailable_models = restrict_to_nous_policy( unavailable_models, _policy_allowed, ) diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 9489692159..c4b9dedfd7 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -565,7 +565,9 @@ def _model_flow_nous(config, current_model="", args=None): from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy _policy_allowed = nous_policy_allowed_ids() - model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed) if not model_ids and not unavailable_models: diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index 3516cdfe24..fe18cb93a6 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -3111,7 +3111,9 @@ def list_authenticated_providers( restrict_to_nous_policy as _nous_restrict, ) - model_ids = _nous_restrict(model_ids, _nous_policy()) + model_ids = _nous_restrict( + model_ids, _nous_policy(), rescue_empty=True, + ) except Exception: pass else: diff --git a/hermes_cli/models.py b/hermes_cli/models.py index e7754fa196..6662b9d3c8 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2657,7 +2657,10 @@ _NOUS_POLICY_APPEND_MAX = 64 def restrict_to_nous_policy( - model_ids: list[str], allowed: Optional[set[str]] + model_ids: list[str], + allowed: Optional[set[str]], + *, + rescue_empty: bool = False, ) -> list[str]: """*model_ids* narrowed to *allowed*, preserving the caller's order. @@ -2681,14 +2684,17 @@ def restrict_to_nous_policy( # An allowlist can admit only models the curated manifest has never heard # of, leaving nothing to intersect and an empty picker — strictly worse than # the unfiltered list, because the models the org may actually use are the - # ones dropped. Fall back to the reachable set itself in exactly that case. + # ones dropped. *rescue_empty* falls back to the reachable set in exactly + # that case, and callers opt in per list: it is meaningful for the list a + # user picks from, and wrong for any list whose emptiness carries meaning. + # An unavailable/gated list is legitimately empty, and rescuing it would + # read that as "nothing survived" and fill it with the whole reachable set. # - # Only when the intersection is empty. A jurisdiction or provider policy - # narrows the catalog without emptying the curated overlap, and appending - # its remainder would push non-curated alphabetical ids into a picker that - # shows a curated order on purpose. Anything omitted is still reachable - # through the picker's custom-model entry. - if not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: + # Bounded, because a jurisdiction or provider policy narrows the catalog + # without shrinking it to an allowlist, and a large alphabetical dump buries + # the curated order the pickers show on purpose. Anything omitted stays + # reachable through the picker's custom-model entry. + if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: return sorted(allowed) return kept diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 41577df7d4..97c5dfe9e2 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7520,7 +7520,9 @@ def get_recommended_default_model(provider: str = ""): # Neither the curated list nor the Portal's recommendations know # what the org may reach, and this endpoint picks the model a user # lands on without choosing it. - model_ids = restrict_to_nous_policy(model_ids, nous_policy_allowed_ids()) + model_ids = restrict_to_nous_policy( + model_ids, nous_policy_allowed_ids(), rescue_empty=True, + ) model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index b91fe10148..b488ca2367 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -63,11 +63,7 @@ class TestRestrictToNousPolicy: ) == ["vendor/model:free"] def test_drops_a_free_sibling_whose_base_is_blocked(self): - """The blocked sibling goes; the model the org may actually use takes - its place rather than leaving the picker empty.""" - assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [ - "other/model" - ] + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] class TestNousPolicyAllowedIds: @@ -200,7 +196,7 @@ class TestAllowlistOutsideTheCuratedList: def test_surfaces_an_allowed_model_the_curated_list_lacks(self): assert restrict_to_nous_policy( - ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"} + ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"}, rescue_empty=True ) == ["amazon/nova-2-lite-v1"] def test_does_not_append_when_the_curated_overlap_is_non_empty(self): @@ -208,7 +204,9 @@ class TestAllowlistOutsideTheCuratedList: emptying the curated overlap. Appending its remainder would push non-curated alphabetical ids into a deliberately curated order.""" kept = restrict_to_nous_policy( - ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"} + ["z/curated", "a/curated"], + {"z/curated", "a/curated", "new/model"}, + rescue_empty=True, ) assert kept == ["z/curated", "a/curated"] @@ -217,19 +215,46 @@ class TestAllowlistOutsideTheCuratedList: the size cap, so a size-only guard let it append.""" curated = ["vendor/one", "vendor/two", "vendor/three"] reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} - assert restrict_to_nous_policy(curated, reachable) == [ + assert restrict_to_nous_policy(curated, reachable, rescue_empty=True) == [ "vendor/one", "vendor/two", ] def test_does_not_append_a_free_sibling_already_covered(self): - assert restrict_to_nous_policy(["vendor/m:free"], {"vendor/m"}) == [ - "vendor/m:free" - ] + assert restrict_to_nous_policy( + ["vendor/m:free"], {"vendor/m"}, rescue_empty=True + ) == ["vendor/m:free"] def test_a_provider_only_policy_does_not_bury_the_curated_order(self): """Such a policy leaves the whole catalog reachable; appending it would drop hundreds of alphabetical ids into the picker.""" curated = ["vendor/one", "vendor/two"] catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) - assert restrict_to_nous_policy(curated, catalog) == curated + assert restrict_to_nous_policy(curated, catalog, rescue_empty=True) == curated + + +class TestRescueIsOptIn: + """The empty-intersection rescue is meaningful only for the list a user + picks from. Any list whose emptiness carries meaning must not get it.""" + + def test_no_rescue_by_default(self): + assert restrict_to_nous_policy([], {"a/one", "b/two"}) == [] + + def test_rescue_only_when_asked(self): + assert restrict_to_nous_policy( + [], {"a/one"}, rescue_empty=True + ) == ["a/one"] + + def test_an_already_empty_unavailable_list_is_never_filled(self): + """Regression: a paid-tier user has no gated models, so the + unavailable list is legitimately empty. Rescuing it read that as + "nothing survived" and pushed the whole reachable set into the picker's + unavailable block.""" + reachable = {f"cn/model-{i}" for i in range(42)} + assert restrict_to_nous_policy([], reachable) == [] + + def test_rescue_does_not_resurrect_a_fully_blocked_list(self): + """A list whose every entry was blocked is a real filter result, not a + signal to show something else — unless the caller asked for the + rescue, which only the selectable list does.""" + assert restrict_to_nous_policy(["x/blocked"], {"y/allowed"}) == [] From a51df3864ea336c82ae40405eca19a8ce6e97fde Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 15:39:01 -0300 Subject: [PATCH 11/21] refactor(nous): trim comments and drop an unused field --- docs/nous-org-model-policy.md | 286 ---------------------------------- 1 file changed, 286 deletions(-) delete mode 100644 docs/nous-org-model-policy.md diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md deleted file mode 100644 index 8b1cbc4573..0000000000 --- a/docs/nous-org-model-policy.md +++ /dev/null @@ -1,286 +0,0 @@ -# Honouring the Nous org model policy in the pickers - -> **Audience:** Contributors touching Nous model selection -> **Source files:** `hermes_cli/auth.py` (`_login_nous`, `fetch_nous_models`, -> `_prompt_model_selection`), `hermes_cli/models.py` (`fetch_models_with_pricing`, -> `get_pricing_for_provider`, `union_with_portal_*`, `partition_nous_models_by_tier`), -> `hermes_cli/model_setup_flows.py` (`_model_flow_nous`), -> `hermes_cli/model_switch.py` (`list_authenticated_providers`), -> `hermes_cli/web_server.py` (`/api/model/recommended-default`), -> `hermes_cli/nous_account.py` (`_info_from_valid_jwt`) -> **Related:** Inference gateway PR #164 (filters `GET /v1/models` by org policy), -> NAS #941 (team admins restrict providers), NAS `openrouter-provider-map` -> (publishes the model→providers map the gateway filter needs) - -## What changed upstream - -A Nous team admin can restrict which models and which serving providers their -org may use. The inference gateway applies that policy to `GET /v1/models`, so -an authenticated catalog read returns only what the caller may actually reach. -Blocked models are **omitted** — the row is skipped, no marker field is added -(`api/src/handlers/models.ts:99-138`). An anonymous read is still allowed and -still returns the full catalog (`api/src/app.ts:309-313` — no auth middleware -on the route). - -Two things bound how urgent this is. - -**The gateway is authoritative and this is cosmetic.** Asking for a hidden -model is refused at request time with `403 model_blocked_by_org_policy` -(`api/src/middleware/model_entitlement_gate.ts:337-356`). The listing fails -open; the request gate fails closed. Nothing here is a security boundary — the -cost of a wrong list is a predictable 403, and the gateway PR states that -tradeoff deliberately. This document is only about the client showing the -right list. - -**It is inert today.** PR #164 is merged, but is switched off until NAS -publishes the policy fields and the provider map, and the admin surface sits -behind the `org-model-policy` Vercel flag. The `openrouter-provider-map` branch -is the publisher half (a daily cron writing `openrouter_model_providers` to the -entitlement Redis). Until that lands, every caller — anonymous and -authenticated — gets the same unfiltered list. **No change here is verifiable -end to end yet; every test mocks the filtered response.** - -## Where we stand - -Four surfaces list Nous models. **None of them is filtered.** - -| surface | builds its list from | filtered | -| --- | --- | --- | -| Login (`_login_nous`, `auth.py:9383`) | `get_curated_nous_model_ids()` ∪ Portal recommendations | no | -| `hermes model` (`_model_flow_nous`, `model_setup_flows.py:399`) | same | no | -| `/model` picker (`list_authenticated_providers`, `model_switch.py:3062`) | same | no | -| Dashboard onboarding (`web_server.py:7486`) | same | no | - -All four seed from the docs-hosted manifest and union the Portal's -`recommended-models` endpoint. Neither source is authenticated, so org policy -has no effect on any list a user picks from. - -`cached_provider_model_ids("nous")` — which *does* reach the authenticated -`fetch_nous_models` — is not consulted by any of them. The `/model` picker -handles nous in its own branch that deliberately bypasses it, and nous cannot -reach the generic pathway at `model_switch.py:2898` because line 2861 skips -every non-`api_key` provider. Its only caller for nous is the background -prefetch (`model_switch.py:2390`), which writes an entry nothing reads. - -Two things that are already fine, and should stay that way: - -- `nous` is **not** in `_MODELS_DEV_PREFERRED`, so no models.dev entries are - merged on top of the live list. -- The nous fallback ladder in `provider_model_ids` is a *chain* (live → - manifest → in-repo snapshot), not a merge, so a successful live fetch is - used exclusively. - ---- - -## Fix 0 — put auth state in the pricing cache key - -**This is a prerequisite for fix 1, and worth landing on its own merits.** - -**Problem.** `fetch_models_with_pricing` caches on the base URL alone, and the -cache check happens *above* the point where the `Authorization` header is built -(`models.py:2404`): - -```python -cache_key = (base_url or "").rstrip("/") -if not force_refresh: - cached = _cached_catalog(cache_key) - if cached is not None: - return cached -... -if api_key: - headers["Authorization"] = f"Bearer {api_key}" -``` - -`_pricing_cache` is process-lifetime with no expiry for a non-empty result -(`models.py:2231-2253`). So whichever read of a given base URL lands first — -authenticated or anonymous — answers every later read in that process, -whatever key it passes. An anonymous read landing first (the auxiliary-model -path in fix 2 is one) makes a later authenticated read return an unfiltered -list without touching the network. A fix built on this cache looks like it -works and does not. - -**Do.** Fold auth state into the cache key. Distinguishing authenticated from -anonymous is enough — the token value need not be in the key, and keeping it -out avoids hashing a secret. - -**Do** update `agent/credits_tracker.py:257`, which reaches into the private -`_pricing_cache` dict assuming one entry per base URL. - -**Test.** An anonymous read followed by an authenticated read of the same base -URL issues two requests and returns two different lists. Independently -testable today, unlike everything below. - -## Fix 1 — narrow each list to the org's policy - -**Problem.** All four surfaces build their list from -`get_curated_nous_model_ids()` unioned with the Portal's `recommended-models` -endpoint. Neither is authenticated, so org policy has no effect on the model a -user picks — which is the model they then use. The Portal endpoint compounds -it: it takes no auth and no parameters, returns one globally CDN-cached payload -for the whole platform, and is invalidated only by admin pricing edits — never -by a policy change. It can put a hidden model straight back into a list. There -is no policy-aware variant of it and no parameter that would make one. - -Each surface, however, already fetches `/v1/models`. -`get_pricing_for_provider("nous")` calls `fetch_models_with_pricing`, which -reads that endpoint and returns `{model_id: {...}}`, and already resolves -credentials (`_resolve_nous_pricing_credentials`), so it is already the -authenticated read. Its keys are the reachable set. - -**Do.** Use that set to *narrow* each list, keeping the curated order. -`nous_policy_allowed_ids()` obtains the set; `restrict_to_nous_policy()` -applies it. Both live in `models.py`, and the fetch reuses the pricing cache -entry the surface already populates, so no surface makes an extra request. - -**Do not** replace a list with the response's keys. Every surface shows the -curated agentic list in curated order deliberately — the live catalog is a -large alphabetical dump of vendor-prefixed models, and swapping it in is the -regression `model_switch.py:3070` records. - -**Do not** treat the policy set as subtract-only either. An allowlist can name -only models the curated manifest has never heard of, and intersecting alone -then empties the picker — strictly worse than an unfiltered list, because the -models the org may use are the ones dropped. Fall back to the reachable set -itself in exactly that case. - -Only when the intersection is empty, only when the set is small enough to be -an allowlist rather than a whole catalog, and **only for the list a user picks -from**. Callers opt in per list. Any list whose emptiness carries meaning must -not get the rescue: a paid-tier user's unavailable list is legitimately empty, -and rescuing it reads that as "nothing survived" and fills the picker's -unavailable block with the entire reachable set. - -A size cap alone does not make this safe — a jurisdiction filter can leave few -enough models to pass it — and neither does gating on an empty intersection, -because an intentionally empty input is indistinguishable from a fully filtered -one. The opt-in is what separates them. - -**Do not** narrow a list on evidence that cannot support it. -`nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — -in three cases, and each matters: - -- **The org has no policy, or the token is too old to say.** Gated on the - `policy_present` claim (fix 4). For an unrestricted org — the common case — - filtering buys nothing and risks dropping a Portal recommendation the - gateway catalog has not caught up on yet. This keeps the change a no-op for - everyone the policy does not apply to. -- **Credential resolution failed**, so the read was anonymous and therefore - unfiltered. A full catalog must not be mistaken for a filtered one. A stated - degradation, not a silent one. -- **The read came back empty**, which is a fetch failure, not an org that may - reach nothing. - -A `:free` sibling is kept when its base model is reachable, mirroring the -gateway, which admits a row when any of its requestable ids passes and treats -anything unknown as a keep — "over-listing costs a 403 from the authoritative -gate, while hiding a row the gate would serve is unrecoverable from the client" -(`api/src/libs/catalog_policy.ts:74-78`). Prefer over-listing here too. - -**Test.** With a policy hiding model X: X is absent from each of the four -lists, and no surface makes more Nous requests than it does today. With no -policy, with credentials broken, or with an empty read, every list is byte-for- -byte what it is today. A model the Portal flags as free but the org hides stays -out; curated ordering survives filtering. - -## Fix 2 — audit the other readers of the pricing map - -**Problem.** `fetch_models_with_pricing` is shared, so any caller that treats -its keys as "the models that exist" inherits whatever authentication the first -caller happened to have. Fix 0 stops the *authentication* from leaking between -callers; this fix is about which callers may treat the map as a source of ids -at all. - -**Do.** Make the map a lookup *for* ids already in the list, never a source of -ids. Two consumers are already correct and should stay that way: -`partition_nous_models_by_tier` only looks up ids it was given, and the -`union_with_portal_*` pair only ever writes into the map — their id-widening -comes from the Portal endpoint (fix 2), not from the map. - -The one that is wrong is `agent/auxiliary_client.py:869-908` -(`_fast_model_from_catalog`), which iterates the map's keys directly as its -candidate list off an anonymous read. Reachable for nous on the titling path, -where it can select a policy-hidden model that then 403s at request time. - -**Test.** With credentials broken so the read falls back to anonymous, no -list grows. - -## Fix 3 — stop prefetching the nous catalog - -**Problem.** The background prefetch calls -`cached_provider_model_ids("nous", force_refresh=True)` -(`model_switch.py:2390`); nous is collected into it because -`_collect_authed_provider_slugs` treats any `auth.json` providers entry as -credentials regardless of `auth_type` (`model_switch.py:2519-2526`). Because -`force_refresh=True` skips the cache read and no nous surface reads the entry, -this is a live authenticated `/v1/models` round trip per picker open written to -a location nothing consults. - -**Do.** Exclude nous from the prefetch and delete the write-only entry. - -This replaces what an earlier draft proposed here — folding `org_id` into -`_credential_fingerprint` and shortening `_PROVIDER_MODELS_STALE_SERVE_MAX` -for nous (a single global constant, `models.py:4204`, with no per-provider -branching today). Both would have hardened a cache that, after fix 1, has no -nous readers to protect. If a future surface routes nous through -`cached_provider_model_ids` again, revisit the fingerprint then: it hashes -env-var values and `auth.json` mtime and carries no org signal -(`models.py:4277`), so two orgs on one machine can serve each other's list. - -**Test.** Opening the `/model` picker makes no Nous `/v1/models` request beyond -the one the displayed list is built from. - -## Fix 4 — the `policy_present` claim - -**Problem.** Under omission a blocked model simply vanishes, which reads as -"Hermes does not support this" rather than "your org disallows it". - -**Do.** Read the `policy_present` claim off the Nous OAuth access token and, -when it is `true`, show a single line stating that the org restricts which -models are available. No enumeration, no per-model marking. - -The claim rides the same JWT as `org_id` -(`access-token-issuer.ts:552,595`, `token_use: "access"`) — the token the -client already decodes — and `_info_from_valid_jwt` already retains every -claim in `raw_claims` (`nous_account.py:600-647`), so surfacing it is one -typed field on `NousPortalAccountInfo` and no new request. - -It is already widened to cover provider-only restrictions, not just model -allowlists (`nous-account-service/src/server/entitlement-snapshot.ts:478-480`). -Two NAS docs still describe it as allowlist-only and list the widening as -pending — they are stale; trust that expression. - -**Do not** enumerate the blocked set. Model policy is allowlist-only — -`denyModels` is a dead column (`nous-account-service/src/server/model-policy.ts:230`) -— so an org that allows five models blocks the entire rest of the catalog. -Graying hundreds of rows is a worse UI than omitting them. An earlier draft -proposed deriving the blocked set by diffing the anonymous and authenticated -reads and feeding it to `_prompt_model_selection`'s `unavailable_models`; that -is the wrong shape twice over, because that picker carries one -`unavailable_message` for the whole list and cannot say "free-tier-gated" and -"policy-hidden" at once. - -**Do not** report the absence of the claim as the absence of a policy. It is -tri-state: `true`, `false`, and absent, where absent means unknown — an older -mint, not an unrestricted org. The gateway rejects a corrupt (non-boolean) -claim outright rather than reading it as "no policy" -(`api/src/middleware/nas_jwt_auth.ts:179`). Show the line only on `true`. - -**Known bound:** the claim is stamped at mint time, so it goes stale until the -next token refresh — the line can lag a policy change by up to the access -token's lifetime. Acceptable, and worth stating rather than rediscovering. - -**Test.** With `policy_present` true the line shows; with it false or absent it -does not. - ---- - -## Order - -Fix 0 first: fix 1 is silently wrong without it, and it is the only piece -testable before NAS switches the feature on. Fix 4's claim gates fix 1, so the -two land together. Fix 1 is the correctness work — without it the policy is -bypassed on every surface a user picks from. Fix 2 keeps the pricing map from -becoming another way to widen a list. Fix 3 is a deletion that fix 1 makes -safe. - -Run tests with `scripts/run_tests.sh` — not bare `pytest`. From 4d482ed344bc0ab507841cfc5efc4cb6bc4cfd0e Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 15:39:23 -0300 Subject: [PATCH 12/21] refactor(nous): trim comments and drop an unused field --- agent/auxiliary_client.py | 11 ++- agent/credits_tracker.py | 5 +- hermes_cli/auth.py | 5 +- hermes_cli/model_setup_flows.py | 5 +- hermes_cli/model_switch.py | 14 ++-- hermes_cli/models.py | 78 +++++++------------ hermes_cli/nous_account.py | 29 ++----- hermes_cli/web_server.py | 5 +- tests/hermes_cli/test_nous_policy_filter.py | 53 ++++--------- tests/hermes_cli/test_nous_policy_surfaces.py | 24 ++---- .../hermes_cli/test_pricing_cache_auth_key.py | 14 +--- 11 files changed, 76 insertions(+), 167 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 534b42184a..e3c87068f1 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -885,10 +885,9 @@ def _fast_model_from_catalog(provider_id: str) -> str: logger.debug("No credentials for %s catalog", provider_id, exc_info=True) if not api_key and provider_id.strip().lower() == "nous": - # Nous is OAuth, so the api-key resolver above raises for it. An - # anonymous read returns the full catalog rather than the one the - # org may reach, and a model picked from it is refused at request - # time with model_blocked_by_org_policy. + # Nous is OAuth, so the resolver above raises for it. An anonymous + # read returns the full catalog, and a model picked from it is + # refused at request time by the org's policy. try: from hermes_cli.models import _resolve_nous_pricing_credentials @@ -913,8 +912,8 @@ def _fast_model_from_catalog(provider_id: str) -> str: ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True) if provider_id.strip().lower() == "nous": - # The catalog's keys are a source of ids here, so the policy has to - # narrow them the same way it narrows the pickers' lists. + # The catalog's keys are a source of ids here, so the policy narrows + # them as it does the pickers' lists. try: from hermes_cli.models import ( nous_policy_allowed_ids, diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 2d0873c563..82e2d53caa 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -254,10 +254,7 @@ def is_free_tier_model(model: str, base_url: str = "") -> bool: try: from hermes_cli.models import _is_model_free, peek_cached_pricing - # The agent's Nous base_url is /v1-suffixed - # (https://inference-api.nousresearch.com/v1) but the catalog fetchers - # key on the pre-/v1 root, and on auth state besides; peek_cached_pricing - # owns both details. + # peek_cached_pricing owns the /v1-suffix and auth-state key details. pricing = peek_cached_pricing(base_url) if not pricing: return False diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index c6a7330e29..6d5bfb8ae1 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9428,9 +9428,8 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) - # The curated list and the Portal's recommendations are both - # unauthenticated, so neither knows what the org may reach. - # Narrow both lists to the policy before they are shown. + # Neither the curated list nor the Portal's recommendations + # know what the org may reach. _policy_allowed = nous_policy_allowed_ids() model_ids = restrict_to_nous_policy( model_ids, _policy_allowed, rescue_empty=True, diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index c4b9dedfd7..90c9dd38d2 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -559,9 +559,8 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing, _nous_portal_url, ) - # The curated list and the Portal's recommendations are both - # unauthenticated, so neither knows what the org may reach. Narrow both - # lists to the policy before they are shown. + # Neither the curated list nor the Portal's recommendations know what the + # org may reach. from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy _policy_allowed = nous_policy_allowed_ids() diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index fe18cb93a6..fa7f4c928f 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2565,11 +2565,9 @@ def _collect_authed_provider_slugs( slugs.append(_cp.slug) seen.add(_cp.slug.lower()) - # Nous is deliberately excluded. Its picker branch builds from the curated - # list rather than cached_provider_model_ids, and nous cannot reach the - # api_key-only unified pathway, so a prefetched entry is written and never - # read — a live authenticated /v1/models round trip per picker open for - # nothing. + # Nous excluded: its picker branch builds from the curated list and it + # cannot reach the api_key-only pathway, so a prefetched entry is written + # and never read. return [s for s in slugs if s != "nous"] @@ -3101,10 +3099,8 @@ def list_authenticated_providers( # curated list alone (still correct, just may lag newly # launched models, exactly like an offline CLI run). pass - # Both the curated list and the Portal's recommendations are - # unauthenticated, so neither knows what the org may reach. Narrow - # to the policy outside the try, so a failed recommendation fetch - # still yields a filtered curated list. + # Outside the try above, so a failed recommendation fetch still + # yields a policy-filtered curated list. try: from hermes_cli.models import ( nous_policy_allowed_ids as _nous_policy, diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 6662b9d3c8..9c81afb132 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2255,18 +2255,16 @@ def _cache_catalog( return result -# A governed endpoint answers an authenticated read with a policy-filtered -# catalog and an anonymous read with the full one, so auth state is part of the -# cache identity. NUL cannot appear in a URL, so the suffix cannot collide with -# a base URL that happens to end this way. +# NUL cannot appear in a URL, so this cannot collide with a real base URL. _PRICING_AUTH_KEY_SUFFIX = "\x00auth" def _pricing_cache_key(url_root: str, api_key: str | None) -> str: - """The ``_pricing_cache`` key for a read of *url_root*. + """Cache key for a read of *url_root*. - Only *whether* a key was supplied participates — never its value, so no - secret reaches the cache key. + A governed endpoint answers an authenticated read with a policy-filtered + catalog and an anonymous one with the full catalog, so the two cannot share + an entry. Only whether a key was supplied participates, never its value. """ return url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root @@ -2274,9 +2272,8 @@ def _pricing_cache_key(url_root: str, api_key: str | None) -> str: def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: """Pricing already cached for *base_url*, or ``{}``. Never fetches. - Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the - catalog fetchers key on. Prefers the authenticated catalog, which is the - one scoped to the caller's org. + Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the fetchers + key on, and prefers the authenticated catalog. """ root = (base_url or "").rstrip("/") if root.endswith("/v1"): @@ -2607,23 +2604,13 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]: def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]: """The Nous model ids the caller's org may reach, or ``None`` to not filter. - The gateway filters ``GET /v1/models`` by the org's model policy for an - authenticated read, omitting blocked rows with no marker field, so the keys - of the authenticated pricing response are the reachable set. This reuses - that response rather than issuing a second round trip. + The gateway omits policy-blocked rows from an authenticated + ``GET /v1/models``, so that response's keys are the reachable set. - Returns ``None`` — meaning "leave the caller's list alone" — in three cases, - each of which would otherwise narrow a list on evidence that cannot support - it: - - * the org carries no policy, or the token is too old to say (see - :func:`~hermes_cli.nous_account.nous_policy_present`). Filtering an - unrestricted org's list buys nothing and risks dropping a model the - Portal recommends before the gateway catalog lists it. - * credential resolution failed, so the read is anonymous and therefore - unfiltered. A full catalog must not be mistaken for a policy-filtered one. - * the read came back empty, which is a fetch failure rather than an org - that may reach nothing. + ``None`` means "leave the caller's list alone", for the three states that + cannot support narrowing one: no policy (or a token too old to say), an + anonymous read whose catalog is unfiltered, and an empty read, which is a + fetch failure rather than an org that may reach nothing. """ try: from hermes_cli.nous_account import nous_policy_present @@ -2638,8 +2625,8 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] return None # Same arguments as get_pricing_for_provider's nous branch, so a caller - # that also asks for pricing shares this cache entry instead of paying for - # a second request. + # asking for pricing too shares this entry instead of paying for a second + # request. pricing = fetch_models_with_pricing( api_key=api_key, base_url=base_url, @@ -2649,10 +2636,8 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] return set(pricing) or None -# Above this many reachable models, an allowed set is treated as catalog-wide -# rather than as an allowlist worth showing in place of an empty picker. NAS -# caps an allowlist at 512, but a set this large is indistinguishable from the -# full catalog for display purposes. +# Past this size an allowed set reads as a whole catalog rather than an +# allowlist, and is not worth showing in place of an empty picker. _NOUS_POLICY_APPEND_MAX = 64 @@ -2664,14 +2649,12 @@ def restrict_to_nous_policy( ) -> list[str]: """*model_ids* narrowed to *allowed*, preserving the caller's order. - A ``None`` or empty *allowed* leaves the list untouched — see - :func:`nous_policy_allowed_ids` for when that happens. + A ``None`` or empty *allowed* leaves the list untouched. - A ``:free`` sibling is kept when its base model is reachable. The gateway - admits a row when any of its requestable ids passes, and treats anything - unknown as a keep on the grounds that over-listing costs a 403 from the - authoritative gate while hiding a row the gate would serve is unrecoverable - from the client. This mirrors that. + A ``:free`` sibling is kept when its base model is reachable, mirroring the + gateway, which admits a row when any of its requestable ids passes. Prefer + over-listing: that costs a 403 from the authoritative gate, while hiding a + row the gate would serve is unrecoverable from the client. """ if not allowed: return list(model_ids) @@ -2681,19 +2664,10 @@ def restrict_to_nous_policy( if mid in allowed or mid.split(":", 1)[0] in allowed ] - # An allowlist can admit only models the curated manifest has never heard - # of, leaving nothing to intersect and an empty picker — strictly worse than - # the unfiltered list, because the models the org may actually use are the - # ones dropped. *rescue_empty* falls back to the reachable set in exactly - # that case, and callers opt in per list: it is meaningful for the list a - # user picks from, and wrong for any list whose emptiness carries meaning. - # An unavailable/gated list is legitimately empty, and rescuing it would - # read that as "nothing survived" and fill it with the whole reachable set. - # - # Bounded, because a jurisdiction or provider policy narrows the catalog - # without shrinking it to an allowlist, and a large alphabetical dump buries - # the curated order the pickers show on purpose. Anything omitted stays - # reachable through the picker's custom-model entry. + # An allowlist can name only models the curated manifest lacks, leaving an + # empty picker — worse than no filter, since the models the org may use are + # the ones dropped. Opt-in per list: an already-empty list (a paid tier's + # gated models) means "nothing to gate", not "nothing survived". if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: return sorted(allowed) return kept diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index c63d090fc4..6eba71c831 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -99,7 +99,6 @@ class NousPortalAccountInfo: subscription: Optional[NousPortalSubscriptionInfo] = None paid_service_access: Optional[bool] = None paid_service_access_info: Optional[NousPaidServiceAccessInfo] = None - policy_present: Optional[bool] = None tool_access: Optional[NousToolAccessInfo] = None raw_claims: Optional[dict[str, Any]] = None raw_account: Optional[dict[str, Any]] = None @@ -400,17 +399,12 @@ def get_nous_portal_account_info( def nous_policy_present() -> Optional[bool]: """Whether the caller's org carries a restrictive model/provider policy. - Read from the ``policy_present`` claim on the Nous OAuth access token, so - this costs no request. ``/api/oauth/account`` does not carry the claim, - which is why this reads the token directly rather than going through - :func:`get_nous_portal_account_info`. + Reads the ``policy_present`` claim off the access token, so it costs no + request; ``/api/oauth/account`` does not carry it. Stamped at mint time, so + it goes stale until the next token refresh. - ``None`` means unknown — an older mint, an unreadable token, or a - non-boolean claim. Unknown is NOT "no policy": callers must not report the - absence of the claim as the absence of a restriction. - - The claim is stamped at mint time, so it goes stale until the next token - refresh. + ``None`` is unknown — an older mint or an unreadable claim — and must not be + reported as the absence of a policy. """ try: from hermes_cli.auth import get_provider_auth_state, _decode_jwt_claims @@ -430,15 +424,9 @@ def nous_policy_present() -> Optional[bool]: def nous_policy_notice() -> str: """A one-line notice for an org that restricts model choice, else ``""``. - Under the gateway's policy filter a blocked model is omitted rather than - marked, which reads as "Hermes does not support this" instead of "your org - disallows it". This says which it is without enumerating anything: model - policy is an allowlist, so an org that admits a handful of models blocks - the whole rest of the catalog, and listing those would be a worse UI than - omitting them. - - Silent unless the claim is explicitly true — absent means an older mint, - not an unrestricted org. + A blocked model is omitted rather than marked, which reads as "Hermes does + not support this". This says which it is without enumerating the blocked + set, which under an allowlist is most of the catalog. """ if nous_policy_present() is not True: return "" @@ -694,7 +682,6 @@ def _info_from_valid_jwt( expires_at=datetime.fromtimestamp(exp, tz=timezone.utc), paid_service_access=paid_access, paid_service_access_info=access_info, - policy_present=_coerce_bool(claims.get("policy_present")), tool_access=_tool_access_from_value(claims.get("tool_access")), raw_claims=dict(claims), ) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 97c5dfe9e2..ec3aae53ec 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7517,9 +7517,8 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing, portal_url ) - # Neither the curated list nor the Portal's recommendations know - # what the org may reach, and this endpoint picks the model a user - # lands on without choosing it. + # This endpoint picks the model a user lands on without choosing + # it, so an unreachable one here is worse than in a picker. model_ids = restrict_to_nous_policy( model_ids, nous_policy_allowed_ids(), rescue_empty=True, ) diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index b488ca2367..35a0de704e 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -1,10 +1,7 @@ """Narrowing the Nous model lists to an org's policy. -The inference gateway omits policy-blocked rows from an authenticated -``GET /v1/models`` with no marker field, so the keys of the authenticated -catalog read are the reachable set. These helpers turn that into a filter the -pickers can apply without a second round trip, and — just as importantly — -decline to filter when the evidence cannot support it. +The gateway omits policy-blocked rows from an authenticated ``GET /v1/models``, +so that response's keys are the reachable set. """ from __future__ import annotations @@ -44,15 +41,12 @@ class TestRestrictToNousPolicy: ) == ["a/one", "c/three"] def test_preserves_curated_order(self): - """The pickers show a curated order deliberately; filtering must not - reorder it into the catalog's alphabetical order.""" curated = ["z/last", "a/first", "m/middle"] allowed = {"a/first", "m/middle", "z/last"} assert restrict_to_nous_policy(curated, allowed) == curated def test_keeps_a_free_sibling_when_its_base_is_reachable(self): - """Portal free recommendations are ``:free`` ids; the gateway admits a - row when any of its requestable ids passes.""" + """Portal free recommendations are ``:free`` ids.""" assert restrict_to_nous_policy(["vendor/model:free"], {"vendor/model"}) == [ "vendor/model:free" ] @@ -115,8 +109,7 @@ class TestNousPolicyAllowedIds: assert calls == [] def test_declines_to_filter_on_an_anonymous_read(self, monkeypatch): - """An anonymous read returns the full catalog; treating it as the - policy-filtered set would silently widen the list to everything.""" + """An anonymous read returns the full, unfiltered catalog.""" self._patch(monkeypatch, policy_present=True, api_key="", pricing={"a/one": {}}) assert nous_policy_allowed_ids() is None @@ -146,7 +139,6 @@ class TestNousPolicyPresent: assert nous_policy_present() is None def test_non_boolean_claim_is_unknown(self, monkeypatch): - """The gateway refuses to read a corrupt claim as "no policy".""" self._patch_token(monkeypatch, _jwt({"policy_present": "yes"})) assert nous_policy_present() is None @@ -160,8 +152,6 @@ class TestNousPolicyPresent: class TestNousPolicyNotice: - """A governed org is told its choice is restricted, rather than left to - read an omitted model as one Hermes does not support.""" def _patch(self, monkeypatch, present): monkeypatch.setattr(account_mod, "nous_policy_present", lambda: present) @@ -172,14 +162,12 @@ class TestNousPolicyNotice: @pytest.mark.parametrize("present", [False, None]) def test_silent_otherwise(self, monkeypatch, present): - """Absent is an older mint, not an unrestricted org — either way there - is nothing truthful to say.""" + """Absent is an older mint, not an unrestricted org.""" self._patch(monkeypatch, present) assert account_mod.nous_policy_notice() == "" def test_names_no_models(self, monkeypatch): - """Policy is an allowlist, so the blocked set is most of the catalog; - the notice must not try to enumerate it.""" + """The blocked set is most of the catalog under an allowlist.""" self._patch(monkeypatch, True) notice = account_mod.nous_policy_notice() assert "/" not in notice, f"looks like it names a model: {notice}" @@ -187,12 +175,8 @@ class TestNousPolicyNotice: class TestAllowlistOutsideTheCuratedList: - """An allowlist can name a model the curated manifest has never heard of. - - Intersecting alone leaves the picker empty in that case — strictly worse - than showing an unfiltered list, because the one model the org may use is - the one that got dropped. - """ + """An allowlist can name only models the curated manifest lacks, which + intersecting alone turns into an empty picker.""" def test_surfaces_an_allowed_model_the_curated_list_lacks(self): assert restrict_to_nous_policy( @@ -200,9 +184,6 @@ class TestAllowlistOutsideTheCuratedList: ) == ["amazon/nova-2-lite-v1"] def test_does_not_append_when_the_curated_overlap_is_non_empty(self): - """A jurisdiction or provider policy narrows the catalog without - emptying the curated overlap. Appending its remainder would push - non-curated alphabetical ids into a deliberately curated order.""" kept = restrict_to_nous_policy( ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"}, @@ -211,8 +192,8 @@ class TestAllowlistOutsideTheCuratedList: assert kept == ["z/curated", "a/curated"] def test_jurisdiction_policy_never_grows_the_list(self): - """Regression: a region filter leaves few enough models to slip under - the size cap, so a size-only guard let it append.""" + """A region filter can slip under the size cap, so the cap alone is not + enough of a guard.""" curated = ["vendor/one", "vendor/two", "vendor/three"] reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} assert restrict_to_nous_policy(curated, reachable, rescue_empty=True) == [ @@ -226,16 +207,13 @@ class TestAllowlistOutsideTheCuratedList: ) == ["vendor/m:free"] def test_a_provider_only_policy_does_not_bury_the_curated_order(self): - """Such a policy leaves the whole catalog reachable; appending it would - drop hundreds of alphabetical ids into the picker.""" curated = ["vendor/one", "vendor/two"] catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) assert restrict_to_nous_policy(curated, catalog, rescue_empty=True) == curated class TestRescueIsOptIn: - """The empty-intersection rescue is meaningful only for the list a user - picks from. Any list whose emptiness carries meaning must not get it.""" + """The rescue is meaningful only for the list a user picks from.""" def test_no_rescue_by_default(self): assert restrict_to_nous_policy([], {"a/one", "b/two"}) == [] @@ -246,15 +224,10 @@ class TestRescueIsOptIn: ) == ["a/one"] def test_an_already_empty_unavailable_list_is_never_filled(self): - """Regression: a paid-tier user has no gated models, so the - unavailable list is legitimately empty. Rescuing it read that as - "nothing survived" and pushed the whole reachable set into the picker's - unavailable block.""" + """A paid tier has no gated models, so this list is legitimately + empty — not a filter result to rescue.""" reachable = {f"cn/model-{i}" for i in range(42)} assert restrict_to_nous_policy([], reachable) == [] def test_rescue_does_not_resurrect_a_fully_blocked_list(self): - """A list whose every entry was blocked is a real filter result, not a - signal to show something else — unless the caller asked for the - rescue, which only the selectable list does.""" assert restrict_to_nous_policy(["x/blocked"], {"y/allowed"}) == [] diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index e59974c511..1fd43725d3 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -1,9 +1,7 @@ """Every Nous model list is narrowed to the org's policy before it is shown. -Four surfaces build a Nous list from the curated manifest unioned with the -Portal's ``recommended-models`` endpoint. Neither source is authenticated, so -without this filter an org's hidden model is offered to the user and then -refused at request time with ``model_blocked_by_org_policy``. +Four surfaces build their list from the curated manifest unioned with the +Portal's ``recommended-models`` endpoint; neither source is authenticated. """ from __future__ import annotations @@ -32,7 +30,6 @@ def no_policy(monkeypatch): class TestLoginNous: - """``_login_nous`` — the model picked at login is the model then used.""" def _run(self, monkeypatch, tmp_path): import hermes_cli.auth as auth_mod @@ -119,8 +116,7 @@ class TestModelSwitchPicker: assert set(CURATED) <= set(row["models"]) def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy): - """The filter sits outside the try that wraps the Portal union, so a - Portal outage still yields a policy-filtered curated list.""" + """The filter sits outside the try wrapping the Portal union.""" def _boom(_p): raise RuntimeError("portal down") @@ -132,8 +128,7 @@ class TestModelSwitchPicker: class TestRecommendedDefaultEndpoint: - """``GET /api/model/recommended-default`` picks a model the user never sees - chosen, so an unreachable one there is worse than in a picker.""" + """This endpoint picks a model the user never sees chosen.""" def _call(self, monkeypatch): import hermes_cli.auth as auth_mod @@ -163,8 +158,7 @@ class TestRecommendedDefaultEndpoint: class TestAuxiliaryFastModel: - """``_fast_model_from_catalog`` treats the catalog's keys as a source of - ids, so an anonymous read there can select a model the gateway refuses.""" + """``_fast_model_from_catalog`` uses the catalog's keys as a source of ids.""" def _pick(self, monkeypatch, *, catalog): import agent.auxiliary_client as aux @@ -184,8 +178,7 @@ class TestAuxiliaryFastModel: return picked, seen def test_reads_the_catalog_with_nous_oauth_credentials(self, monkeypatch, no_policy): - """The api-key resolver raises for OAuth providers; without a fallback - the read goes out anonymous and returns the unfiltered catalog.""" + """The api-key resolver raises for OAuth providers.""" _, seen = self._pick(monkeypatch, catalog=["vendor/haiku-fast"]) assert seen["api_key"] == "sk-nous" @@ -204,8 +197,8 @@ class TestAuxiliaryFastModel: class TestNousPrefetch: - """The nous disk-cache entry is write-only: its picker branch builds from - the curated list, so prefetching it is a round trip for nothing.""" + """The nous disk-cache entry is write-only, so prefetching it is a round + trip for nothing.""" def test_nous_is_not_collected_for_prefetch(self, monkeypatch): import hermes_cli.auth as auth_mod @@ -220,7 +213,6 @@ class TestNousPrefetch: class TestPolicyNoticeIsShown: - """The notice reaches the two flows where a user picks a model.""" def test_login_prints_it(self, monkeypatch, tmp_path, policy, capsys): import hermes_cli.nous_account as account_mod diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py index a1ae120365..d67c8d6e3a 100644 --- a/tests/hermes_cli/test_pricing_cache_auth_key.py +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -1,10 +1,8 @@ """``_pricing_cache`` keys on auth state, not just the base URL. -A governed endpoint (Nous ``/v1/models`` filtered by an org's model policy) -answers an authenticated read with a narrower catalog than an anonymous one. -Keyed on the base URL alone, whichever read landed first in a process answered -every later one — so an authenticated caller could be handed the full, -unfiltered catalog without a request going out. +Nous ``/v1/models`` answers an authenticated read with a policy-filtered +catalog and an anonymous one with the full catalog, so the two must not share +a cache entry. """ from __future__ import annotations @@ -60,8 +58,6 @@ def catalog(monkeypatch): def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): - """The bug: an anonymous read landing first must not answer the next - authenticated read out of cache.""" anon = fetch_models_with_pricing(api_key="", base_url=BASE) authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) @@ -72,7 +68,6 @@ def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): def test_anonymous_read_is_not_answered_by_an_authenticated_one(catalog): - """And the reverse direction, so neither entry can shadow the other.""" authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) anon = fetch_models_with_pricing(api_key="", base_url=BASE) @@ -108,12 +103,11 @@ class TestPeekCachedPricing: assert peek_cached_pricing(BASE) == {} def test_accepts_a_v1_suffixed_url(self, catalog): - """The agent holds a /v1-suffixed base URL; the fetchers key on the root.""" + """The agent holds a /v1-suffixed base URL; fetchers key on the root.""" fetch_models_with_pricing(api_key="sk-test", base_url=BASE) assert sorted(peek_cached_pricing(BASE + "/v1")) == sorted(_FILTERED) def test_prefers_the_authenticated_catalog(self, catalog): - """It is the one scoped to the caller's org.""" fetch_models_with_pricing(api_key="", base_url=BASE) fetch_models_with_pricing(api_key="sk-test", base_url=BASE) assert sorted(peek_cached_pricing(BASE)) == sorted(_FILTERED) From 705a10850d7fcf3db04f6a83432ab41ff8755aba Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 17:00:26 -0300 Subject: [PATCH 13/21] refactor(nous): trim comments and drop unused code --- hermes_cli/models.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 9c81afb132..e2bd3a6682 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2259,16 +2259,6 @@ def _cache_catalog( _PRICING_AUTH_KEY_SUFFIX = "\x00auth" -def _pricing_cache_key(url_root: str, api_key: str | None) -> str: - """Cache key for a read of *url_root*. - - A governed endpoint answers an authenticated read with a policy-filtered - catalog and an anonymous one with the full catalog, so the two cannot share - an entry. Only whether a key was supplied participates, never its value. - """ - return url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root - - def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: """Pricing already cached for *base_url*, or ``{}``. Never fetches. @@ -2434,7 +2424,11 @@ def fetch_models_with_pricing( ``original``. """ url_root = (base_url or "").rstrip("/") - cache_key = _pricing_cache_key(url_root, api_key) + # A governed endpoint answers an authenticated read with a policy-filtered + # catalog and an anonymous one with the full catalog, so the two cannot + # share an entry. Only whether a key was supplied participates, never its + # value. + cache_key = url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root if not force_refresh: cached = _cached_catalog(cache_key) if cached is not None: From e89f0087b477d472383a029ab09fdb11cb257198 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Mon, 31 Aug 2026 15:29:53 -0300 Subject: [PATCH 14/21] fix(models): key the pricing cache per credential, not per auth state --- hermes_cli/models.py | 38 +++++++---- tests/hermes_cli/test_nous_policy_filter.py | 40 ++++-------- .../hermes_cli/test_pricing_cache_auth_key.py | 65 ++++++++++++++----- 3 files changed, 85 insertions(+), 58 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index e2bd3a6682..c5a3d1e5d0 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2256,23 +2256,39 @@ def _cache_catalog( # NUL cannot appear in a URL, so this cannot collide with a real base URL. -_PRICING_AUTH_KEY_SUFFIX = "\x00auth" +_PRICING_AUTH_KEY_PREFIX = "\x00auth:" + + +def _pricing_auth_fingerprint(api_key: str | None) -> str: + """Key suffix identifying the credential a catalog was read with. + + A governed endpoint answers each token with the catalog its org may reach, + so two credentials cannot share an entry. blake2b for cache-key + fingerprinting only, same rationale as :func:`_custom_endpoint_fingerprint`. + """ + if not api_key: + return "" + import hashlib + + digest = hashlib.blake2b(api_key.encode("utf-8", errors="replace"), digest_size=8) + return _PRICING_AUTH_KEY_PREFIX + digest.hexdigest() def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: """Pricing already cached for *base_url*, or ``{}``. Never fetches. Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the fetchers - key on, and prefers the authenticated catalog. + key on, and prefers an authenticated catalog. Scans rather than rebuilding a + key, because callers hold a base URL but no credential. """ root = (base_url or "").rstrip("/") if root.endswith("/v1"): root = root[:-3].rstrip("/") - for key in (root + _PRICING_AUTH_KEY_SUFFIX, root): - cached = _pricing_cache.get(key) - if cached: + authed_prefix = root + _PRICING_AUTH_KEY_PREFIX + for key, cached in _pricing_cache.items(): + if cached and key.startswith(authed_prefix): return cached - return {} + return _pricing_cache.get(root) or {} def _format_price_per_mtok(per_token_str: str) -> str: @@ -2411,8 +2427,8 @@ def fetch_models_with_pricing( ) -> dict[str, dict[str, Any]]: """Fetch ``/v1/models`` and return ``{model_id: {prompt, completion, ...}}``. - Results are cached per *base_url* and per auth state, so repeated calls - are free and an authenticated read never answers an anonymous one. + Results are cached per *base_url* and per credential, so repeated calls are + free and one caller's catalog never answers another's read. Works with any OpenRouter-compatible endpoint (OpenRouter, Nous Portal). When *include_sale_original* is true (Nous Portal only) and the gateway @@ -2424,11 +2440,7 @@ def fetch_models_with_pricing( ``original``. """ url_root = (base_url or "").rstrip("/") - # A governed endpoint answers an authenticated read with a policy-filtered - # catalog and an anonymous one with the full catalog, so the two cannot - # share an entry. Only whether a key was supplied participates, never its - # value. - cache_key = url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root + cache_key = url_root + _pricing_auth_fingerprint(api_key) if not force_refresh: cached = _cached_catalog(cache_key) if cached is not None: diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index 35a0de704e..2de0597964 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -13,7 +13,11 @@ import pytest import hermes_cli.models as models_mod import hermes_cli.nous_account as account_mod -from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy +from hermes_cli.models import ( + _NOUS_POLICY_APPEND_MAX, + nous_policy_allowed_ids, + restrict_to_nous_policy, +) from hermes_cli.nous_account import nous_policy_present @@ -191,43 +195,21 @@ class TestAllowlistOutsideTheCuratedList: ) assert kept == ["z/curated", "a/curated"] - def test_jurisdiction_policy_never_grows_the_list(self): - """A region filter can slip under the size cap, so the cap alone is not - enough of a guard.""" - curated = ["vendor/one", "vendor/two", "vendor/three"] - reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} - assert restrict_to_nous_policy(curated, reachable, rescue_empty=True) == [ - "vendor/one", - "vendor/two", - ] - - def test_does_not_append_a_free_sibling_already_covered(self): - assert restrict_to_nous_policy( - ["vendor/m:free"], {"vendor/m"}, rescue_empty=True - ) == ["vendor/m:free"] - - def test_a_provider_only_policy_does_not_bury_the_curated_order(self): - curated = ["vendor/one", "vendor/two"] - catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) - assert restrict_to_nous_policy(curated, catalog, rescue_empty=True) == curated + def test_does_not_rescue_a_catalog_sized_allowed_set(self): + """Past the cap the set reads as a whole catalog, and dumping it would + bury the curated order the pickers show on purpose.""" + oversized = {f"cn/model-{i}" for i in range(_NOUS_POLICY_APPEND_MAX + 1)} + assert restrict_to_nous_policy(["vendor/one"], oversized, rescue_empty=True) == [] class TestRescueIsOptIn: """The rescue is meaningful only for the list a user picks from.""" - def test_no_rescue_by_default(self): - assert restrict_to_nous_policy([], {"a/one", "b/two"}) == [] - def test_rescue_only_when_asked(self): - assert restrict_to_nous_policy( - [], {"a/one"}, rescue_empty=True - ) == ["a/one"] + assert restrict_to_nous_policy([], {"a/one"}, rescue_empty=True) == ["a/one"] def test_an_already_empty_unavailable_list_is_never_filled(self): """A paid tier has no gated models, so this list is legitimately empty — not a filter result to rescue.""" reachable = {f"cn/model-{i}" for i in range(42)} assert restrict_to_nous_policy([], reachable) == [] - - def test_rescue_does_not_resurrect_a_fully_blocked_list(self): - assert restrict_to_nous_policy(["x/blocked"], {"y/allowed"}) == [] diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py index d67c8d6e3a..df3946ac39 100644 --- a/tests/hermes_cli/test_pricing_cache_auth_key.py +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -1,8 +1,7 @@ -"""``_pricing_cache`` keys on auth state, not just the base URL. +"""``_pricing_cache`` keys on the credential, not just the base URL. -Nous ``/v1/models`` answers an authenticated read with a policy-filtered -catalog and an anonymous one with the full catalog, so the two must not share -a cache entry. +Nous ``/v1/models`` answers each caller with the catalog their org may reach, +so an anonymous read, and two different tokens, must not share a cache entry. """ from __future__ import annotations @@ -57,23 +56,57 @@ def catalog(monkeypatch): return requests -def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): +@pytest.fixture +def per_org_catalog(monkeypatch): + """Serve each token the catalog its own org may reach.""" + requests: list[str | None] = [] + + def _fake_urlopen(req, timeout=8.0): + auth = req.get_header("Authorization") + requests.append(auth) + org = "a" if auth == "Bearer tok-a" else "b" + payload = { + "data": [ + { + "id": f"org-{org}/only", + "pricing": {"prompt": "0.000002", "completion": "0.00001"}, + } + ] + } + resp = MagicMock() + resp.read.return_value = json.dumps(payload).encode() + resp.__enter__ = lambda self: self + resp.__exit__ = lambda *a: False + return resp + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen) + return requests + + +def test_one_token_does_not_receive_another_tokens_catalog(per_org_catalog): + """Two orgs in one process — a long-lived gateway or desktop backend after + a profile switch or re-login.""" + a = fetch_models_with_pricing(api_key="tok-a", base_url=BASE) + b = fetch_models_with_pricing(api_key="tok-b", base_url=BASE) + + assert list(a) == ["org-a/only"] + assert list(b) == ["org-b/only"], "token B was handed token A's catalog" + assert len(per_org_catalog) == 2, "token B must reach the network" + + +def test_credential_value_does_not_appear_in_the_cache_key(): + """Guards against keying on the raw token.""" + assert "sk-super-secret" not in models_mod._pricing_auth_fingerprint("sk-super-secret") + + +def test_anonymous_and_authenticated_reads_are_separate(catalog): + """Also pins the header: anonymous must send none.""" anon = fetch_models_with_pricing(api_key="", base_url=BASE) authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) assert sorted(anon) == sorted(_FULL) assert sorted(authed) == sorted(_FILTERED) - assert len(catalog) == 2, "the authenticated read must reach the network" - assert catalog[0] is None and catalog[1] == "Bearer sk-test" - - -def test_anonymous_read_is_not_answered_by_an_authenticated_one(catalog): - authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) - anon = fetch_models_with_pricing(api_key="", base_url=BASE) - - assert sorted(authed) == sorted(_FILTERED) - assert sorted(anon) == sorted(_FULL) - assert len(catalog) == 2 + assert catalog == [None, "Bearer sk-test"] @pytest.mark.parametrize("api_key", ["sk-test", ""]) From 6e20ec4101f8178cfcfdf781a26803ff46e13229 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Mon, 31 Aug 2026 15:40:49 -0300 Subject: [PATCH 15/21] fix(nous): apply the org policy before the free/paid tier split Rescuing an empty list after partitioning put paid models back into a free-tier user's selectable list, and the dashboard could pick one as the silent default. Narrowing first also drops the separate unavailable-list filter. --- hermes_cli/auth.py | 18 +++++++-------- hermes_cli/model_setup_flows.py | 24 +++++++++++--------- hermes_cli/web_server.py | 18 ++++++++++----- tests/hermes_cli/test_nous_policy_filter.py | 25 +++++++++++++++++++++ 4 files changed, 60 insertions(+), 25 deletions(-) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 6d5bfb8ae1..e24bbe6355 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9392,6 +9392,9 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: # purchases are reflected immediately. free_tier = check_nous_free_tier(force_fresh=True) _portal_for_recs = auth_state.get("portal_base_url", "") + # Narrow before the tier split, so a rescued id still has to + # pass the free/paid predicate. + _policy_allowed = nous_policy_allowed_ids() if free_tier: try: from hermes_cli.nous_account import ( @@ -9417,6 +9420,9 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, _portal_for_recs, ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) model_ids, unavailable_models = partition_nous_models_by_tier( model_ids, pricing, free_tier=True, ) @@ -9428,15 +9434,9 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) - # Neither the curated list nor the Portal's recommendations - # know what the org may reach. - _policy_allowed = nous_policy_allowed_ids() - model_ids = restrict_to_nous_policy( - model_ids, _policy_allowed, rescue_empty=True, - ) - unavailable_models = restrict_to_nous_policy( - unavailable_models, _policy_allowed, - ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) _portal = auth_state.get("portal_base_url", "") if model_ids: from hermes_cli.nous_account import nous_policy_notice diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 90c9dd38d2..c24274591d 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -531,6 +531,14 @@ def _model_flow_nous(config, current_model="", args=None): # of CLI release cadence. unavailable_models: list[str] = [] unavailable_message = "" + + # Neither the curated list nor the Portal's recommendations know what the + # org may reach. Narrow before the tier split, so an id the policy rescues + # still has to pass the free/paid predicate instead of going around it. + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + _policy_allowed = nous_policy_allowed_ids() + if free_tier: try: from hermes_cli.nous_account import ( @@ -551,6 +559,9 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, _nous_portal_url, ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) model_ids, unavailable_models = partition_nous_models_by_tier( model_ids, pricing, free_tier=True ) @@ -558,16 +569,9 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _nous_portal_url, ) - - # Neither the curated list nor the Portal's recommendations know what the - # org may reach. - from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy - - _policy_allowed = nous_policy_allowed_ids() - model_ids = restrict_to_nous_policy( - model_ids, _policy_allowed, rescue_empty=True, - ) - unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) if not model_ids and not unavailable_models: print("No models available for Nous Portal after filtering.") diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index ec3aae53ec..4c66f9e9b3 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7505,10 +7505,19 @@ def get_recommended_default_model(provider: str = ""): except Exception: portal_url = "" + # This endpoint picks the model a user lands on without choosing it, + # so an unreachable one here is worse than in a picker. Narrow before + # the tier split, so a rescued id still has to pass the free/paid + # predicate. + _policy_allowed = nous_policy_allowed_ids() + if free_tier: model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, portal_url ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) model_ids, _unavailable = partition_nous_models_by_tier( model_ids, pricing, free_tier=True ) @@ -7516,12 +7525,9 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, portal_url ) - - # This endpoint picks the model a user lands on without choosing - # it, so an unreachable one here is worse than in a picker. - model_ids = restrict_to_nous_policy( - model_ids, nous_policy_allowed_ids(), rescue_empty=True, - ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index 2de0597964..5dc6c60d28 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -213,3 +213,28 @@ class TestRescueIsOptIn: empty — not a filter result to rescue.""" reachable = {f"cn/model-{i}" for i in range(42)} assert restrict_to_nous_policy([], reachable) == [] + + +class TestPolicyRunsBeforeTierSplit: + """A rescued id must still pass the free/paid predicate. + + Rescuing after the tier split put paid models back into a free-tier user's + selectable list, and the same id into both lists at once. + """ + + def test_a_rescued_paid_model_stays_unavailable_for_a_free_tier_user(self): + from hermes_cli.models import partition_nous_models_by_tier + + pricing = { + "vendor/free": {"prompt": "0", "completion": "0"}, + "vendor/paid": {"prompt": "0.000002", "completion": "0.00001"}, + } + narrowed = restrict_to_nous_policy( + list(pricing), {"vendor/paid"}, rescue_empty=True + ) + selectable, unavailable = partition_nous_models_by_tier( + narrowed, pricing, free_tier=True + ) + + assert selectable == [] + assert unavailable == ["vendor/paid"] From e681decfae9afd019baec0af9e7f68dde548cdc8 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Mon, 31 Aug 2026 15:58:27 -0300 Subject: [PATCH 16/21] fix(aux): policy-check the whole auxiliary model ladder Only the catalog step was filtered. With no fast-family match in the allowed catalog it returned empty and the ladder fell through to a public recommendation, which could hand titling a model the org blocks. --- agent/auxiliary_client.py | 37 +++++++++---- tests/hermes_cli/test_nous_policy_surfaces.py | 55 +++++++++++++++++++ 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index e3c87068f1..a85fb25731 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -931,6 +931,18 @@ def _fast_model_from_catalog(provider_id: str) -> str: return "" +def _nous_policy_blocks(model_id: str) -> bool: + """True when the org's model policy does not admit *model_id*.""" + try: + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + allowed = nous_policy_allowed_ids() + return bool(allowed) and not restrict_to_nous_policy([model_id], allowed) + except Exception: + logger.debug("Nous policy check unavailable", exc_info=True) + return False + + # Default auxiliary models for direct API-key providers (cheap/fast for side tasks) def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) -> str: """Return the cheap auxiliary model for a provider. @@ -958,21 +970,26 @@ def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) except Exception: pass + picked = "" if prefer_fast: - catalog_pick = _fast_model_from_catalog(provider_id) - if catalog_pick: - return catalog_pick - if profile is not None: + picked = _fast_model_from_catalog(provider_id) + if not picked and profile is not None: try: - live = profile.resolve_aux_model() - if live: - return live + picked = profile.resolve_aux_model() or "" except Exception: logger.debug("resolve_aux_model failed for %s", provider_id, exc_info=True) - if profile is not None and profile.default_aux_model: - return profile.default_aux_model - return _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "") + if not picked and profile is not None and profile.default_aux_model: + picked = profile.default_aux_model + if not picked: + picked = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "") + + # Steps 2-4 are policy-blind: resolve_aux_model queries a public + # recommendation and the rest are hardcoded. A blocked pick is refused at + # request time, so drop it and let the caller keep the main model. + if picked and provider_id.strip().lower() == "nous" and _nous_policy_blocks(picked): + return "" + return picked diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index 1fd43725d3..8e30f0efc8 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -229,3 +229,58 @@ class TestPolicyNoticeIsShown: monkeypatch.setattr(account_mod, "nous_policy_present", lambda: False) TestLoginNous()._run(monkeypatch, tmp_path) assert "restricts which models" not in capsys.readouterr().out + + +class TestAuxFallbackRespectsPolicy: + """Steps 2-4 of the aux ladder are policy-blind: `resolve_aux_model` queries + a public recommendation and the rest are hardcoded.""" + + def _patch(self, monkeypatch, *, allowed, recommended): + import agent.auxiliary_client as aux + import providers + + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: allowed) + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("sk", "https://inference.example.com"), + ) + # No fast-family match, so the catalog step yields nothing. + monkeypatch.setattr( + models_mod, "fetch_models_with_pricing", + lambda **_k: {"vendor/allowed-large": {}}, + ) + + class _Profile: + default_aux_model = "" + + def resolve_aux_model(self, **_k): + return recommended + + monkeypatch.setattr(providers, "get_provider_profile", lambda _p: _Profile()) + return aux + + def test_blocked_recommendation_is_not_used(self, monkeypatch): + aux = self._patch( + monkeypatch, allowed={"vendor/allowed-large"}, + recommended="vendor/blocked-haiku", + ) + assert aux._get_aux_model_for_provider("nous", prefer_fast=True) == "" + + def test_allowed_recommendation_still_used(self, monkeypatch): + aux = self._patch( + monkeypatch, allowed={"vendor/allowed-large", "vendor/ok-haiku"}, + recommended="vendor/ok-haiku", + ) + assert ( + aux._get_aux_model_for_provider("nous", prefer_fast=True) + == "vendor/ok-haiku" + ) + + def test_ungoverned_org_is_unaffected(self, monkeypatch): + aux = self._patch( + monkeypatch, allowed=None, recommended="vendor/anything" + ) + assert ( + aux._get_aux_model_for_provider("nous", prefer_fast=True) + == "vendor/anything" + ) From 79972c67814a8c5ef478edbd7f1c607e0fb5748c Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Mon, 31 Aug 2026 16:18:07 -0300 Subject: [PATCH 17/21] fix(models): expire the Nous catalog so policy changes land MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cached catalog was held for the life of the process, so a long-lived gateway or desktop kept offering models the org had since blocked until restart. Opt-in TTL — other providers keep no-expiry caching. --- hermes_cli/models.py | 28 ++++++++++++--- .../hermes_cli/test_pricing_cache_auth_key.py | 34 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index c5a3d1e5d0..32fff0244d 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2242,12 +2242,22 @@ def _cached_catalog(cache_key: str) -> Optional[dict[str, dict[str, Any]]]: def _cache_catalog( - cache_key: str, result: dict[str, dict[str, Any]] + cache_key: str, + result: dict[str, dict[str, Any]], + ttl_seconds: Optional[float] = None, ) -> dict[str, dict[str, Any]]: - """Cache a catalog result, giving an empty one an expiry.""" + """Cache a catalog result, giving an empty one an expiry. + + *ttl_seconds* expires a non-empty result too. Only a catalog whose contents + depend on server-side state the client cannot observe needs it — an org's + model policy can change while a long-lived process holds the entry. + """ _pricing_cache[cache_key] = result if result: - _pricing_cache_retry_after.pop(cache_key, None) + if ttl_seconds: + _pricing_cache_retry_after[cache_key] = time.monotonic() + ttl_seconds + else: + _pricing_cache_retry_after.pop(cache_key, None) else: _pricing_cache_retry_after[cache_key] = ( time.monotonic() + _FAILED_CATALOG_TTL_SECONDS @@ -2424,6 +2434,7 @@ def fetch_models_with_pricing( *, force_refresh: bool = False, include_sale_original: bool = False, + cache_ttl_seconds: Optional[float] = None, ) -> dict[str, dict[str, Any]]: """Fetch ``/v1/models`` and return ``{model_id: {prompt, completion, ...}}``. @@ -2497,7 +2508,7 @@ def fetch_models_with_pricing( entry["original"] = orig_entry result[mid] = entry - return _cache_catalog(cache_key, result) + return _cache_catalog(cache_key, result, cache_ttl_seconds) def fetch_ai_gateway_pricing( @@ -2638,6 +2649,7 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] base_url=base_url, force_refresh=force_refresh, include_sale_original=True, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, ) return set(pricing) or None @@ -2646,6 +2658,13 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] # allowlist, and is not worth showing in place of an empty picker. _NOUS_POLICY_APPEND_MAX = 64 +# How long a Nous catalog stays trusted. Its contents depend on the org's +# policy, which an admin can change at any time and the client cannot observe, +# so a long-lived process must re-ask instead of holding the first answer for +# its whole life. Other providers' catalogs carry no such state and keep the +# default no-expiry caching. +_NOUS_CATALOG_TTL_SECONDS = 300.0 + def restrict_to_nous_policy( model_ids: list[str], @@ -2705,6 +2724,7 @@ def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> d force_refresh=force_refresh, # Sale chrome (pricing.original) is Nous Portal-only. include_sale_original=True, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, ) return {} diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py index df3946ac39..9292f376f3 100644 --- a/tests/hermes_cli/test_pricing_cache_auth_key.py +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -152,3 +152,37 @@ class TestPeekCachedPricing: def test_never_fetches(self, catalog): peek_cached_pricing(BASE) assert catalog == [] + + +class TestNousCatalogExpiry: + """A Nous catalog reflects the org's policy, which an admin can change while + a long-lived process holds the entry.""" + + def test_entry_expires_so_a_policy_change_is_picked_up(self, catalog, monkeypatch): + from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS + + fetch_models_with_pricing( + api_key="sk-test", base_url=BASE, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + assert len(catalog) == 1 + + now = models_mod.time.monotonic() + monkeypatch.setattr( + models_mod.time, "monotonic", + lambda: now + _NOUS_CATALOG_TTL_SECONDS + 1, + ) + fetch_models_with_pricing( + api_key="sk-test", base_url=BASE, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + assert len(catalog) == 2, "expired entry should be re-read" + + def test_no_ttl_keeps_the_entry_indefinitely(self, catalog, monkeypatch): + """Other providers' catalogs carry no policy and must not start + re-fetching.""" + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + now = models_mod.time.monotonic() + monkeypatch.setattr(models_mod.time, "monotonic", lambda: now + 86_400) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert len(catalog) == 1 From 9d5bb7a8073ffa21ec049d6fb213152a3e6ed4d7 Mon Sep 17 00:00:00 2001 From: pefontana Date: Tue, 1 Sep 2026 12:41:28 -0300 Subject: [PATCH 18/21] map mariano.nicolini@lambdaclass.com to entropidelic The contributor check failed because the PR author's commit email had no mapping under contributors/emails/. --- contributors/emails/mariano.nicolini@lambdaclass.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/mariano.nicolini@lambdaclass.com diff --git a/contributors/emails/mariano.nicolini@lambdaclass.com b/contributors/emails/mariano.nicolini@lambdaclass.com new file mode 100644 index 0000000000..e0ed36846c --- /dev/null +++ b/contributors/emails/mariano.nicolini@lambdaclass.com @@ -0,0 +1 @@ +entropidelic From d7520b2822e0e2ec2063877b1bf960b628760643 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Tue, 1 Sep 2026 16:34:48 -0300 Subject: [PATCH 19/21] fix(aux): seed the shared Nous catalog entry with the pickers' arguments --- agent/auxiliary_client.py | 18 ++++++++++++--- tests/hermes_cli/test_nous_policy_surfaces.py | 23 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index a85fb25731..7e2c382494 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -865,6 +865,7 @@ def _fast_model_from_catalog(provider_id: str) -> str: network path — the underlying fetch is memory+disk cached with a last-known-good fallback. """ + is_nous = provider_id.strip().lower() == "nous" try: from hermes_cli.auth import resolve_api_key_provider_credentials from hermes_cli.models import fetch_models_with_pricing @@ -884,7 +885,7 @@ def _fast_model_from_catalog(provider_id: str) -> str: # fetch below still works for the catalogs that allow it. logger.debug("No credentials for %s catalog", provider_id, exc_info=True) - if not api_key and provider_id.strip().lower() == "nous": + if not api_key and is_nous: # Nous is OAuth, so the resolver above raises for it. An anonymous # read returns the full catalog, and a model picked from it is # refused at request time by the org's policy. @@ -903,15 +904,26 @@ def _fast_model_from_catalog(provider_id: str) -> str: # fetch_models_with_pricing appends its own /v1/models. if base_url.endswith("/v1"): base_url = base_url[:-3] + # Same entry the pickers use, so the Nous-only arguments must match + # theirs: seeding it here without them costs the picker its sale chrome + # and leaves the policy catalog with no expiry. + _nous_kwargs = {} + if is_nous: + from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS + + _nous_kwargs = { + "include_sale_original": True, + "cache_ttl_seconds": _NOUS_CATALOG_TTL_SECONDS, + } catalog = fetch_models_with_pricing( - api_key=api_key or None, base_url=base_url, timeout=3.0 + api_key=api_key or None, base_url=base_url, timeout=3.0, **_nous_kwargs ) or {} except Exception: logger.debug("Fast-model catalog lookup failed for %s", provider_id, exc_info=True) return "" ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True) - if provider_id.strip().lower() == "nous": + if is_nous: # The catalog's keys are a source of ids here, so the policy narrows # them as it does the pickers' lists. try: diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index 8e30f0efc8..04146a617f 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -284,3 +284,26 @@ class TestAuxFallbackRespectsPolicy: aux._get_aux_model_for_provider("nous", prefer_fast=True) == "vendor/anything" ) + + +def test_titling_seeds_the_shared_catalog_entry_like_the_pickers(monkeypatch): + """The aux catalog read shares the pickers' cache entry, so seeding it + without the Nous-only arguments costs the picker its sale chrome and leaves + the policy catalog with no expiry.""" + import agent.auxiliary_client as aux + + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("tok", "https://inference.example.com"), + ) + seen: dict = {} + + def _fake_fetch(**kwargs): + seen.update(kwargs) + return {"vendor/haiku": {}} + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + aux._fast_model_from_catalog("nous") + + assert seen.get("include_sale_original") is True + assert seen.get("cache_ttl_seconds") == models_mod._NOUS_CATALOG_TTL_SECONDS From 3c4e84c1663107a0305247c59b2cfbea4c2d18b2 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Tue, 1 Sep 2026 16:35:47 -0300 Subject: [PATCH 20/21] fix(models): peek past expired and superseded pricing entries --- hermes_cli/models.py | 14 +++++++----- .../hermes_cli/test_pricing_cache_auth_key.py | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 32fff0244d..8448baf1b1 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2289,16 +2289,20 @@ def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the fetchers key on, and prefers an authenticated catalog. Scans rather than rebuilding a - key, because callers hold a base URL but no credential. + key, because callers hold a base URL but no credential — newest first, and + skipping expired entries, so a rotated credential does not keep answering + from the catalog its predecessor read. """ root = (base_url or "").rstrip("/") if root.endswith("/v1"): root = root[:-3].rstrip("/") authed_prefix = root + _PRICING_AUTH_KEY_PREFIX - for key, cached in _pricing_cache.items(): - if cached and key.startswith(authed_prefix): - return cached - return _pricing_cache.get(root) or {} + for key in reversed(list(_pricing_cache)): + if key.startswith(authed_prefix): + cached = _cached_catalog(key) + if cached: + return cached + return _cached_catalog(root) or {} def _format_price_per_mtok(per_token_str: str) -> str: diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py index 9292f376f3..d9846097fd 100644 --- a/tests/hermes_cli/test_pricing_cache_auth_key.py +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -186,3 +186,25 @@ class TestNousCatalogExpiry: monkeypatch.setattr(models_mod.time, "monotonic", lambda: now + 86_400) fetch_models_with_pricing(api_key="sk-test", base_url=BASE) assert len(catalog) == 1 + + def test_peek_prefers_the_newest_credential(self, per_org_catalog): + """After a rotation the older entry is still resident and, being + insertion-ordered, comes first.""" + fetch_models_with_pricing(api_key="tok-a", base_url=BASE, cache_ttl_seconds=300) + fetch_models_with_pricing(api_key="tok-b", base_url=BASE, cache_ttl_seconds=300) + assert list(peek_cached_pricing(BASE)) == ["org-b/only"] + + def test_peek_skips_an_expired_entry(self, catalog, monkeypatch): + """Reading _pricing_cache directly walked straight past the TTL.""" + from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS + + fetch_models_with_pricing( + api_key="sk-test", base_url=BASE, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + now = models_mod.time.monotonic() + monkeypatch.setattr( + models_mod.time, "monotonic", + lambda: now + _NOUS_CATALOG_TTL_SECONDS + 1, + ) + assert peek_cached_pricing(BASE) == {} From f48e61bb99625b48cf9436ccbee3ec805466feb1 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Tue, 1 Sep 2026 16:37:36 -0300 Subject: [PATCH 21/21] fix(nous): show the policy notice only when the filter narrowed the list --- hermes_cli/auth.py | 7 ++++++- hermes_cli/model_setup_flows.py | 7 ++++++- hermes_cli/nous_account.py | 10 +++++++--- tests/hermes_cli/test_nous_policy_filter.py | 12 +++++++++--- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index e24bbe6355..c5e790b956 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9395,6 +9395,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: # Narrow before the tier split, so a rescued id still has to # pass the free/paid predicate. _policy_allowed = nous_policy_allowed_ids() + _policy_narrowed = False if free_tier: try: from hermes_cli.nous_account import ( @@ -9420,9 +9421,11 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, _portal_for_recs, ) + _before_policy = model_ids model_ids = restrict_to_nous_policy( model_ids, _policy_allowed, rescue_empty=True, ) + _policy_narrowed = model_ids != _before_policy model_ids, unavailable_models = partition_nous_models_by_tier( model_ids, pricing, free_tier=True, ) @@ -9434,14 +9437,16 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) + _before_policy = model_ids model_ids = restrict_to_nous_policy( model_ids, _policy_allowed, rescue_empty=True, ) + _policy_narrowed = model_ids != _before_policy _portal = auth_state.get("portal_base_url", "") if model_ids: from hermes_cli.nous_account import nous_policy_notice - _policy_notice = nous_policy_notice() + _policy_notice = nous_policy_notice(removed=_policy_narrowed) if _policy_notice: print(_policy_notice) print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index c24274591d..827caa4370 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -538,6 +538,7 @@ def _model_flow_nous(config, current_model="", args=None): from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy _policy_allowed = nous_policy_allowed_ids() + _policy_narrowed = False if free_tier: try: @@ -559,9 +560,11 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, _nous_portal_url, ) + _before_policy = model_ids model_ids = restrict_to_nous_policy( model_ids, _policy_allowed, rescue_empty=True, ) + _policy_narrowed = model_ids != _before_policy model_ids, unavailable_models = partition_nous_models_by_tier( model_ids, pricing, free_tier=True ) @@ -569,9 +572,11 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _nous_portal_url, ) + _before_policy = model_ids model_ids = restrict_to_nous_policy( model_ids, _policy_allowed, rescue_empty=True, ) + _policy_narrowed = model_ids != _before_policy if not model_ids and not unavailable_models: print("No models available for Nous Portal after filtering.") @@ -588,7 +593,7 @@ def _model_flow_nous(config, current_model="", args=None): from hermes_cli.nous_account import nous_policy_notice - _policy_notice = nous_policy_notice() + _policy_notice = nous_policy_notice(removed=_policy_narrowed) if _policy_notice: print(_policy_notice) print( diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index 6eba71c831..30247ff661 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -421,14 +421,18 @@ def nous_policy_present() -> Optional[bool]: return None -def nous_policy_notice() -> str: - """A one-line notice for an org that restricts model choice, else ``""``. +def nous_policy_notice(*, removed: bool) -> str: + """A one-line notice for a list the org's policy narrowed, else ``""``. A blocked model is omitted rather than marked, which reads as "Hermes does not support this". This says which it is without enumerating the blocked set, which under an allowlist is most of the catalog. + + *removed* is whether the filter actually dropped anything. The catalog read + fails open — an anonymous or empty one narrows nothing — so the claim alone + would label a full list as filtered. """ - if nous_policy_present() is not True: + if not removed or nous_policy_present() is not True: return "" return ( "Your organization restricts which models are available — " diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index 5dc6c60d28..948885603f 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -162,18 +162,24 @@ class TestNousPolicyNotice: def test_shows_a_line_for_a_governed_org(self, monkeypatch): self._patch(monkeypatch, True) - assert "restricts which models" in account_mod.nous_policy_notice() + assert "restricts which models" in account_mod.nous_policy_notice(removed=True) @pytest.mark.parametrize("present", [False, None]) def test_silent_otherwise(self, monkeypatch, present): """Absent is an older mint, not an unrestricted org.""" self._patch(monkeypatch, present) - assert account_mod.nous_policy_notice() == "" + assert account_mod.nous_policy_notice(removed=True) == "" + + def test_silent_when_the_filter_removed_nothing(self, monkeypatch): + """The catalog read fails open, so a governed org can still end up with + a full list — saying it was filtered would be false.""" + self._patch(monkeypatch, True) + assert account_mod.nous_policy_notice(removed=False) == "" def test_names_no_models(self, monkeypatch): """The blocked set is most of the catalog under an allowlist.""" self._patch(monkeypatch, True) - notice = account_mod.nous_policy_notice() + notice = account_mod.nous_policy_notice(removed=True) assert "/" not in notice, f"looks like it names a model: {notice}" assert len(notice.splitlines()) == 1