From b1ea9196f7ed87dc4d919b528c3b006bdfd174c3 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:13:53 -0300 Subject: [PATCH] fix(nous): narrow every model list to the org's policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four surfaces list Nous models, and none of them was filtered. All four seed from the docs-hosted curated manifest and union the Portal's `recommended-models` endpoint; neither source is authenticated, so org policy had no effect on the model a user picks — which is the model they then use. The Portal endpoint compounds it, serving one globally CDN-cached payload for the whole platform, invalidated only by admin pricing edits and never by a policy change, so it can put a hidden model straight back into a list. Narrow all four against the authenticated catalog: - `_login_nous`, which chooses the model the session starts on - `_model_flow_nous`, the `hermes model` picker - `list_authenticated_providers`, the `/model` picker - `/api/model/recommended-default`, dashboard onboarding The list stays curated and curated-ordered — the policy set only ever subtracts. Replacing a list with the catalog's keys would swap a curated agentic list for a large alphabetical dump of vendor-prefixed models, which is the regression the picker's nous branch already exists to avoid. The `/model` picker's filter sits outside the try that wraps the Portal union, so a Portal outage still yields a policy-filtered curated list. `_login_nous` and `_model_flow_nous` also narrow their unavailable lists, so a policy-hidden model is not offered as a free-tier upsell either. For an org with no policy — the common case — the filter is a no-op and every list is what it was. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/auth.py | 9 + hermes_cli/model_setup_flows.py | 9 + hermes_cli/model_switch.py | 13 ++ hermes_cli/web_server.py | 7 + tests/hermes_cli/test_nous_policy_surfaces.py | 162 ++++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100644 tests/hermes_cli/test_nous_policy_surfaces.py diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 8d3a3d13ae..f97f91bca5 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9377,6 +9377,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: from hermes_cli.models import ( get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, partition_nous_models_by_tier, + nous_policy_allowed_ids, restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -9427,6 +9428,14 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) + # The curated list and the Portal's recommendations are both + # unauthenticated, so neither knows what the org may reach. + # Narrow both lists to the policy before they are shown. + _policy_allowed = nous_policy_allowed_ids() + model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + unavailable_models = restrict_to_nous_policy( + unavailable_models, _policy_allowed, + ) _portal = auth_state.get("portal_base_url", "") if model_ids: print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index fbb6f35c94..90075912a1 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -559,6 +559,15 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing, _nous_portal_url, ) + # The curated list and the Portal's recommendations are both + # unauthenticated, so neither knows what the org may reach. Narrow both + # lists to the policy before they are shown. + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + _policy_allowed = nous_policy_allowed_ids() + model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed) + if not model_ids and not unavailable_models: print("No models available for Nous Portal after filtering.") return diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index d4689df874..cb7f9cf6df 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -3096,6 +3096,19 @@ def list_authenticated_providers( # curated list alone (still correct, just may lag newly # launched models, exactly like an offline CLI run). pass + # Both the curated list and the Portal's recommendations are + # unauthenticated, so neither knows what the org may reach. Narrow + # to the policy outside the try, so a failed recommendation fetch + # still yields a filtered curated list. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids as _nous_policy, + restrict_to_nous_policy as _nous_restrict, + ) + + model_ids = _nous_restrict(model_ids, _nous_policy()) + except Exception: + pass else: # Unified pathway — see Section 1 rationale. Fall back to the # curated dict (with models.dev merge for preferred providers) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 73b2014e8e..41577df7d4 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7485,8 +7485,10 @@ def get_recommended_default_model(provider: str = ""): get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, + nous_policy_allowed_ids, partition_nous_models_by_tier, pick_silent_default_model, + restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -7515,6 +7517,11 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing, portal_url ) + # Neither the curated list nor the Portal's recommendations know + # what the org may reach, and this endpoint picks the model a user + # lands on without choosing it. + model_ids = restrict_to_nous_policy(model_ids, nous_policy_allowed_ids()) + model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} except Exception: diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py new file mode 100644 index 0000000000..c8f4cc9d18 --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -0,0 +1,162 @@ +"""Every Nous model list is narrowed to the org's policy before it is shown. + +Four surfaces build a Nous list from the curated manifest unioned with the +Portal's ``recommended-models`` endpoint. Neither source is authenticated, so +without this filter an org's hidden model is offered to the user and then +refused at request time with ``model_blocked_by_org_policy``. +""" + +from __future__ import annotations + +import argparse + +import pytest + +import hermes_cli.models as models_mod + +CURATED = ["vendor/allowed", "vendor/blocked"] +ALLOWED = {"vendor/allowed"} + + +@pytest.fixture +def policy(monkeypatch): + """An org whose policy admits only ``vendor/allowed``.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: ALLOWED) + return ALLOWED + + +@pytest.fixture +def no_policy(monkeypatch): + """An unrestricted org — lists must come through untouched.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: None) + + +class TestLoginNous: + """``_login_nous`` — the model picked at login is the model then used.""" + + def _run(self, monkeypatch, tmp_path): + import hermes_cli.auth as auth_mod + import hermes_cli.nous_subscription as ns + + seen: dict = {} + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr( + auth_mod, + "_nous_device_code_login", + lambda **_k: { + "access_token": "tok", + "agent_key": "key", + "inference_base_url": "https://inference.example.com", + "portal_base_url": "https://portal.example.com", + "refresh_token": "r", + "token_expires_at": 9999999999, + }, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(ns, "prompt_enable_tool_gateway", lambda _c: None) + + def _capture(model_ids, **kwargs): + seen["model_ids"] = list(model_ids) + return None + + monkeypatch.setattr(auth_mod, "_prompt_model_selection", _capture) + + args = argparse.Namespace( + portal_url=None, inference_url=None, client_id=None, scope=None, + no_browser=True, timeout=15.0, ca_bundle=None, insecure=False, + ) + auth_mod._login_nous(args, auth_mod.PROVIDER_REGISTRY["nous"]) + return seen + + def test_hidden_model_is_not_offered(self, monkeypatch, tmp_path, policy): + assert self._run(monkeypatch, tmp_path).get("model_ids") == ["vendor/allowed"] + + def test_unrestricted_org_sees_the_full_curated_list( + self, monkeypatch, tmp_path, no_policy + ): + assert self._run(monkeypatch, tmp_path).get("model_ids") == CURATED + + +class TestModelSwitchPicker: + """The ``/model`` picker's nous branch (``list_authenticated_providers``).""" + + def _rows(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, + "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + rows = ms.list_authenticated_providers(max_models=10) + return next((r for r in rows if r["slug"] == "nous"), None) + + def test_hidden_model_is_filtered(self, monkeypatch, policy): + row = self._rows(monkeypatch) + assert row is not None, "nous row should be listed" + assert "vendor/blocked" not in row["models"] + assert "vendor/allowed" in row["models"] + + def test_unrestricted_org_keeps_both(self, monkeypatch, no_policy): + row = self._rows(monkeypatch) + assert row is not None + assert set(CURATED) <= set(row["models"]) + + def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy): + """The filter sits outside the try that wraps the Portal union, so a + Portal outage still yields a policy-filtered curated list.""" + + def _boom(_p): + raise RuntimeError("portal down") + + monkeypatch.setattr(models_mod, "get_pricing_for_provider", _boom) + row = self._rows(monkeypatch) + assert row is not None + assert "vendor/blocked" not in row["models"] + + +class TestRecommendedDefaultEndpoint: + """``GET /api/model/recommended-default`` picks a model the user never sees + chosen, so an unreachable one there is worse than in a picker.""" + + def _call(self, monkeypatch): + import hermes_cli.auth as auth_mod + from hermes_cli.web_server import get_recommended_default_model + + # Blocked first, so an unfiltered list would make it the silent + # default — otherwise this passes whether or not the filter runs. + monkeypatch.setattr( + models_mod, "get_curated_nous_model_ids", + lambda: ["vendor/blocked", "vendor/allowed"], + ) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(auth_mod, "get_provider_auth_state", lambda _p: {}) + return get_recommended_default_model(provider="nous") + + def test_hidden_model_is_never_the_silent_default(self, monkeypatch, policy): + assert self._call(monkeypatch)["model"] == "vendor/allowed" + + def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy): + assert self._call(monkeypatch)["model"] == "vendor/blocked"