fix(nous): narrow every model list to the org's policy
Four surfaces list Nous models, and none of them was filtered. All four seed from the docs-hosted curated manifest and union the Portal's `recommended-models` endpoint; neither source is authenticated, so org policy had no effect on the model a user picks — which is the model they then use. The Portal endpoint compounds it, serving one globally CDN-cached payload for the whole platform, invalidated only by admin pricing edits and never by a policy change, so it can put a hidden model straight back into a list. Narrow all four against the authenticated catalog: - `_login_nous`, which chooses the model the session starts on - `_model_flow_nous`, the `hermes model` picker - `list_authenticated_providers`, the `/model` picker - `/api/model/recommended-default`, dashboard onboarding The list stays curated and curated-ordered — the policy set only ever subtracts. Replacing a list with the catalog's keys would swap a curated agentic list for a large alphabetical dump of vendor-prefixed models, which is the regression the picker's nous branch already exists to avoid. The `/model` picker's filter sits outside the try that wraps the Portal union, so a Portal outage still yields a policy-filtered curated list. `_login_nous` and `_model_flow_nous` also narrow their unavailable lists, so a policy-hidden model is not offered as a free-tier upsell either. For an org with no policy — the common case — the filter is a no-op and every list is what it was. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -9377,6 +9377,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None:
|
||||
from hermes_cli.models import (
|
||||
get_curated_nous_model_ids, get_pricing_for_provider,
|
||||
check_nous_free_tier, partition_nous_models_by_tier,
|
||||
nous_policy_allowed_ids, restrict_to_nous_policy,
|
||||
union_with_portal_free_recommendations,
|
||||
union_with_portal_paid_recommendations,
|
||||
)
|
||||
@@ -9427,6 +9428,14 @@ def _login_nous(args, pconfig: ProviderConfig) -> None:
|
||||
model_ids, pricing = union_with_portal_paid_recommendations(
|
||||
model_ids, pricing, _portal_for_recs,
|
||||
)
|
||||
# The curated list and the Portal's recommendations are both
|
||||
# unauthenticated, so neither knows what the org may reach.
|
||||
# Narrow both lists to the policy before they are shown.
|
||||
_policy_allowed = nous_policy_allowed_ids()
|
||||
model_ids = restrict_to_nous_policy(model_ids, _policy_allowed)
|
||||
unavailable_models = restrict_to_nous_policy(
|
||||
unavailable_models, _policy_allowed,
|
||||
)
|
||||
_portal = auth_state.get("portal_base_url", "")
|
||||
if model_ids:
|
||||
print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.")
|
||||
|
||||
@@ -559,6 +559,15 @@ def _model_flow_nous(config, current_model="", args=None):
|
||||
model_ids, pricing, _nous_portal_url,
|
||||
)
|
||||
|
||||
# The curated list and the Portal's recommendations are both
|
||||
# unauthenticated, so neither knows what the org may reach. Narrow both
|
||||
# lists to the policy before they are shown.
|
||||
from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy
|
||||
|
||||
_policy_allowed = nous_policy_allowed_ids()
|
||||
model_ids = restrict_to_nous_policy(model_ids, _policy_allowed)
|
||||
unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed)
|
||||
|
||||
if not model_ids and not unavailable_models:
|
||||
print("No models available for Nous Portal after filtering.")
|
||||
return
|
||||
|
||||
@@ -3096,6 +3096,19 @@ def list_authenticated_providers(
|
||||
# curated list alone (still correct, just may lag newly
|
||||
# launched models, exactly like an offline CLI run).
|
||||
pass
|
||||
# Both the curated list and the Portal's recommendations are
|
||||
# unauthenticated, so neither knows what the org may reach. Narrow
|
||||
# to the policy outside the try, so a failed recommendation fetch
|
||||
# still yields a filtered curated list.
|
||||
try:
|
||||
from hermes_cli.models import (
|
||||
nous_policy_allowed_ids as _nous_policy,
|
||||
restrict_to_nous_policy as _nous_restrict,
|
||||
)
|
||||
|
||||
model_ids = _nous_restrict(model_ids, _nous_policy())
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# Unified pathway — see Section 1 rationale. Fall back to the
|
||||
# curated dict (with models.dev merge for preferred providers)
|
||||
|
||||
@@ -7485,8 +7485,10 @@ def get_recommended_default_model(provider: str = ""):
|
||||
get_curated_nous_model_ids,
|
||||
get_pricing_for_provider,
|
||||
check_nous_free_tier,
|
||||
nous_policy_allowed_ids,
|
||||
partition_nous_models_by_tier,
|
||||
pick_silent_default_model,
|
||||
restrict_to_nous_policy,
|
||||
union_with_portal_free_recommendations,
|
||||
union_with_portal_paid_recommendations,
|
||||
)
|
||||
@@ -7515,6 +7517,11 @@ def get_recommended_default_model(provider: str = ""):
|
||||
model_ids, pricing, portal_url
|
||||
)
|
||||
|
||||
# Neither the curated list nor the Portal's recommendations know
|
||||
# what the org may reach, and this endpoint picks the model a user
|
||||
# lands on without choosing it.
|
||||
model_ids = restrict_to_nous_policy(model_ids, nous_policy_allowed_ids())
|
||||
|
||||
model = pick_silent_default_model(model_ids, provider="nous")
|
||||
return {"provider": "nous", "model": model, "free_tier": bool(free_tier)}
|
||||
except Exception:
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Every Nous model list is narrowed to the org's policy before it is shown.
|
||||
|
||||
Four surfaces build a Nous list from the curated manifest unioned with the
|
||||
Portal's ``recommended-models`` endpoint. Neither source is authenticated, so
|
||||
without this filter an org's hidden model is offered to the user and then
|
||||
refused at request time with ``model_blocked_by_org_policy``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import pytest
|
||||
|
||||
import hermes_cli.models as models_mod
|
||||
|
||||
CURATED = ["vendor/allowed", "vendor/blocked"]
|
||||
ALLOWED = {"vendor/allowed"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def policy(monkeypatch):
|
||||
"""An org whose policy admits only ``vendor/allowed``."""
|
||||
monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: ALLOWED)
|
||||
return ALLOWED
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_policy(monkeypatch):
|
||||
"""An unrestricted org — lists must come through untouched."""
|
||||
monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: None)
|
||||
|
||||
|
||||
class TestLoginNous:
|
||||
"""``_login_nous`` — the model picked at login is the model then used."""
|
||||
|
||||
def _run(self, monkeypatch, tmp_path):
|
||||
import hermes_cli.auth as auth_mod
|
||||
import hermes_cli.nous_subscription as ns
|
||||
|
||||
seen: dict = {}
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.setattr(
|
||||
auth_mod,
|
||||
"_nous_device_code_login",
|
||||
lambda **_k: {
|
||||
"access_token": "tok",
|
||||
"agent_key": "key",
|
||||
"inference_base_url": "https://inference.example.com",
|
||||
"portal_base_url": "https://portal.example.com",
|
||||
"refresh_token": "r",
|
||||
"token_expires_at": 9999999999,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED))
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {})
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"union_with_portal_paid_recommendations",
|
||||
lambda ids, pricing, _portal: (list(ids), pricing),
|
||||
)
|
||||
monkeypatch.setattr(ns, "prompt_enable_tool_gateway", lambda _c: None)
|
||||
|
||||
def _capture(model_ids, **kwargs):
|
||||
seen["model_ids"] = list(model_ids)
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(auth_mod, "_prompt_model_selection", _capture)
|
||||
|
||||
args = argparse.Namespace(
|
||||
portal_url=None, inference_url=None, client_id=None, scope=None,
|
||||
no_browser=True, timeout=15.0, ca_bundle=None, insecure=False,
|
||||
)
|
||||
auth_mod._login_nous(args, auth_mod.PROVIDER_REGISTRY["nous"])
|
||||
return seen
|
||||
|
||||
def test_hidden_model_is_not_offered(self, monkeypatch, tmp_path, policy):
|
||||
assert self._run(monkeypatch, tmp_path).get("model_ids") == ["vendor/allowed"]
|
||||
|
||||
def test_unrestricted_org_sees_the_full_curated_list(
|
||||
self, monkeypatch, tmp_path, no_policy
|
||||
):
|
||||
assert self._run(monkeypatch, tmp_path).get("model_ids") == CURATED
|
||||
|
||||
|
||||
class TestModelSwitchPicker:
|
||||
"""The ``/model`` picker's nous branch (``list_authenticated_providers``)."""
|
||||
|
||||
def _rows(self, monkeypatch):
|
||||
import hermes_cli.auth as auth_mod
|
||||
import hermes_cli.model_switch as ms
|
||||
|
||||
monkeypatch.setattr(
|
||||
auth_mod,
|
||||
"_load_auth_store",
|
||||
lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}},
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED))
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {})
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"union_with_portal_paid_recommendations",
|
||||
lambda ids, pricing, _portal: (list(ids), pricing),
|
||||
)
|
||||
rows = ms.list_authenticated_providers(max_models=10)
|
||||
return next((r for r in rows if r["slug"] == "nous"), None)
|
||||
|
||||
def test_hidden_model_is_filtered(self, monkeypatch, policy):
|
||||
row = self._rows(monkeypatch)
|
||||
assert row is not None, "nous row should be listed"
|
||||
assert "vendor/blocked" not in row["models"]
|
||||
assert "vendor/allowed" in row["models"]
|
||||
|
||||
def test_unrestricted_org_keeps_both(self, monkeypatch, no_policy):
|
||||
row = self._rows(monkeypatch)
|
||||
assert row is not None
|
||||
assert set(CURATED) <= set(row["models"])
|
||||
|
||||
def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy):
|
||||
"""The filter sits outside the try that wraps the Portal union, so a
|
||||
Portal outage still yields a policy-filtered curated list."""
|
||||
|
||||
def _boom(_p):
|
||||
raise RuntimeError("portal down")
|
||||
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", _boom)
|
||||
row = self._rows(monkeypatch)
|
||||
assert row is not None
|
||||
assert "vendor/blocked" not in row["models"]
|
||||
|
||||
|
||||
class TestRecommendedDefaultEndpoint:
|
||||
"""``GET /api/model/recommended-default`` picks a model the user never sees
|
||||
chosen, so an unreachable one there is worse than in a picker."""
|
||||
|
||||
def _call(self, monkeypatch):
|
||||
import hermes_cli.auth as auth_mod
|
||||
from hermes_cli.web_server import get_recommended_default_model
|
||||
|
||||
# Blocked first, so an unfiltered list would make it the silent
|
||||
# default — otherwise this passes whether or not the filter runs.
|
||||
monkeypatch.setattr(
|
||||
models_mod, "get_curated_nous_model_ids",
|
||||
lambda: ["vendor/blocked", "vendor/allowed"],
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {})
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"union_with_portal_paid_recommendations",
|
||||
lambda ids, pricing, _portal: (list(ids), pricing),
|
||||
)
|
||||
monkeypatch.setattr(auth_mod, "get_provider_auth_state", lambda _p: {})
|
||||
return get_recommended_default_model(provider="nous")
|
||||
|
||||
def test_hidden_model_is_never_the_silent_default(self, monkeypatch, policy):
|
||||
assert self._call(monkeypatch)["model"] == "vendor/allowed"
|
||||
|
||||
def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy):
|
||||
assert self._call(monkeypatch)["model"] == "vendor/blocked"
|
||||
Reference in New Issue
Block a user