diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 6bee187258..a6ae7d2271 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -993,6 +993,7 @@ def _fast_model_from_catalog(provider_id: str) -> str: network path — the underlying fetch is memory+disk cached with a last-known-good fallback. """ + is_nous = provider_id.strip().lower() == "nous" try: from hermes_cli.auth import resolve_api_key_provider_credentials from hermes_cli.models import fetch_models_with_pricing @@ -1012,6 +1013,17 @@ def _fast_model_from_catalog(provider_id: str) -> str: # fetch below still works for the catalogs that allow it. logger.debug("No credentials for %s catalog", provider_id, exc_info=True) + if not api_key and is_nous: + # Nous is OAuth, so the resolver above raises for it. An anonymous + # read returns the full catalog, and a model picked from it is + # refused at request time by the org's policy. + try: + from hermes_cli.models import _resolve_nous_pricing_credentials + + api_key, base_url = _resolve_nous_pricing_credentials() + except Exception: + logger.debug("No Nous credentials for catalog", exc_info=True) + if not base_url: base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "") base_url = base_url.rstrip("/") @@ -1020,14 +1032,37 @@ def _fast_model_from_catalog(provider_id: str) -> str: # fetch_models_with_pricing appends its own /v1/models. if base_url.endswith("/v1"): base_url = base_url[:-3] + # Same entry the pickers use, so the Nous-only arguments must match + # theirs: seeding it here without them costs the picker its sale chrome + # and leaves the policy catalog with no expiry. + _nous_kwargs = {} + if is_nous: + from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS + + _nous_kwargs = { + "include_sale_original": True, + "cache_ttl_seconds": _NOUS_CATALOG_TTL_SECONDS, + } catalog = fetch_models_with_pricing( - api_key=api_key or None, base_url=base_url, timeout=3.0 + api_key=api_key or None, base_url=base_url, timeout=3.0, **_nous_kwargs ) or {} except Exception: logger.debug("Fast-model catalog lookup failed for %s", provider_id, exc_info=True) return "" ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True) + if is_nous: + # The catalog's keys are a source of ids here, so the policy narrows + # them as it does the pickers' lists. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids, + restrict_to_nous_policy, + ) + + ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids()) + except Exception: + logger.debug("Nous policy filter unavailable", exc_info=True) for family in _FAST_MODEL_FAMILIES: for model_id in ids: lowered = model_id.lower() @@ -1036,6 +1071,18 @@ def _fast_model_from_catalog(provider_id: str) -> str: return "" +def _nous_policy_blocks(model_id: str) -> bool: + """True when the org's model policy does not admit *model_id*.""" + try: + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + allowed = nous_policy_allowed_ids() + return bool(allowed) and not restrict_to_nous_policy([model_id], allowed) + except Exception: + logger.debug("Nous policy check unavailable", exc_info=True) + return False + + # Default auxiliary models for direct API-key providers (cheap/fast for side tasks) def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) -> str: """Return the cheap auxiliary model for a provider. @@ -1063,21 +1110,26 @@ def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) except Exception: pass + picked = "" if prefer_fast: - catalog_pick = _fast_model_from_catalog(provider_id) - if catalog_pick: - return catalog_pick - if profile is not None: + picked = _fast_model_from_catalog(provider_id) + if not picked and profile is not None: try: - live = profile.resolve_aux_model() - if live: - return live + picked = profile.resolve_aux_model() or "" except Exception: logger.debug("resolve_aux_model failed for %s", provider_id, exc_info=True) - if profile is not None and profile.default_aux_model: - return profile.default_aux_model - return _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "") + if not picked and profile is not None and profile.default_aux_model: + picked = profile.default_aux_model + if not picked: + picked = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "") + + # Steps 2-4 are policy-blind: resolve_aux_model queries a public + # recommendation and the rest are hardcoded. A blocked pick is refused at + # request time, so drop it and let the caller keep the main model. + if picked and provider_id.strip().lower() == "nous" and _nous_policy_blocks(picked): + return "" + return picked diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 39c74ea58b..82e2d53caa 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -252,15 +252,10 @@ def is_free_tier_model(model: str, base_url: str = "") -> bool: if not base_url: return False try: - from hermes_cli.models import _is_model_free, _pricing_cache + from hermes_cli.models import _is_model_free, peek_cached_pricing - # Mirror get_pricing_for_provider's key normalization: the agent's - # Nous base_url is /v1-suffixed (https://inference-api.nousresearch.com/v1) - # but the picker keys _pricing_cache on the pre-/v1 root. - key = base_url.rstrip("/") - if key.endswith("/v1"): - key = key[:-3].rstrip("/") - pricing = _pricing_cache.get(key) + # peek_cached_pricing owns the /v1-suffix and auth-state key details. + pricing = peek_cached_pricing(base_url) if not pricing: return False return _is_model_free(model, pricing) diff --git a/contributors/emails/mariano.nicolini@lambdaclass.com b/contributors/emails/mariano.nicolini@lambdaclass.com new file mode 100644 index 0000000000..e0ed36846c --- /dev/null +++ b/contributors/emails/mariano.nicolini@lambdaclass.com @@ -0,0 +1 @@ +entropidelic diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 3392157b00..6f823fa63b 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9460,6 +9460,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: from hermes_cli.models import ( get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, partition_nous_models_by_tier, + nous_policy_allowed_ids, restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -9474,6 +9475,10 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: # purchases are reflected immediately. free_tier = check_nous_free_tier(force_fresh=True) _portal_for_recs = auth_state.get("portal_base_url", "") + # Narrow before the tier split, so a rescued id still has to + # pass the free/paid predicate. + _policy_allowed = nous_policy_allowed_ids() + _policy_narrowed = False if free_tier: try: from hermes_cli.nous_account import ( @@ -9499,6 +9504,11 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, _portal_for_recs, ) + _before_policy = model_ids + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) + _policy_narrowed = model_ids != _before_policy model_ids, unavailable_models = partition_nous_models_by_tier( model_ids, pricing, free_tier=True, ) @@ -9510,8 +9520,18 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) + _before_policy = model_ids + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) + _policy_narrowed = model_ids != _before_policy _portal = auth_state.get("portal_base_url", "") if model_ids: + from hermes_cli.nous_account import nous_policy_notice + + _policy_notice = nous_policy_notice(removed=_policy_narrowed) + if _policy_notice: + print(_policy_notice) print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") selected_model = _prompt_model_selection( model_ids, pricing=pricing, diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 44aab0ca6c..adda8b6a6a 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -539,6 +539,15 @@ def _model_flow_nous(config, current_model="", args=None): # of CLI release cadence. unavailable_models: list[str] = [] unavailable_message = "" + + # Neither the curated list nor the Portal's recommendations know what the + # org may reach. Narrow before the tier split, so an id the policy rescues + # still has to pass the free/paid predicate instead of going around it. + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + _policy_allowed = nous_policy_allowed_ids() + _policy_narrowed = False + if free_tier: try: from hermes_cli.nous_account import ( @@ -559,6 +568,11 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, _nous_portal_url, ) + _before_policy = model_ids + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) + _policy_narrowed = model_ids != _before_policy model_ids, unavailable_models = partition_nous_models_by_tier( model_ids, pricing, free_tier=True ) @@ -566,6 +580,11 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _nous_portal_url, ) + _before_policy = model_ids + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) + _policy_narrowed = model_ids != _before_policy if not model_ids and not unavailable_models: print("No models available for Nous Portal after filtering.") @@ -580,6 +599,11 @@ def _model_flow_nous(config, current_model="", args=None): print(unavailable_message or f"Upgrade at {_url} to access paid models.") return + from hermes_cli.nous_account import nous_policy_notice + + _policy_notice = nous_policy_notice(removed=_policy_narrowed) + if _policy_notice: + print(_policy_notice) print( f'Showing {len(model_ids)} curated models — use "Enter custom model name" for others.' ) diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index e8bd679c8f..5f9c11d085 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2934,7 +2934,10 @@ def _collect_authed_provider_slugs( slugs.append(_cp.slug) seen.add(_cp.slug.lower()) - return slugs + # Nous excluded: its picker branch builds from the curated list and it + # cannot reach the api_key-only pathway, so a prefetched entry is written + # and never read. + return [s for s in slugs if s != "nous"] def list_authenticated_providers( @@ -3465,6 +3468,19 @@ def list_authenticated_providers( # curated list alone (still correct, just may lag newly # launched models, exactly like an offline CLI run). pass + # Outside the try above, so a failed recommendation fetch still + # yields a policy-filtered curated list. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids as _nous_policy, + restrict_to_nous_policy as _nous_restrict, + ) + + model_ids = _nous_restrict( + model_ids, _nous_policy(), rescue_empty=True, + ) + except Exception: + pass else: # Unified pathway — see Section 1 rationale. Fall back to the # curated dict (with models.dev merge for preferred providers) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 926bd1b844..86aa7f1e41 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2345,12 +2345,22 @@ def _cached_catalog(cache_key: str) -> Optional[dict[str, dict[str, Any]]]: def _cache_catalog( - cache_key: str, result: dict[str, dict[str, Any]] + cache_key: str, + result: dict[str, dict[str, Any]], + ttl_seconds: Optional[float] = None, ) -> dict[str, dict[str, Any]]: - """Cache a catalog result, giving an empty one an expiry.""" + """Cache a catalog result, giving an empty one an expiry. + + *ttl_seconds* expires a non-empty result too. Only a catalog whose contents + depend on server-side state the client cannot observe needs it — an org's + model policy can change while a long-lived process holds the entry. + """ _pricing_cache[cache_key] = result if result: - _pricing_cache_retry_after.pop(cache_key, None) + if ttl_seconds: + _pricing_cache_retry_after[cache_key] = time.monotonic() + ttl_seconds + else: + _pricing_cache_retry_after.pop(cache_key, None) else: _pricing_cache_retry_after[cache_key] = ( time.monotonic() + _FAILED_CATALOG_TTL_SECONDS @@ -2358,6 +2368,46 @@ def _cache_catalog( return result +# NUL cannot appear in a URL, so this cannot collide with a real base URL. +_PRICING_AUTH_KEY_PREFIX = "\x00auth:" + + +def _pricing_auth_fingerprint(api_key: str | None) -> str: + """Key suffix identifying the credential a catalog was read with. + + A governed endpoint answers each token with the catalog its org may reach, + so two credentials cannot share an entry. blake2b for cache-key + fingerprinting only, same rationale as :func:`_custom_endpoint_fingerprint`. + """ + if not api_key: + return "" + import hashlib + + digest = hashlib.blake2b(api_key.encode("utf-8", errors="replace"), digest_size=8) + return _PRICING_AUTH_KEY_PREFIX + digest.hexdigest() + + +def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: + """Pricing already cached for *base_url*, or ``{}``. Never fetches. + + Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the fetchers + key on, and prefers an authenticated catalog. Scans rather than rebuilding a + key, because callers hold a base URL but no credential — newest first, and + skipping expired entries, so a rotated credential does not keep answering + from the catalog its predecessor read. + """ + root = (base_url or "").rstrip("/") + if root.endswith("/v1"): + root = root[:-3].rstrip("/") + authed_prefix = root + _PRICING_AUTH_KEY_PREFIX + for key in reversed(list(_pricing_cache)): + if key.startswith(authed_prefix): + cached = _cached_catalog(key) + if cached: + return cached + return _cached_catalog(root) or {} + + def _format_price_per_mtok(per_token_str: str) -> str: """Convert a per-token price string to a human-friendly $/Mtok string. @@ -2491,10 +2541,12 @@ def fetch_models_with_pricing( *, force_refresh: bool = False, include_sale_original: bool = False, + cache_ttl_seconds: Optional[float] = None, ) -> dict[str, dict[str, Any]]: """Fetch ``/v1/models`` and return ``{model_id: {prompt, completion, ...}}``. - Results are cached per *base_url* so repeated calls are free. + Results are cached per *base_url* and per credential, so repeated calls are + free and one caller's catalog never answers another's read. Works with any OpenRouter-compatible endpoint (OpenRouter, Nous Portal). When *include_sale_original* is true (Nous Portal only) and the gateway @@ -2505,13 +2557,14 @@ def fetch_models_with_pricing( ``{prompt, completion}`` shape even if a response happens to nest ``original``. """ - cache_key = (base_url or "").rstrip("/") + url_root = (base_url or "").rstrip("/") + cache_key = url_root + _pricing_auth_fingerprint(api_key) if not force_refresh: cached = _cached_catalog(cache_key) if cached is not None: return cached - url = cache_key + "/v1/models" + url = url_root + "/v1/models" headers: dict[str, str] = { "Accept": "application/json", "User-Agent": _HERMES_USER_AGENT, @@ -2562,7 +2615,7 @@ def fetch_models_with_pricing( entry["original"] = orig_entry result[mid] = entry - return _cache_catalog(cache_key, result) + return _cache_catalog(cache_key, result, cache_ttl_seconds) def fetch_ai_gateway_pricing( @@ -2672,6 +2725,86 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]: return (api_key, base_url) +def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]: + """The Nous model ids the caller's org may reach, or ``None`` to not filter. + + The gateway omits policy-blocked rows from an authenticated + ``GET /v1/models``, so that response's keys are the reachable set. + + ``None`` means "leave the caller's list alone", for the three states that + cannot support narrowing one: no policy (or a token too old to say), an + anonymous read whose catalog is unfiltered, and an empty read, which is a + fetch failure rather than an org that may reach nothing. + """ + try: + from hermes_cli.nous_account import nous_policy_present + + if nous_policy_present() is not True: + return None + except Exception: + return None + + api_key, base_url = _resolve_nous_pricing_credentials() + if not api_key or not base_url: + return None + + # Same arguments as get_pricing_for_provider's nous branch, so a caller + # asking for pricing too shares this entry instead of paying for a second + # request. + pricing = fetch_models_with_pricing( + api_key=api_key, + base_url=base_url, + force_refresh=force_refresh, + include_sale_original=True, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + return set(pricing) or None + + +# Past this size an allowed set reads as a whole catalog rather than an +# allowlist, and is not worth showing in place of an empty picker. +_NOUS_POLICY_APPEND_MAX = 64 + +# How long a Nous catalog stays trusted. Its contents depend on the org's +# policy, which an admin can change at any time and the client cannot observe, +# so a long-lived process must re-ask instead of holding the first answer for +# its whole life. Other providers' catalogs carry no such state and keep the +# default no-expiry caching. +_NOUS_CATALOG_TTL_SECONDS = 300.0 + + +def restrict_to_nous_policy( + model_ids: list[str], + allowed: Optional[set[str]], + *, + rescue_empty: bool = False, +) -> list[str]: + """*model_ids* narrowed to *allowed*, preserving the caller's order. + + A ``None`` or empty *allowed* leaves the list untouched. + + A ``:free`` sibling is kept when its base model is reachable, mirroring the + gateway, which admits a row when any of its requestable ids passes. Prefer + over-listing: that costs a 403 from the authoritative gate, while hiding a + row the gate would serve is unrecoverable from the client. + """ + if not allowed: + return list(model_ids) + kept = [ + mid + for mid in model_ids + if mid in allowed or mid.split(":", 1)[0] in allowed + ] + + # An allowlist can name only models the curated manifest lacks, leaving an + # empty picker — worse than no filter, since the models the org may use are + # the ones dropped. Opt-in per list: an already-empty list (a paid tier's + # gated models) means "nothing to gate", not "nothing survived". + if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: + return sorted(allowed) + return kept + + def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita).""" normalized = normalize_provider(provider) @@ -2698,6 +2831,7 @@ def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> d force_refresh=force_refresh, # Sale chrome (pricing.original) is Nous Portal-only. include_sale_original=True, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, ) return {} diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index 654487e684..30247ff661 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -396,6 +396,50 @@ def get_nous_portal_account_info( ) +def nous_policy_present() -> Optional[bool]: + """Whether the caller's org carries a restrictive model/provider policy. + + Reads the ``policy_present`` claim off the access token, so it costs no + request; ``/api/oauth/account`` does not carry it. Stamped at mint time, so + it goes stale until the next token refresh. + + ``None`` is unknown — an older mint or an unreadable claim — and must not be + reported as the absence of a policy. + """ + try: + from hermes_cli.auth import get_provider_auth_state, _decode_jwt_claims + + state = get_provider_auth_state("nous") or {} + access_token = state.get("access_token") + if not isinstance(access_token, str) or not access_token.strip(): + return None + claims = _decode_jwt_claims(access_token) + if not claims: + return None + return _coerce_bool(claims.get("policy_present")) + except Exception: + return None + + +def nous_policy_notice(*, removed: bool) -> str: + """A one-line notice for a list the org's policy narrowed, else ``""``. + + A blocked model is omitted rather than marked, which reads as "Hermes does + not support this". This says which it is without enumerating the blocked + set, which under an allowlist is most of the catalog. + + *removed* is whether the filter actually dropped anything. The catalog read + fails open — an anonymous or empty one narrows nothing — so the claim alone + would label a full list as filtered. + """ + if not removed or nous_policy_present() is not True: + return "" + return ( + "Your organization restricts which models are available — " + "models outside its policy are not listed." + ) + + def _fresh_account_info( *, state: dict[str, Any], diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 3c368dd62c..7c36cb0935 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7543,8 +7543,10 @@ def get_recommended_default_model(provider: str = ""): get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, + nous_policy_allowed_ids, partition_nous_models_by_tier, pick_silent_default_model, + restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -7561,10 +7563,19 @@ def get_recommended_default_model(provider: str = ""): except Exception: portal_url = "" + # This endpoint picks the model a user lands on without choosing it, + # so an unreachable one here is worse than in a picker. Narrow before + # the tier split, so a rescued id still has to pass the free/paid + # predicate. + _policy_allowed = nous_policy_allowed_ids() + if free_tier: model_ids, pricing = union_with_portal_free_recommendations( model_ids, pricing, portal_url ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) model_ids, _unavailable = partition_nous_models_by_tier( model_ids, pricing, free_tier=True ) @@ -7572,6 +7583,9 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, portal_url ) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py new file mode 100644 index 0000000000..948885603f --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -0,0 +1,246 @@ +"""Narrowing the Nous model lists to an org's policy. + +The gateway omits policy-blocked rows from an authenticated ``GET /v1/models``, +so that response's keys are the reachable set. +""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +import hermes_cli.models as models_mod +import hermes_cli.nous_account as account_mod +from hermes_cli.models import ( + _NOUS_POLICY_APPEND_MAX, + nous_policy_allowed_ids, + restrict_to_nous_policy, +) +from hermes_cli.nous_account import nous_policy_present + + +def _jwt(claims: dict) -> str: + def seg(obj): + raw = json.dumps(obj).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + return f"{seg({'alg': 'RS256'})}.{seg(claims)}.sig" + + +class TestRestrictToNousPolicy: + def test_none_leaves_the_list_untouched(self): + ids = ["a/one", "b/two"] + assert restrict_to_nous_policy(ids, None) == ids + + def test_empty_set_leaves_the_list_untouched(self): + """Empty is a failed read, not an org that may reach nothing.""" + ids = ["a/one", "b/two"] + assert restrict_to_nous_policy(ids, set()) == ids + + def test_drops_ids_outside_the_policy(self): + assert restrict_to_nous_policy( + ["a/one", "b/two", "c/three"], {"a/one", "c/three"} + ) == ["a/one", "c/three"] + + def test_preserves_curated_order(self): + curated = ["z/last", "a/first", "m/middle"] + allowed = {"a/first", "m/middle", "z/last"} + assert restrict_to_nous_policy(curated, allowed) == curated + + def test_keeps_a_free_sibling_when_its_base_is_reachable(self): + """Portal free recommendations are ``:free`` ids.""" + assert restrict_to_nous_policy(["vendor/model:free"], {"vendor/model"}) == [ + "vendor/model:free" + ] + + def test_keeps_a_free_id_listed_in_its_own_right(self): + assert restrict_to_nous_policy( + ["vendor/model:free"], {"vendor/model:free"} + ) == ["vendor/model:free"] + + def test_drops_a_free_sibling_whose_base_is_blocked(self): + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] + + +class TestNousPolicyAllowedIds: + @pytest.fixture(autouse=True) + def _clear_cache(self): + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + yield + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + + def _patch(self, monkeypatch, *, policy_present, api_key="sk-test", pricing=None): + calls = [] + monkeypatch.setattr( + account_mod, "nous_policy_present", lambda: policy_present + ) + monkeypatch.setattr( + models_mod, + "_resolve_nous_pricing_credentials", + lambda: (api_key, "https://inference.example.com"), + ) + + def _fake_fetch(**kwargs): + calls.append(kwargs) + return pricing if pricing is not None else {} + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + return calls + + def test_returns_the_authenticated_catalog_keys(self, monkeypatch): + calls = self._patch( + monkeypatch, + policy_present=True, + pricing={"a/one": {}, "b/two": {}}, + ) + assert nous_policy_allowed_ids() == {"a/one", "b/two"} + assert len(calls) == 1 + assert calls[0]["api_key"] == "sk-test" + + def test_declines_to_filter_an_unrestricted_org(self, monkeypatch): + calls = self._patch(monkeypatch, policy_present=False, pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + assert calls == [], "an unrestricted org should not pay for the read" + + def test_declines_to_filter_when_the_claim_is_unknown(self, monkeypatch): + """Absent is an older mint, not an unrestricted org.""" + calls = self._patch(monkeypatch, policy_present=None, pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + assert calls == [] + + def test_declines_to_filter_on_an_anonymous_read(self, monkeypatch): + """An anonymous read returns the full, unfiltered catalog.""" + self._patch(monkeypatch, policy_present=True, api_key="", pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + + def test_declines_to_filter_on_an_empty_read(self, monkeypatch): + """A failed fetch must not read as an org that may reach nothing.""" + self._patch(monkeypatch, policy_present=True, pricing={}) + assert nous_policy_allowed_ids() is None + + +class TestNousPolicyPresent: + def _patch_token(self, monkeypatch, token): + import hermes_cli.auth as auth_mod + + monkeypatch.setattr( + auth_mod, + "get_provider_auth_state", + lambda _p: {"access_token": token} if token is not None else {}, + ) + + @pytest.mark.parametrize("claim,expected", [(True, True), (False, False)]) + def test_reads_the_claim(self, monkeypatch, claim, expected): + self._patch_token(monkeypatch, _jwt({"policy_present": claim})) + assert nous_policy_present() is expected + + def test_absent_claim_is_unknown_not_false(self, monkeypatch): + self._patch_token(monkeypatch, _jwt({"org_id": "org_1"})) + assert nous_policy_present() is None + + def test_non_boolean_claim_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, _jwt({"policy_present": "yes"})) + assert nous_policy_present() is None + + def test_no_token_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, None) + assert nous_policy_present() is None + + def test_undecodable_token_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, "not-a-jwt") + assert nous_policy_present() is None + + +class TestNousPolicyNotice: + + def _patch(self, monkeypatch, present): + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: present) + + def test_shows_a_line_for_a_governed_org(self, monkeypatch): + self._patch(monkeypatch, True) + assert "restricts which models" in account_mod.nous_policy_notice(removed=True) + + @pytest.mark.parametrize("present", [False, None]) + def test_silent_otherwise(self, monkeypatch, present): + """Absent is an older mint, not an unrestricted org.""" + self._patch(monkeypatch, present) + assert account_mod.nous_policy_notice(removed=True) == "" + + def test_silent_when_the_filter_removed_nothing(self, monkeypatch): + """The catalog read fails open, so a governed org can still end up with + a full list — saying it was filtered would be false.""" + self._patch(monkeypatch, True) + assert account_mod.nous_policy_notice(removed=False) == "" + + def test_names_no_models(self, monkeypatch): + """The blocked set is most of the catalog under an allowlist.""" + self._patch(monkeypatch, True) + notice = account_mod.nous_policy_notice(removed=True) + assert "/" not in notice, f"looks like it names a model: {notice}" + assert len(notice.splitlines()) == 1 + + +class TestAllowlistOutsideTheCuratedList: + """An allowlist can name only models the curated manifest lacks, which + intersecting alone turns into an empty picker.""" + + def test_surfaces_an_allowed_model_the_curated_list_lacks(self): + assert restrict_to_nous_policy( + ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"}, rescue_empty=True + ) == ["amazon/nova-2-lite-v1"] + + def test_does_not_append_when_the_curated_overlap_is_non_empty(self): + kept = restrict_to_nous_policy( + ["z/curated", "a/curated"], + {"z/curated", "a/curated", "new/model"}, + rescue_empty=True, + ) + assert kept == ["z/curated", "a/curated"] + + def test_does_not_rescue_a_catalog_sized_allowed_set(self): + """Past the cap the set reads as a whole catalog, and dumping it would + bury the curated order the pickers show on purpose.""" + oversized = {f"cn/model-{i}" for i in range(_NOUS_POLICY_APPEND_MAX + 1)} + assert restrict_to_nous_policy(["vendor/one"], oversized, rescue_empty=True) == [] + + +class TestRescueIsOptIn: + """The rescue is meaningful only for the list a user picks from.""" + + def test_rescue_only_when_asked(self): + assert restrict_to_nous_policy([], {"a/one"}, rescue_empty=True) == ["a/one"] + + def test_an_already_empty_unavailable_list_is_never_filled(self): + """A paid tier has no gated models, so this list is legitimately + empty — not a filter result to rescue.""" + reachable = {f"cn/model-{i}" for i in range(42)} + assert restrict_to_nous_policy([], reachable) == [] + + +class TestPolicyRunsBeforeTierSplit: + """A rescued id must still pass the free/paid predicate. + + Rescuing after the tier split put paid models back into a free-tier user's + selectable list, and the same id into both lists at once. + """ + + def test_a_rescued_paid_model_stays_unavailable_for_a_free_tier_user(self): + from hermes_cli.models import partition_nous_models_by_tier + + pricing = { + "vendor/free": {"prompt": "0", "completion": "0"}, + "vendor/paid": {"prompt": "0.000002", "completion": "0.00001"}, + } + narrowed = restrict_to_nous_policy( + list(pricing), {"vendor/paid"}, rescue_empty=True + ) + selectable, unavailable = partition_nous_models_by_tier( + narrowed, pricing, free_tier=True + ) + + assert selectable == [] + assert unavailable == ["vendor/paid"] diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py new file mode 100644 index 0000000000..04146a617f --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -0,0 +1,309 @@ +"""Every Nous model list is narrowed to the org's policy before it is shown. + +Four surfaces build their list from the curated manifest unioned with the +Portal's ``recommended-models`` endpoint; neither source is authenticated. +""" + +from __future__ import annotations + +import argparse + +import pytest + +import hermes_cli.models as models_mod + +CURATED = ["vendor/allowed", "vendor/blocked"] +ALLOWED = {"vendor/allowed"} + + +@pytest.fixture +def policy(monkeypatch): + """An org whose policy admits only ``vendor/allowed``.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: ALLOWED) + return ALLOWED + + +@pytest.fixture +def no_policy(monkeypatch): + """An unrestricted org — lists must come through untouched.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: None) + + +class TestLoginNous: + + def _run(self, monkeypatch, tmp_path): + import hermes_cli.auth as auth_mod + import hermes_cli.nous_subscription as ns + + seen: dict = {} + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr( + auth_mod, + "_nous_device_code_login", + lambda **_k: { + "access_token": "tok", + "agent_key": "key", + "inference_base_url": "https://inference.example.com", + "portal_base_url": "https://portal.example.com", + "refresh_token": "r", + "token_expires_at": 9999999999, + }, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(ns, "prompt_enable_tool_gateway", lambda _c: None) + + def _capture(model_ids, **kwargs): + seen["model_ids"] = list(model_ids) + return None + + monkeypatch.setattr(auth_mod, "_prompt_model_selection", _capture) + + args = argparse.Namespace( + portal_url=None, inference_url=None, client_id=None, scope=None, + no_browser=True, timeout=15.0, ca_bundle=None, insecure=False, + ) + auth_mod._login_nous(args, auth_mod.PROVIDER_REGISTRY["nous"]) + return seen + + def test_hidden_model_is_not_offered(self, monkeypatch, tmp_path, policy): + assert self._run(monkeypatch, tmp_path).get("model_ids") == ["vendor/allowed"] + + def test_unrestricted_org_sees_the_full_curated_list( + self, monkeypatch, tmp_path, no_policy + ): + assert self._run(monkeypatch, tmp_path).get("model_ids") == CURATED + + +class TestModelSwitchPicker: + """The ``/model`` picker's nous branch (``list_authenticated_providers``).""" + + def _rows(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, + "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + rows = ms.list_authenticated_providers(max_models=10) + return next((r for r in rows if r["slug"] == "nous"), None) + + def test_hidden_model_is_filtered(self, monkeypatch, policy): + row = self._rows(monkeypatch) + assert row is not None, "nous row should be listed" + assert "vendor/blocked" not in row["models"] + assert "vendor/allowed" in row["models"] + + def test_unrestricted_org_keeps_both(self, monkeypatch, no_policy): + row = self._rows(monkeypatch) + assert row is not None + assert set(CURATED) <= set(row["models"]) + + def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy): + """The filter sits outside the try wrapping the Portal union.""" + + def _boom(_p): + raise RuntimeError("portal down") + + monkeypatch.setattr(models_mod, "get_pricing_for_provider", _boom) + row = self._rows(monkeypatch) + assert row is not None + assert "vendor/blocked" not in row["models"] + + +class TestRecommendedDefaultEndpoint: + """This endpoint picks a model the user never sees chosen.""" + + def _call(self, monkeypatch): + import hermes_cli.auth as auth_mod + from hermes_cli.web_server import get_recommended_default_model + + # Blocked first, so an unfiltered list would make it the silent + # default — otherwise this passes whether or not the filter runs. + monkeypatch.setattr( + models_mod, "get_curated_nous_model_ids", + lambda: ["vendor/blocked", "vendor/allowed"], + ) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(auth_mod, "get_provider_auth_state", lambda _p: {}) + return get_recommended_default_model(provider="nous") + + def test_hidden_model_is_never_the_silent_default(self, monkeypatch, policy): + assert self._call(monkeypatch)["model"] == "vendor/allowed" + + def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy): + assert self._call(monkeypatch)["model"] == "vendor/blocked" + + +class TestAuxiliaryFastModel: + """``_fast_model_from_catalog`` uses the catalog's keys as a source of ids.""" + + def _pick(self, monkeypatch, *, catalog): + import agent.auxiliary_client as aux + + seen: dict = {} + + def _fake_fetch(*, api_key=None, base_url="", timeout=8.0, **_k): + seen["api_key"] = api_key + return {mid: {} for mid in catalog} + + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("sk-nous", "https://inference.example.com"), + ) + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + picked = aux._fast_model_from_catalog("nous") + return picked, seen + + def test_reads_the_catalog_with_nous_oauth_credentials(self, monkeypatch, no_policy): + """The api-key resolver raises for OAuth providers.""" + _, seen = self._pick(monkeypatch, catalog=["vendor/haiku-fast"]) + assert seen["api_key"] == "sk-nous" + + def test_hidden_model_is_not_selected(self, monkeypatch, policy): + import agent.auxiliary_client as aux + + monkeypatch.setattr( + models_mod, "nous_policy_allowed_ids", lambda **_k: {"vendor/allowed"} + ) + monkeypatch.setattr(aux, "_FAST_MODEL_FAMILIES", ("vendor/",)) + monkeypatch.setattr(aux, "_FAST_MODEL_EXCLUDE", ()) + picked, _ = self._pick( + monkeypatch, catalog=["vendor/blocked", "vendor/allowed"] + ) + assert picked == "vendor/allowed" + + +class TestNousPrefetch: + """The nous disk-cache entry is write-only, so prefetching it is a round + trip for nothing.""" + + def test_nous_is_not_collected_for_prefetch(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + slugs = ms._collect_authed_provider_slugs({}, {"nous": list(CURATED)}, []) + assert "nous" not in slugs + + +class TestPolicyNoticeIsShown: + + def test_login_prints_it(self, monkeypatch, tmp_path, policy, capsys): + import hermes_cli.nous_account as account_mod + + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: True) + TestLoginNous()._run(monkeypatch, tmp_path) + assert "restricts which models" in capsys.readouterr().out + + def test_login_silent_for_an_ungoverned_org( + self, monkeypatch, tmp_path, no_policy, capsys + ): + import hermes_cli.nous_account as account_mod + + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: False) + TestLoginNous()._run(monkeypatch, tmp_path) + assert "restricts which models" not in capsys.readouterr().out + + +class TestAuxFallbackRespectsPolicy: + """Steps 2-4 of the aux ladder are policy-blind: `resolve_aux_model` queries + a public recommendation and the rest are hardcoded.""" + + def _patch(self, monkeypatch, *, allowed, recommended): + import agent.auxiliary_client as aux + import providers + + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: allowed) + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("sk", "https://inference.example.com"), + ) + # No fast-family match, so the catalog step yields nothing. + monkeypatch.setattr( + models_mod, "fetch_models_with_pricing", + lambda **_k: {"vendor/allowed-large": {}}, + ) + + class _Profile: + default_aux_model = "" + + def resolve_aux_model(self, **_k): + return recommended + + monkeypatch.setattr(providers, "get_provider_profile", lambda _p: _Profile()) + return aux + + def test_blocked_recommendation_is_not_used(self, monkeypatch): + aux = self._patch( + monkeypatch, allowed={"vendor/allowed-large"}, + recommended="vendor/blocked-haiku", + ) + assert aux._get_aux_model_for_provider("nous", prefer_fast=True) == "" + + def test_allowed_recommendation_still_used(self, monkeypatch): + aux = self._patch( + monkeypatch, allowed={"vendor/allowed-large", "vendor/ok-haiku"}, + recommended="vendor/ok-haiku", + ) + assert ( + aux._get_aux_model_for_provider("nous", prefer_fast=True) + == "vendor/ok-haiku" + ) + + def test_ungoverned_org_is_unaffected(self, monkeypatch): + aux = self._patch( + monkeypatch, allowed=None, recommended="vendor/anything" + ) + assert ( + aux._get_aux_model_for_provider("nous", prefer_fast=True) + == "vendor/anything" + ) + + +def test_titling_seeds_the_shared_catalog_entry_like_the_pickers(monkeypatch): + """The aux catalog read shares the pickers' cache entry, so seeding it + without the Nous-only arguments costs the picker its sale chrome and leaves + the policy catalog with no expiry.""" + import agent.auxiliary_client as aux + + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("tok", "https://inference.example.com"), + ) + seen: dict = {} + + def _fake_fetch(**kwargs): + seen.update(kwargs) + return {"vendor/haiku": {}} + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + aux._fast_model_from_catalog("nous") + + assert seen.get("include_sale_original") is True + assert seen.get("cache_ttl_seconds") == models_mod._NOUS_CATALOG_TTL_SECONDS diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py new file mode 100644 index 0000000000..d9846097fd --- /dev/null +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -0,0 +1,210 @@ +"""``_pricing_cache`` keys on the credential, not just the base URL. + +Nous ``/v1/models`` answers each caller with the catalog their org may reach, +so an anonymous read, and two different tokens, must not share a cache entry. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest + +import hermes_cli.models as models_mod +from hermes_cli.models import fetch_models_with_pricing, peek_cached_pricing + +BASE = "https://inference-api.example.com" + +# What the endpoint serves anonymously vs. to a policy-restricted caller. +_FULL = ["vendor/allowed", "vendor/blocked"] +_FILTERED = ["vendor/allowed"] + + +@pytest.fixture(autouse=True) +def _clear_pricing_cache(): + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + yield + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + + +@pytest.fixture +def catalog(monkeypatch): + """Serve the filtered catalog to an authenticated read, the full one to an + anonymous read, and record every request.""" + requests: list[str | None] = [] + + def _fake_urlopen(req, timeout=8.0): + auth = req.get_header("Authorization") + requests.append(auth) + ids = _FILTERED if auth else _FULL + payload = { + "data": [ + {"id": mid, "pricing": {"prompt": "0.000002", "completion": "0.00001"}} + for mid in ids + ] + } + resp = MagicMock() + resp.read.return_value = json.dumps(payload).encode() + resp.__enter__ = lambda self: self + resp.__exit__ = lambda *a: False + return resp + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen) + return requests + + +@pytest.fixture +def per_org_catalog(monkeypatch): + """Serve each token the catalog its own org may reach.""" + requests: list[str | None] = [] + + def _fake_urlopen(req, timeout=8.0): + auth = req.get_header("Authorization") + requests.append(auth) + org = "a" if auth == "Bearer tok-a" else "b" + payload = { + "data": [ + { + "id": f"org-{org}/only", + "pricing": {"prompt": "0.000002", "completion": "0.00001"}, + } + ] + } + resp = MagicMock() + resp.read.return_value = json.dumps(payload).encode() + resp.__enter__ = lambda self: self + resp.__exit__ = lambda *a: False + return resp + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen) + return requests + + +def test_one_token_does_not_receive_another_tokens_catalog(per_org_catalog): + """Two orgs in one process — a long-lived gateway or desktop backend after + a profile switch or re-login.""" + a = fetch_models_with_pricing(api_key="tok-a", base_url=BASE) + b = fetch_models_with_pricing(api_key="tok-b", base_url=BASE) + + assert list(a) == ["org-a/only"] + assert list(b) == ["org-b/only"], "token B was handed token A's catalog" + assert len(per_org_catalog) == 2, "token B must reach the network" + + +def test_credential_value_does_not_appear_in_the_cache_key(): + """Guards against keying on the raw token.""" + assert "sk-super-secret" not in models_mod._pricing_auth_fingerprint("sk-super-secret") + + +def test_anonymous_and_authenticated_reads_are_separate(catalog): + """Also pins the header: anonymous must send none.""" + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + + assert sorted(anon) == sorted(_FULL) + assert sorted(authed) == sorted(_FILTERED) + assert catalog == [None, "Bearer sk-test"] + + +@pytest.mark.parametrize("api_key", ["sk-test", ""]) +def test_repeated_read_still_hits_the_cache(catalog, api_key): + """Widening the key must not cost the caching it was there for.""" + first = fetch_models_with_pricing(api_key=api_key, base_url=BASE) + second = fetch_models_with_pricing(api_key=api_key, base_url=BASE) + + assert first == second + assert len(catalog) == 1, "second read should be served from cache" + + +def test_force_refresh_replaces_only_its_own_entry(catalog): + """A forced authenticated re-read must leave the anonymous entry intact.""" + fetch_models_with_pricing(api_key="", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE, force_refresh=True) + + assert len(catalog) == 3 + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + assert sorted(anon) == sorted(_FULL) + assert len(catalog) == 3, "the anonymous entry should have survived" + + +class TestPeekCachedPricing: + def test_returns_empty_when_nothing_cached(self): + assert peek_cached_pricing(BASE) == {} + + def test_accepts_a_v1_suffixed_url(self, catalog): + """The agent holds a /v1-suffixed base URL; fetchers key on the root.""" + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert sorted(peek_cached_pricing(BASE + "/v1")) == sorted(_FILTERED) + + def test_prefers_the_authenticated_catalog(self, catalog): + fetch_models_with_pricing(api_key="", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert sorted(peek_cached_pricing(BASE)) == sorted(_FILTERED) + + def test_falls_back_to_the_anonymous_catalog(self, catalog): + fetch_models_with_pricing(api_key="", base_url=BASE) + assert sorted(peek_cached_pricing(BASE)) == sorted(_FULL) + + def test_never_fetches(self, catalog): + peek_cached_pricing(BASE) + assert catalog == [] + + +class TestNousCatalogExpiry: + """A Nous catalog reflects the org's policy, which an admin can change while + a long-lived process holds the entry.""" + + def test_entry_expires_so_a_policy_change_is_picked_up(self, catalog, monkeypatch): + from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS + + fetch_models_with_pricing( + api_key="sk-test", base_url=BASE, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + assert len(catalog) == 1 + + now = models_mod.time.monotonic() + monkeypatch.setattr( + models_mod.time, "monotonic", + lambda: now + _NOUS_CATALOG_TTL_SECONDS + 1, + ) + fetch_models_with_pricing( + api_key="sk-test", base_url=BASE, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + assert len(catalog) == 2, "expired entry should be re-read" + + def test_no_ttl_keeps_the_entry_indefinitely(self, catalog, monkeypatch): + """Other providers' catalogs carry no policy and must not start + re-fetching.""" + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + now = models_mod.time.monotonic() + monkeypatch.setattr(models_mod.time, "monotonic", lambda: now + 86_400) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert len(catalog) == 1 + + def test_peek_prefers_the_newest_credential(self, per_org_catalog): + """After a rotation the older entry is still resident and, being + insertion-ordered, comes first.""" + fetch_models_with_pricing(api_key="tok-a", base_url=BASE, cache_ttl_seconds=300) + fetch_models_with_pricing(api_key="tok-b", base_url=BASE, cache_ttl_seconds=300) + assert list(peek_cached_pricing(BASE)) == ["org-b/only"] + + def test_peek_skips_an_expired_entry(self, catalog, monkeypatch): + """Reading _pricing_cache directly walked straight past the TTL.""" + from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS + + fetch_models_with_pricing( + api_key="sk-test", base_url=BASE, + cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS, + ) + now = models_mod.time.monotonic() + monkeypatch.setattr( + models_mod.time, "monotonic", + lambda: now + _NOUS_CATALOG_TTL_SECONDS + 1, + ) + assert peek_cached_pricing(BASE) == {}