Merge pull request #97354 from NousResearch/fix/nous-org-model-policy
fix(nous): honour the org model policy in the model pickers
This commit is contained in:
+63
-11
@@ -993,6 +993,7 @@ def _fast_model_from_catalog(provider_id: str) -> str:
|
||||
network path — the underlying fetch is memory+disk cached with a
|
||||
last-known-good fallback.
|
||||
"""
|
||||
is_nous = provider_id.strip().lower() == "nous"
|
||||
try:
|
||||
from hermes_cli.auth import resolve_api_key_provider_credentials
|
||||
from hermes_cli.models import fetch_models_with_pricing
|
||||
@@ -1012,6 +1013,17 @@ def _fast_model_from_catalog(provider_id: str) -> str:
|
||||
# fetch below still works for the catalogs that allow it.
|
||||
logger.debug("No credentials for %s catalog", provider_id, exc_info=True)
|
||||
|
||||
if not api_key and is_nous:
|
||||
# Nous is OAuth, so the resolver above raises for it. An anonymous
|
||||
# read returns the full catalog, and a model picked from it is
|
||||
# refused at request time by the org's policy.
|
||||
try:
|
||||
from hermes_cli.models import _resolve_nous_pricing_credentials
|
||||
|
||||
api_key, base_url = _resolve_nous_pricing_credentials()
|
||||
except Exception:
|
||||
logger.debug("No Nous credentials for catalog", exc_info=True)
|
||||
|
||||
if not base_url:
|
||||
base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "")
|
||||
base_url = base_url.rstrip("/")
|
||||
@@ -1020,14 +1032,37 @@ def _fast_model_from_catalog(provider_id: str) -> str:
|
||||
# fetch_models_with_pricing appends its own /v1/models.
|
||||
if base_url.endswith("/v1"):
|
||||
base_url = base_url[:-3]
|
||||
# Same entry the pickers use, so the Nous-only arguments must match
|
||||
# theirs: seeding it here without them costs the picker its sale chrome
|
||||
# and leaves the policy catalog with no expiry.
|
||||
_nous_kwargs = {}
|
||||
if is_nous:
|
||||
from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS
|
||||
|
||||
_nous_kwargs = {
|
||||
"include_sale_original": True,
|
||||
"cache_ttl_seconds": _NOUS_CATALOG_TTL_SECONDS,
|
||||
}
|
||||
catalog = fetch_models_with_pricing(
|
||||
api_key=api_key or None, base_url=base_url, timeout=3.0
|
||||
api_key=api_key or None, base_url=base_url, timeout=3.0, **_nous_kwargs
|
||||
) or {}
|
||||
except Exception:
|
||||
logger.debug("Fast-model catalog lookup failed for %s", provider_id, exc_info=True)
|
||||
return ""
|
||||
|
||||
ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True)
|
||||
if is_nous:
|
||||
# The catalog's keys are a source of ids here, so the policy narrows
|
||||
# them as it does the pickers' lists.
|
||||
try:
|
||||
from hermes_cli.models import (
|
||||
nous_policy_allowed_ids,
|
||||
restrict_to_nous_policy,
|
||||
)
|
||||
|
||||
ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids())
|
||||
except Exception:
|
||||
logger.debug("Nous policy filter unavailable", exc_info=True)
|
||||
for family in _FAST_MODEL_FAMILIES:
|
||||
for model_id in ids:
|
||||
lowered = model_id.lower()
|
||||
@@ -1036,6 +1071,18 @@ def _fast_model_from_catalog(provider_id: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _nous_policy_blocks(model_id: str) -> bool:
|
||||
"""True when the org's model policy does not admit *model_id*."""
|
||||
try:
|
||||
from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy
|
||||
|
||||
allowed = nous_policy_allowed_ids()
|
||||
return bool(allowed) and not restrict_to_nous_policy([model_id], allowed)
|
||||
except Exception:
|
||||
logger.debug("Nous policy check unavailable", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
# Default auxiliary models for direct API-key providers (cheap/fast for side tasks)
|
||||
def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) -> str:
|
||||
"""Return the cheap auxiliary model for a provider.
|
||||
@@ -1063,21 +1110,26 @@ def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
picked = ""
|
||||
if prefer_fast:
|
||||
catalog_pick = _fast_model_from_catalog(provider_id)
|
||||
if catalog_pick:
|
||||
return catalog_pick
|
||||
if profile is not None:
|
||||
picked = _fast_model_from_catalog(provider_id)
|
||||
if not picked and profile is not None:
|
||||
try:
|
||||
live = profile.resolve_aux_model()
|
||||
if live:
|
||||
return live
|
||||
picked = profile.resolve_aux_model() or ""
|
||||
except Exception:
|
||||
logger.debug("resolve_aux_model failed for %s", provider_id, exc_info=True)
|
||||
|
||||
if profile is not None and profile.default_aux_model:
|
||||
return profile.default_aux_model
|
||||
return _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "")
|
||||
if not picked and profile is not None and profile.default_aux_model:
|
||||
picked = profile.default_aux_model
|
||||
if not picked:
|
||||
picked = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "")
|
||||
|
||||
# Steps 2-4 are policy-blind: resolve_aux_model queries a public
|
||||
# recommendation and the rest are hardcoded. A blocked pick is refused at
|
||||
# request time, so drop it and let the caller keep the main model.
|
||||
if picked and provider_id.strip().lower() == "nous" and _nous_policy_blocks(picked):
|
||||
return ""
|
||||
return picked
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -252,15 +252,10 @@ def is_free_tier_model(model: str, base_url: str = "") -> bool:
|
||||
if not base_url:
|
||||
return False
|
||||
try:
|
||||
from hermes_cli.models import _is_model_free, _pricing_cache
|
||||
from hermes_cli.models import _is_model_free, peek_cached_pricing
|
||||
|
||||
# Mirror get_pricing_for_provider's key normalization: the agent's
|
||||
# Nous base_url is /v1-suffixed (https://inference-api.nousresearch.com/v1)
|
||||
# but the picker keys _pricing_cache on the pre-/v1 root.
|
||||
key = base_url.rstrip("/")
|
||||
if key.endswith("/v1"):
|
||||
key = key[:-3].rstrip("/")
|
||||
pricing = _pricing_cache.get(key)
|
||||
# peek_cached_pricing owns the /v1-suffix and auth-state key details.
|
||||
pricing = peek_cached_pricing(base_url)
|
||||
if not pricing:
|
||||
return False
|
||||
return _is_model_free(model, pricing)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
entropidelic
|
||||
@@ -9460,6 +9460,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None:
|
||||
from hermes_cli.models import (
|
||||
get_curated_nous_model_ids, get_pricing_for_provider,
|
||||
check_nous_free_tier, partition_nous_models_by_tier,
|
||||
nous_policy_allowed_ids, restrict_to_nous_policy,
|
||||
union_with_portal_free_recommendations,
|
||||
union_with_portal_paid_recommendations,
|
||||
)
|
||||
@@ -9474,6 +9475,10 @@ def _login_nous(args, pconfig: ProviderConfig) -> None:
|
||||
# purchases are reflected immediately.
|
||||
free_tier = check_nous_free_tier(force_fresh=True)
|
||||
_portal_for_recs = auth_state.get("portal_base_url", "")
|
||||
# Narrow before the tier split, so a rescued id still has to
|
||||
# pass the free/paid predicate.
|
||||
_policy_allowed = nous_policy_allowed_ids()
|
||||
_policy_narrowed = False
|
||||
if free_tier:
|
||||
try:
|
||||
from hermes_cli.nous_account import (
|
||||
@@ -9499,6 +9504,11 @@ def _login_nous(args, pconfig: ProviderConfig) -> None:
|
||||
model_ids, pricing = union_with_portal_free_recommendations(
|
||||
model_ids, pricing, _portal_for_recs,
|
||||
)
|
||||
_before_policy = model_ids
|
||||
model_ids = restrict_to_nous_policy(
|
||||
model_ids, _policy_allowed, rescue_empty=True,
|
||||
)
|
||||
_policy_narrowed = model_ids != _before_policy
|
||||
model_ids, unavailable_models = partition_nous_models_by_tier(
|
||||
model_ids, pricing, free_tier=True,
|
||||
)
|
||||
@@ -9510,8 +9520,18 @@ def _login_nous(args, pconfig: ProviderConfig) -> None:
|
||||
model_ids, pricing = union_with_portal_paid_recommendations(
|
||||
model_ids, pricing, _portal_for_recs,
|
||||
)
|
||||
_before_policy = model_ids
|
||||
model_ids = restrict_to_nous_policy(
|
||||
model_ids, _policy_allowed, rescue_empty=True,
|
||||
)
|
||||
_policy_narrowed = model_ids != _before_policy
|
||||
_portal = auth_state.get("portal_base_url", "")
|
||||
if model_ids:
|
||||
from hermes_cli.nous_account import nous_policy_notice
|
||||
|
||||
_policy_notice = nous_policy_notice(removed=_policy_narrowed)
|
||||
if _policy_notice:
|
||||
print(_policy_notice)
|
||||
print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.")
|
||||
selected_model = _prompt_model_selection(
|
||||
model_ids, pricing=pricing,
|
||||
|
||||
@@ -539,6 +539,15 @@ def _model_flow_nous(config, current_model="", args=None):
|
||||
# of CLI release cadence.
|
||||
unavailable_models: list[str] = []
|
||||
unavailable_message = ""
|
||||
|
||||
# Neither the curated list nor the Portal's recommendations know what the
|
||||
# org may reach. Narrow before the tier split, so an id the policy rescues
|
||||
# still has to pass the free/paid predicate instead of going around it.
|
||||
from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy
|
||||
|
||||
_policy_allowed = nous_policy_allowed_ids()
|
||||
_policy_narrowed = False
|
||||
|
||||
if free_tier:
|
||||
try:
|
||||
from hermes_cli.nous_account import (
|
||||
@@ -559,6 +568,11 @@ def _model_flow_nous(config, current_model="", args=None):
|
||||
model_ids, pricing = union_with_portal_free_recommendations(
|
||||
model_ids, pricing, _nous_portal_url,
|
||||
)
|
||||
_before_policy = model_ids
|
||||
model_ids = restrict_to_nous_policy(
|
||||
model_ids, _policy_allowed, rescue_empty=True,
|
||||
)
|
||||
_policy_narrowed = model_ids != _before_policy
|
||||
model_ids, unavailable_models = partition_nous_models_by_tier(
|
||||
model_ids, pricing, free_tier=True
|
||||
)
|
||||
@@ -566,6 +580,11 @@ def _model_flow_nous(config, current_model="", args=None):
|
||||
model_ids, pricing = union_with_portal_paid_recommendations(
|
||||
model_ids, pricing, _nous_portal_url,
|
||||
)
|
||||
_before_policy = model_ids
|
||||
model_ids = restrict_to_nous_policy(
|
||||
model_ids, _policy_allowed, rescue_empty=True,
|
||||
)
|
||||
_policy_narrowed = model_ids != _before_policy
|
||||
|
||||
if not model_ids and not unavailable_models:
|
||||
print("No models available for Nous Portal after filtering.")
|
||||
@@ -580,6 +599,11 @@ def _model_flow_nous(config, current_model="", args=None):
|
||||
print(unavailable_message or f"Upgrade at {_url} to access paid models.")
|
||||
return
|
||||
|
||||
from hermes_cli.nous_account import nous_policy_notice
|
||||
|
||||
_policy_notice = nous_policy_notice(removed=_policy_narrowed)
|
||||
if _policy_notice:
|
||||
print(_policy_notice)
|
||||
print(
|
||||
f'Showing {len(model_ids)} curated models — use "Enter custom model name" for others.'
|
||||
)
|
||||
|
||||
@@ -2934,7 +2934,10 @@ def _collect_authed_provider_slugs(
|
||||
slugs.append(_cp.slug)
|
||||
seen.add(_cp.slug.lower())
|
||||
|
||||
return slugs
|
||||
# Nous excluded: its picker branch builds from the curated list and it
|
||||
# cannot reach the api_key-only pathway, so a prefetched entry is written
|
||||
# and never read.
|
||||
return [s for s in slugs if s != "nous"]
|
||||
|
||||
|
||||
def list_authenticated_providers(
|
||||
@@ -3465,6 +3468,19 @@ def list_authenticated_providers(
|
||||
# curated list alone (still correct, just may lag newly
|
||||
# launched models, exactly like an offline CLI run).
|
||||
pass
|
||||
# Outside the try above, so a failed recommendation fetch still
|
||||
# yields a policy-filtered curated list.
|
||||
try:
|
||||
from hermes_cli.models import (
|
||||
nous_policy_allowed_ids as _nous_policy,
|
||||
restrict_to_nous_policy as _nous_restrict,
|
||||
)
|
||||
|
||||
model_ids = _nous_restrict(
|
||||
model_ids, _nous_policy(), rescue_empty=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# Unified pathway — see Section 1 rationale. Fall back to the
|
||||
# curated dict (with models.dev merge for preferred providers)
|
||||
|
||||
+141
-7
@@ -2345,12 +2345,22 @@ def _cached_catalog(cache_key: str) -> Optional[dict[str, dict[str, Any]]]:
|
||||
|
||||
|
||||
def _cache_catalog(
|
||||
cache_key: str, result: dict[str, dict[str, Any]]
|
||||
cache_key: str,
|
||||
result: dict[str, dict[str, Any]],
|
||||
ttl_seconds: Optional[float] = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Cache a catalog result, giving an empty one an expiry."""
|
||||
"""Cache a catalog result, giving an empty one an expiry.
|
||||
|
||||
*ttl_seconds* expires a non-empty result too. Only a catalog whose contents
|
||||
depend on server-side state the client cannot observe needs it — an org's
|
||||
model policy can change while a long-lived process holds the entry.
|
||||
"""
|
||||
_pricing_cache[cache_key] = result
|
||||
if result:
|
||||
_pricing_cache_retry_after.pop(cache_key, None)
|
||||
if ttl_seconds:
|
||||
_pricing_cache_retry_after[cache_key] = time.monotonic() + ttl_seconds
|
||||
else:
|
||||
_pricing_cache_retry_after.pop(cache_key, None)
|
||||
else:
|
||||
_pricing_cache_retry_after[cache_key] = (
|
||||
time.monotonic() + _FAILED_CATALOG_TTL_SECONDS
|
||||
@@ -2358,6 +2368,46 @@ def _cache_catalog(
|
||||
return result
|
||||
|
||||
|
||||
# NUL cannot appear in a URL, so this cannot collide with a real base URL.
|
||||
_PRICING_AUTH_KEY_PREFIX = "\x00auth:"
|
||||
|
||||
|
||||
def _pricing_auth_fingerprint(api_key: str | None) -> str:
|
||||
"""Key suffix identifying the credential a catalog was read with.
|
||||
|
||||
A governed endpoint answers each token with the catalog its org may reach,
|
||||
so two credentials cannot share an entry. blake2b for cache-key
|
||||
fingerprinting only, same rationale as :func:`_custom_endpoint_fingerprint`.
|
||||
"""
|
||||
if not api_key:
|
||||
return ""
|
||||
import hashlib
|
||||
|
||||
digest = hashlib.blake2b(api_key.encode("utf-8", errors="replace"), digest_size=8)
|
||||
return _PRICING_AUTH_KEY_PREFIX + digest.hexdigest()
|
||||
|
||||
|
||||
def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]:
|
||||
"""Pricing already cached for *base_url*, or ``{}``. Never fetches.
|
||||
|
||||
Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the fetchers
|
||||
key on, and prefers an authenticated catalog. Scans rather than rebuilding a
|
||||
key, because callers hold a base URL but no credential — newest first, and
|
||||
skipping expired entries, so a rotated credential does not keep answering
|
||||
from the catalog its predecessor read.
|
||||
"""
|
||||
root = (base_url or "").rstrip("/")
|
||||
if root.endswith("/v1"):
|
||||
root = root[:-3].rstrip("/")
|
||||
authed_prefix = root + _PRICING_AUTH_KEY_PREFIX
|
||||
for key in reversed(list(_pricing_cache)):
|
||||
if key.startswith(authed_prefix):
|
||||
cached = _cached_catalog(key)
|
||||
if cached:
|
||||
return cached
|
||||
return _cached_catalog(root) or {}
|
||||
|
||||
|
||||
def _format_price_per_mtok(per_token_str: str) -> str:
|
||||
"""Convert a per-token price string to a human-friendly $/Mtok string.
|
||||
|
||||
@@ -2491,10 +2541,12 @@ def fetch_models_with_pricing(
|
||||
*,
|
||||
force_refresh: bool = False,
|
||||
include_sale_original: bool = False,
|
||||
cache_ttl_seconds: Optional[float] = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Fetch ``/v1/models`` and return ``{model_id: {prompt, completion, ...}}``.
|
||||
|
||||
Results are cached per *base_url* so repeated calls are free.
|
||||
Results are cached per *base_url* and per credential, so repeated calls are
|
||||
free and one caller's catalog never answers another's read.
|
||||
Works with any OpenRouter-compatible endpoint (OpenRouter, Nous Portal).
|
||||
|
||||
When *include_sale_original* is true (Nous Portal only) and the gateway
|
||||
@@ -2505,13 +2557,14 @@ def fetch_models_with_pricing(
|
||||
``{prompt, completion}`` shape even if a response happens to nest
|
||||
``original``.
|
||||
"""
|
||||
cache_key = (base_url or "").rstrip("/")
|
||||
url_root = (base_url or "").rstrip("/")
|
||||
cache_key = url_root + _pricing_auth_fingerprint(api_key)
|
||||
if not force_refresh:
|
||||
cached = _cached_catalog(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
url = cache_key + "/v1/models"
|
||||
url = url_root + "/v1/models"
|
||||
headers: dict[str, str] = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": _HERMES_USER_AGENT,
|
||||
@@ -2562,7 +2615,7 @@ def fetch_models_with_pricing(
|
||||
entry["original"] = orig_entry
|
||||
result[mid] = entry
|
||||
|
||||
return _cache_catalog(cache_key, result)
|
||||
return _cache_catalog(cache_key, result, cache_ttl_seconds)
|
||||
|
||||
|
||||
def fetch_ai_gateway_pricing(
|
||||
@@ -2672,6 +2725,86 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]:
|
||||
return (api_key, base_url)
|
||||
|
||||
|
||||
def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]:
|
||||
"""The Nous model ids the caller's org may reach, or ``None`` to not filter.
|
||||
|
||||
The gateway omits policy-blocked rows from an authenticated
|
||||
``GET /v1/models``, so that response's keys are the reachable set.
|
||||
|
||||
``None`` means "leave the caller's list alone", for the three states that
|
||||
cannot support narrowing one: no policy (or a token too old to say), an
|
||||
anonymous read whose catalog is unfiltered, and an empty read, which is a
|
||||
fetch failure rather than an org that may reach nothing.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.nous_account import nous_policy_present
|
||||
|
||||
if nous_policy_present() is not True:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
api_key, base_url = _resolve_nous_pricing_credentials()
|
||||
if not api_key or not base_url:
|
||||
return None
|
||||
|
||||
# Same arguments as get_pricing_for_provider's nous branch, so a caller
|
||||
# asking for pricing too shares this entry instead of paying for a second
|
||||
# request.
|
||||
pricing = fetch_models_with_pricing(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
force_refresh=force_refresh,
|
||||
include_sale_original=True,
|
||||
cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS,
|
||||
)
|
||||
return set(pricing) or None
|
||||
|
||||
|
||||
# Past this size an allowed set reads as a whole catalog rather than an
|
||||
# allowlist, and is not worth showing in place of an empty picker.
|
||||
_NOUS_POLICY_APPEND_MAX = 64
|
||||
|
||||
# How long a Nous catalog stays trusted. Its contents depend on the org's
|
||||
# policy, which an admin can change at any time and the client cannot observe,
|
||||
# so a long-lived process must re-ask instead of holding the first answer for
|
||||
# its whole life. Other providers' catalogs carry no such state and keep the
|
||||
# default no-expiry caching.
|
||||
_NOUS_CATALOG_TTL_SECONDS = 300.0
|
||||
|
||||
|
||||
def restrict_to_nous_policy(
|
||||
model_ids: list[str],
|
||||
allowed: Optional[set[str]],
|
||||
*,
|
||||
rescue_empty: bool = False,
|
||||
) -> list[str]:
|
||||
"""*model_ids* narrowed to *allowed*, preserving the caller's order.
|
||||
|
||||
A ``None`` or empty *allowed* leaves the list untouched.
|
||||
|
||||
A ``:free`` sibling is kept when its base model is reachable, mirroring the
|
||||
gateway, which admits a row when any of its requestable ids passes. Prefer
|
||||
over-listing: that costs a 403 from the authoritative gate, while hiding a
|
||||
row the gate would serve is unrecoverable from the client.
|
||||
"""
|
||||
if not allowed:
|
||||
return list(model_ids)
|
||||
kept = [
|
||||
mid
|
||||
for mid in model_ids
|
||||
if mid in allowed or mid.split(":", 1)[0] in allowed
|
||||
]
|
||||
|
||||
# An allowlist can name only models the curated manifest lacks, leaving an
|
||||
# empty picker — worse than no filter, since the models the org may use are
|
||||
# the ones dropped. Opt-in per list: an already-empty list (a paid tier's
|
||||
# gated models) means "nothing to gate", not "nothing survived".
|
||||
if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX:
|
||||
return sorted(allowed)
|
||||
return kept
|
||||
|
||||
|
||||
def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]:
|
||||
"""Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita)."""
|
||||
normalized = normalize_provider(provider)
|
||||
@@ -2698,6 +2831,7 @@ def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> d
|
||||
force_refresh=force_refresh,
|
||||
# Sale chrome (pricing.original) is Nous Portal-only.
|
||||
include_sale_original=True,
|
||||
cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS,
|
||||
)
|
||||
return {}
|
||||
|
||||
|
||||
@@ -396,6 +396,50 @@ def get_nous_portal_account_info(
|
||||
)
|
||||
|
||||
|
||||
def nous_policy_present() -> Optional[bool]:
|
||||
"""Whether the caller's org carries a restrictive model/provider policy.
|
||||
|
||||
Reads the ``policy_present`` claim off the access token, so it costs no
|
||||
request; ``/api/oauth/account`` does not carry it. Stamped at mint time, so
|
||||
it goes stale until the next token refresh.
|
||||
|
||||
``None`` is unknown — an older mint or an unreadable claim — and must not be
|
||||
reported as the absence of a policy.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.auth import get_provider_auth_state, _decode_jwt_claims
|
||||
|
||||
state = get_provider_auth_state("nous") or {}
|
||||
access_token = state.get("access_token")
|
||||
if not isinstance(access_token, str) or not access_token.strip():
|
||||
return None
|
||||
claims = _decode_jwt_claims(access_token)
|
||||
if not claims:
|
||||
return None
|
||||
return _coerce_bool(claims.get("policy_present"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def nous_policy_notice(*, removed: bool) -> str:
|
||||
"""A one-line notice for a list the org's policy narrowed, else ``""``.
|
||||
|
||||
A blocked model is omitted rather than marked, which reads as "Hermes does
|
||||
not support this". This says which it is without enumerating the blocked
|
||||
set, which under an allowlist is most of the catalog.
|
||||
|
||||
*removed* is whether the filter actually dropped anything. The catalog read
|
||||
fails open — an anonymous or empty one narrows nothing — so the claim alone
|
||||
would label a full list as filtered.
|
||||
"""
|
||||
if not removed or nous_policy_present() is not True:
|
||||
return ""
|
||||
return (
|
||||
"Your organization restricts which models are available — "
|
||||
"models outside its policy are not listed."
|
||||
)
|
||||
|
||||
|
||||
def _fresh_account_info(
|
||||
*,
|
||||
state: dict[str, Any],
|
||||
|
||||
@@ -7543,8 +7543,10 @@ def get_recommended_default_model(provider: str = ""):
|
||||
get_curated_nous_model_ids,
|
||||
get_pricing_for_provider,
|
||||
check_nous_free_tier,
|
||||
nous_policy_allowed_ids,
|
||||
partition_nous_models_by_tier,
|
||||
pick_silent_default_model,
|
||||
restrict_to_nous_policy,
|
||||
union_with_portal_free_recommendations,
|
||||
union_with_portal_paid_recommendations,
|
||||
)
|
||||
@@ -7561,10 +7563,19 @@ def get_recommended_default_model(provider: str = ""):
|
||||
except Exception:
|
||||
portal_url = ""
|
||||
|
||||
# This endpoint picks the model a user lands on without choosing it,
|
||||
# so an unreachable one here is worse than in a picker. Narrow before
|
||||
# the tier split, so a rescued id still has to pass the free/paid
|
||||
# predicate.
|
||||
_policy_allowed = nous_policy_allowed_ids()
|
||||
|
||||
if free_tier:
|
||||
model_ids, pricing = union_with_portal_free_recommendations(
|
||||
model_ids, pricing, portal_url
|
||||
)
|
||||
model_ids = restrict_to_nous_policy(
|
||||
model_ids, _policy_allowed, rescue_empty=True,
|
||||
)
|
||||
model_ids, _unavailable = partition_nous_models_by_tier(
|
||||
model_ids, pricing, free_tier=True
|
||||
)
|
||||
@@ -7572,6 +7583,9 @@ def get_recommended_default_model(provider: str = ""):
|
||||
model_ids, pricing = union_with_portal_paid_recommendations(
|
||||
model_ids, pricing, portal_url
|
||||
)
|
||||
model_ids = restrict_to_nous_policy(
|
||||
model_ids, _policy_allowed, rescue_empty=True,
|
||||
)
|
||||
|
||||
model = pick_silent_default_model(model_ids, provider="nous")
|
||||
return {"provider": "nous", "model": model, "free_tier": bool(free_tier)}
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Narrowing the Nous model lists to an org's policy.
|
||||
|
||||
The gateway omits policy-blocked rows from an authenticated ``GET /v1/models``,
|
||||
so that response's keys are the reachable set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import hermes_cli.models as models_mod
|
||||
import hermes_cli.nous_account as account_mod
|
||||
from hermes_cli.models import (
|
||||
_NOUS_POLICY_APPEND_MAX,
|
||||
nous_policy_allowed_ids,
|
||||
restrict_to_nous_policy,
|
||||
)
|
||||
from hermes_cli.nous_account import nous_policy_present
|
||||
|
||||
|
||||
def _jwt(claims: dict) -> str:
|
||||
def seg(obj):
|
||||
raw = json.dumps(obj).encode()
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
||||
|
||||
return f"{seg({'alg': 'RS256'})}.{seg(claims)}.sig"
|
||||
|
||||
|
||||
class TestRestrictToNousPolicy:
|
||||
def test_none_leaves_the_list_untouched(self):
|
||||
ids = ["a/one", "b/two"]
|
||||
assert restrict_to_nous_policy(ids, None) == ids
|
||||
|
||||
def test_empty_set_leaves_the_list_untouched(self):
|
||||
"""Empty is a failed read, not an org that may reach nothing."""
|
||||
ids = ["a/one", "b/two"]
|
||||
assert restrict_to_nous_policy(ids, set()) == ids
|
||||
|
||||
def test_drops_ids_outside_the_policy(self):
|
||||
assert restrict_to_nous_policy(
|
||||
["a/one", "b/two", "c/three"], {"a/one", "c/three"}
|
||||
) == ["a/one", "c/three"]
|
||||
|
||||
def test_preserves_curated_order(self):
|
||||
curated = ["z/last", "a/first", "m/middle"]
|
||||
allowed = {"a/first", "m/middle", "z/last"}
|
||||
assert restrict_to_nous_policy(curated, allowed) == curated
|
||||
|
||||
def test_keeps_a_free_sibling_when_its_base_is_reachable(self):
|
||||
"""Portal free recommendations are ``:free`` ids."""
|
||||
assert restrict_to_nous_policy(["vendor/model:free"], {"vendor/model"}) == [
|
||||
"vendor/model:free"
|
||||
]
|
||||
|
||||
def test_keeps_a_free_id_listed_in_its_own_right(self):
|
||||
assert restrict_to_nous_policy(
|
||||
["vendor/model:free"], {"vendor/model:free"}
|
||||
) == ["vendor/model:free"]
|
||||
|
||||
def test_drops_a_free_sibling_whose_base_is_blocked(self):
|
||||
assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == []
|
||||
|
||||
|
||||
class TestNousPolicyAllowedIds:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cache(self):
|
||||
models_mod._pricing_cache.clear()
|
||||
models_mod._pricing_cache_retry_after.clear()
|
||||
yield
|
||||
models_mod._pricing_cache.clear()
|
||||
models_mod._pricing_cache_retry_after.clear()
|
||||
|
||||
def _patch(self, monkeypatch, *, policy_present, api_key="sk-test", pricing=None):
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
account_mod, "nous_policy_present", lambda: policy_present
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"_resolve_nous_pricing_credentials",
|
||||
lambda: (api_key, "https://inference.example.com"),
|
||||
)
|
||||
|
||||
def _fake_fetch(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return pricing if pricing is not None else {}
|
||||
|
||||
monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch)
|
||||
return calls
|
||||
|
||||
def test_returns_the_authenticated_catalog_keys(self, monkeypatch):
|
||||
calls = self._patch(
|
||||
monkeypatch,
|
||||
policy_present=True,
|
||||
pricing={"a/one": {}, "b/two": {}},
|
||||
)
|
||||
assert nous_policy_allowed_ids() == {"a/one", "b/two"}
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["api_key"] == "sk-test"
|
||||
|
||||
def test_declines_to_filter_an_unrestricted_org(self, monkeypatch):
|
||||
calls = self._patch(monkeypatch, policy_present=False, pricing={"a/one": {}})
|
||||
assert nous_policy_allowed_ids() is None
|
||||
assert calls == [], "an unrestricted org should not pay for the read"
|
||||
|
||||
def test_declines_to_filter_when_the_claim_is_unknown(self, monkeypatch):
|
||||
"""Absent is an older mint, not an unrestricted org."""
|
||||
calls = self._patch(monkeypatch, policy_present=None, pricing={"a/one": {}})
|
||||
assert nous_policy_allowed_ids() is None
|
||||
assert calls == []
|
||||
|
||||
def test_declines_to_filter_on_an_anonymous_read(self, monkeypatch):
|
||||
"""An anonymous read returns the full, unfiltered catalog."""
|
||||
self._patch(monkeypatch, policy_present=True, api_key="", pricing={"a/one": {}})
|
||||
assert nous_policy_allowed_ids() is None
|
||||
|
||||
def test_declines_to_filter_on_an_empty_read(self, monkeypatch):
|
||||
"""A failed fetch must not read as an org that may reach nothing."""
|
||||
self._patch(monkeypatch, policy_present=True, pricing={})
|
||||
assert nous_policy_allowed_ids() is None
|
||||
|
||||
|
||||
class TestNousPolicyPresent:
|
||||
def _patch_token(self, monkeypatch, token):
|
||||
import hermes_cli.auth as auth_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
auth_mod,
|
||||
"get_provider_auth_state",
|
||||
lambda _p: {"access_token": token} if token is not None else {},
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("claim,expected", [(True, True), (False, False)])
|
||||
def test_reads_the_claim(self, monkeypatch, claim, expected):
|
||||
self._patch_token(monkeypatch, _jwt({"policy_present": claim}))
|
||||
assert nous_policy_present() is expected
|
||||
|
||||
def test_absent_claim_is_unknown_not_false(self, monkeypatch):
|
||||
self._patch_token(monkeypatch, _jwt({"org_id": "org_1"}))
|
||||
assert nous_policy_present() is None
|
||||
|
||||
def test_non_boolean_claim_is_unknown(self, monkeypatch):
|
||||
self._patch_token(monkeypatch, _jwt({"policy_present": "yes"}))
|
||||
assert nous_policy_present() is None
|
||||
|
||||
def test_no_token_is_unknown(self, monkeypatch):
|
||||
self._patch_token(monkeypatch, None)
|
||||
assert nous_policy_present() is None
|
||||
|
||||
def test_undecodable_token_is_unknown(self, monkeypatch):
|
||||
self._patch_token(monkeypatch, "not-a-jwt")
|
||||
assert nous_policy_present() is None
|
||||
|
||||
|
||||
class TestNousPolicyNotice:
|
||||
|
||||
def _patch(self, monkeypatch, present):
|
||||
monkeypatch.setattr(account_mod, "nous_policy_present", lambda: present)
|
||||
|
||||
def test_shows_a_line_for_a_governed_org(self, monkeypatch):
|
||||
self._patch(monkeypatch, True)
|
||||
assert "restricts which models" in account_mod.nous_policy_notice(removed=True)
|
||||
|
||||
@pytest.mark.parametrize("present", [False, None])
|
||||
def test_silent_otherwise(self, monkeypatch, present):
|
||||
"""Absent is an older mint, not an unrestricted org."""
|
||||
self._patch(monkeypatch, present)
|
||||
assert account_mod.nous_policy_notice(removed=True) == ""
|
||||
|
||||
def test_silent_when_the_filter_removed_nothing(self, monkeypatch):
|
||||
"""The catalog read fails open, so a governed org can still end up with
|
||||
a full list — saying it was filtered would be false."""
|
||||
self._patch(monkeypatch, True)
|
||||
assert account_mod.nous_policy_notice(removed=False) == ""
|
||||
|
||||
def test_names_no_models(self, monkeypatch):
|
||||
"""The blocked set is most of the catalog under an allowlist."""
|
||||
self._patch(monkeypatch, True)
|
||||
notice = account_mod.nous_policy_notice(removed=True)
|
||||
assert "/" not in notice, f"looks like it names a model: {notice}"
|
||||
assert len(notice.splitlines()) == 1
|
||||
|
||||
|
||||
class TestAllowlistOutsideTheCuratedList:
|
||||
"""An allowlist can name only models the curated manifest lacks, which
|
||||
intersecting alone turns into an empty picker."""
|
||||
|
||||
def test_surfaces_an_allowed_model_the_curated_list_lacks(self):
|
||||
assert restrict_to_nous_policy(
|
||||
["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"}, rescue_empty=True
|
||||
) == ["amazon/nova-2-lite-v1"]
|
||||
|
||||
def test_does_not_append_when_the_curated_overlap_is_non_empty(self):
|
||||
kept = restrict_to_nous_policy(
|
||||
["z/curated", "a/curated"],
|
||||
{"z/curated", "a/curated", "new/model"},
|
||||
rescue_empty=True,
|
||||
)
|
||||
assert kept == ["z/curated", "a/curated"]
|
||||
|
||||
def test_does_not_rescue_a_catalog_sized_allowed_set(self):
|
||||
"""Past the cap the set reads as a whole catalog, and dumping it would
|
||||
bury the curated order the pickers show on purpose."""
|
||||
oversized = {f"cn/model-{i}" for i in range(_NOUS_POLICY_APPEND_MAX + 1)}
|
||||
assert restrict_to_nous_policy(["vendor/one"], oversized, rescue_empty=True) == []
|
||||
|
||||
|
||||
class TestRescueIsOptIn:
|
||||
"""The rescue is meaningful only for the list a user picks from."""
|
||||
|
||||
def test_rescue_only_when_asked(self):
|
||||
assert restrict_to_nous_policy([], {"a/one"}, rescue_empty=True) == ["a/one"]
|
||||
|
||||
def test_an_already_empty_unavailable_list_is_never_filled(self):
|
||||
"""A paid tier has no gated models, so this list is legitimately
|
||||
empty — not a filter result to rescue."""
|
||||
reachable = {f"cn/model-{i}" for i in range(42)}
|
||||
assert restrict_to_nous_policy([], reachable) == []
|
||||
|
||||
|
||||
class TestPolicyRunsBeforeTierSplit:
|
||||
"""A rescued id must still pass the free/paid predicate.
|
||||
|
||||
Rescuing after the tier split put paid models back into a free-tier user's
|
||||
selectable list, and the same id into both lists at once.
|
||||
"""
|
||||
|
||||
def test_a_rescued_paid_model_stays_unavailable_for_a_free_tier_user(self):
|
||||
from hermes_cli.models import partition_nous_models_by_tier
|
||||
|
||||
pricing = {
|
||||
"vendor/free": {"prompt": "0", "completion": "0"},
|
||||
"vendor/paid": {"prompt": "0.000002", "completion": "0.00001"},
|
||||
}
|
||||
narrowed = restrict_to_nous_policy(
|
||||
list(pricing), {"vendor/paid"}, rescue_empty=True
|
||||
)
|
||||
selectable, unavailable = partition_nous_models_by_tier(
|
||||
narrowed, pricing, free_tier=True
|
||||
)
|
||||
|
||||
assert selectable == []
|
||||
assert unavailable == ["vendor/paid"]
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Every Nous model list is narrowed to the org's policy before it is shown.
|
||||
|
||||
Four surfaces build their list from the curated manifest unioned with the
|
||||
Portal's ``recommended-models`` endpoint; neither source is authenticated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import pytest
|
||||
|
||||
import hermes_cli.models as models_mod
|
||||
|
||||
CURATED = ["vendor/allowed", "vendor/blocked"]
|
||||
ALLOWED = {"vendor/allowed"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def policy(monkeypatch):
|
||||
"""An org whose policy admits only ``vendor/allowed``."""
|
||||
monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: ALLOWED)
|
||||
return ALLOWED
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_policy(monkeypatch):
|
||||
"""An unrestricted org — lists must come through untouched."""
|
||||
monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: None)
|
||||
|
||||
|
||||
class TestLoginNous:
|
||||
|
||||
def _run(self, monkeypatch, tmp_path):
|
||||
import hermes_cli.auth as auth_mod
|
||||
import hermes_cli.nous_subscription as ns
|
||||
|
||||
seen: dict = {}
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.setattr(
|
||||
auth_mod,
|
||||
"_nous_device_code_login",
|
||||
lambda **_k: {
|
||||
"access_token": "tok",
|
||||
"agent_key": "key",
|
||||
"inference_base_url": "https://inference.example.com",
|
||||
"portal_base_url": "https://portal.example.com",
|
||||
"refresh_token": "r",
|
||||
"token_expires_at": 9999999999,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED))
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {})
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"union_with_portal_paid_recommendations",
|
||||
lambda ids, pricing, _portal: (list(ids), pricing),
|
||||
)
|
||||
monkeypatch.setattr(ns, "prompt_enable_tool_gateway", lambda _c: None)
|
||||
|
||||
def _capture(model_ids, **kwargs):
|
||||
seen["model_ids"] = list(model_ids)
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(auth_mod, "_prompt_model_selection", _capture)
|
||||
|
||||
args = argparse.Namespace(
|
||||
portal_url=None, inference_url=None, client_id=None, scope=None,
|
||||
no_browser=True, timeout=15.0, ca_bundle=None, insecure=False,
|
||||
)
|
||||
auth_mod._login_nous(args, auth_mod.PROVIDER_REGISTRY["nous"])
|
||||
return seen
|
||||
|
||||
def test_hidden_model_is_not_offered(self, monkeypatch, tmp_path, policy):
|
||||
assert self._run(monkeypatch, tmp_path).get("model_ids") == ["vendor/allowed"]
|
||||
|
||||
def test_unrestricted_org_sees_the_full_curated_list(
|
||||
self, monkeypatch, tmp_path, no_policy
|
||||
):
|
||||
assert self._run(monkeypatch, tmp_path).get("model_ids") == CURATED
|
||||
|
||||
|
||||
class TestModelSwitchPicker:
|
||||
"""The ``/model`` picker's nous branch (``list_authenticated_providers``)."""
|
||||
|
||||
def _rows(self, monkeypatch):
|
||||
import hermes_cli.auth as auth_mod
|
||||
import hermes_cli.model_switch as ms
|
||||
|
||||
monkeypatch.setattr(
|
||||
auth_mod,
|
||||
"_load_auth_store",
|
||||
lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}},
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED))
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {})
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"union_with_portal_paid_recommendations",
|
||||
lambda ids, pricing, _portal: (list(ids), pricing),
|
||||
)
|
||||
rows = ms.list_authenticated_providers(max_models=10)
|
||||
return next((r for r in rows if r["slug"] == "nous"), None)
|
||||
|
||||
def test_hidden_model_is_filtered(self, monkeypatch, policy):
|
||||
row = self._rows(monkeypatch)
|
||||
assert row is not None, "nous row should be listed"
|
||||
assert "vendor/blocked" not in row["models"]
|
||||
assert "vendor/allowed" in row["models"]
|
||||
|
||||
def test_unrestricted_org_keeps_both(self, monkeypatch, no_policy):
|
||||
row = self._rows(monkeypatch)
|
||||
assert row is not None
|
||||
assert set(CURATED) <= set(row["models"])
|
||||
|
||||
def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy):
|
||||
"""The filter sits outside the try wrapping the Portal union."""
|
||||
|
||||
def _boom(_p):
|
||||
raise RuntimeError("portal down")
|
||||
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", _boom)
|
||||
row = self._rows(monkeypatch)
|
||||
assert row is not None
|
||||
assert "vendor/blocked" not in row["models"]
|
||||
|
||||
|
||||
class TestRecommendedDefaultEndpoint:
|
||||
"""This endpoint picks a model the user never sees chosen."""
|
||||
|
||||
def _call(self, monkeypatch):
|
||||
import hermes_cli.auth as auth_mod
|
||||
from hermes_cli.web_server import get_recommended_default_model
|
||||
|
||||
# Blocked first, so an unfiltered list would make it the silent
|
||||
# default — otherwise this passes whether or not the filter runs.
|
||||
monkeypatch.setattr(
|
||||
models_mod, "get_curated_nous_model_ids",
|
||||
lambda: ["vendor/blocked", "vendor/allowed"],
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {})
|
||||
monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None)
|
||||
monkeypatch.setattr(
|
||||
models_mod,
|
||||
"union_with_portal_paid_recommendations",
|
||||
lambda ids, pricing, _portal: (list(ids), pricing),
|
||||
)
|
||||
monkeypatch.setattr(auth_mod, "get_provider_auth_state", lambda _p: {})
|
||||
return get_recommended_default_model(provider="nous")
|
||||
|
||||
def test_hidden_model_is_never_the_silent_default(self, monkeypatch, policy):
|
||||
assert self._call(monkeypatch)["model"] == "vendor/allowed"
|
||||
|
||||
def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy):
|
||||
assert self._call(monkeypatch)["model"] == "vendor/blocked"
|
||||
|
||||
|
||||
class TestAuxiliaryFastModel:
|
||||
"""``_fast_model_from_catalog`` uses the catalog's keys as a source of ids."""
|
||||
|
||||
def _pick(self, monkeypatch, *, catalog):
|
||||
import agent.auxiliary_client as aux
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
def _fake_fetch(*, api_key=None, base_url="", timeout=8.0, **_k):
|
||||
seen["api_key"] = api_key
|
||||
return {mid: {} for mid in catalog}
|
||||
|
||||
monkeypatch.setattr(
|
||||
models_mod, "_resolve_nous_pricing_credentials",
|
||||
lambda: ("sk-nous", "https://inference.example.com"),
|
||||
)
|
||||
monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch)
|
||||
picked = aux._fast_model_from_catalog("nous")
|
||||
return picked, seen
|
||||
|
||||
def test_reads_the_catalog_with_nous_oauth_credentials(self, monkeypatch, no_policy):
|
||||
"""The api-key resolver raises for OAuth providers."""
|
||||
_, seen = self._pick(monkeypatch, catalog=["vendor/haiku-fast"])
|
||||
assert seen["api_key"] == "sk-nous"
|
||||
|
||||
def test_hidden_model_is_not_selected(self, monkeypatch, policy):
|
||||
import agent.auxiliary_client as aux
|
||||
|
||||
monkeypatch.setattr(
|
||||
models_mod, "nous_policy_allowed_ids", lambda **_k: {"vendor/allowed"}
|
||||
)
|
||||
monkeypatch.setattr(aux, "_FAST_MODEL_FAMILIES", ("vendor/",))
|
||||
monkeypatch.setattr(aux, "_FAST_MODEL_EXCLUDE", ())
|
||||
picked, _ = self._pick(
|
||||
monkeypatch, catalog=["vendor/blocked", "vendor/allowed"]
|
||||
)
|
||||
assert picked == "vendor/allowed"
|
||||
|
||||
|
||||
class TestNousPrefetch:
|
||||
"""The nous disk-cache entry is write-only, so prefetching it is a round
|
||||
trip for nothing."""
|
||||
|
||||
def test_nous_is_not_collected_for_prefetch(self, monkeypatch):
|
||||
import hermes_cli.auth as auth_mod
|
||||
import hermes_cli.model_switch as ms
|
||||
|
||||
monkeypatch.setattr(
|
||||
auth_mod, "_load_auth_store",
|
||||
lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}},
|
||||
)
|
||||
slugs = ms._collect_authed_provider_slugs({}, {"nous": list(CURATED)}, [])
|
||||
assert "nous" not in slugs
|
||||
|
||||
|
||||
class TestPolicyNoticeIsShown:
|
||||
|
||||
def test_login_prints_it(self, monkeypatch, tmp_path, policy, capsys):
|
||||
import hermes_cli.nous_account as account_mod
|
||||
|
||||
monkeypatch.setattr(account_mod, "nous_policy_present", lambda: True)
|
||||
TestLoginNous()._run(monkeypatch, tmp_path)
|
||||
assert "restricts which models" in capsys.readouterr().out
|
||||
|
||||
def test_login_silent_for_an_ungoverned_org(
|
||||
self, monkeypatch, tmp_path, no_policy, capsys
|
||||
):
|
||||
import hermes_cli.nous_account as account_mod
|
||||
|
||||
monkeypatch.setattr(account_mod, "nous_policy_present", lambda: False)
|
||||
TestLoginNous()._run(monkeypatch, tmp_path)
|
||||
assert "restricts which models" not in capsys.readouterr().out
|
||||
|
||||
|
||||
class TestAuxFallbackRespectsPolicy:
|
||||
"""Steps 2-4 of the aux ladder are policy-blind: `resolve_aux_model` queries
|
||||
a public recommendation and the rest are hardcoded."""
|
||||
|
||||
def _patch(self, monkeypatch, *, allowed, recommended):
|
||||
import agent.auxiliary_client as aux
|
||||
import providers
|
||||
|
||||
monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: allowed)
|
||||
monkeypatch.setattr(
|
||||
models_mod, "_resolve_nous_pricing_credentials",
|
||||
lambda: ("sk", "https://inference.example.com"),
|
||||
)
|
||||
# No fast-family match, so the catalog step yields nothing.
|
||||
monkeypatch.setattr(
|
||||
models_mod, "fetch_models_with_pricing",
|
||||
lambda **_k: {"vendor/allowed-large": {}},
|
||||
)
|
||||
|
||||
class _Profile:
|
||||
default_aux_model = ""
|
||||
|
||||
def resolve_aux_model(self, **_k):
|
||||
return recommended
|
||||
|
||||
monkeypatch.setattr(providers, "get_provider_profile", lambda _p: _Profile())
|
||||
return aux
|
||||
|
||||
def test_blocked_recommendation_is_not_used(self, monkeypatch):
|
||||
aux = self._patch(
|
||||
monkeypatch, allowed={"vendor/allowed-large"},
|
||||
recommended="vendor/blocked-haiku",
|
||||
)
|
||||
assert aux._get_aux_model_for_provider("nous", prefer_fast=True) == ""
|
||||
|
||||
def test_allowed_recommendation_still_used(self, monkeypatch):
|
||||
aux = self._patch(
|
||||
monkeypatch, allowed={"vendor/allowed-large", "vendor/ok-haiku"},
|
||||
recommended="vendor/ok-haiku",
|
||||
)
|
||||
assert (
|
||||
aux._get_aux_model_for_provider("nous", prefer_fast=True)
|
||||
== "vendor/ok-haiku"
|
||||
)
|
||||
|
||||
def test_ungoverned_org_is_unaffected(self, monkeypatch):
|
||||
aux = self._patch(
|
||||
monkeypatch, allowed=None, recommended="vendor/anything"
|
||||
)
|
||||
assert (
|
||||
aux._get_aux_model_for_provider("nous", prefer_fast=True)
|
||||
== "vendor/anything"
|
||||
)
|
||||
|
||||
|
||||
def test_titling_seeds_the_shared_catalog_entry_like_the_pickers(monkeypatch):
|
||||
"""The aux catalog read shares the pickers' cache entry, so seeding it
|
||||
without the Nous-only arguments costs the picker its sale chrome and leaves
|
||||
the policy catalog with no expiry."""
|
||||
import agent.auxiliary_client as aux
|
||||
|
||||
monkeypatch.setattr(
|
||||
models_mod, "_resolve_nous_pricing_credentials",
|
||||
lambda: ("tok", "https://inference.example.com"),
|
||||
)
|
||||
seen: dict = {}
|
||||
|
||||
def _fake_fetch(**kwargs):
|
||||
seen.update(kwargs)
|
||||
return {"vendor/haiku": {}}
|
||||
|
||||
monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch)
|
||||
aux._fast_model_from_catalog("nous")
|
||||
|
||||
assert seen.get("include_sale_original") is True
|
||||
assert seen.get("cache_ttl_seconds") == models_mod._NOUS_CATALOG_TTL_SECONDS
|
||||
@@ -0,0 +1,210 @@
|
||||
"""``_pricing_cache`` keys on the credential, not just the base URL.
|
||||
|
||||
Nous ``/v1/models`` answers each caller with the catalog their org may reach,
|
||||
so an anonymous read, and two different tokens, must not share a cache entry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import hermes_cli.models as models_mod
|
||||
from hermes_cli.models import fetch_models_with_pricing, peek_cached_pricing
|
||||
|
||||
BASE = "https://inference-api.example.com"
|
||||
|
||||
# What the endpoint serves anonymously vs. to a policy-restricted caller.
|
||||
_FULL = ["vendor/allowed", "vendor/blocked"]
|
||||
_FILTERED = ["vendor/allowed"]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_pricing_cache():
|
||||
models_mod._pricing_cache.clear()
|
||||
models_mod._pricing_cache_retry_after.clear()
|
||||
yield
|
||||
models_mod._pricing_cache.clear()
|
||||
models_mod._pricing_cache_retry_after.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def catalog(monkeypatch):
|
||||
"""Serve the filtered catalog to an authenticated read, the full one to an
|
||||
anonymous read, and record every request."""
|
||||
requests: list[str | None] = []
|
||||
|
||||
def _fake_urlopen(req, timeout=8.0):
|
||||
auth = req.get_header("Authorization")
|
||||
requests.append(auth)
|
||||
ids = _FILTERED if auth else _FULL
|
||||
payload = {
|
||||
"data": [
|
||||
{"id": mid, "pricing": {"prompt": "0.000002", "completion": "0.00001"}}
|
||||
for mid in ids
|
||||
]
|
||||
}
|
||||
resp = MagicMock()
|
||||
resp.read.return_value = json.dumps(payload).encode()
|
||||
resp.__enter__ = lambda self: self
|
||||
resp.__exit__ = lambda *a: False
|
||||
return resp
|
||||
|
||||
monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen)
|
||||
return requests
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def per_org_catalog(monkeypatch):
|
||||
"""Serve each token the catalog its own org may reach."""
|
||||
requests: list[str | None] = []
|
||||
|
||||
def _fake_urlopen(req, timeout=8.0):
|
||||
auth = req.get_header("Authorization")
|
||||
requests.append(auth)
|
||||
org = "a" if auth == "Bearer tok-a" else "b"
|
||||
payload = {
|
||||
"data": [
|
||||
{
|
||||
"id": f"org-{org}/only",
|
||||
"pricing": {"prompt": "0.000002", "completion": "0.00001"},
|
||||
}
|
||||
]
|
||||
}
|
||||
resp = MagicMock()
|
||||
resp.read.return_value = json.dumps(payload).encode()
|
||||
resp.__enter__ = lambda self: self
|
||||
resp.__exit__ = lambda *a: False
|
||||
return resp
|
||||
|
||||
monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen)
|
||||
return requests
|
||||
|
||||
|
||||
def test_one_token_does_not_receive_another_tokens_catalog(per_org_catalog):
|
||||
"""Two orgs in one process — a long-lived gateway or desktop backend after
|
||||
a profile switch or re-login."""
|
||||
a = fetch_models_with_pricing(api_key="tok-a", base_url=BASE)
|
||||
b = fetch_models_with_pricing(api_key="tok-b", base_url=BASE)
|
||||
|
||||
assert list(a) == ["org-a/only"]
|
||||
assert list(b) == ["org-b/only"], "token B was handed token A's catalog"
|
||||
assert len(per_org_catalog) == 2, "token B must reach the network"
|
||||
|
||||
|
||||
def test_credential_value_does_not_appear_in_the_cache_key():
|
||||
"""Guards against keying on the raw token."""
|
||||
assert "sk-super-secret" not in models_mod._pricing_auth_fingerprint("sk-super-secret")
|
||||
|
||||
|
||||
def test_anonymous_and_authenticated_reads_are_separate(catalog):
|
||||
"""Also pins the header: anonymous must send none."""
|
||||
anon = fetch_models_with_pricing(api_key="", base_url=BASE)
|
||||
authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE)
|
||||
|
||||
assert sorted(anon) == sorted(_FULL)
|
||||
assert sorted(authed) == sorted(_FILTERED)
|
||||
assert catalog == [None, "Bearer sk-test"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("api_key", ["sk-test", ""])
|
||||
def test_repeated_read_still_hits_the_cache(catalog, api_key):
|
||||
"""Widening the key must not cost the caching it was there for."""
|
||||
first = fetch_models_with_pricing(api_key=api_key, base_url=BASE)
|
||||
second = fetch_models_with_pricing(api_key=api_key, base_url=BASE)
|
||||
|
||||
assert first == second
|
||||
assert len(catalog) == 1, "second read should be served from cache"
|
||||
|
||||
|
||||
def test_force_refresh_replaces_only_its_own_entry(catalog):
|
||||
"""A forced authenticated re-read must leave the anonymous entry intact."""
|
||||
fetch_models_with_pricing(api_key="", base_url=BASE)
|
||||
fetch_models_with_pricing(api_key="sk-test", base_url=BASE)
|
||||
fetch_models_with_pricing(api_key="sk-test", base_url=BASE, force_refresh=True)
|
||||
|
||||
assert len(catalog) == 3
|
||||
anon = fetch_models_with_pricing(api_key="", base_url=BASE)
|
||||
assert sorted(anon) == sorted(_FULL)
|
||||
assert len(catalog) == 3, "the anonymous entry should have survived"
|
||||
|
||||
|
||||
class TestPeekCachedPricing:
|
||||
def test_returns_empty_when_nothing_cached(self):
|
||||
assert peek_cached_pricing(BASE) == {}
|
||||
|
||||
def test_accepts_a_v1_suffixed_url(self, catalog):
|
||||
"""The agent holds a /v1-suffixed base URL; fetchers key on the root."""
|
||||
fetch_models_with_pricing(api_key="sk-test", base_url=BASE)
|
||||
assert sorted(peek_cached_pricing(BASE + "/v1")) == sorted(_FILTERED)
|
||||
|
||||
def test_prefers_the_authenticated_catalog(self, catalog):
|
||||
fetch_models_with_pricing(api_key="", base_url=BASE)
|
||||
fetch_models_with_pricing(api_key="sk-test", base_url=BASE)
|
||||
assert sorted(peek_cached_pricing(BASE)) == sorted(_FILTERED)
|
||||
|
||||
def test_falls_back_to_the_anonymous_catalog(self, catalog):
|
||||
fetch_models_with_pricing(api_key="", base_url=BASE)
|
||||
assert sorted(peek_cached_pricing(BASE)) == sorted(_FULL)
|
||||
|
||||
def test_never_fetches(self, catalog):
|
||||
peek_cached_pricing(BASE)
|
||||
assert catalog == []
|
||||
|
||||
|
||||
class TestNousCatalogExpiry:
|
||||
"""A Nous catalog reflects the org's policy, which an admin can change while
|
||||
a long-lived process holds the entry."""
|
||||
|
||||
def test_entry_expires_so_a_policy_change_is_picked_up(self, catalog, monkeypatch):
|
||||
from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS
|
||||
|
||||
fetch_models_with_pricing(
|
||||
api_key="sk-test", base_url=BASE,
|
||||
cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS,
|
||||
)
|
||||
assert len(catalog) == 1
|
||||
|
||||
now = models_mod.time.monotonic()
|
||||
monkeypatch.setattr(
|
||||
models_mod.time, "monotonic",
|
||||
lambda: now + _NOUS_CATALOG_TTL_SECONDS + 1,
|
||||
)
|
||||
fetch_models_with_pricing(
|
||||
api_key="sk-test", base_url=BASE,
|
||||
cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS,
|
||||
)
|
||||
assert len(catalog) == 2, "expired entry should be re-read"
|
||||
|
||||
def test_no_ttl_keeps_the_entry_indefinitely(self, catalog, monkeypatch):
|
||||
"""Other providers' catalogs carry no policy and must not start
|
||||
re-fetching."""
|
||||
fetch_models_with_pricing(api_key="sk-test", base_url=BASE)
|
||||
now = models_mod.time.monotonic()
|
||||
monkeypatch.setattr(models_mod.time, "monotonic", lambda: now + 86_400)
|
||||
fetch_models_with_pricing(api_key="sk-test", base_url=BASE)
|
||||
assert len(catalog) == 1
|
||||
|
||||
def test_peek_prefers_the_newest_credential(self, per_org_catalog):
|
||||
"""After a rotation the older entry is still resident and, being
|
||||
insertion-ordered, comes first."""
|
||||
fetch_models_with_pricing(api_key="tok-a", base_url=BASE, cache_ttl_seconds=300)
|
||||
fetch_models_with_pricing(api_key="tok-b", base_url=BASE, cache_ttl_seconds=300)
|
||||
assert list(peek_cached_pricing(BASE)) == ["org-b/only"]
|
||||
|
||||
def test_peek_skips_an_expired_entry(self, catalog, monkeypatch):
|
||||
"""Reading _pricing_cache directly walked straight past the TTL."""
|
||||
from hermes_cli.models import _NOUS_CATALOG_TTL_SECONDS
|
||||
|
||||
fetch_models_with_pricing(
|
||||
api_key="sk-test", base_url=BASE,
|
||||
cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS,
|
||||
)
|
||||
now = models_mod.time.monotonic()
|
||||
monkeypatch.setattr(
|
||||
models_mod.time, "monotonic",
|
||||
lambda: now + _NOUS_CATALOG_TTL_SECONDS + 1,
|
||||
)
|
||||
assert peek_cached_pricing(BASE) == {}
|
||||
Reference in New Issue
Block a user