diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 51643ed5ca..ba240171a9 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -2451,66 +2451,114 @@ _CODEX_OAUTH_STALE_ADVERTISED_CTX = 272_000 # large window. Never sent on the wire. CODEX_CONTEXT_VARIANT_SUFFIX = "-900k" +# The ONLY base slugs eligible for a ``-900k`` variant: routable, +# live-verified models. gpt-5.6 family-prefix matching is deliberately NOT +# used here — it would synthesize dead variants for ``-pro`` slugs (the +# Codex backend 400s them) and accept arbitrary future descendants that +# were never probed. Dated snapshots of the routable 5.6 bases are allowed +# via _CODEX_900K_SNAPSHOT_RE. +_CODEX_900K_ELIGIBLE_BASES = frozenset({ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-5.4", # exact; gpt-5.4-mini enforces 272K + "gpt-daybreak-blue-latest", # verified Sol alias +}) +_CODEX_900K_SNAPSHOT_BASES = ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") +_CODEX_900K_SNAPSHOT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def _bare_codex_slug(model: Optional[str]) -> str: + """Lowercased slug with any ``vendor/`` namespace removed. + + Display/auxiliary callers pass ids like ``openai/gpt-5.6-sol-900k``; + the main-agent path normalizes the namespace away earlier, but this + resolver must accept both shapes (#92797 review). + """ + return (model or "").strip().lower().rsplit("/", 1)[-1] + + +def is_codex_900k_base(model: Optional[str]) -> bool: + """True when *model* (a BASE slug, no suffix) may carry a ``-900k`` variant. + + Single source of truth for the eligibility check — used by picker + synthesis, context resolution, `/model` validation, and wire stripping + so the four sites can never drift apart. + """ + slug = _bare_codex_slug(model) + if not slug or slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX): + return False + if slug in _CODEX_900K_ELIGIBLE_BASES: + return True + # Dated snapshots of the routable 5.6 bases (gpt-5.6-sol-2026-07-09). + for base in _CODEX_900K_SNAPSHOT_BASES: + if slug.startswith(base + "-") and _CODEX_900K_SNAPSHOT_RE.match( + slug[len(base) + 1:] + ): + return True + return False + def is_codex_context_variant(model: Optional[str]) -> bool: - """True when the model id carries the Hermes ``-900k`` opt-in suffix.""" - return (model or "").strip().lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX) + """True when the model id is a VALID ``-900k`` opt-in variant. + + Requires both the suffix and an eligible base — ``gpt-5.5-900k`` is not + a variant, it's an invalid alias. + """ + slug = _bare_codex_slug(model) + if not slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX): + return False + return is_codex_900k_base(slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]) def strip_codex_context_variant_suffix(model: Optional[str]) -> str: - """Return the wire-safe slug with any ``-900k`` opt-in suffix removed. + """Return the wire-safe slug with a VALID ``-900k`` suffix removed. The suffix is a Hermes picker alias (``gpt-5.6-sol-900k``); the Codex - backend only knows the base slug. Case-insensitive; non-variant ids are - returned unchanged. + backend only knows the base slug. Stripping is conditional on base + eligibility: an ineligible alias like ``gpt-5.5-900k`` is returned + unchanged so it fails honestly at the API instead of silently running + as a different model. Case-insensitive; preserves any ``vendor/`` + namespace prefix. """ raw = (model or "").strip() - if raw.lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX): - return raw[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] + if not raw.lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX): + return raw + base = raw[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] + if is_codex_900k_base(base): + return base return raw def has_codex_context_variant(model_bare: str) -> bool: - """True when a Codex BASE slug has a live-verified ``-900k`` variant. + """True when a Codex BASE slug should get a synthetic ``-900k`` entry. - Used by the model pickers to decide which base slugs get a synthetic - ``-900k`` entry. Exact table first, then family prefixes — - mirrors ``_verified_codex_ctx_for_slug`` minus the suffix requirement. + Thin alias over :func:`is_codex_900k_base` kept for the picker call + sites' readability. """ - slug = (model_bare or "").strip().lower() - if not slug or slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX): - return False - if slug in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT: - return True - for key in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES: - if slug == key or slug.startswith(key + "-") or slug.startswith(key + "."): - return True - return False + return is_codex_900k_base(model_bare) def _verified_codex_ctx_for_slug(model_bare: str) -> Optional[int]: """Return the live-verified Codex cap for an OPTED-IN slug, or ``None``. - The large window is opt-in: only ``-900k``-suffixed picker variants + The large window is opt-in: only VALID ``-900k`` picker variants (e.g. ``gpt-5.6-sol-900k``) resolve to the verified cap. Base slugs keep the advertised 272K so the cheaper default limit applies unless - the user explicitly selects the large-context variant. - - After stripping the suffix: exact slugs first, then family prefixes - (````, ``-``, ``.``) so dated snapshots of a verified - family inherit the bump. + the user explicitly selects the large-context variant; ineligible + aliases (``gpt-5.5-900k``) never resolve here. """ - slug = (model_bare or "").strip().lower() + slug = _bare_codex_slug(model_bare) if not slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX): return None - slug = slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] - if not slug: + base = slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] + if not is_codex_900k_base(base): return None - exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(slug) + exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(base) if exact is not None: return exact for key, ctx in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES.items(): - if slug == key or slug.startswith(key + "-") or slug.startswith(key + "."): + if base == key or base.startswith(key + "-") or base.startswith(key + "."): return ctx return None @@ -2660,8 +2708,11 @@ def _resolve_codex_oauth_context_length_with_source( return ctx, source # ``-900k`` variants are Hermes picker aliases — the Codex catalog only - # knows the base slug, so resolve against the stripped id. - lookup_bare = strip_codex_context_variant_suffix(model_bare) + # knows the base slug, so resolve against the stripped id. Also drop any + # ``vendor/`` namespace (``openai/gpt-5.6-sol-900k``): the main-agent + # path normalizes it away before reaching here, but display/auxiliary + # callers pass it through (#92797 review). + lookup_bare = _bare_codex_slug(strip_codex_context_variant_suffix(model_bare)) if access_token: live, fresh_probe = _fetch_codex_oauth_context_lengths_with_source(access_token) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index f638be4419..b939780b4e 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -6558,6 +6558,45 @@ def validate_requested_model( catalog_models = provider_model_ids(normalized) except Exception: catalog_models = [] + # Ineligible ``-900k`` aliases (e.g. `gpt-5.5-900k`) must be rejected + # BEFORE the hidden-slug soft-accept below: the suffix is a Hermes + # picker convention, so an unknown `*-900k` name can never be a real + # hidden provider slug — soft-accepting one silently runs at 272K on + # a different model than the user thinks (#92797 review). + if normalized == "openai-codex": + from agent.model_metadata import ( + CODEX_CONTEXT_VARIANT_SUFFIX, + is_codex_context_variant, + ) + _req_lower = requested_for_lookup.strip().lower() + if ( + _req_lower.endswith(CODEX_CONTEXT_VARIANT_SUFFIX) + and requested_for_lookup not in set(catalog_models) + ): + if is_codex_context_variant(requested_for_lookup): + # Valid variant that a stale catalog hasn't synthesized + # yet. Accept it directly — falling through would let the + # typo auto-corrector "fix" it to the base slug and + # silently drop the large-context opt-in. + return { + "accepted": True, + "persist": True, + "recognized": True, + "message": None, + } + _base_guess = requested_for_lookup[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] + return { + "accepted": False, + "persist": False, + "recognized": False, + "message": ( + f"`{requested}` is not a valid large-context variant — " + f"`{_base_guess}` enforces the standard 272K window on " + f"Codex, so no `-900k` option exists for it. Pick the " + f"base model, or a verified variant from the `/model` " + f"picker (e.g. `gpt-5.6-sol-900k`)." + ), + } if catalog_models: if requested_for_lookup in set(catalog_models): return { diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index a0f3b1fb6b..3a91b968bc 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -666,6 +666,64 @@ class TestCodexOAuthContextLength: ) assert ctx == 272_000 + # Table-driven eligibility contract (#92797 review): one predicate + # (is_codex_900k_base) drives picker synthesis, context resolution, + # validation, and wire stripping — this table pins all of them. + # (model_id, is_valid_variant, expected_ctx, expected_wire_model) + _900K_TABLE = [ + ("gpt-5.6-sol-900k", True, 900_000, "gpt-5.6-sol"), + ("gpt-5.6-terra-900k", True, 900_000, "gpt-5.6-terra"), + ("gpt-5.6-luna-900k", True, 900_000, "gpt-5.6-luna"), + ("gpt-5.4-900k", True, 900_000, "gpt-5.4"), + ("gpt-daybreak-blue-latest-900k", True, 900_000, "gpt-daybreak-blue-latest"), + # dated snapshot of a routable 5.6 base + ("gpt-5.6-sol-2026-07-09-900k", True, 900_000, "gpt-5.6-sol-2026-07-09"), + # vendor-namespaced variant (display/aux callers) resolves too + ("openai/gpt-5.6-sol-900k", True, 900_000, "openai/gpt-5.6-sol"), + # -pro slugs are not routable on Codex OAuth: never a valid variant, + # never stripped (fails honestly at the API instead) + ("gpt-5.6-sol-pro-900k", False, 272_000, "gpt-5.6-sol-pro-900k"), + # genuine 272K enforcers get no variant + ("gpt-5.5-900k", False, 272_000, "gpt-5.5-900k"), + ("gpt-5.4-mini-900k", False, 272_000, "gpt-5.4-mini-900k"), + # arbitrary future family descendants are not auto-eligible + ("gpt-5.6-nova-900k", False, 272_000, "gpt-5.6-nova-900k"), + ] + + @pytest.mark.parametrize("model_id,valid,expected_ctx,wire", _900K_TABLE) + def test_900k_eligibility_table(self, model_id, valid, expected_ctx, wire): + from agent.model_metadata import ( + get_model_context_length, + is_codex_context_variant, + strip_codex_context_variant_suffix, + ) + + assert is_codex_context_variant(model_id) is valid + assert strip_codex_context_variant_suffix(model_id) == wire + + bare = model_id.rsplit("/", 1)[-1] + catalog_slug = strip_codex_context_variant_suffix(bare) + if catalog_slug.endswith("-900k"): + # invalid alias — catalog advertises the underlying family slug + catalog_slug = catalog_slug[: -len("-900k")] + fake_response = MagicMock() + fake_response.status_code = 200 + fake_response.json.return_value = { + "models": [{"slug": catalog_slug, "context_window": 272_000}] + } + import agent.model_metadata as mm + mm._codex_oauth_context_cache = {} + with patch("agent.model_metadata.requests.get", return_value=fake_response), \ + patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata.save_context_length"): + ctx = get_model_context_length( + model=model_id, + base_url="https://chatgpt.com/backend-api/codex", + api_key="fake-token", + provider="openai-codex", + ) + assert ctx == expected_ctx + diff --git a/tests/hermes_cli/test_codex_models.py b/tests/hermes_cli/test_codex_models.py index be27a4e62a..50a5d51d64 100644 --- a/tests/hermes_cli/test_codex_models.py +++ b/tests/hermes_cli/test_codex_models.py @@ -52,6 +52,18 @@ def test_picker_synthesizes_900k_variants_for_verified_slugs(): assert "gpt-5.3-codex-900k" not in model_ids +def test_picker_never_synthesizes_900k_for_pro_or_unknown_slugs(): + """Eligibility is an exact predicate, not a family-prefix match: + ``-pro`` slugs are not routable on Codex OAuth (backend 400s them) and + unknown future descendants were never probed — neither may gain a + synthetic ``-900k`` entry (#92797 review).""" + from hermes_cli.codex_models import _finalize_codex_models + + out = _finalize_codex_models(["gpt-5.6-sol-pro", "gpt-5.6-nova"]) + assert "gpt-5.6-sol-pro-900k" not in out + assert "gpt-5.6-nova-900k" not in out + + def test_setup_wizard_codex_import_resolves(): diff --git a/tests/hermes_cli/test_model_validation.py b/tests/hermes_cli/test_model_validation.py index aa3bc4c3a2..966eae73e7 100644 --- a/tests/hermes_cli/test_model_validation.py +++ b/tests/hermes_cli/test_model_validation.py @@ -1,5 +1,6 @@ """Tests for provider-aware `/model` validation in hermes_cli.models.""" +import pytest from unittest.mock import MagicMock, patch from hermes_cli.models import ( @@ -504,6 +505,35 @@ class TestValidateCodexAutoCorrection: assert result["message"] is None +class TestValidateCodex900kVariants: + """`-900k` is a Hermes picker convention: valid variants come from the + catalog; ineligible aliases are hard-rejected BEFORE the hidden-slug + soft-accept (#92797 review).""" + + _CATALOG = ["gpt-5.6-sol", "gpt-5.6-sol-900k", "gpt-5.5", "gpt-5.4-mini"] + + def test_catalog_listed_variant_accepted(self): + with patch("hermes_cli.models.provider_model_ids", return_value=self._CATALOG): + result = validate_requested_model("gpt-5.6-sol-900k", "openai-codex") + assert result["accepted"] is True + assert result["recognized"] is True + + @pytest.mark.parametrize("alias", ["gpt-5.5-900k", "gpt-5.4-mini-900k", "gpt-5.6-sol-pro-900k"]) + def test_ineligible_900k_alias_rejected_not_soft_accepted(self, alias): + with patch("hermes_cli.models.provider_model_ids", return_value=self._CATALOG): + result = validate_requested_model(alias, "openai-codex") + assert result["accepted"] is False + assert result["persist"] is False + assert "272K" in result["message"] + + def test_valid_variant_missing_from_catalog_still_accepted(self): + """A verified variant not yet in the (possibly stale) catalog is + accepted via the eligibility predicate, not the soft-accept.""" + with patch("hermes_cli.models.provider_model_ids", return_value=["gpt-5.6-sol"]): + result = validate_requested_model("gpt-5.6-sol-900k", "openai-codex") + assert result["accepted"] is True + + # -- probe_api_models — Cloudflare UA mitigation -------------------------------- class TestProbeApiModelsUserAgent: