fix(model): single exact eligibility predicate for -900k variants; reject ineligible aliases
Review findings on #92797 (@100yenadmin): - is_codex_900k_base() is now the single source of truth used by picker synthesis, context resolution, /model validation, and wire stripping. Eligibility is an exact table (sol/terra/luna, gpt-5.4, daybreak alias) plus date-shaped 5.6 snapshots — family-prefix matching removed, so non-routable -pro slugs and unknown descendants never gain variants. - strip_codex_context_variant_suffix() strips conditionally: ineligible aliases (gpt-5.5-900k) are returned unchanged and fail honestly at the API instead of silently running as the base model at 272K. - validate_requested_model() rejects ineligible *-900k aliases before the hidden-slug soft-accept, and accepts valid variants missing from a stale catalog without letting the typo auto-corrector eat the suffix. - Codex context resolver drops vendor/ namespaces, so openai/gpt-5.6-sol-900k resolves to 900K like the bare id. - Table-driven regression covering eligible bases/snapshots/namespaced ids and rejected -pro/-mini/5.5/unknown aliases, asserting context AND wire model.
This commit is contained in:
Teknium
+84
-33
@@ -2451,66 +2451,114 @@ _CODEX_OAUTH_STALE_ADVERTISED_CTX = 272_000
|
||||
# large window. Never sent on the wire.
|
||||
CODEX_CONTEXT_VARIANT_SUFFIX = "-900k"
|
||||
|
||||
# The ONLY base slugs eligible for a ``-900k`` variant: routable,
|
||||
# live-verified models. gpt-5.6 family-prefix matching is deliberately NOT
|
||||
# used here — it would synthesize dead variants for ``-pro`` slugs (the
|
||||
# Codex backend 400s them) and accept arbitrary future descendants that
|
||||
# were never probed. Dated snapshots of the routable 5.6 bases are allowed
|
||||
# via _CODEX_900K_SNAPSHOT_RE.
|
||||
_CODEX_900K_ELIGIBLE_BASES = frozenset({
|
||||
"gpt-5.6-sol",
|
||||
"gpt-5.6-terra",
|
||||
"gpt-5.6-luna",
|
||||
"gpt-5.4", # exact; gpt-5.4-mini enforces 272K
|
||||
"gpt-daybreak-blue-latest", # verified Sol alias
|
||||
})
|
||||
_CODEX_900K_SNAPSHOT_BASES = ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna")
|
||||
_CODEX_900K_SNAPSHOT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
|
||||
def _bare_codex_slug(model: Optional[str]) -> str:
|
||||
"""Lowercased slug with any ``vendor/`` namespace removed.
|
||||
|
||||
Display/auxiliary callers pass ids like ``openai/gpt-5.6-sol-900k``;
|
||||
the main-agent path normalizes the namespace away earlier, but this
|
||||
resolver must accept both shapes (#92797 review).
|
||||
"""
|
||||
return (model or "").strip().lower().rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def is_codex_900k_base(model: Optional[str]) -> bool:
|
||||
"""True when *model* (a BASE slug, no suffix) may carry a ``-900k`` variant.
|
||||
|
||||
Single source of truth for the eligibility check — used by picker
|
||||
synthesis, context resolution, `/model` validation, and wire stripping
|
||||
so the four sites can never drift apart.
|
||||
"""
|
||||
slug = _bare_codex_slug(model)
|
||||
if not slug or slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
|
||||
return False
|
||||
if slug in _CODEX_900K_ELIGIBLE_BASES:
|
||||
return True
|
||||
# Dated snapshots of the routable 5.6 bases (gpt-5.6-sol-2026-07-09).
|
||||
for base in _CODEX_900K_SNAPSHOT_BASES:
|
||||
if slug.startswith(base + "-") and _CODEX_900K_SNAPSHOT_RE.match(
|
||||
slug[len(base) + 1:]
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_codex_context_variant(model: Optional[str]) -> bool:
|
||||
"""True when the model id carries the Hermes ``-900k`` opt-in suffix."""
|
||||
return (model or "").strip().lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX)
|
||||
"""True when the model id is a VALID ``-900k`` opt-in variant.
|
||||
|
||||
Requires both the suffix and an eligible base — ``gpt-5.5-900k`` is not
|
||||
a variant, it's an invalid alias.
|
||||
"""
|
||||
slug = _bare_codex_slug(model)
|
||||
if not slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
|
||||
return False
|
||||
return is_codex_900k_base(slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)])
|
||||
|
||||
|
||||
def strip_codex_context_variant_suffix(model: Optional[str]) -> str:
|
||||
"""Return the wire-safe slug with any ``-900k`` opt-in suffix removed.
|
||||
"""Return the wire-safe slug with a VALID ``-900k`` suffix removed.
|
||||
|
||||
The suffix is a Hermes picker alias (``gpt-5.6-sol-900k``); the Codex
|
||||
backend only knows the base slug. Case-insensitive; non-variant ids are
|
||||
returned unchanged.
|
||||
backend only knows the base slug. Stripping is conditional on base
|
||||
eligibility: an ineligible alias like ``gpt-5.5-900k`` is returned
|
||||
unchanged so it fails honestly at the API instead of silently running
|
||||
as a different model. Case-insensitive; preserves any ``vendor/``
|
||||
namespace prefix.
|
||||
"""
|
||||
raw = (model or "").strip()
|
||||
if raw.lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
|
||||
return raw[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]
|
||||
if not raw.lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
|
||||
return raw
|
||||
base = raw[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]
|
||||
if is_codex_900k_base(base):
|
||||
return base
|
||||
return raw
|
||||
|
||||
|
||||
def has_codex_context_variant(model_bare: str) -> bool:
|
||||
"""True when a Codex BASE slug has a live-verified ``-900k`` variant.
|
||||
"""True when a Codex BASE slug should get a synthetic ``-900k`` entry.
|
||||
|
||||
Used by the model pickers to decide which base slugs get a synthetic
|
||||
``<slug>-900k`` entry. Exact table first, then family prefixes —
|
||||
mirrors ``_verified_codex_ctx_for_slug`` minus the suffix requirement.
|
||||
Thin alias over :func:`is_codex_900k_base` kept for the picker call
|
||||
sites' readability.
|
||||
"""
|
||||
slug = (model_bare or "").strip().lower()
|
||||
if not slug or slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
|
||||
return False
|
||||
if slug in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT:
|
||||
return True
|
||||
for key in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES:
|
||||
if slug == key or slug.startswith(key + "-") or slug.startswith(key + "."):
|
||||
return True
|
||||
return False
|
||||
return is_codex_900k_base(model_bare)
|
||||
|
||||
|
||||
def _verified_codex_ctx_for_slug(model_bare: str) -> Optional[int]:
|
||||
"""Return the live-verified Codex cap for an OPTED-IN slug, or ``None``.
|
||||
|
||||
The large window is opt-in: only ``-900k``-suffixed picker variants
|
||||
The large window is opt-in: only VALID ``-900k`` picker variants
|
||||
(e.g. ``gpt-5.6-sol-900k``) resolve to the verified cap. Base slugs
|
||||
keep the advertised 272K so the cheaper default limit applies unless
|
||||
the user explicitly selects the large-context variant.
|
||||
|
||||
After stripping the suffix: exact slugs first, then family prefixes
|
||||
(``<key>``, ``<key>-``, ``<key>.``) so dated snapshots of a verified
|
||||
family inherit the bump.
|
||||
the user explicitly selects the large-context variant; ineligible
|
||||
aliases (``gpt-5.5-900k``) never resolve here.
|
||||
"""
|
||||
slug = (model_bare or "").strip().lower()
|
||||
slug = _bare_codex_slug(model_bare)
|
||||
if not slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
|
||||
return None
|
||||
slug = slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]
|
||||
if not slug:
|
||||
base = slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]
|
||||
if not is_codex_900k_base(base):
|
||||
return None
|
||||
exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(slug)
|
||||
exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(base)
|
||||
if exact is not None:
|
||||
return exact
|
||||
for key, ctx in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES.items():
|
||||
if slug == key or slug.startswith(key + "-") or slug.startswith(key + "."):
|
||||
if base == key or base.startswith(key + "-") or base.startswith(key + "."):
|
||||
return ctx
|
||||
return None
|
||||
|
||||
@@ -2660,8 +2708,11 @@ def _resolve_codex_oauth_context_length_with_source(
|
||||
return ctx, source
|
||||
|
||||
# ``-900k`` variants are Hermes picker aliases — the Codex catalog only
|
||||
# knows the base slug, so resolve against the stripped id.
|
||||
lookup_bare = strip_codex_context_variant_suffix(model_bare)
|
||||
# knows the base slug, so resolve against the stripped id. Also drop any
|
||||
# ``vendor/`` namespace (``openai/gpt-5.6-sol-900k``): the main-agent
|
||||
# path normalizes it away before reaching here, but display/auxiliary
|
||||
# callers pass it through (#92797 review).
|
||||
lookup_bare = _bare_codex_slug(strip_codex_context_variant_suffix(model_bare))
|
||||
|
||||
if access_token:
|
||||
live, fresh_probe = _fetch_codex_oauth_context_lengths_with_source(access_token)
|
||||
|
||||
@@ -6558,6 +6558,45 @@ def validate_requested_model(
|
||||
catalog_models = provider_model_ids(normalized)
|
||||
except Exception:
|
||||
catalog_models = []
|
||||
# Ineligible ``-900k`` aliases (e.g. `gpt-5.5-900k`) must be rejected
|
||||
# BEFORE the hidden-slug soft-accept below: the suffix is a Hermes
|
||||
# picker convention, so an unknown `*-900k` name can never be a real
|
||||
# hidden provider slug — soft-accepting one silently runs at 272K on
|
||||
# a different model than the user thinks (#92797 review).
|
||||
if normalized == "openai-codex":
|
||||
from agent.model_metadata import (
|
||||
CODEX_CONTEXT_VARIANT_SUFFIX,
|
||||
is_codex_context_variant,
|
||||
)
|
||||
_req_lower = requested_for_lookup.strip().lower()
|
||||
if (
|
||||
_req_lower.endswith(CODEX_CONTEXT_VARIANT_SUFFIX)
|
||||
and requested_for_lookup not in set(catalog_models)
|
||||
):
|
||||
if is_codex_context_variant(requested_for_lookup):
|
||||
# Valid variant that a stale catalog hasn't synthesized
|
||||
# yet. Accept it directly — falling through would let the
|
||||
# typo auto-corrector "fix" it to the base slug and
|
||||
# silently drop the large-context opt-in.
|
||||
return {
|
||||
"accepted": True,
|
||||
"persist": True,
|
||||
"recognized": True,
|
||||
"message": None,
|
||||
}
|
||||
_base_guess = requested_for_lookup[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]
|
||||
return {
|
||||
"accepted": False,
|
||||
"persist": False,
|
||||
"recognized": False,
|
||||
"message": (
|
||||
f"`{requested}` is not a valid large-context variant — "
|
||||
f"`{_base_guess}` enforces the standard 272K window on "
|
||||
f"Codex, so no `-900k` option exists for it. Pick the "
|
||||
f"base model, or a verified variant from the `/model` "
|
||||
f"picker (e.g. `gpt-5.6-sol-900k`)."
|
||||
),
|
||||
}
|
||||
if catalog_models:
|
||||
if requested_for_lookup in set(catalog_models):
|
||||
return {
|
||||
|
||||
@@ -666,6 +666,64 @@ class TestCodexOAuthContextLength:
|
||||
)
|
||||
assert ctx == 272_000
|
||||
|
||||
# Table-driven eligibility contract (#92797 review): one predicate
|
||||
# (is_codex_900k_base) drives picker synthesis, context resolution,
|
||||
# validation, and wire stripping — this table pins all of them.
|
||||
# (model_id, is_valid_variant, expected_ctx, expected_wire_model)
|
||||
_900K_TABLE = [
|
||||
("gpt-5.6-sol-900k", True, 900_000, "gpt-5.6-sol"),
|
||||
("gpt-5.6-terra-900k", True, 900_000, "gpt-5.6-terra"),
|
||||
("gpt-5.6-luna-900k", True, 900_000, "gpt-5.6-luna"),
|
||||
("gpt-5.4-900k", True, 900_000, "gpt-5.4"),
|
||||
("gpt-daybreak-blue-latest-900k", True, 900_000, "gpt-daybreak-blue-latest"),
|
||||
# dated snapshot of a routable 5.6 base
|
||||
("gpt-5.6-sol-2026-07-09-900k", True, 900_000, "gpt-5.6-sol-2026-07-09"),
|
||||
# vendor-namespaced variant (display/aux callers) resolves too
|
||||
("openai/gpt-5.6-sol-900k", True, 900_000, "openai/gpt-5.6-sol"),
|
||||
# -pro slugs are not routable on Codex OAuth: never a valid variant,
|
||||
# never stripped (fails honestly at the API instead)
|
||||
("gpt-5.6-sol-pro-900k", False, 272_000, "gpt-5.6-sol-pro-900k"),
|
||||
# genuine 272K enforcers get no variant
|
||||
("gpt-5.5-900k", False, 272_000, "gpt-5.5-900k"),
|
||||
("gpt-5.4-mini-900k", False, 272_000, "gpt-5.4-mini-900k"),
|
||||
# arbitrary future family descendants are not auto-eligible
|
||||
("gpt-5.6-nova-900k", False, 272_000, "gpt-5.6-nova-900k"),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize("model_id,valid,expected_ctx,wire", _900K_TABLE)
|
||||
def test_900k_eligibility_table(self, model_id, valid, expected_ctx, wire):
|
||||
from agent.model_metadata import (
|
||||
get_model_context_length,
|
||||
is_codex_context_variant,
|
||||
strip_codex_context_variant_suffix,
|
||||
)
|
||||
|
||||
assert is_codex_context_variant(model_id) is valid
|
||||
assert strip_codex_context_variant_suffix(model_id) == wire
|
||||
|
||||
bare = model_id.rsplit("/", 1)[-1]
|
||||
catalog_slug = strip_codex_context_variant_suffix(bare)
|
||||
if catalog_slug.endswith("-900k"):
|
||||
# invalid alias — catalog advertises the underlying family slug
|
||||
catalog_slug = catalog_slug[: -len("-900k")]
|
||||
fake_response = MagicMock()
|
||||
fake_response.status_code = 200
|
||||
fake_response.json.return_value = {
|
||||
"models": [{"slug": catalog_slug, "context_window": 272_000}]
|
||||
}
|
||||
import agent.model_metadata as mm
|
||||
mm._codex_oauth_context_cache = {}
|
||||
with patch("agent.model_metadata.requests.get", return_value=fake_response), \
|
||||
patch("agent.model_metadata.get_cached_context_length", return_value=None), \
|
||||
patch("agent.model_metadata.save_context_length"):
|
||||
ctx = get_model_context_length(
|
||||
model=model_id,
|
||||
base_url="https://chatgpt.com/backend-api/codex",
|
||||
api_key="fake-token",
|
||||
provider="openai-codex",
|
||||
)
|
||||
assert ctx == expected_ctx
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -52,6 +52,18 @@ def test_picker_synthesizes_900k_variants_for_verified_slugs():
|
||||
assert "gpt-5.3-codex-900k" not in model_ids
|
||||
|
||||
|
||||
def test_picker_never_synthesizes_900k_for_pro_or_unknown_slugs():
|
||||
"""Eligibility is an exact predicate, not a family-prefix match:
|
||||
``-pro`` slugs are not routable on Codex OAuth (backend 400s them) and
|
||||
unknown future descendants were never probed — neither may gain a
|
||||
synthetic ``-900k`` entry (#92797 review)."""
|
||||
from hermes_cli.codex_models import _finalize_codex_models
|
||||
|
||||
out = _finalize_codex_models(["gpt-5.6-sol-pro", "gpt-5.6-nova"])
|
||||
assert "gpt-5.6-sol-pro-900k" not in out
|
||||
assert "gpt-5.6-nova-900k" not in out
|
||||
|
||||
|
||||
|
||||
|
||||
def test_setup_wizard_codex_import_resolves():
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Tests for provider-aware `/model` validation in hermes_cli.models."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from hermes_cli.models import (
|
||||
@@ -504,6 +505,35 @@ class TestValidateCodexAutoCorrection:
|
||||
assert result["message"] is None
|
||||
|
||||
|
||||
class TestValidateCodex900kVariants:
|
||||
"""`-900k` is a Hermes picker convention: valid variants come from the
|
||||
catalog; ineligible aliases are hard-rejected BEFORE the hidden-slug
|
||||
soft-accept (#92797 review)."""
|
||||
|
||||
_CATALOG = ["gpt-5.6-sol", "gpt-5.6-sol-900k", "gpt-5.5", "gpt-5.4-mini"]
|
||||
|
||||
def test_catalog_listed_variant_accepted(self):
|
||||
with patch("hermes_cli.models.provider_model_ids", return_value=self._CATALOG):
|
||||
result = validate_requested_model("gpt-5.6-sol-900k", "openai-codex")
|
||||
assert result["accepted"] is True
|
||||
assert result["recognized"] is True
|
||||
|
||||
@pytest.mark.parametrize("alias", ["gpt-5.5-900k", "gpt-5.4-mini-900k", "gpt-5.6-sol-pro-900k"])
|
||||
def test_ineligible_900k_alias_rejected_not_soft_accepted(self, alias):
|
||||
with patch("hermes_cli.models.provider_model_ids", return_value=self._CATALOG):
|
||||
result = validate_requested_model(alias, "openai-codex")
|
||||
assert result["accepted"] is False
|
||||
assert result["persist"] is False
|
||||
assert "272K" in result["message"]
|
||||
|
||||
def test_valid_variant_missing_from_catalog_still_accepted(self):
|
||||
"""A verified variant not yet in the (possibly stale) catalog is
|
||||
accepted via the eligibility predicate, not the soft-accept."""
|
||||
with patch("hermes_cli.models.provider_model_ids", return_value=["gpt-5.6-sol"]):
|
||||
result = validate_requested_model("gpt-5.6-sol-900k", "openai-codex")
|
||||
assert result["accepted"] is True
|
||||
|
||||
|
||||
# -- probe_api_models — Cloudflare UA mitigation --------------------------------
|
||||
|
||||
class TestProbeApiModelsUserAgent:
|
||||
|
||||
Reference in New Issue
Block a user