From e583d68c14b7448144d785eb2ecc696b7a1ca7de Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:36:06 +0530 Subject: [PATCH] perf(models): stop revalidating a fresh provider cache just because it lists Astra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``gated_cache`` bypassed both the fresh-hit and the stale-while-revalidate branches of ``cached_provider_model_ids`` whenever the disk entry contained Astra. For an entitled account that is every entry: live discovery rewrites the entry with Astra → next call is gated again → a blocking /models round-trip on every picker open, for exactly the providers the user is most likely on. The parallel prefetch's staleness check didn't know about the gate either, so it skipped the slug and the blocking call landed in the serial picker loop the prefetch exists to avoid. Gate on provenance, not contents: only live discovery ever writes Astra into a same-credential entry (the static/offline paths filter it out), so a fresh entry IS the entitlement record. The one filter that matters stays — a stale entry served because the refresh failed drops Astra, and the entry itself is left intact so the next successful fetch restores it. The regression test now pins both halves: fresh entry served with Astra and zero live calls; failed refresh past the fresh window serves the entry minus Astra. --- hermes_cli/models.py | 12 ++++-------- tests/agent/test_astra_baseline_runtime.py | 9 +++++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 9b2e3c8463..eb9bea8ee8 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -1536,14 +1536,8 @@ def cached_provider_model_ids( fp = _credential_fingerprint(normalized) entry = cache.get(normalized) now = time.time() - # Even a credential-matched fresh disk cache predates the current account's - # entitlement check. Revalidate gated entries before offering them again. - cached_models = entry.get("models") if isinstance(entry, dict) else None - gated_cache = isinstance(cached_models, list) and any( - _model_requires_account_discovery(normalized, model) for model in cached_models - ) - if not force_refresh and not gated_cache and _cache_entry_valid(entry, fp, allow_empty=is_ollama): + if not force_refresh and _cache_entry_valid(entry, fp, allow_empty=is_ollama): age = now - entry["at"] if age < ttl_seconds: return list(entry["models"]) @@ -1570,7 +1564,9 @@ def cached_provider_model_ids( if same_creds and isinstance(entry.get("models"), list) and entry["models"]: return list(entry["models"]) return [] - # Live returned nothing: a stale same-fingerprint entry beats an empty result. + # Live returned nothing: a stale same-fingerprint entry beats an empty result — minus account-gated + # models, which only a successful discovery may advertise (the entry itself is untouched, so the + # next successful fetch restores them). if _cache_entry_valid(entry, fp): return [model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)] return [] diff --git a/tests/agent/test_astra_baseline_runtime.py b/tests/agent/test_astra_baseline_runtime.py index b3591ad11a..b1fad0e133 100644 --- a/tests/agent/test_astra_baseline_runtime.py +++ b/tests/agent/test_astra_baseline_runtime.py @@ -77,7 +77,6 @@ def test_astra_900k_opt_in_preserves_live_limits_and_wire_contract(monkeypatch, kwargs = ResponsesApiTransport().build_kwargs(model="gpt-6-astra-900k", **params) assert kwargs["model"] == "gpt-6-astra" assert kwargs == ResponsesApiTransport().build_kwargs(model="gpt-6-astra", **params) - assert "prompt_cache_options" not in kwargs @pytest.mark.parametrize("provider,model", [ @@ -99,7 +98,13 @@ def test_picker_revalidates_cached_astra_and_never_injects_saved_entitlement(mon return list(live) monkeypatch.setattr(models, "provider_model_ids", discover) - assert models.cached_provider_model_ids(provider) == ["gpt-5.6-sol"] + # Only live discovery writes Astra into a same-credential entry, so a fresh entry IS the + # entitlement record: served without a round-trip (a per-open revalidation defeated the cache). + assert model in models.cached_provider_model_ids(provider) + assert calls == [] + # After the entry ages past the fresh window, a failed refresh must not resurrect Astra from it. + monkeypatch.setattr(models, "_PROVIDER_MODELS_STALE_SERVE_MAX", 0) + assert models.cached_provider_model_ids(provider, ttl_seconds=0) == ["gpt-5.6-sol"] assert calls == [provider] row = {"slug": provider, "is_current": True, "models": ["gpt-5.6-sol"], "total_models": 1} assert model not in _finalize_picker_rows([row], {}, model)[0]["models"]