perf(models): stop revalidating a fresh provider cache just because it lists Astra

``gated_cache`` bypassed both the fresh-hit and the stale-while-revalidate branches of
``cached_provider_model_ids`` whenever the disk entry contained Astra. For an entitled
account that is every entry: live discovery rewrites the entry with Astra → next call is gated
again → a blocking /models round-trip on every picker open, for exactly the providers the user
is most likely on. The parallel prefetch's staleness check didn't know about the gate either, so
it skipped the slug and the blocking call landed in the serial picker loop the prefetch exists to
avoid.

Gate on provenance, not contents: only live discovery ever writes Astra into a same-credential
entry (the static/offline paths filter it out), so a fresh entry IS the entitlement record. The
one filter that matters stays — a stale entry served because the refresh failed drops Astra, and
the entry itself is left intact so the next successful fetch restores it.

The regression test now pins both halves: fresh entry served with Astra and zero live calls;
failed refresh past the fresh window serves the entry minus Astra.
This commit is contained in:
kshitijk4poor
2026-09-07 21:36:06 +05:30
committed by kshitij
parent b6852995ed
commit e583d68c14
2 changed files with 11 additions and 10 deletions
+4 -8
View File
@@ -1536,14 +1536,8 @@ def cached_provider_model_ids(
fp = _credential_fingerprint(normalized)
entry = cache.get(normalized)
now = time.time()
# Even a credential-matched fresh disk cache predates the current account's
# entitlement check. Revalidate gated entries before offering them again.
cached_models = entry.get("models") if isinstance(entry, dict) else None
gated_cache = isinstance(cached_models, list) and any(
_model_requires_account_discovery(normalized, model) for model in cached_models
)
if not force_refresh and not gated_cache and _cache_entry_valid(entry, fp, allow_empty=is_ollama):
if not force_refresh and _cache_entry_valid(entry, fp, allow_empty=is_ollama):
age = now - entry["at"]
if age < ttl_seconds:
return list(entry["models"])
@@ -1570,7 +1564,9 @@ def cached_provider_model_ids(
if same_creds and isinstance(entry.get("models"), list) and entry["models"]:
return list(entry["models"])
return []
# Live returned nothing: a stale same-fingerprint entry beats an empty result.
# Live returned nothing: a stale same-fingerprint entry beats an empty result — minus account-gated
# models, which only a successful discovery may advertise (the entry itself is untouched, so the
# next successful fetch restores them).
if _cache_entry_valid(entry, fp):
return [model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)]
return []
+7 -2
View File
@@ -77,7 +77,6 @@ def test_astra_900k_opt_in_preserves_live_limits_and_wire_contract(monkeypatch,
kwargs = ResponsesApiTransport().build_kwargs(model="gpt-6-astra-900k", **params)
assert kwargs["model"] == "gpt-6-astra"
assert kwargs == ResponsesApiTransport().build_kwargs(model="gpt-6-astra", **params)
assert "prompt_cache_options" not in kwargs
@pytest.mark.parametrize("provider,model", [
@@ -99,7 +98,13 @@ def test_picker_revalidates_cached_astra_and_never_injects_saved_entitlement(mon
return list(live)
monkeypatch.setattr(models, "provider_model_ids", discover)
assert models.cached_provider_model_ids(provider) == ["gpt-5.6-sol"]
# Only live discovery writes Astra into a same-credential entry, so a fresh entry IS the
# entitlement record: served without a round-trip (a per-open revalidation defeated the cache).
assert model in models.cached_provider_model_ids(provider)
assert calls == []
# After the entry ages past the fresh window, a failed refresh must not resurrect Astra from it.
monkeypatch.setattr(models, "_PROVIDER_MODELS_STALE_SERVE_MAX", 0)
assert models.cached_provider_model_ids(provider, ttl_seconds=0) == ["gpt-5.6-sol"]
assert calls == [provider]
row = {"slug": provider, "is_current": True, "models": ["gpt-5.6-sol"], "total_models": 1}
assert model not in _finalize_picker_rows([row], {}, model)[0]["models"]