fix(agent): isolate custom endpoint billing health
This commit is contained in:
+91
-44
@@ -2883,8 +2883,8 @@ def _get_provider_chain() -> List[tuple]:
|
||||
# "Recently 402'd" unhealthy-provider cache: a depleted provider stays so for hours, so hiding it
|
||||
# for a TTL saves an RTT per aux call. In-process only (profiles may use different keys).
|
||||
_AUX_UNHEALTHY_TTL_SECONDS = 600 # 10 minutes
|
||||
_aux_unhealthy_until: Dict[str, float] = {}
|
||||
_aux_unhealthy_logged_at: Dict[str, float] = {}
|
||||
_aux_unhealthy_until: Dict[Any, float] = {}
|
||||
_aux_unhealthy_logged_at: Dict[Any, float] = {}
|
||||
# resolved_provider / explicit-config names → chain labels.
|
||||
_AUX_UNHEALTHY_LABEL_ALIASES = {
|
||||
"openrouter": "openrouter", "nous": "nous", "custom": "local/custom",
|
||||
@@ -2900,14 +2900,26 @@ def _normalize_chain_label(provider: str) -> str:
|
||||
return _AUX_UNHEALTHY_LABEL_ALIASES.get(p, p)
|
||||
|
||||
|
||||
def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None:
|
||||
"""Hide ``provider`` from chain iteration until the TTL expires (after a confirmed payment error)."""
|
||||
def _unhealthy_cache_key(provider: str, base_url: Optional[str] = None) -> Any:
|
||||
"""Provider-wide key, or endpoint-specific key for an explicit custom endpoint."""
|
||||
label = _normalize_chain_label(provider)
|
||||
endpoint = str(base_url or "").strip().lower().rstrip("/")
|
||||
if endpoint and (label == "local/custom" or label.startswith("custom:")):
|
||||
return label, endpoint
|
||||
return label
|
||||
|
||||
|
||||
def _mark_provider_unhealthy(
|
||||
provider: str, ttl: Optional[float] = None, *, base_url: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Hide one provider endpoint until the TTL expires after a confirmed payment error."""
|
||||
label = _normalize_chain_label(provider)
|
||||
if not label:
|
||||
return
|
||||
key = _unhealthy_cache_key(label, base_url)
|
||||
ttl = _AUX_UNHEALTHY_TTL_SECONDS if ttl is None else ttl
|
||||
expires_at = time.time() + ttl
|
||||
_aux_unhealthy_until[label] = expires_at
|
||||
_aux_unhealthy_until[key] = expires_at
|
||||
logger.warning(
|
||||
"Auxiliary: marking %s unhealthy for %ds (payment / credit error). "
|
||||
"Subsequent auxiliary calls will skip it until %s.",
|
||||
@@ -2915,26 +2927,30 @@ def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None
|
||||
)
|
||||
|
||||
|
||||
def _is_provider_unhealthy(label: str) -> bool:
|
||||
"""True iff ``label`` is unhealthy and unexpired; lazily evicts expired entries."""
|
||||
def _is_provider_unhealthy(label: str, base_url: Optional[str] = None) -> bool:
|
||||
"""True iff this provider endpoint is unhealthy and unexpired; lazily evicts expired entries."""
|
||||
if not label:
|
||||
return False
|
||||
expires_at = _aux_unhealthy_until.get(label)
|
||||
key = _unhealthy_cache_key(label, base_url)
|
||||
expires_at = _aux_unhealthy_until.get(key)
|
||||
if expires_at is None:
|
||||
return False
|
||||
if time.time() >= expires_at:
|
||||
_aux_unhealthy_until.pop(label, None)
|
||||
_aux_unhealthy_logged_at.pop(label, None)
|
||||
_aux_unhealthy_until.pop(key, None)
|
||||
_aux_unhealthy_logged_at.pop(key, None)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _log_skip_unhealthy(label: str, task: Optional[str] = None) -> None:
|
||||
def _log_skip_unhealthy(
|
||||
label: str, task: Optional[str] = None, *, base_url: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Log a skipped unhealthy provider at most once per minute per label."""
|
||||
now = time.time()
|
||||
if now - _aux_unhealthy_logged_at.get(label, 0.0) >= 60:
|
||||
_aux_unhealthy_logged_at[label] = now
|
||||
expires_at = _aux_unhealthy_until.get(label, now)
|
||||
key = _unhealthy_cache_key(label, base_url)
|
||||
if now - _aux_unhealthy_logged_at.get(key, 0.0) >= 60:
|
||||
_aux_unhealthy_logged_at[key] = now
|
||||
expires_at = _aux_unhealthy_until.get(key, now)
|
||||
logger.info(
|
||||
"Auxiliary %s: skipping %s (recently returned payment error, retry in %ds)",
|
||||
task or "call", label, max(0, int(expires_at - now)),
|
||||
@@ -3749,7 +3765,8 @@ async def _call_fallback_candidate_async(
|
||||
|
||||
|
||||
def _try_payment_fallback(
|
||||
failed_provider: str, task: str = None, reason: str = "payment error"
|
||||
failed_provider: str, task: str = None, reason: str = "payment error", *,
|
||||
failed_base_url: str = "", failure_scope: Any = None,
|
||||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||||
"""Try the auto-detection chain after a payment/credit or connection error, skipping the failed
|
||||
provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "")."""
|
||||
@@ -3759,12 +3776,16 @@ def _try_payment_fallback(
|
||||
if main_provider and main_provider.lower() in skip:
|
||||
skip_labels.add(main_provider.lower())
|
||||
skip_chain_labels = {_normalize_chain_label(s) for s in skip_labels}
|
||||
skip_backend = _failed_backend_skip(
|
||||
failed_provider, None, failed_base_url=failed_base_url, failure_scope=failure_scope)
|
||||
tried = []
|
||||
for label, try_fn in _get_provider_chain():
|
||||
if label in skip_chain_labels:
|
||||
candidate_base_url = _current_custom_base_url() if label == "local/custom" else ""
|
||||
if (not failed_base_url and label in skip_chain_labels) or skip_backend(
|
||||
label, None, candidate_base_url):
|
||||
continue
|
||||
if _is_provider_unhealthy(label):
|
||||
_log_skip_unhealthy(label, task)
|
||||
if _is_provider_unhealthy(label, candidate_base_url):
|
||||
_log_skip_unhealthy(label, task, base_url=candidate_base_url)
|
||||
tried.append(f"{label} (unhealthy)")
|
||||
continue
|
||||
client, model = try_fn()
|
||||
@@ -3778,14 +3799,18 @@ def _try_payment_fallback(
|
||||
return None, None, ""
|
||||
|
||||
|
||||
def _failed_backend_skip(failed_provider: str, failed_model: Optional[str]) -> Callable[..., bool]:
|
||||
def _failed_backend_skip(
|
||||
failed_provider: str, failed_model: Optional[str], *, failed_base_url: str = "",
|
||||
failure_scope: Any = None,
|
||||
) -> Callable[..., bool]:
|
||||
"""Predicate ``skip(provider, model, base_url="")`` → True when a candidate must be skipped for the failed
|
||||
route. Scope: ``failed_model`` → model-scoped (only that deployment; timeout/connection/rate-limit);
|
||||
None → credential-wide (whole provider; auth/payment)."""
|
||||
from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate
|
||||
skip_model = (failed_model or "").strip().lower() or None
|
||||
failed_ident = BackendIdentity.build(provider=failed_provider, model=skip_model)
|
||||
failure_scope = FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL
|
||||
failed_ident = BackendIdentity.build(
|
||||
provider=failed_provider, model=skip_model, base_url=failed_base_url)
|
||||
failure_scope = failure_scope or (FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL)
|
||||
|
||||
def _skip(provider: str, model: Optional[str], base_url: str = "") -> bool:
|
||||
return should_skip_candidate(
|
||||
@@ -3796,7 +3821,7 @@ def _failed_backend_skip(failed_provider: str, failed_model: Optional[str]) -> C
|
||||
|
||||
def _try_main_agent_model_fallback(
|
||||
failed_provider: str, task: str = None, reason: str = "error",
|
||||
failed_model: Optional[str] = None,
|
||||
failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
|
||||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||||
"""Last-resort fallback to the main agent provider + model after the configured chain is exhausted.
|
||||
``failed_model`` scoping per ``_failed_backend_skip``; same-URL custom endpoints serve many models,
|
||||
@@ -3811,10 +3836,13 @@ def _try_main_agent_model_fallback(
|
||||
main_provider, main_model = _agg_provider, _agg_model
|
||||
if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
|
||||
return None, None, ""
|
||||
if _failed_backend_skip(failed_provider, failed_model)(main_provider, main_model):
|
||||
main_base_url = _current_custom_base_url() if _normalize_chain_label(main_provider) == "local/custom" else ""
|
||||
if _failed_backend_skip(
|
||||
failed_provider, failed_model, failed_base_url=failed_base_url,
|
||||
failure_scope=failure_scope)(main_provider, main_model, main_base_url):
|
||||
return None, None, ""
|
||||
if _is_provider_unhealthy(main_provider):
|
||||
_log_skip_unhealthy(main_provider, task)
|
||||
if _is_provider_unhealthy(main_provider, main_base_url):
|
||||
_log_skip_unhealthy(main_provider, task, base_url=main_base_url)
|
||||
return None, None, ""
|
||||
try:
|
||||
client, resolved_model = resolve_provider_client(provider=main_provider, model=main_model)
|
||||
@@ -3881,7 +3909,8 @@ def _context_too_small(
|
||||
|
||||
|
||||
def _try_configured_fallback_chain(
|
||||
task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None
|
||||
task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None, *,
|
||||
failed_base_url: str = "", failure_scope: Any = None,
|
||||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||||
"""Try auxiliary.<task>.fallback_chain entries in order (each needs ``provider``; model/base_url/api_key optional).
|
||||
``failed_model`` scoping per ``_failed_backend_skip`` (sibling models on the same provider still
|
||||
@@ -3891,7 +3920,8 @@ def _try_configured_fallback_chain(
|
||||
chain = _get_auxiliary_task_config(task).get("fallback_chain")
|
||||
if not chain or not isinstance(chain, list):
|
||||
return None, None, ""
|
||||
skip = _failed_backend_skip(failed_provider, failed_model)
|
||||
skip = _failed_backend_skip(
|
||||
failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
|
||||
tried = []
|
||||
min_ctx = _task_minimum_context_length(task)
|
||||
for i, entry in enumerate(chain):
|
||||
@@ -3901,7 +3931,12 @@ def _try_configured_fallback_chain(
|
||||
if not fb_provider:
|
||||
continue
|
||||
fb_model_raw = str(entry.get("model", "")).strip()
|
||||
if skip(fb_provider, fb_model_raw, str(entry.get("base_url") or "")):
|
||||
fb_base_url = str(entry.get("base_url") or "")
|
||||
if skip(fb_provider, fb_model_raw, fb_base_url):
|
||||
continue
|
||||
if _is_provider_unhealthy(fb_provider, fb_base_url):
|
||||
_log_skip_unhealthy(fb_provider, task, base_url=fb_base_url)
|
||||
tried.append(f"fallback_chain[{i}]({fb_provider}) (unhealthy)")
|
||||
continue
|
||||
fb_model = fb_model_raw or None
|
||||
label = f"fallback_chain[{i}]({fb_provider})"
|
||||
@@ -3960,7 +3995,8 @@ def _resolve_fallback_entry(entry: Dict[str, Any]) -> Tuple[Optional[Any], Optio
|
||||
|
||||
|
||||
def _try_main_fallback_chain(
|
||||
task: Optional[str], failed_provider: str = "", reason: str = "error"
|
||||
task: Optional[str], failed_provider: str = "", reason: str = "error", *,
|
||||
failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
|
||||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||||
"""Top-level main-agent fallback chain for a ``provider: auto`` auxiliary call: auto tasks honour the
|
||||
user's main fallback policy before the built-in discovery chain; read via ``get_fallback_chain`` so
|
||||
@@ -3974,7 +4010,8 @@ def _try_main_fallback_chain(
|
||||
return None, None, ""
|
||||
if not chain:
|
||||
return None, None, ""
|
||||
skip = {p for p in ((failed_provider or "").strip().lower(), (_read_main_provider() or "").strip().lower(), "auto") if p}
|
||||
skip = _failed_backend_skip(
|
||||
failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
|
||||
tried: List[str] = []
|
||||
min_ctx = _task_minimum_context_length(task)
|
||||
for i, entry in enumerate(chain):
|
||||
@@ -3986,11 +4023,12 @@ def _try_main_fallback_chain(
|
||||
continue
|
||||
fb_norm = fb_provider.lower()
|
||||
label = f"fallback_providers[{i}]({fb_provider})"
|
||||
if fb_norm in skip:
|
||||
fb_base_url = str(entry.get("base_url") or "")
|
||||
if fb_norm == "auto" or skip(fb_provider, fb_model, fb_base_url):
|
||||
tried.append(f"{label} (skipped)")
|
||||
continue
|
||||
if _is_provider_unhealthy(fb_norm):
|
||||
_log_skip_unhealthy(fb_norm, task)
|
||||
if _is_provider_unhealthy(fb_norm, fb_base_url):
|
||||
_log_skip_unhealthy(fb_norm, task, base_url=fb_base_url)
|
||||
tried.append(f"{label} (unhealthy)")
|
||||
continue
|
||||
try:
|
||||
@@ -4093,8 +4131,8 @@ def _try_main_provider_route(
|
||||
explicit_api_key = runtime_api_key
|
||||
# Skip if the main provider was recently 402'd (unhealthy TTL bounds the bypass).
|
||||
main_chain_label = _normalize_chain_label(resolved_provider)
|
||||
if main_chain_label and _is_provider_unhealthy(main_chain_label):
|
||||
_log_skip_unhealthy(main_chain_label)
|
||||
if main_chain_label and _is_provider_unhealthy(main_chain_label, explicit_base_url):
|
||||
_log_skip_unhealthy(main_chain_label, base_url=explicit_base_url)
|
||||
return None
|
||||
client, resolved = resolve_provider_client(
|
||||
resolved_provider, main_model, explicit_base_url=explicit_base_url,
|
||||
@@ -4110,8 +4148,9 @@ def _try_discovery_chain() -> Tuple[Optional[OpenAI], Optional[str], str]:
|
||||
"""Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers."""
|
||||
tried = []
|
||||
for label, try_fn in _get_provider_chain():
|
||||
if _is_provider_unhealthy(label):
|
||||
_log_skip_unhealthy(label)
|
||||
candidate_base_url = _current_custom_base_url() if label == "local/custom" else ""
|
||||
if _is_provider_unhealthy(label, candidate_base_url):
|
||||
_log_skip_unhealthy(label, base_url=candidate_base_url)
|
||||
tried.append(f"{label} (unhealthy)")
|
||||
continue
|
||||
client, model = try_fn()
|
||||
@@ -6788,23 +6827,30 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
|
||||
# it instead of paying another doomed RTT.
|
||||
_mark_provider_unhealthy(
|
||||
_recoverable_pool_provider(resolved_provider, route.client, main_runtime=route.main_runtime)
|
||||
or resolved_provider)
|
||||
or resolved_provider, base_url=route.base_info)
|
||||
logger.info("Auxiliary %s%s: %s on %s (%s), trying fallback",
|
||||
task or "call", tag, reason, resolved_provider, first_err)
|
||||
# Skip only the failed model for model-specific failures; 401/402 are provider-wide, so
|
||||
# keep skipping the whole provider.
|
||||
# auth keeps skipping the credential surface, while billing is scoped to the endpoint:
|
||||
# separate custom URLs can carry separate credentials (or no billing relationship at all).
|
||||
_chain_failed_model = None if reason in ("auth error", "payment error") else route.final_model
|
||||
from agent.backend_identity import FailureScope
|
||||
_chain_failure_scope = FailureScope.ENDPOINT if reason == "payment error" else None
|
||||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model)
|
||||
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
|
||||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||||
if fb_client is None and is_auto:
|
||||
fb_client, fb_model, fb_label = _try_main_fallback_chain(
|
||||
task, resolved_provider or "auto", reason=reason)
|
||||
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
|
||||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||||
if fb_client is None:
|
||||
fb_client, fb_model, fb_label = _try_payment_fallback(
|
||||
resolved_provider, task, reason=reason)
|
||||
resolved_provider, task, reason=reason, failed_base_url=route.base_info,
|
||||
failure_scope=_chain_failure_scope)
|
||||
elif fb_client is None:
|
||||
fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
|
||||
resolved_provider, task, reason=reason, failed_model=_chain_failed_model)
|
||||
resolved_provider, task, reason=reason, failed_model=_chain_failed_model,
|
||||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||||
if fb_client is not None:
|
||||
# Second pass: the candidate credential was stale and quarantined — walk the discovery
|
||||
# chain once more (unhealthy entries are skipped).
|
||||
@@ -6815,7 +6861,8 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
|
||||
return fb_resp
|
||||
if _pass == 0:
|
||||
fb_client, fb_model, fb_label = _try_payment_fallback(
|
||||
resolved_provider, task, reason="stale fallback credential")
|
||||
resolved_provider, task, reason="stale fallback credential",
|
||||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||||
if fb_client is None:
|
||||
break
|
||||
# All fallback layers exhausted — one user-visible warning, then re-raise.
|
||||
|
||||
@@ -4193,6 +4193,48 @@ class TestAuxUnhealthyCache:
|
||||
# After the 402, OpenRouter is in the unhealthy cache.
|
||||
assert _is_provider_unhealthy("openrouter") is True
|
||||
|
||||
def test_custom_billing_failure_keeps_distinct_endpoint_eligible(self):
|
||||
"""A hosted custom endpoint's billing state must not quarantine a local custom endpoint."""
|
||||
from agent.auxiliary_client import call_llm, _is_provider_unhealthy
|
||||
|
||||
hosted_url = "https://hosted.example/v1"
|
||||
local_url = "http://127.0.0.1:8080/v1"
|
||||
payment_error = Exception("Payment Required: weekly usage limit")
|
||||
payment_error.status_code = 402
|
||||
|
||||
hosted_client = MagicMock(base_url=hosted_url)
|
||||
hosted_client.chat.completions.create.side_effect = payment_error
|
||||
local_client = MagicMock(base_url=local_url)
|
||||
local_client.chat.completions.create.return_value = _DummyResponse("local-ok")
|
||||
fallback_entry = {
|
||||
"provider": "custom", "model": "local-model", "base_url": local_url,
|
||||
"api_key": "local",
|
||||
}
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("custom", "hosted-model", hosted_url, "hosted", None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._get_cached_client",
|
||||
return_value=(hosted_client, "hosted-model"),
|
||||
), patch(
|
||||
"agent.auxiliary_client._get_auxiliary_task_config",
|
||||
return_value={"fallback_chain": [fallback_entry]},
|
||||
), patch(
|
||||
"agent.auxiliary_client._resolve_fallback_entry",
|
||||
return_value=(local_client, "local-model"),
|
||||
):
|
||||
response = call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "summarize"}],
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "local-ok"
|
||||
assert _is_provider_unhealthy("custom", hosted_url) is True
|
||||
assert _is_provider_unhealthy("custom", local_url) is False
|
||||
assert hosted_client.chat.completions.create.call_count == 1
|
||||
assert local_client.chat.completions.create.call_count == 1
|
||||
|
||||
|
||||
# ── auxiliary_max_tokens_param ──────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user