fix(agent): isolate custom endpoint billing health

This commit is contained in:
fangliquanflq
2026-09-06 20:24:54 +08:00
committed by Teknium
parent 22a03c830f
commit b40998bc3c
2 changed files with 133 additions and 44 deletions
+91 -44
View File
@@ -2883,8 +2883,8 @@ def _get_provider_chain() -> List[tuple]:
# "Recently 402'd" unhealthy-provider cache: a depleted provider stays so for hours, so hiding it
# for a TTL saves an RTT per aux call. In-process only (profiles may use different keys).
_AUX_UNHEALTHY_TTL_SECONDS = 600 # 10 minutes
_aux_unhealthy_until: Dict[str, float] = {}
_aux_unhealthy_logged_at: Dict[str, float] = {}
_aux_unhealthy_until: Dict[Any, float] = {}
_aux_unhealthy_logged_at: Dict[Any, float] = {}
# resolved_provider / explicit-config names → chain labels.
_AUX_UNHEALTHY_LABEL_ALIASES = {
"openrouter": "openrouter", "nous": "nous", "custom": "local/custom",
@@ -2900,14 +2900,26 @@ def _normalize_chain_label(provider: str) -> str:
return _AUX_UNHEALTHY_LABEL_ALIASES.get(p, p)
def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None:
"""Hide ``provider`` from chain iteration until the TTL expires (after a confirmed payment error)."""
def _unhealthy_cache_key(provider: str, base_url: Optional[str] = None) -> Any:
"""Provider-wide key, or endpoint-specific key for an explicit custom endpoint."""
label = _normalize_chain_label(provider)
endpoint = str(base_url or "").strip().lower().rstrip("/")
if endpoint and (label == "local/custom" or label.startswith("custom:")):
return label, endpoint
return label
def _mark_provider_unhealthy(
provider: str, ttl: Optional[float] = None, *, base_url: Optional[str] = None,
) -> None:
"""Hide one provider endpoint until the TTL expires after a confirmed payment error."""
label = _normalize_chain_label(provider)
if not label:
return
key = _unhealthy_cache_key(label, base_url)
ttl = _AUX_UNHEALTHY_TTL_SECONDS if ttl is None else ttl
expires_at = time.time() + ttl
_aux_unhealthy_until[label] = expires_at
_aux_unhealthy_until[key] = expires_at
logger.warning(
"Auxiliary: marking %s unhealthy for %ds (payment / credit error). "
"Subsequent auxiliary calls will skip it until %s.",
@@ -2915,26 +2927,30 @@ def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None
)
def _is_provider_unhealthy(label: str) -> bool:
"""True iff ``label`` is unhealthy and unexpired; lazily evicts expired entries."""
def _is_provider_unhealthy(label: str, base_url: Optional[str] = None) -> bool:
"""True iff this provider endpoint is unhealthy and unexpired; lazily evicts expired entries."""
if not label:
return False
expires_at = _aux_unhealthy_until.get(label)
key = _unhealthy_cache_key(label, base_url)
expires_at = _aux_unhealthy_until.get(key)
if expires_at is None:
return False
if time.time() >= expires_at:
_aux_unhealthy_until.pop(label, None)
_aux_unhealthy_logged_at.pop(label, None)
_aux_unhealthy_until.pop(key, None)
_aux_unhealthy_logged_at.pop(key, None)
return False
return True
def _log_skip_unhealthy(label: str, task: Optional[str] = None) -> None:
def _log_skip_unhealthy(
label: str, task: Optional[str] = None, *, base_url: Optional[str] = None,
) -> None:
"""Log a skipped unhealthy provider at most once per minute per label."""
now = time.time()
if now - _aux_unhealthy_logged_at.get(label, 0.0) >= 60:
_aux_unhealthy_logged_at[label] = now
expires_at = _aux_unhealthy_until.get(label, now)
key = _unhealthy_cache_key(label, base_url)
if now - _aux_unhealthy_logged_at.get(key, 0.0) >= 60:
_aux_unhealthy_logged_at[key] = now
expires_at = _aux_unhealthy_until.get(key, now)
logger.info(
"Auxiliary %s: skipping %s (recently returned payment error, retry in %ds)",
task or "call", label, max(0, int(expires_at - now)),
@@ -3749,7 +3765,8 @@ async def _call_fallback_candidate_async(
def _try_payment_fallback(
failed_provider: str, task: str = None, reason: str = "payment error"
failed_provider: str, task: str = None, reason: str = "payment error", *,
failed_base_url: str = "", failure_scope: Any = None,
) -> Tuple[Optional[Any], Optional[str], str]:
"""Try the auto-detection chain after a payment/credit or connection error, skipping the failed
provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "")."""
@@ -3759,12 +3776,16 @@ def _try_payment_fallback(
if main_provider and main_provider.lower() in skip:
skip_labels.add(main_provider.lower())
skip_chain_labels = {_normalize_chain_label(s) for s in skip_labels}
skip_backend = _failed_backend_skip(
failed_provider, None, failed_base_url=failed_base_url, failure_scope=failure_scope)
tried = []
for label, try_fn in _get_provider_chain():
if label in skip_chain_labels:
candidate_base_url = _current_custom_base_url() if label == "local/custom" else ""
if (not failed_base_url and label in skip_chain_labels) or skip_backend(
label, None, candidate_base_url):
continue
if _is_provider_unhealthy(label):
_log_skip_unhealthy(label, task)
if _is_provider_unhealthy(label, candidate_base_url):
_log_skip_unhealthy(label, task, base_url=candidate_base_url)
tried.append(f"{label} (unhealthy)")
continue
client, model = try_fn()
@@ -3778,14 +3799,18 @@ def _try_payment_fallback(
return None, None, ""
def _failed_backend_skip(failed_provider: str, failed_model: Optional[str]) -> Callable[..., bool]:
def _failed_backend_skip(
failed_provider: str, failed_model: Optional[str], *, failed_base_url: str = "",
failure_scope: Any = None,
) -> Callable[..., bool]:
"""Predicate ``skip(provider, model, base_url="")`` → True when a candidate must be skipped for the failed
route. Scope: ``failed_model`` → model-scoped (only that deployment; timeout/connection/rate-limit);
None → credential-wide (whole provider; auth/payment)."""
from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate
skip_model = (failed_model or "").strip().lower() or None
failed_ident = BackendIdentity.build(provider=failed_provider, model=skip_model)
failure_scope = FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL
failed_ident = BackendIdentity.build(
provider=failed_provider, model=skip_model, base_url=failed_base_url)
failure_scope = failure_scope or (FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL)
def _skip(provider: str, model: Optional[str], base_url: str = "") -> bool:
return should_skip_candidate(
@@ -3796,7 +3821,7 @@ def _failed_backend_skip(failed_provider: str, failed_model: Optional[str]) -> C
def _try_main_agent_model_fallback(
failed_provider: str, task: str = None, reason: str = "error",
failed_model: Optional[str] = None,
failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
) -> Tuple[Optional[Any], Optional[str], str]:
"""Last-resort fallback to the main agent provider + model after the configured chain is exhausted.
``failed_model`` scoping per ``_failed_backend_skip``; same-URL custom endpoints serve many models,
@@ -3811,10 +3836,13 @@ def _try_main_agent_model_fallback(
main_provider, main_model = _agg_provider, _agg_model
if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
return None, None, ""
if _failed_backend_skip(failed_provider, failed_model)(main_provider, main_model):
main_base_url = _current_custom_base_url() if _normalize_chain_label(main_provider) == "local/custom" else ""
if _failed_backend_skip(
failed_provider, failed_model, failed_base_url=failed_base_url,
failure_scope=failure_scope)(main_provider, main_model, main_base_url):
return None, None, ""
if _is_provider_unhealthy(main_provider):
_log_skip_unhealthy(main_provider, task)
if _is_provider_unhealthy(main_provider, main_base_url):
_log_skip_unhealthy(main_provider, task, base_url=main_base_url)
return None, None, ""
try:
client, resolved_model = resolve_provider_client(provider=main_provider, model=main_model)
@@ -3881,7 +3909,8 @@ def _context_too_small(
def _try_configured_fallback_chain(
task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None
task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None, *,
failed_base_url: str = "", failure_scope: Any = None,
) -> Tuple[Optional[Any], Optional[str], str]:
"""Try auxiliary.<task>.fallback_chain entries in order (each needs ``provider``; model/base_url/api_key optional).
``failed_model`` scoping per ``_failed_backend_skip`` (sibling models on the same provider still
@@ -3891,7 +3920,8 @@ def _try_configured_fallback_chain(
chain = _get_auxiliary_task_config(task).get("fallback_chain")
if not chain or not isinstance(chain, list):
return None, None, ""
skip = _failed_backend_skip(failed_provider, failed_model)
skip = _failed_backend_skip(
failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
tried = []
min_ctx = _task_minimum_context_length(task)
for i, entry in enumerate(chain):
@@ -3901,7 +3931,12 @@ def _try_configured_fallback_chain(
if not fb_provider:
continue
fb_model_raw = str(entry.get("model", "")).strip()
if skip(fb_provider, fb_model_raw, str(entry.get("base_url") or "")):
fb_base_url = str(entry.get("base_url") or "")
if skip(fb_provider, fb_model_raw, fb_base_url):
continue
if _is_provider_unhealthy(fb_provider, fb_base_url):
_log_skip_unhealthy(fb_provider, task, base_url=fb_base_url)
tried.append(f"fallback_chain[{i}]({fb_provider}) (unhealthy)")
continue
fb_model = fb_model_raw or None
label = f"fallback_chain[{i}]({fb_provider})"
@@ -3960,7 +3995,8 @@ def _resolve_fallback_entry(entry: Dict[str, Any]) -> Tuple[Optional[Any], Optio
def _try_main_fallback_chain(
task: Optional[str], failed_provider: str = "", reason: str = "error"
task: Optional[str], failed_provider: str = "", reason: str = "error", *,
failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
) -> Tuple[Optional[Any], Optional[str], str]:
"""Top-level main-agent fallback chain for a ``provider: auto`` auxiliary call: auto tasks honour the
user's main fallback policy before the built-in discovery chain; read via ``get_fallback_chain`` so
@@ -3974,7 +4010,8 @@ def _try_main_fallback_chain(
return None, None, ""
if not chain:
return None, None, ""
skip = {p for p in ((failed_provider or "").strip().lower(), (_read_main_provider() or "").strip().lower(), "auto") if p}
skip = _failed_backend_skip(
failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
tried: List[str] = []
min_ctx = _task_minimum_context_length(task)
for i, entry in enumerate(chain):
@@ -3986,11 +4023,12 @@ def _try_main_fallback_chain(
continue
fb_norm = fb_provider.lower()
label = f"fallback_providers[{i}]({fb_provider})"
if fb_norm in skip:
fb_base_url = str(entry.get("base_url") or "")
if fb_norm == "auto" or skip(fb_provider, fb_model, fb_base_url):
tried.append(f"{label} (skipped)")
continue
if _is_provider_unhealthy(fb_norm):
_log_skip_unhealthy(fb_norm, task)
if _is_provider_unhealthy(fb_norm, fb_base_url):
_log_skip_unhealthy(fb_norm, task, base_url=fb_base_url)
tried.append(f"{label} (unhealthy)")
continue
try:
@@ -4093,8 +4131,8 @@ def _try_main_provider_route(
explicit_api_key = runtime_api_key
# Skip if the main provider was recently 402'd (unhealthy TTL bounds the bypass).
main_chain_label = _normalize_chain_label(resolved_provider)
if main_chain_label and _is_provider_unhealthy(main_chain_label):
_log_skip_unhealthy(main_chain_label)
if main_chain_label and _is_provider_unhealthy(main_chain_label, explicit_base_url):
_log_skip_unhealthy(main_chain_label, base_url=explicit_base_url)
return None
client, resolved = resolve_provider_client(
resolved_provider, main_model, explicit_base_url=explicit_base_url,
@@ -4110,8 +4148,9 @@ def _try_discovery_chain() -> Tuple[Optional[OpenAI], Optional[str], str]:
"""Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers."""
tried = []
for label, try_fn in _get_provider_chain():
if _is_provider_unhealthy(label):
_log_skip_unhealthy(label)
candidate_base_url = _current_custom_base_url() if label == "local/custom" else ""
if _is_provider_unhealthy(label, candidate_base_url):
_log_skip_unhealthy(label, base_url=candidate_base_url)
tried.append(f"{label} (unhealthy)")
continue
client, model = try_fn()
@@ -6788,23 +6827,30 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
# it instead of paying another doomed RTT.
_mark_provider_unhealthy(
_recoverable_pool_provider(resolved_provider, route.client, main_runtime=route.main_runtime)
or resolved_provider)
or resolved_provider, base_url=route.base_info)
logger.info("Auxiliary %s%s: %s on %s (%s), trying fallback",
task or "call", tag, reason, resolved_provider, first_err)
# Skip only the failed model for model-specific failures; 401/402 are provider-wide, so
# keep skipping the whole provider.
# auth keeps skipping the credential surface, while billing is scoped to the endpoint:
# separate custom URLs can carry separate credentials (or no billing relationship at all).
_chain_failed_model = None if reason in ("auth error", "payment error") else route.final_model
from agent.backend_identity import FailureScope
_chain_failure_scope = FailureScope.ENDPOINT if reason == "payment error" else None
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model)
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
if fb_client is None and is_auto:
fb_client, fb_model, fb_label = _try_main_fallback_chain(
task, resolved_provider or "auto", reason=reason)
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
if fb_client is None:
fb_client, fb_model, fb_label = _try_payment_fallback(
resolved_provider, task, reason=reason)
resolved_provider, task, reason=reason, failed_base_url=route.base_info,
failure_scope=_chain_failure_scope)
elif fb_client is None:
fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
resolved_provider, task, reason=reason, failed_model=_chain_failed_model)
resolved_provider, task, reason=reason, failed_model=_chain_failed_model,
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
if fb_client is not None:
# Second pass: the candidate credential was stale and quarantined — walk the discovery
# chain once more (unhealthy entries are skipped).
@@ -6815,7 +6861,8 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
return fb_resp
if _pass == 0:
fb_client, fb_model, fb_label = _try_payment_fallback(
resolved_provider, task, reason="stale fallback credential")
resolved_provider, task, reason="stale fallback credential",
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
if fb_client is None:
break
# All fallback layers exhausted — one user-visible warning, then re-raise.
+42
View File
@@ -4193,6 +4193,48 @@ class TestAuxUnhealthyCache:
# After the 402, OpenRouter is in the unhealthy cache.
assert _is_provider_unhealthy("openrouter") is True
def test_custom_billing_failure_keeps_distinct_endpoint_eligible(self):
"""A hosted custom endpoint's billing state must not quarantine a local custom endpoint."""
from agent.auxiliary_client import call_llm, _is_provider_unhealthy
hosted_url = "https://hosted.example/v1"
local_url = "http://127.0.0.1:8080/v1"
payment_error = Exception("Payment Required: weekly usage limit")
payment_error.status_code = 402
hosted_client = MagicMock(base_url=hosted_url)
hosted_client.chat.completions.create.side_effect = payment_error
local_client = MagicMock(base_url=local_url)
local_client.chat.completions.create.return_value = _DummyResponse("local-ok")
fallback_entry = {
"provider": "custom", "model": "local-model", "base_url": local_url,
"api_key": "local",
}
with patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("custom", "hosted-model", hosted_url, "hosted", None),
), patch(
"agent.auxiliary_client._get_cached_client",
return_value=(hosted_client, "hosted-model"),
), patch(
"agent.auxiliary_client._get_auxiliary_task_config",
return_value={"fallback_chain": [fallback_entry]},
), patch(
"agent.auxiliary_client._resolve_fallback_entry",
return_value=(local_client, "local-model"),
):
response = call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
assert response.choices[0].message.content == "local-ok"
assert _is_provider_unhealthy("custom", hosted_url) is True
assert _is_provider_unhealthy("custom", local_url) is False
assert hosted_client.chat.completions.create.call_count == 1
assert local_client.chat.completions.create.call_count == 1
# ── auxiliary_max_tokens_param ──────────────────────────────────────────────