From b40998bc3c47f04c44ea01f6247e26c965399ee7 Mon Sep 17 00:00:00 2001 From: fangliquanflq Date: Sun, 6 Sep 2026 20:24:54 +0800 Subject: [PATCH] fix(agent): isolate custom endpoint billing health --- agent/auxiliary_client.py | 135 ++++++++++++++++++--------- tests/agent/test_auxiliary_client.py | 42 +++++++++ 2 files changed, 133 insertions(+), 44 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 34a12bf2cc..3ec6241712 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -2883,8 +2883,8 @@ def _get_provider_chain() -> List[tuple]: # "Recently 402'd" unhealthy-provider cache: a depleted provider stays so for hours, so hiding it # for a TTL saves an RTT per aux call. In-process only (profiles may use different keys). _AUX_UNHEALTHY_TTL_SECONDS = 600 # 10 minutes -_aux_unhealthy_until: Dict[str, float] = {} -_aux_unhealthy_logged_at: Dict[str, float] = {} +_aux_unhealthy_until: Dict[Any, float] = {} +_aux_unhealthy_logged_at: Dict[Any, float] = {} # resolved_provider / explicit-config names → chain labels. _AUX_UNHEALTHY_LABEL_ALIASES = { "openrouter": "openrouter", "nous": "nous", "custom": "local/custom", @@ -2900,14 +2900,26 @@ def _normalize_chain_label(provider: str) -> str: return _AUX_UNHEALTHY_LABEL_ALIASES.get(p, p) -def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None: - """Hide ``provider`` from chain iteration until the TTL expires (after a confirmed payment error).""" +def _unhealthy_cache_key(provider: str, base_url: Optional[str] = None) -> Any: + """Provider-wide key, or endpoint-specific key for an explicit custom endpoint.""" + label = _normalize_chain_label(provider) + endpoint = str(base_url or "").strip().lower().rstrip("/") + if endpoint and (label == "local/custom" or label.startswith("custom:")): + return label, endpoint + return label + + +def _mark_provider_unhealthy( + provider: str, ttl: Optional[float] = None, *, base_url: Optional[str] = None, +) -> None: + """Hide one provider endpoint until the TTL expires after a confirmed payment error.""" label = _normalize_chain_label(provider) if not label: return + key = _unhealthy_cache_key(label, base_url) ttl = _AUX_UNHEALTHY_TTL_SECONDS if ttl is None else ttl expires_at = time.time() + ttl - _aux_unhealthy_until[label] = expires_at + _aux_unhealthy_until[key] = expires_at logger.warning( "Auxiliary: marking %s unhealthy for %ds (payment / credit error). " "Subsequent auxiliary calls will skip it until %s.", @@ -2915,26 +2927,30 @@ def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None ) -def _is_provider_unhealthy(label: str) -> bool: - """True iff ``label`` is unhealthy and unexpired; lazily evicts expired entries.""" +def _is_provider_unhealthy(label: str, base_url: Optional[str] = None) -> bool: + """True iff this provider endpoint is unhealthy and unexpired; lazily evicts expired entries.""" if not label: return False - expires_at = _aux_unhealthy_until.get(label) + key = _unhealthy_cache_key(label, base_url) + expires_at = _aux_unhealthy_until.get(key) if expires_at is None: return False if time.time() >= expires_at: - _aux_unhealthy_until.pop(label, None) - _aux_unhealthy_logged_at.pop(label, None) + _aux_unhealthy_until.pop(key, None) + _aux_unhealthy_logged_at.pop(key, None) return False return True -def _log_skip_unhealthy(label: str, task: Optional[str] = None) -> None: +def _log_skip_unhealthy( + label: str, task: Optional[str] = None, *, base_url: Optional[str] = None, +) -> None: """Log a skipped unhealthy provider at most once per minute per label.""" now = time.time() - if now - _aux_unhealthy_logged_at.get(label, 0.0) >= 60: - _aux_unhealthy_logged_at[label] = now - expires_at = _aux_unhealthy_until.get(label, now) + key = _unhealthy_cache_key(label, base_url) + if now - _aux_unhealthy_logged_at.get(key, 0.0) >= 60: + _aux_unhealthy_logged_at[key] = now + expires_at = _aux_unhealthy_until.get(key, now) logger.info( "Auxiliary %s: skipping %s (recently returned payment error, retry in %ds)", task or "call", label, max(0, int(expires_at - now)), @@ -3749,7 +3765,8 @@ async def _call_fallback_candidate_async( def _try_payment_fallback( - failed_provider: str, task: str = None, reason: str = "payment error" + failed_provider: str, task: str = None, reason: str = "payment error", *, + failed_base_url: str = "", failure_scope: Any = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Try the auto-detection chain after a payment/credit or connection error, skipping the failed provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "").""" @@ -3759,12 +3776,16 @@ def _try_payment_fallback( if main_provider and main_provider.lower() in skip: skip_labels.add(main_provider.lower()) skip_chain_labels = {_normalize_chain_label(s) for s in skip_labels} + skip_backend = _failed_backend_skip( + failed_provider, None, failed_base_url=failed_base_url, failure_scope=failure_scope) tried = [] for label, try_fn in _get_provider_chain(): - if label in skip_chain_labels: + candidate_base_url = _current_custom_base_url() if label == "local/custom" else "" + if (not failed_base_url and label in skip_chain_labels) or skip_backend( + label, None, candidate_base_url): continue - if _is_provider_unhealthy(label): - _log_skip_unhealthy(label, task) + if _is_provider_unhealthy(label, candidate_base_url): + _log_skip_unhealthy(label, task, base_url=candidate_base_url) tried.append(f"{label} (unhealthy)") continue client, model = try_fn() @@ -3778,14 +3799,18 @@ def _try_payment_fallback( return None, None, "" -def _failed_backend_skip(failed_provider: str, failed_model: Optional[str]) -> Callable[..., bool]: +def _failed_backend_skip( + failed_provider: str, failed_model: Optional[str], *, failed_base_url: str = "", + failure_scope: Any = None, +) -> Callable[..., bool]: """Predicate ``skip(provider, model, base_url="")`` → True when a candidate must be skipped for the failed route. Scope: ``failed_model`` → model-scoped (only that deployment; timeout/connection/rate-limit); None → credential-wide (whole provider; auth/payment).""" from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate skip_model = (failed_model or "").strip().lower() or None - failed_ident = BackendIdentity.build(provider=failed_provider, model=skip_model) - failure_scope = FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL + failed_ident = BackendIdentity.build( + provider=failed_provider, model=skip_model, base_url=failed_base_url) + failure_scope = failure_scope or (FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL) def _skip(provider: str, model: Optional[str], base_url: str = "") -> bool: return should_skip_candidate( @@ -3796,7 +3821,7 @@ def _failed_backend_skip(failed_provider: str, failed_model: Optional[str]) -> C def _try_main_agent_model_fallback( failed_provider: str, task: str = None, reason: str = "error", - failed_model: Optional[str] = None, + failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Last-resort fallback to the main agent provider + model after the configured chain is exhausted. ``failed_model`` scoping per ``_failed_backend_skip``; same-URL custom endpoints serve many models, @@ -3811,10 +3836,13 @@ def _try_main_agent_model_fallback( main_provider, main_model = _agg_provider, _agg_model if not main_provider or not main_model or main_provider.lower() in {"auto", ""}: return None, None, "" - if _failed_backend_skip(failed_provider, failed_model)(main_provider, main_model): + main_base_url = _current_custom_base_url() if _normalize_chain_label(main_provider) == "local/custom" else "" + if _failed_backend_skip( + failed_provider, failed_model, failed_base_url=failed_base_url, + failure_scope=failure_scope)(main_provider, main_model, main_base_url): return None, None, "" - if _is_provider_unhealthy(main_provider): - _log_skip_unhealthy(main_provider, task) + if _is_provider_unhealthy(main_provider, main_base_url): + _log_skip_unhealthy(main_provider, task, base_url=main_base_url) return None, None, "" try: client, resolved_model = resolve_provider_client(provider=main_provider, model=main_model) @@ -3881,7 +3909,8 @@ def _context_too_small( def _try_configured_fallback_chain( - task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None + task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None, *, + failed_base_url: str = "", failure_scope: Any = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Try auxiliary..fallback_chain entries in order (each needs ``provider``; model/base_url/api_key optional). ``failed_model`` scoping per ``_failed_backend_skip`` (sibling models on the same provider still @@ -3891,7 +3920,8 @@ def _try_configured_fallback_chain( chain = _get_auxiliary_task_config(task).get("fallback_chain") if not chain or not isinstance(chain, list): return None, None, "" - skip = _failed_backend_skip(failed_provider, failed_model) + skip = _failed_backend_skip( + failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope) tried = [] min_ctx = _task_minimum_context_length(task) for i, entry in enumerate(chain): @@ -3901,7 +3931,12 @@ def _try_configured_fallback_chain( if not fb_provider: continue fb_model_raw = str(entry.get("model", "")).strip() - if skip(fb_provider, fb_model_raw, str(entry.get("base_url") or "")): + fb_base_url = str(entry.get("base_url") or "") + if skip(fb_provider, fb_model_raw, fb_base_url): + continue + if _is_provider_unhealthy(fb_provider, fb_base_url): + _log_skip_unhealthy(fb_provider, task, base_url=fb_base_url) + tried.append(f"fallback_chain[{i}]({fb_provider}) (unhealthy)") continue fb_model = fb_model_raw or None label = f"fallback_chain[{i}]({fb_provider})" @@ -3960,7 +3995,8 @@ def _resolve_fallback_entry(entry: Dict[str, Any]) -> Tuple[Optional[Any], Optio def _try_main_fallback_chain( - task: Optional[str], failed_provider: str = "", reason: str = "error" + task: Optional[str], failed_provider: str = "", reason: str = "error", *, + failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Top-level main-agent fallback chain for a ``provider: auto`` auxiliary call: auto tasks honour the user's main fallback policy before the built-in discovery chain; read via ``get_fallback_chain`` so @@ -3974,7 +4010,8 @@ def _try_main_fallback_chain( return None, None, "" if not chain: return None, None, "" - skip = {p for p in ((failed_provider or "").strip().lower(), (_read_main_provider() or "").strip().lower(), "auto") if p} + skip = _failed_backend_skip( + failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope) tried: List[str] = [] min_ctx = _task_minimum_context_length(task) for i, entry in enumerate(chain): @@ -3986,11 +4023,12 @@ def _try_main_fallback_chain( continue fb_norm = fb_provider.lower() label = f"fallback_providers[{i}]({fb_provider})" - if fb_norm in skip: + fb_base_url = str(entry.get("base_url") or "") + if fb_norm == "auto" or skip(fb_provider, fb_model, fb_base_url): tried.append(f"{label} (skipped)") continue - if _is_provider_unhealthy(fb_norm): - _log_skip_unhealthy(fb_norm, task) + if _is_provider_unhealthy(fb_norm, fb_base_url): + _log_skip_unhealthy(fb_norm, task, base_url=fb_base_url) tried.append(f"{label} (unhealthy)") continue try: @@ -4093,8 +4131,8 @@ def _try_main_provider_route( explicit_api_key = runtime_api_key # Skip if the main provider was recently 402'd (unhealthy TTL bounds the bypass). main_chain_label = _normalize_chain_label(resolved_provider) - if main_chain_label and _is_provider_unhealthy(main_chain_label): - _log_skip_unhealthy(main_chain_label) + if main_chain_label and _is_provider_unhealthy(main_chain_label, explicit_base_url): + _log_skip_unhealthy(main_chain_label, base_url=explicit_base_url) return None client, resolved = resolve_provider_client( resolved_provider, main_model, explicit_base_url=explicit_base_url, @@ -4110,8 +4148,9 @@ def _try_discovery_chain() -> Tuple[Optional[OpenAI], Optional[str], str]: """Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers.""" tried = [] for label, try_fn in _get_provider_chain(): - if _is_provider_unhealthy(label): - _log_skip_unhealthy(label) + candidate_base_url = _current_custom_base_url() if label == "local/custom" else "" + if _is_provider_unhealthy(label, candidate_base_url): + _log_skip_unhealthy(label, base_url=candidate_base_url) tried.append(f"{label} (unhealthy)") continue client, model = try_fn() @@ -6788,23 +6827,30 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute): # it instead of paying another doomed RTT. _mark_provider_unhealthy( _recoverable_pool_provider(resolved_provider, route.client, main_runtime=route.main_runtime) - or resolved_provider) + or resolved_provider, base_url=route.base_info) logger.info("Auxiliary %s%s: %s on %s (%s), trying fallback", task or "call", tag, reason, resolved_provider, first_err) # Skip only the failed model for model-specific failures; 401/402 are provider-wide, so - # keep skipping the whole provider. + # auth keeps skipping the credential surface, while billing is scoped to the endpoint: + # separate custom URLs can carry separate credentials (or no billing relationship at all). _chain_failed_model = None if reason in ("auth error", "payment error") else route.final_model + from agent.backend_identity import FailureScope + _chain_failure_scope = FailureScope.ENDPOINT if reason == "payment error" else None fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model) + task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model, + failed_base_url=route.base_info, failure_scope=_chain_failure_scope) if fb_client is None and is_auto: fb_client, fb_model, fb_label = _try_main_fallback_chain( - task, resolved_provider or "auto", reason=reason) + task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model, + failed_base_url=route.base_info, failure_scope=_chain_failure_scope) if fb_client is None: fb_client, fb_model, fb_label = _try_payment_fallback( - resolved_provider, task, reason=reason) + resolved_provider, task, reason=reason, failed_base_url=route.base_info, + failure_scope=_chain_failure_scope) elif fb_client is None: fb_client, fb_model, fb_label = _try_main_agent_model_fallback( - resolved_provider, task, reason=reason, failed_model=_chain_failed_model) + resolved_provider, task, reason=reason, failed_model=_chain_failed_model, + failed_base_url=route.base_info, failure_scope=_chain_failure_scope) if fb_client is not None: # Second pass: the candidate credential was stale and quarantined — walk the discovery # chain once more (unhealthy entries are skipped). @@ -6815,7 +6861,8 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute): return fb_resp if _pass == 0: fb_client, fb_model, fb_label = _try_payment_fallback( - resolved_provider, task, reason="stale fallback credential") + resolved_provider, task, reason="stale fallback credential", + failed_base_url=route.base_info, failure_scope=_chain_failure_scope) if fb_client is None: break # All fallback layers exhausted — one user-visible warning, then re-raise. diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 3271af6080..80a1c47a61 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -4193,6 +4193,48 @@ class TestAuxUnhealthyCache: # After the 402, OpenRouter is in the unhealthy cache. assert _is_provider_unhealthy("openrouter") is True + def test_custom_billing_failure_keeps_distinct_endpoint_eligible(self): + """A hosted custom endpoint's billing state must not quarantine a local custom endpoint.""" + from agent.auxiliary_client import call_llm, _is_provider_unhealthy + + hosted_url = "https://hosted.example/v1" + local_url = "http://127.0.0.1:8080/v1" + payment_error = Exception("Payment Required: weekly usage limit") + payment_error.status_code = 402 + + hosted_client = MagicMock(base_url=hosted_url) + hosted_client.chat.completions.create.side_effect = payment_error + local_client = MagicMock(base_url=local_url) + local_client.chat.completions.create.return_value = _DummyResponse("local-ok") + fallback_entry = { + "provider": "custom", "model": "local-model", "base_url": local_url, + "api_key": "local", + } + + with patch( + "agent.auxiliary_client._resolve_task_provider_model", + return_value=("custom", "hosted-model", hosted_url, "hosted", None), + ), patch( + "agent.auxiliary_client._get_cached_client", + return_value=(hosted_client, "hosted-model"), + ), patch( + "agent.auxiliary_client._get_auxiliary_task_config", + return_value={"fallback_chain": [fallback_entry]}, + ), patch( + "agent.auxiliary_client._resolve_fallback_entry", + return_value=(local_client, "local-model"), + ): + response = call_llm( + task="compression", + messages=[{"role": "user", "content": "summarize"}], + ) + + assert response.choices[0].message.content == "local-ok" + assert _is_provider_unhealthy("custom", hosted_url) is True + assert _is_provider_unhealthy("custom", local_url) is False + assert hosted_client.chat.completions.create.call_count == 1 + assert local_client.chat.completions.create.call_count == 1 + # ── auxiliary_max_tokens_param ──────────────────────────────────────────────