From e3ff3e78d13337ffae19388aa155ed9acb67ffc0 Mon Sep 17 00:00:00 2001 From: fangliquanflq Date: Sun, 6 Sep 2026 21:21:44 +0800 Subject: [PATCH] fix(agent): quarantine failed fallback destination --- agent/auxiliary_client.py | 20 +++++++++++--- tests/agent/test_auxiliary_client.py | 39 +++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 4687801cf5..b4edb01458 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -3675,9 +3675,12 @@ def _plan_fallback_candidate( return destination, _fallback_request_kwargs(destination, **common), _rebuild -def _quarantine_fallback_candidate(task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *, tag: str = "") -> None: +def _quarantine_fallback_candidate( + task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *, + base_url: str = "", tag: str = "", +) -> None: """Refresh unavailable or still 401s: token is dead. Quarantine the candidate so the caller moves on.""" - _mark_provider_unhealthy(fb_provider or fb_label) + _mark_provider_unhealthy(fb_provider or fb_label, base_url=base_url) logger.warning("Auxiliary %s%s: fallback candidate %s has a stale/unrefreshable " "credential (%s) — skipping to next fallback", task or "call", tag, fb_label, fb_err) @@ -3738,13 +3741,17 @@ def _call_fallback_candidate_sync( if not _is_auth_error(fb_err): raise fb_provider, retry = _plan_fallback_auth_retry(destination, rebuild, async_mode=False) + failed_destination = destination if retry is not None: + failed_destination = retry[2] try: return _send(*retry) except Exception as retry_err: if not _is_auth_error(retry_err): raise - _quarantine_fallback_candidate(task, fb_label, fb_provider, fb_err) + _quarantine_fallback_candidate( + task, fb_label, fb_provider, fb_err, base_url=failed_destination.base_url, + ) return None @@ -3772,13 +3779,18 @@ async def _call_fallback_candidate_async( if not _is_auth_error(fb_err): raise fb_provider, retry = _plan_fallback_auth_retry(destination, rebuild, async_mode=True) + failed_destination = destination if retry is not None: + failed_destination = retry[2] try: return await _send(*retry) except Exception as retry_err: if not _is_auth_error(retry_err): raise - _quarantine_fallback_candidate(task, fb_label, fb_provider, fb_err, tag=" (async)") + _quarantine_fallback_candidate( + task, fb_label, fb_provider, fb_err, + base_url=failed_destination.base_url, tag=" (async)", + ) return None diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 972a15d196..a5875c7504 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -1797,7 +1797,9 @@ class TestStaleFallbackCandidateSkip: assert result.choices[0].message.content == "openrouter-serves" assert mock_fb.call_count == 2 assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential" - mock_mark.assert_called_once_with("anthropic") + mock_mark.assert_called_once_with( + "anthropic", base_url="https://api.anthropic.com", + ) assert stale_fb.chat.completions.create.call_count == 1 assert healthy_fb.chat.completions.create.call_count == 1 @@ -4235,6 +4237,41 @@ class TestAuxUnhealthyCache: assert hosted_client.chat.completions.create.call_count == 1 assert local_client.chat.completions.create.call_count == 1 + def test_custom_fallback_auth_failure_quarantines_failed_endpoint(self): + """Terminal auth failure quarantines the fallback URL, not the active custom URL.""" + from agent.auxiliary_client import ( + _call_fallback_candidate_sync, + _is_provider_unhealthy, + ) + + hosted_url = "https://hosted.example/v1" + local_url = "http://127.0.0.1:8080/v1" + hosted_client = MagicMock(base_url=hosted_url) + hosted_client.chat.completions.create.side_effect = _AuxAuth401("expired hosted key") + + with patch( + "agent.auxiliary_client._current_custom_base_url", return_value=local_url, + ), patch( + "agent.auxiliary_client._refresh_provider_credentials", return_value=False, + ): + result = _call_fallback_candidate_sync( + hosted_client, + "hosted-model", + "fallback_chain[0](custom)", + task="session_search", + messages=[{"role": "user", "content": "search"}], + temperature=None, + max_tokens=None, + tools=None, + effective_timeout=30.0, + effective_extra_body={}, + reasoning_config=None, + ) + + assert result is None + assert _is_provider_unhealthy("custom", hosted_url) is True + assert _is_provider_unhealthy("custom", local_url) is False + def test_named_custom_main_route_honors_endpoint_quarantine(self): """A named custom main route is skipped before its client is resolved.""" from agent.auxiliary_client import _mark_provider_unhealthy, _try_main_provider_route