fix(agent): quarantine failed fallback destination

This commit is contained in:
fangliquanflq
2026-09-06 21:21:44 +08:00
committed by Teknium
parent e033bbaa3c
commit e3ff3e78d1
2 changed files with 54 additions and 5 deletions
+16 -4
View File
@@ -3675,9 +3675,12 @@ def _plan_fallback_candidate(
return destination, _fallback_request_kwargs(destination, **common), _rebuild
def _quarantine_fallback_candidate(task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *, tag: str = "") -> None:
def _quarantine_fallback_candidate(
task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *,
base_url: str = "", tag: str = "",
) -> None:
"""Refresh unavailable or still 401s: token is dead. Quarantine the candidate so the caller moves on."""
_mark_provider_unhealthy(fb_provider or fb_label)
_mark_provider_unhealthy(fb_provider or fb_label, base_url=base_url)
logger.warning("Auxiliary %s%s: fallback candidate %s has a stale/unrefreshable "
"credential (%s) — skipping to next fallback", task or "call", tag, fb_label, fb_err)
@@ -3738,13 +3741,17 @@ def _call_fallback_candidate_sync(
if not _is_auth_error(fb_err):
raise
fb_provider, retry = _plan_fallback_auth_retry(destination, rebuild, async_mode=False)
failed_destination = destination
if retry is not None:
failed_destination = retry[2]
try:
return _send(*retry)
except Exception as retry_err:
if not _is_auth_error(retry_err):
raise
_quarantine_fallback_candidate(task, fb_label, fb_provider, fb_err)
_quarantine_fallback_candidate(
task, fb_label, fb_provider, fb_err, base_url=failed_destination.base_url,
)
return None
@@ -3772,13 +3779,18 @@ async def _call_fallback_candidate_async(
if not _is_auth_error(fb_err):
raise
fb_provider, retry = _plan_fallback_auth_retry(destination, rebuild, async_mode=True)
failed_destination = destination
if retry is not None:
failed_destination = retry[2]
try:
return await _send(*retry)
except Exception as retry_err:
if not _is_auth_error(retry_err):
raise
_quarantine_fallback_candidate(task, fb_label, fb_provider, fb_err, tag=" (async)")
_quarantine_fallback_candidate(
task, fb_label, fb_provider, fb_err,
base_url=failed_destination.base_url, tag=" (async)",
)
return None
+38 -1
View File
@@ -1797,7 +1797,9 @@ class TestStaleFallbackCandidateSkip:
assert result.choices[0].message.content == "openrouter-serves"
assert mock_fb.call_count == 2
assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential"
mock_mark.assert_called_once_with("anthropic")
mock_mark.assert_called_once_with(
"anthropic", base_url="https://api.anthropic.com",
)
assert stale_fb.chat.completions.create.call_count == 1
assert healthy_fb.chat.completions.create.call_count == 1
@@ -4235,6 +4237,41 @@ class TestAuxUnhealthyCache:
assert hosted_client.chat.completions.create.call_count == 1
assert local_client.chat.completions.create.call_count == 1
def test_custom_fallback_auth_failure_quarantines_failed_endpoint(self):
"""Terminal auth failure quarantines the fallback URL, not the active custom URL."""
from agent.auxiliary_client import (
_call_fallback_candidate_sync,
_is_provider_unhealthy,
)
hosted_url = "https://hosted.example/v1"
local_url = "http://127.0.0.1:8080/v1"
hosted_client = MagicMock(base_url=hosted_url)
hosted_client.chat.completions.create.side_effect = _AuxAuth401("expired hosted key")
with patch(
"agent.auxiliary_client._current_custom_base_url", return_value=local_url,
), patch(
"agent.auxiliary_client._refresh_provider_credentials", return_value=False,
):
result = _call_fallback_candidate_sync(
hosted_client,
"hosted-model",
"fallback_chain[0](custom)",
task="session_search",
messages=[{"role": "user", "content": "search"}],
temperature=None,
max_tokens=None,
tools=None,
effective_timeout=30.0,
effective_extra_body={},
reasoning_config=None,
)
assert result is None
assert _is_provider_unhealthy("custom", hosted_url) is True
assert _is_provider_unhealthy("custom", local_url) is False
def test_named_custom_main_route_honors_endpoint_quarantine(self):
"""A named custom main route is skipped before its client is resolved."""
from agent.auxiliary_client import _mark_provider_unhealthy, _try_main_provider_route