fix(agent): quarantine failed fallback destination
This commit is contained in:
@@ -3675,9 +3675,12 @@ def _plan_fallback_candidate(
|
||||
return destination, _fallback_request_kwargs(destination, **common), _rebuild
|
||||
|
||||
|
||||
def _quarantine_fallback_candidate(task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *, tag: str = "") -> None:
|
||||
def _quarantine_fallback_candidate(
|
||||
task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *,
|
||||
base_url: str = "", tag: str = "",
|
||||
) -> None:
|
||||
"""Refresh unavailable or still 401s: token is dead. Quarantine the candidate so the caller moves on."""
|
||||
_mark_provider_unhealthy(fb_provider or fb_label)
|
||||
_mark_provider_unhealthy(fb_provider or fb_label, base_url=base_url)
|
||||
logger.warning("Auxiliary %s%s: fallback candidate %s has a stale/unrefreshable "
|
||||
"credential (%s) — skipping to next fallback", task or "call", tag, fb_label, fb_err)
|
||||
|
||||
@@ -3738,13 +3741,17 @@ def _call_fallback_candidate_sync(
|
||||
if not _is_auth_error(fb_err):
|
||||
raise
|
||||
fb_provider, retry = _plan_fallback_auth_retry(destination, rebuild, async_mode=False)
|
||||
failed_destination = destination
|
||||
if retry is not None:
|
||||
failed_destination = retry[2]
|
||||
try:
|
||||
return _send(*retry)
|
||||
except Exception as retry_err:
|
||||
if not _is_auth_error(retry_err):
|
||||
raise
|
||||
_quarantine_fallback_candidate(task, fb_label, fb_provider, fb_err)
|
||||
_quarantine_fallback_candidate(
|
||||
task, fb_label, fb_provider, fb_err, base_url=failed_destination.base_url,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -3772,13 +3779,18 @@ async def _call_fallback_candidate_async(
|
||||
if not _is_auth_error(fb_err):
|
||||
raise
|
||||
fb_provider, retry = _plan_fallback_auth_retry(destination, rebuild, async_mode=True)
|
||||
failed_destination = destination
|
||||
if retry is not None:
|
||||
failed_destination = retry[2]
|
||||
try:
|
||||
return await _send(*retry)
|
||||
except Exception as retry_err:
|
||||
if not _is_auth_error(retry_err):
|
||||
raise
|
||||
_quarantine_fallback_candidate(task, fb_label, fb_provider, fb_err, tag=" (async)")
|
||||
_quarantine_fallback_candidate(
|
||||
task, fb_label, fb_provider, fb_err,
|
||||
base_url=failed_destination.base_url, tag=" (async)",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -1797,7 +1797,9 @@ class TestStaleFallbackCandidateSkip:
|
||||
assert result.choices[0].message.content == "openrouter-serves"
|
||||
assert mock_fb.call_count == 2
|
||||
assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential"
|
||||
mock_mark.assert_called_once_with("anthropic")
|
||||
mock_mark.assert_called_once_with(
|
||||
"anthropic", base_url="https://api.anthropic.com",
|
||||
)
|
||||
assert stale_fb.chat.completions.create.call_count == 1
|
||||
assert healthy_fb.chat.completions.create.call_count == 1
|
||||
|
||||
@@ -4235,6 +4237,41 @@ class TestAuxUnhealthyCache:
|
||||
assert hosted_client.chat.completions.create.call_count == 1
|
||||
assert local_client.chat.completions.create.call_count == 1
|
||||
|
||||
def test_custom_fallback_auth_failure_quarantines_failed_endpoint(self):
|
||||
"""Terminal auth failure quarantines the fallback URL, not the active custom URL."""
|
||||
from agent.auxiliary_client import (
|
||||
_call_fallback_candidate_sync,
|
||||
_is_provider_unhealthy,
|
||||
)
|
||||
|
||||
hosted_url = "https://hosted.example/v1"
|
||||
local_url = "http://127.0.0.1:8080/v1"
|
||||
hosted_client = MagicMock(base_url=hosted_url)
|
||||
hosted_client.chat.completions.create.side_effect = _AuxAuth401("expired hosted key")
|
||||
|
||||
with patch(
|
||||
"agent.auxiliary_client._current_custom_base_url", return_value=local_url,
|
||||
), patch(
|
||||
"agent.auxiliary_client._refresh_provider_credentials", return_value=False,
|
||||
):
|
||||
result = _call_fallback_candidate_sync(
|
||||
hosted_client,
|
||||
"hosted-model",
|
||||
"fallback_chain[0](custom)",
|
||||
task="session_search",
|
||||
messages=[{"role": "user", "content": "search"}],
|
||||
temperature=None,
|
||||
max_tokens=None,
|
||||
tools=None,
|
||||
effective_timeout=30.0,
|
||||
effective_extra_body={},
|
||||
reasoning_config=None,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert _is_provider_unhealthy("custom", hosted_url) is True
|
||||
assert _is_provider_unhealthy("custom", local_url) is False
|
||||
|
||||
def test_named_custom_main_route_honors_endpoint_quarantine(self):
|
||||
"""A named custom main route is skipped before its client is resolved."""
|
||||
from agent.auxiliary_client import _mark_provider_unhealthy, _try_main_provider_route
|
||||
|
||||
Reference in New Issue
Block a user