fix(cron): fall back on transient DNS during provider resolve
Agent crons resolve OAuth credentials before the agent loop. A short macOS/WARP DNS blip raised httpx.ConnectError ([Errno 8] nodename nor servname provided) from xai-oauth token refresh, and the scheduler only walked fallback_providers on AuthError — so Daily Focus Kickoff died even when XAI_API_KEY / Anthropic were healthy. Treat ConnectError/DNS OSError (and cause-chain equivalents) like AuthError when selecting the fallback chain. Keep provider+model atomic. Regression test covers the ConnectError path.
This commit is contained in:
Andrew Fiebert
+97
-10
@@ -3747,6 +3747,78 @@ DRIFT_SKIP_MARKER = "[drift_skip]"
|
||||
DRIFT_SKIP_SILENT_MARKER = "[drift_skip:silent]"
|
||||
|
||||
|
||||
|
||||
def _is_transient_provider_resolve_error(exc: BaseException) -> bool:
|
||||
"""True when primary provider resolution failed for a transient network reason.
|
||||
|
||||
Agent crons resolve OAuth credentials (token refresh / discovery) before the
|
||||
agent loop starts. A short DNS outage (Cloudflare WARP / macOS resolver blip)
|
||||
surfaces as httpx/httpcore ConnectError or raw OSError errno 8 ("nodename nor
|
||||
servname provided") and must be eligible for ``fallback_providers`` the same
|
||||
way AuthError already is — otherwise a healthy XAI_API_KEY / Anthropic rung
|
||||
never gets tried and the whole job dies before the first model call.
|
||||
"""
|
||||
# Walk the cause chain; scheduler wraps raw transport errors.
|
||||
seen: set[int] = set()
|
||||
cur: Optional[BaseException] = exc
|
||||
while cur is not None and id(cur) not in seen:
|
||||
seen.add(id(cur))
|
||||
name = type(cur).__name__
|
||||
module = type(cur).__module__ or ""
|
||||
msg = str(cur).lower()
|
||||
# Explicit transport classes from httpx/httpcore/aiohttp.
|
||||
if name in {
|
||||
"ConnectError",
|
||||
"ConnectTimeout",
|
||||
"ReadTimeout",
|
||||
"WriteTimeout",
|
||||
"PoolTimeout",
|
||||
"NetworkError",
|
||||
"TimeoutException",
|
||||
"ClientConnectorError",
|
||||
"ClientConnectorDNSError",
|
||||
"ServerTimeoutError",
|
||||
"ClientOSError",
|
||||
}:
|
||||
return True
|
||||
if "httpx" in module or "httpcore" in module or "aiohttp" in module:
|
||||
if any(
|
||||
needle in msg
|
||||
for needle in (
|
||||
"nodename nor servname",
|
||||
"name or service not known",
|
||||
"temporary failure in name resolution",
|
||||
"failed to resolve",
|
||||
"connection refused",
|
||||
"network is unreachable",
|
||||
"timed out",
|
||||
"timeout",
|
||||
)
|
||||
):
|
||||
return True
|
||||
if isinstance(cur, OSError):
|
||||
# errno 8 = EAI_NONAME on macOS ("nodename nor servname provided")
|
||||
err_no = getattr(cur, "errno", None)
|
||||
if err_no in {8, 7, 11, 51, 60, 61, 65}: # common resolve/conn failures
|
||||
return True
|
||||
if any(
|
||||
needle in msg
|
||||
for needle in (
|
||||
"nodename nor servname",
|
||||
"name or service not known",
|
||||
"temporary failure in name resolution",
|
||||
"network is unreachable",
|
||||
)
|
||||
):
|
||||
return True
|
||||
# Bare RuntimeError/Exception that already carries the DNS text
|
||||
# (format_runtime_provider_error sometimes surfaces the raw message).
|
||||
if "nodename nor servname" in msg or "name or service not known" in msg:
|
||||
return True
|
||||
cur = cur.__cause__ or cur.__context__
|
||||
return False
|
||||
|
||||
|
||||
def _cron_preflight_enabled(cfg: dict) -> bool:
|
||||
"""Whether cron pre-dispatch configuration validation is enabled.
|
||||
|
||||
@@ -4716,15 +4788,33 @@ def run_job(
|
||||
str(runtime.get("provider") or "").strip().lower()
|
||||
or primary_provider_for_drift
|
||||
)
|
||||
except AuthError as auth_exc:
|
||||
# Primary provider auth failed — try each configured provider/model
|
||||
# pair atomically. Keeping the primary model while changing only the
|
||||
# provider can silently route a paid GPT model through OpenRouter.
|
||||
except Exception as resolve_exc:
|
||||
# Primary provider resolution failed. Walk fallback_providers for:
|
||||
# 1) AuthError (missing/expired credential)
|
||||
# 2) Transient network/DNS failures during OAuth refresh or
|
||||
# discovery (e.g. macOS morning DNS blip → httpx.ConnectError
|
||||
# "[Errno 8] nodename nor servname provided").
|
||||
# Previously only AuthError tried the chain; a ConnectError during
|
||||
# xai-oauth token refresh killed agent crons even when XAI_API_KEY
|
||||
# / Anthropic fallbacks were healthy (Daily Focus Kickoff 2026-08-11).
|
||||
# Keeping provider+model atomic still applies — never swap only the
|
||||
# provider while retaining a paid primary model.
|
||||
is_auth = isinstance(resolve_exc, AuthError)
|
||||
is_transient_net = _is_transient_provider_resolve_error(resolve_exc)
|
||||
if not (is_auth or is_transient_net):
|
||||
raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc
|
||||
|
||||
primary_provider_for_drift = (
|
||||
str(getattr(auth_exc, "provider", "") or "").strip().lower()
|
||||
str(getattr(resolve_exc, "provider", "") or "").strip().lower()
|
||||
or primary_provider_for_drift
|
||||
)
|
||||
logger.warning("Job '%s': primary auth failed (%s), trying fallback", job_id, auth_exc)
|
||||
reason = "auth" if is_auth else "transient network"
|
||||
logger.warning(
|
||||
"Job '%s': primary provider resolve failed (%s: %s), trying fallback",
|
||||
job_id,
|
||||
reason,
|
||||
resolve_exc,
|
||||
)
|
||||
fb_list = get_fallback_chain(_cfg)
|
||||
runtime = None
|
||||
for entry in fb_list:
|
||||
@@ -4758,10 +4848,7 @@ def run_job(
|
||||
except Exception as fb_exc:
|
||||
logger.debug("Job '%s': fallback %s failed: %s", job_id, fb_provider, fb_exc)
|
||||
if runtime is None:
|
||||
raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc
|
||||
except Exception as exc:
|
||||
message = format_runtime_provider_error(exc)
|
||||
raise RuntimeError(message) from exc
|
||||
raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc
|
||||
|
||||
reasoning_config = resolve_reasoning_config(
|
||||
_cfg if isinstance(_cfg, dict) else {}, str(model)
|
||||
|
||||
@@ -1082,6 +1082,70 @@ class TestRunJobConfigEnvVarExpansion:
|
||||
)
|
||||
|
||||
|
||||
def test_transient_dns_fallback_switches_provider_and_model_together(self, tmp_path):
|
||||
"""DNS blip during primary OAuth resolve must still walk fallback_providers.
|
||||
|
||||
Regression for Daily Focus Kickoff 2026-08-11: xai-oauth token refresh
|
||||
raised httpx.ConnectError ([Errno 8] nodename nor servname provided)
|
||||
and the scheduler only tried fallbacks on AuthError, so the job died
|
||||
before XAI_API_KEY / Anthropic could rescue it.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
"model:\n"
|
||||
" default: grok-4.5\n"
|
||||
" provider: xai-oauth\n"
|
||||
"fallback_providers:\n"
|
||||
" - provider: xai\n"
|
||||
" model: grok-4.5\n"
|
||||
" - provider: anthropic\n"
|
||||
" model: claude-opus-5\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
job = {
|
||||
"id": "dns-fallback",
|
||||
"name": "dns fallback",
|
||||
"prompt": "hi",
|
||||
"provider": "xai-oauth",
|
||||
"model": "grok-4.5",
|
||||
}
|
||||
fake_db = MagicMock()
|
||||
requested = []
|
||||
|
||||
def resolve_runtime(**kwargs):
|
||||
requested.append(kwargs.get("requested"))
|
||||
if kwargs.get("requested") in (None, "xai-oauth"):
|
||||
raise httpx.ConnectError(
|
||||
"[Errno 8] nodename nor servname provided, or not known"
|
||||
)
|
||||
# First fallback rung (xai API key) succeeds.
|
||||
assert kwargs["requested"] == "xai"
|
||||
assert kwargs["target_model"] == "grok-4.5"
|
||||
return {**self._RUNTIME, "provider": "xai", "api_mode": "chat_completions"}
|
||||
|
||||
with patch("cron.scheduler._hermes_home", tmp_path), \
|
||||
patch("cron.scheduler._resolve_origin", return_value=None), \
|
||||
patch("hermes_cli.env_loader.load_hermes_dotenv"), \
|
||||
patch("hermes_cli.env_loader.reset_secret_source_cache"), \
|
||||
patch("hermes_state.SessionDB", return_value=fake_db), \
|
||||
patch("hermes_cli.runtime_provider.resolve_runtime_provider",
|
||||
side_effect=resolve_runtime), \
|
||||
patch("tools.mcp_tool.discover_mcp_tools", return_value=[]), \
|
||||
patch("run_agent.AIAgent") as mock_agent_cls:
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run_conversation.return_value = {"final_response": "ok"}
|
||||
mock_agent_cls.return_value = mock_agent
|
||||
success, _, _, error = run_job(job)
|
||||
|
||||
assert success is True, error
|
||||
assert error is None
|
||||
assert requested == ["xai-oauth", "xai"]
|
||||
kwargs = mock_agent_cls.call_args.kwargs
|
||||
assert kwargs["provider"] == "xai"
|
||||
assert kwargs["model"] == "grok-4.5"
|
||||
|
||||
|
||||
def test_auth_fallback_switches_provider_and_model_together(self, tmp_path):
|
||||
"""Codex auth failure must produce OpenRouter+GLM, never OpenRouter+GPT."""
|
||||
from hermes_cli.auth import AuthError
|
||||
|
||||
Reference in New Issue
Block a user