refactor(agent/client_lifecycle): collapse token/credential guards, one-line log calls, compact docstrings and comments
This commit is contained in:
+126
-206
@@ -1,9 +1,6 @@
|
||||
"""OpenAI/Anthropic wire-client lifecycle + credential refresh for ``AIAgent``.
|
||||
|
||||
``ClientLifecycleMixin`` owns the shared primary client, the single-slot per-request client caches
|
||||
(owner-thread close vs stranger-thread abort), credential refresh/rotation, and route-derived default
|
||||
headers. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
|
||||
"""
|
||||
"""OpenAI/Anthropic wire-client lifecycle + credential refresh for ``AIAgent`` (``ClientLifecycleMixin``):
|
||||
shared primary client, single-slot per-request client caches (owner-thread close vs stranger-thread abort),
|
||||
credential refresh/rotation, route-derived default headers. Extracted from ``run_agent.py``, MRO unchanged."""
|
||||
import logging
|
||||
import threading
|
||||
from contextlib import suppress
|
||||
@@ -13,12 +10,8 @@ from agent.lazy_forward import forward as _forward, forward_static as _forward_s
|
||||
from hermes_cli.timeouts import get_provider_request_timeout
|
||||
from utils import base_url_host_matches, env_float
|
||||
|
||||
# Same logger name as the origin module so log records / caplog filters are unchanged.
|
||||
logger = logging.getLogger("run_agent")
|
||||
|
||||
# Qwen Portal mimics the QwenCode CLI for portal.qwen.ai.
|
||||
_QWEN_CODE_VERSION = "0.14.1"
|
||||
|
||||
logger = logging.getLogger("run_agent") # origin module's logger name: log records / caplog filters unchanged
|
||||
_QWEN_CODE_VERSION = "0.14.1" # Qwen Portal mimics the QwenCode CLI
|
||||
# Per-request cache slot attribute names (OpenAI-style and Anthropic clients).
|
||||
_OPENAI_SLOT = "_request_client_cache"
|
||||
_ANTHROPIC_SLOT = "_request_anthropic_client_cache"
|
||||
@@ -51,8 +44,7 @@ _ROUTE_DEFAULT_HEADERS = (
|
||||
("api.kimi.com", lambda self, url: dict(_lazy_attr("agent.auxiliary_client", "_AI_GATEWAY_HEADERS"))),
|
||||
("portal.qwen.ai", lambda self, url: _qwen_portal_headers()),
|
||||
("chatgpt.com", lambda self, url: _lazy_attr("agent.codex_headers", "codex_cloudflare_headers")(
|
||||
self._client_kwargs.get("api_key", ""), base_url=url,
|
||||
)),
|
||||
self._client_kwargs.get("api_key", ""), base_url=url)),
|
||||
# Covers provider=xai and provider=xai-oauth (api.x.ai).
|
||||
("x.ai", lambda self, url: _lazy_attr("tools.xai_http", "hermes_xai_default_headers")()),
|
||||
)
|
||||
@@ -70,8 +62,6 @@ def _valid_credential_pair(api_key: Any, base_url: Any) -> bool:
|
||||
|
||||
|
||||
class ClientLifecycleMixin:
|
||||
"""Wire-client construction, caching, teardown and credential refresh (see module docstring)."""
|
||||
|
||||
def _client_log_context(self) -> str:
|
||||
thread = threading.current_thread()
|
||||
return (
|
||||
@@ -89,15 +79,13 @@ class ClientLifecycleMixin:
|
||||
|
||||
@staticmethod
|
||||
def _is_openai_client_closed(client: Any) -> bool:
|
||||
"""``is_closed`` is a bool property on httpx.Client but a method on openai.OpenAI; a bare getattr
|
||||
returned the always-truthy bound method and recreated the client on every call."""
|
||||
"""``is_closed`` is a property on httpx.Client but a method on openai.OpenAI (a bare getattr is always truthy)."""
|
||||
from unittest.mock import Mock
|
||||
if isinstance(client, Mock):
|
||||
return False
|
||||
is_closed = getattr(client, "is_closed", None)
|
||||
if is_closed is not None and (is_closed() if callable(is_closed) else bool(is_closed)):
|
||||
return True
|
||||
return bool(getattr(getattr(client, "_client", None), "is_closed", False))
|
||||
closed = is_closed is not None and (is_closed() if callable(is_closed) else bool(is_closed))
|
||||
return bool(closed) or bool(getattr(getattr(client, "_client", None), "is_closed", False))
|
||||
|
||||
@staticmethod
|
||||
def _build_keepalive_http_client(base_url: str = "", *, verify: Any = True) -> Any:
|
||||
@@ -115,27 +103,20 @@ class ClientLifecycleMixin:
|
||||
def _close_openai_client(self, client: Any, *, reason: str, shared: bool) -> None:
|
||||
if client is None:
|
||||
return
|
||||
ctx = self._client_log_context()
|
||||
# Force-close TCP sockets first (no CLOSE-WAIT accumulation), then the graceful SDK close.
|
||||
force_closed = self._force_close_tcp_sockets(client)
|
||||
try:
|
||||
client.close()
|
||||
logger.info(
|
||||
"OpenAI client closed (%s, shared=%s, tcp_force_closed=%d) %s",
|
||||
reason, shared, force_closed, self._client_log_context(),
|
||||
)
|
||||
logger.info("OpenAI client closed (%s, shared=%s, tcp_force_closed=%d) %s", reason, shared, force_closed, ctx)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"OpenAI client close failed (%s, shared=%s) %s error=%s",
|
||||
reason, shared, self._client_log_context(), exc,
|
||||
)
|
||||
logger.debug("OpenAI client close failed (%s, shared=%s) %s error=%s", reason, shared, ctx, exc)
|
||||
|
||||
def _retire_shared_openai_client(self, client: Any, *, reason: str) -> None:
|
||||
"""Ownership-safe retirement of a replaced shared OpenAI client.
|
||||
"""Ownership-safe retirement of a replaced shared OpenAI client: ``shutdown()`` sockets only, FDs to GC.
|
||||
|
||||
``close()`` releases raw FDs from the calling thread; the shared client has no owning thread and
|
||||
other threads may still hold its fd in an SSL BIO — a recycled fd then gets a TLS record written
|
||||
into an unrelated file (SQLite-header corruption). So only ``shutdown()`` the sockets (FD-safe
|
||||
from any thread) and let GC release the FDs once every borrower has unwound.
|
||||
``close()`` releases FDs from the calling thread while other threads may still hold the fd in an SSL BIO;
|
||||
a recycled fd then gets a TLS record written into an unrelated file (SQLite-header corruption).
|
||||
"""
|
||||
if client is None:
|
||||
return
|
||||
@@ -146,16 +127,13 @@ class ClientLifecycleMixin:
|
||||
reason, shutdown_count, self._client_log_context(),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Shared OpenAI client retire failed (%s) %s error=%s", reason, self._client_log_context(), exc,
|
||||
)
|
||||
logger.debug("Shared OpenAI client retire failed (%s) %s error=%s", reason, self._client_log_context(), exc)
|
||||
|
||||
def _drain_transports_after_abandonment(self, *, reason: str) -> int:
|
||||
"""FD-safe transport drain for an abandoned (timed-out) worker; returns sockets shut down.
|
||||
|
||||
The worker may be blocked in an OpenSSL read; hard-closing from the timeout thread releases FDs
|
||||
under a live BIO (native corruption / SIGSEGV). Only ``shutdown()`` so the read settles with EOF
|
||||
and the worker closes itself.
|
||||
The worker may be blocked in an OpenSSL read; hard-closing from the timeout thread releases FDs under a
|
||||
live BIO (native corruption / SIGSEGV). Only ``shutdown()`` so the read sees EOF and the worker closes itself.
|
||||
"""
|
||||
drained = 0
|
||||
# Shared primary client (codex-direct / MoA stream on it directly).
|
||||
@@ -203,8 +181,7 @@ class ClientLifecycleMixin:
|
||||
with self._openai_client_lock():
|
||||
old_client = getattr(self, "client", None)
|
||||
try:
|
||||
# MoA is a virtual provider whose ``client`` is an in-process facade, not an SDK client;
|
||||
# generic rebuild paths (rotation, timeout, dead-connection cleanup) must preserve that.
|
||||
# MoA's ``client`` is an in-process facade, not an SDK client; generic rebuilds must preserve that.
|
||||
if (getattr(self, "provider", "") or "").strip().lower() == "moa":
|
||||
from agent.moa_loop import build_moa_facade
|
||||
new_client = build_moa_facade(self, self.model)
|
||||
@@ -212,13 +189,11 @@ class ClientLifecycleMixin:
|
||||
new_client = self._create_openai_client(self._client_kwargs, reason=reason, shared=True)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to rebuild shared primary client (%s) %s error=%s",
|
||||
reason, self._client_log_context(), exc,
|
||||
"Failed to rebuild shared primary client (%s) %s error=%s", reason, self._client_log_context(), exc,
|
||||
)
|
||||
return False
|
||||
self.client = new_client
|
||||
# Never hard-close the replaced shared client: the caller may not own the thread still unwinding
|
||||
# on the old pool. Retire: sockets shut down, FD release deferred to GC.
|
||||
# Never hard-close the replaced shared client (another thread may still be unwinding on the old pool).
|
||||
self._retire_shared_openai_client(old_client, reason=f"replace:{reason}")
|
||||
return True
|
||||
|
||||
@@ -227,27 +202,23 @@ class ClientLifecycleMixin:
|
||||
client = getattr(self, "client", None)
|
||||
if client is not None and not self._is_openai_client_closed(client):
|
||||
return client
|
||||
old_client = client
|
||||
try:
|
||||
new_client = self._create_openai_client(self._client_kwargs, reason=reason, shared=True)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to recreate closed OpenAI client (%s) %s error=%s",
|
||||
reason, self._client_log_context(), exc,
|
||||
"Failed to recreate closed OpenAI client (%s) %s error=%s", reason, self._client_log_context(), exc,
|
||||
)
|
||||
raise RuntimeError("Failed to recreate closed OpenAI client") from exc
|
||||
self.client = new_client
|
||||
logger.warning(
|
||||
"Detected closed shared OpenAI client; recreated before use (%s) %s",
|
||||
reason, self._client_log_context(),
|
||||
"Detected closed shared OpenAI client; recreated before use (%s) %s", reason, self._client_log_context(),
|
||||
)
|
||||
self._close_openai_client(old_client, reason=f"replace:{reason}", shared=True)
|
||||
self._close_openai_client(client, reason=f"replace:{reason}", shared=True)
|
||||
return new_client
|
||||
|
||||
@staticmethod
|
||||
def _api_kwargs_have_image_parts(api_kwargs: dict) -> bool:
|
||||
"""True when the outbound request still contains native image parts (Chat ``messages`` or
|
||||
Responses ``input``)."""
|
||||
"""True when the outbound request still has native image parts (Chat ``messages`` / Responses ``input``)."""
|
||||
if not isinstance(api_kwargs, dict):
|
||||
return False
|
||||
|
||||
@@ -265,17 +236,15 @@ class ClientLifecycleMixin:
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ per-request client slots
|
||||
# One single-slot cache per client kind: {"client", "key", "poisoned", "in_use"}. Reuse keeps the
|
||||
# warm httpx pool between sequential calls; ``in_use`` keeps two concurrent calls off one pool;
|
||||
# ``poisoned`` marks a pool whose sockets were shut from a stranger thread (never reuse it).
|
||||
|
||||
# Close reasons reported by a request worker's own finally after a response — the only closes
|
||||
# that come from the FD-owning thread AND attest a healthy pool. Poisoning still wins.
|
||||
# One single-slot cache per client kind: {"client", "key", "poisoned", "in_use"}. Reuse keeps the warm httpx
|
||||
# pool between sequential calls; ``in_use`` keeps concurrent calls off one pool; ``poisoned`` marks a pool
|
||||
# whose sockets were shut from a stranger thread (never reuse it).
|
||||
# Reuse reasons: closes from the FD-owning worker's own finally after a response — the only closes that
|
||||
# attest a healthy pool. Poisoning still wins.
|
||||
_REQUEST_CLIENT_REUSE_REASONS = frozenset({"request_complete", "stream_request_complete"})
|
||||
|
||||
def _request_slot(self, slot_attr: str) -> dict:
|
||||
# Lazy init — tests build agents via AIAgent.__new__ without __init__.
|
||||
cache = getattr(self, slot_attr, None)
|
||||
cache = getattr(self, slot_attr, None) # lazy: tests build agents via AIAgent.__new__ without __init__
|
||||
if cache is None:
|
||||
setattr(self, slot_attr, cache := {})
|
||||
_reset_slot(cache)
|
||||
@@ -291,8 +260,8 @@ class ClientLifecycleMixin:
|
||||
if not cache["poisoned"] and cache["key"] == key and not self._is_openai_client_closed(cached):
|
||||
cache["in_use"] = True
|
||||
return cached, None
|
||||
# Key changed, poisoned by a cross-thread abort, or externally closed — rebuild. Safe to
|
||||
# close the stale one from this thread: in_use was False, so no worker owns its FDs.
|
||||
# Key changed / poisoned / externally closed — rebuild. in_use was False, so closing the stale
|
||||
# client from this thread is FD-safe (no worker owns it).
|
||||
_reset_slot(cache)
|
||||
return None, cached
|
||||
|
||||
@@ -318,25 +287,22 @@ class ClientLifecycleMixin:
|
||||
"""Teardown: empty the slot and return ``(client, was_in_use)``."""
|
||||
with self._openai_client_lock():
|
||||
cache = getattr(self, slot_attr, None)
|
||||
client = cache["client"] if cache else None
|
||||
in_use = bool(cache["in_use"]) if cache else False
|
||||
client, in_use = (cache["client"], bool(cache["in_use"])) if cache else (None, False)
|
||||
if cache is not None:
|
||||
_reset_slot(cache)
|
||||
return client, in_use
|
||||
|
||||
def _abort_request_slot_client(self, slot_attr: str, client: Any, *, reason: str) -> None:
|
||||
"""Cross-thread abort: shut sockets down without releasing FDs.
|
||||
"""Cross-thread abort (interrupt loop, stale detector): ``shutdown(SHUT_RDWR)`` without releasing FDs.
|
||||
|
||||
For stranger-thread callers (interrupt loop, stale detector). ``close()`` from a non-owning
|
||||
thread races the live SSL BIO and corrupts unrelated FDs; ``shutdown(SHUT_RDWR)`` unblocks the
|
||||
owner's recv/send so it closes from its own context. The slot is poisoned so the pool is never reused.
|
||||
``close()`` from a non-owning thread races the live SSL BIO and corrupts unrelated FDs; shutdown unblocks
|
||||
the owner's recv/send so it closes from its own context. The slot is poisoned so the pool is never reused.
|
||||
"""
|
||||
if client is None:
|
||||
return
|
||||
if slot_attr == _ANTHROPIC_SLOT:
|
||||
label, context = "Anthropic", self._anthropic_log_context()
|
||||
else:
|
||||
label, context = "OpenAI", self._client_log_context()
|
||||
anthropic = slot_attr == _ANTHROPIC_SLOT
|
||||
label = "Anthropic" if anthropic else "OpenAI"
|
||||
context = self._anthropic_log_context() if anthropic else self._client_log_context()
|
||||
with self._openai_client_lock():
|
||||
cache = self._request_slot(slot_attr)
|
||||
if cache["client"] is client:
|
||||
@@ -359,8 +325,8 @@ class ClientLifecycleMixin:
|
||||
return primary_client
|
||||
with self._openai_client_lock():
|
||||
request_kwargs = dict(self._client_kwargs)
|
||||
# Per-request clients must not run the SDK retry loop: the outer loop owns retries/rotation/
|
||||
# fallback, and SDK retries stretch a hung request ~3x past our stale detector.
|
||||
# No SDK retry loop: the outer loop owns retries/rotation/fallback, and SDK retries stretch a hung
|
||||
# request ~3x past our stale detector.
|
||||
request_kwargs["max_retries"] = 0
|
||||
is_copilot = base_url_host_matches(str(request_kwargs.get("base_url", "")), "githubcopilot.com")
|
||||
if is_copilot and self._api_kwargs_have_image_parts(api_kwargs or {}):
|
||||
@@ -372,7 +338,7 @@ class ClientLifecycleMixin:
|
||||
if stale is not None:
|
||||
self._close_openai_client(stale, reason=f"reuse_evict:{reason}", shared=False)
|
||||
client = self._create_openai_client(request_kwargs, reason=reason, shared=False)
|
||||
# Snapshot nested dicts (default_headers) so an aliased inner object can't compare equal after mutation.
|
||||
# Snapshot nested dicts (default_headers) so an aliased inner object can't mutate the cache key.
|
||||
snapshot = {k: dict(v) if isinstance(v, dict) else v for k, v in request_kwargs.items()}
|
||||
self._store_request_slot(_OPENAI_SLOT, client, snapshot)
|
||||
return client
|
||||
@@ -387,8 +353,8 @@ class ClientLifecycleMixin:
|
||||
if client is None:
|
||||
return
|
||||
if in_use:
|
||||
# A worker has this client checked out; close() here would release FDs from a stranger
|
||||
# thread. Abort the sockets; the worker's own finally does the real close.
|
||||
# Checked out by a worker: close() here would release FDs from a stranger thread. Abort the
|
||||
# sockets; the worker's own finally does the real close.
|
||||
self._abort_request_openai_client(client, reason=f"{reason}_in_flight")
|
||||
else:
|
||||
self._close_openai_client(client, reason=reason, shared=False)
|
||||
@@ -397,8 +363,7 @@ class ClientLifecycleMixin:
|
||||
self._abort_request_slot_client(_OPENAI_SLOT, client, reason=reason)
|
||||
|
||||
def _request_anthropic_client_key(self) -> tuple:
|
||||
"""Cache key covering everything that forces a fresh client: credential rotation, base URL /
|
||||
region, timeout (model switch), and the 1M-context beta flag."""
|
||||
"""Cache key over everything forcing a fresh client: credential, base URL/region, timeout, 1M-beta flag."""
|
||||
if getattr(self, "provider", None) == "bedrock":
|
||||
return ("bedrock", getattr(self, "_bedrock_region", "us-east-1") or "us-east-1")
|
||||
return (
|
||||
@@ -417,18 +382,16 @@ class ClientLifecycleMixin:
|
||||
return _is_oauth_token(token) if self.provider == "anthropic" else False
|
||||
|
||||
def _build_anthropic_client_for_key(self, key: tuple) -> Any:
|
||||
from agent.anthropic_adapter import build_anthropic_bedrock_client, build_anthropic_client
|
||||
if key[0] == "bedrock":
|
||||
from agent.anthropic_adapter import build_anthropic_bedrock_client
|
||||
return build_anthropic_bedrock_client(key[1])
|
||||
from agent.anthropic_adapter import build_anthropic_client
|
||||
return build_anthropic_client(key[1], key[2], timeout=key[3], drop_context_1m_beta=key[4])
|
||||
|
||||
def _create_request_anthropic_client(self, *, reason: str) -> Any:
|
||||
"""Build (or reuse) a request-local Anthropic client for one in-flight call.
|
||||
|
||||
The stale/interrupt watchdog must never ``close()`` the client a worker is still reading (fd
|
||||
recycled under a live SSL BIO → TLS record in a SQLite header). A per-request client lets the
|
||||
stranger ``shutdown()`` while the owner closes. Mirrors ``_rebuild_anthropic_client`` construction.
|
||||
The watchdog must never ``close()`` a client a worker is still reading (fd recycled under a live SSL BIO →
|
||||
TLS record in a SQLite header); per-request clients let the stranger ``shutdown()`` while the owner closes.
|
||||
"""
|
||||
if self.api_mode == "anthropic_messages":
|
||||
self._try_refresh_anthropic_client_credentials()
|
||||
@@ -439,15 +402,12 @@ class ClientLifecycleMixin:
|
||||
if stale is not None:
|
||||
self._close_request_anthropic_client(stale, reason=f"reuse_evict:{reason}")
|
||||
client = self._build_anthropic_client_for_key(key)
|
||||
logger.debug(
|
||||
"Anthropic request client created (%s, shared=False) %s", reason, self._anthropic_log_context(),
|
||||
)
|
||||
logger.debug("Anthropic request client created (%s, shared=False) %s", reason, self._anthropic_log_context())
|
||||
self._store_request_slot(_ANTHROPIC_SLOT, client, key)
|
||||
return client
|
||||
|
||||
def _close_request_anthropic_client(self, client: Any, *, reason: str) -> None:
|
||||
"""Owner-thread close: clean finish keeps the pool warm in the slot; any other outcome
|
||||
force-closes the TCP sockets (CLOSE-WAIT hygiene) then does the graceful SDK close."""
|
||||
"""Owner-thread close: clean finish keeps the pool warm; otherwise force-close sockets (CLOSE-WAIT) + SDK close."""
|
||||
if client is None or self._release_request_slot(_ANTHROPIC_SLOT, client, reason):
|
||||
return
|
||||
try:
|
||||
@@ -456,8 +416,7 @@ class ClientLifecycleMixin:
|
||||
logger.info("Anthropic client closed (%s, shared=False) %s", reason, self._anthropic_log_context())
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Anthropic client close failed (%s, shared=False) %s error=%s",
|
||||
reason, self._anthropic_log_context(), exc,
|
||||
"Anthropic client close failed (%s, shared=False) %s error=%s", reason, self._anthropic_log_context(), exc,
|
||||
)
|
||||
|
||||
def _close_cached_request_anthropic_client(self, *, reason: str) -> None:
|
||||
@@ -465,8 +424,7 @@ class ClientLifecycleMixin:
|
||||
client, in_use = self._take_request_slot(_ANTHROPIC_SLOT)
|
||||
if client is None:
|
||||
return
|
||||
if in_use:
|
||||
# A worker thread has this client checked out — same reasoning as the OpenAI teardown hook.
|
||||
if in_use: # checked out by a worker — same reasoning as the OpenAI teardown hook
|
||||
self._abort_request_anthropic_client(client, reason=f"{reason}_in_flight")
|
||||
return
|
||||
with suppress(Exception):
|
||||
@@ -477,7 +435,6 @@ class ClientLifecycleMixin:
|
||||
self._abort_request_slot_client(_ANTHROPIC_SLOT, client, reason=reason)
|
||||
|
||||
# ------------------------------------------------------------------ credential refresh
|
||||
|
||||
def _sync_client_kwargs_credentials(self) -> None:
|
||||
"""Mirror ``self.api_key`` / ``self.base_url`` into the OpenAI-style client kwargs."""
|
||||
self._client_kwargs["api_key"] = self.api_key
|
||||
@@ -485,22 +442,21 @@ class ClientLifecycleMixin:
|
||||
|
||||
def _adopt_openai_credentials(self, api_key: str, base_url: str, *, reason: str) -> bool:
|
||||
"""Apply a fresh key/base_url to the OpenAI-style kwargs and rebuild the shared client."""
|
||||
self.api_key = api_key.strip()
|
||||
self.base_url = base_url.strip().rstrip("/")
|
||||
self.api_key, self.base_url = api_key.strip(), base_url.strip().rstrip("/")
|
||||
self._sync_client_kwargs_credentials()
|
||||
return self._replace_primary_openai_client(reason=reason)
|
||||
|
||||
def _try_refresh_codex_client_credentials(self, *, force: bool = True) -> bool:
|
||||
if self.api_mode != "codex_responses" or self.provider not in {"openai-codex", "xai-oauth"}:
|
||||
return False
|
||||
# Guard against silent account swap: a non-singleton credential (manual pool entry, explicit
|
||||
# api_key=) must not be replaced by the device_code singleton's tokens. The pool's reactive
|
||||
# recovery owns that case; this singleton fallback MUST only fire on singleton tokens.
|
||||
# No silent account swap: a non-singleton credential (manual pool entry, explicit api_key=) must not be
|
||||
# replaced by the device_code singleton's tokens — the pool's reactive recovery owns that case.
|
||||
try:
|
||||
if self.provider == "openai-codex":
|
||||
from hermes_cli.auth import resolve_codex_runtime_credentials as resolve
|
||||
else:
|
||||
from hermes_cli.auth import resolve_xai_oauth_runtime_credentials as resolve
|
||||
from hermes_cli import auth as _auth
|
||||
resolve = (
|
||||
_auth.resolve_codex_runtime_credentials if self.provider == "openai-codex"
|
||||
else _auth.resolve_xai_oauth_runtime_credentials
|
||||
)
|
||||
singleton_now = resolve(refresh_if_expiring=False)
|
||||
except Exception as exc:
|
||||
logger.debug("%s singleton read failed: %s", self.provider, exc)
|
||||
@@ -509,10 +465,8 @@ class ClientLifecycleMixin:
|
||||
old_key = str(self.api_key or "").strip()
|
||||
if singleton_key and old_key and singleton_key != old_key:
|
||||
logger.debug(
|
||||
"%s singleton tokens differ from the active api_key; "
|
||||
"skipping singleton force-refresh to avoid silent account swap. "
|
||||
"Reactive credential rotation should go through the pool.",
|
||||
self.provider,
|
||||
"%s singleton tokens differ from the active api_key; skipping singleton force-refresh to avoid "
|
||||
"silent account swap. Reactive credential rotation should go through the pool.", self.provider,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
@@ -525,22 +479,18 @@ class ClientLifecycleMixin:
|
||||
return False
|
||||
# No NEW token minted (the resolver returns the same stale token when refresh fails) → False.
|
||||
if old_key and api_key.strip() == old_key:
|
||||
logger.debug(
|
||||
"%s credential refresh returned the same token; refresh likely failed silently", self.provider,
|
||||
)
|
||||
logger.debug("%s credential refresh returned the same token; refresh likely failed silently", self.provider)
|
||||
return False
|
||||
return self._adopt_openai_credentials(api_key, base_url, reason=f"{self.provider}_credential_refresh")
|
||||
|
||||
def _try_refresh_nous_client_credentials(self, *, force: bool = True) -> bool:
|
||||
# Portal serves anthropic/* on the native Messages route, so either client kind may hold the
|
||||
# expiring invoke JWT.
|
||||
# Portal serves anthropic/* on the native Messages route, so either client kind may hold the expiring JWT.
|
||||
if self.provider != "nous" or self.api_mode not in ("chat_completions", "anthropic_messages"):
|
||||
return False
|
||||
try:
|
||||
from hermes_cli.auth import resolve_nous_runtime_credentials
|
||||
creds = resolve_nous_runtime_credentials(
|
||||
timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=force,
|
||||
)
|
||||
timeout = env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15)
|
||||
creds = resolve_nous_runtime_credentials(timeout_seconds=timeout, force_refresh=force)
|
||||
except Exception as exc:
|
||||
logger.debug("Nous credential refresh failed: %s", exc)
|
||||
return False
|
||||
@@ -548,10 +498,8 @@ class ClientLifecycleMixin:
|
||||
if not _valid_credential_pair(api_key, base_url):
|
||||
return False
|
||||
if self.api_mode == "anthropic_messages":
|
||||
self.api_key = api_key.strip()
|
||||
self.base_url = base_url.strip().rstrip("/")
|
||||
self._anthropic_api_key = self.api_key
|
||||
self._anthropic_base_url = self.base_url
|
||||
self.api_key, self.base_url = api_key.strip(), base_url.strip().rstrip("/")
|
||||
self._anthropic_api_key, self._anthropic_base_url = self.api_key, self.base_url
|
||||
self._rebuild_anthropic_client()
|
||||
return True
|
||||
# Nous requests should not inherit OpenRouter-only attribution headers.
|
||||
@@ -559,10 +507,9 @@ class ClientLifecycleMixin:
|
||||
return self._adopt_openai_credentials(api_key, base_url, reason="nous_credential_refresh")
|
||||
|
||||
def _resolve_env_credentials(self) -> Optional[tuple]:
|
||||
"""Read the current ``.env``-sourced ``(api_key, base_url, default_base)`` for this provider.
|
||||
"""Current ``.env``-sourced ``(api_key, base_url, default_base)`` for this provider, or ``None``.
|
||||
|
||||
Covers registry api-key providers and named custom providers with ``key_env``; ``None`` when
|
||||
nothing env-sourced applies.
|
||||
Covers registry api-key providers and named custom providers with ``key_env``.
|
||||
"""
|
||||
try:
|
||||
from agent.credential_pool import get_env_prefer_dotenv
|
||||
@@ -575,9 +522,8 @@ class ClientLifecycleMixin:
|
||||
api_key = next((k for k in (get_env_prefer_dotenv(v).strip() for v in pconfig.api_key_env_vars) if k), "")
|
||||
if not api_key:
|
||||
return None
|
||||
env_url = ""
|
||||
if pconfig.base_url_env_var:
|
||||
env_url = get_env_prefer_dotenv(pconfig.base_url_env_var).strip().rstrip("/")
|
||||
url_var = pconfig.base_url_env_var
|
||||
env_url = get_env_prefer_dotenv(url_var).strip().rstrip("/") if url_var else ""
|
||||
default_base = (pconfig.inference_base_url or "").strip().rstrip("/")
|
||||
base_url = env_url or default_base
|
||||
if self.provider in ("kimi-coding", "zai"):
|
||||
@@ -585,18 +531,15 @@ class ClientLifecycleMixin:
|
||||
resolver = _auth._resolve_kimi_base_url if self.provider == "kimi-coding" else _auth._resolve_zai_base_url
|
||||
base_url = resolver(api_key, pconfig.inference_base_url, env_url).rstrip("/")
|
||||
elif self.provider == "custom":
|
||||
# Named custom provider: identity in config, credential in the env var named by key_env;
|
||||
# entries without key_env have nothing env-sourced to watch.
|
||||
# Named custom provider: identity in config, credential in key_env; no key_env → nothing to watch.
|
||||
try:
|
||||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||||
except ImportError:
|
||||
return None
|
||||
custom_provider = _get_named_custom_provider(getattr(self, "requested_provider", "") or "")
|
||||
key_env = str((custom_provider or {}).get("key_env") or "").strip()
|
||||
if not custom_provider or not key_env:
|
||||
return None
|
||||
api_key = get_env_prefer_dotenv(key_env).strip()
|
||||
if not api_key:
|
||||
api_key = get_env_prefer_dotenv(key_env).strip() if key_env else ""
|
||||
if not custom_provider or not api_key:
|
||||
return None
|
||||
# Custom providers pin base_url in config, so only key edits are adopted here.
|
||||
base_url = default_base = str(custom_provider.get("base_url") or "").strip().rstrip("/")
|
||||
@@ -613,8 +556,8 @@ class ClientLifecycleMixin:
|
||||
current_base = (self.base_url or "").strip().rstrip("/")
|
||||
unchanged = base_url == current_base and api_key == self.api_key
|
||||
if prev is None:
|
||||
# First look — adopt only the boot-default case; anything else is unattributable on turn one.
|
||||
# A pool-rotated key is not a boot-time env adoption; don't stomp it.
|
||||
# First look: adopt only the boot-default case (anything else is unattributable on turn one), and
|
||||
# never stomp a pool-rotated key.
|
||||
pool_rotated = (
|
||||
api_key != self.api_key and getattr(self, "_credential_pool", None) is not None
|
||||
and getattr(self, "_credential_pool_entry_id", None)
|
||||
@@ -624,11 +567,8 @@ class ClientLifecycleMixin:
|
||||
return (base_url, api_key) != prev and current_base in {default_base, prev[0]} and not unchanged
|
||||
|
||||
def _try_refresh_env_client_credentials(self) -> bool:
|
||||
"""Adopt ``~/.hermes/.env`` credential/base-url edits at the turn boundary.
|
||||
|
||||
A Settings save updates ``.env`` but a live worker keeps init-time values, so an open chat kept
|
||||
calling the old endpoint. See ``_should_adopt_env_credentials`` for the adoption rule.
|
||||
"""
|
||||
"""Adopt ``~/.hermes/.env`` credential/base-url edits at the turn boundary (a Settings save updates ``.env``
|
||||
but a live worker keeps init-time values). Adoption rule: ``_should_adopt_env_credentials``."""
|
||||
if self.api_mode != "chat_completions" or getattr(self, "_fallback_activated", False):
|
||||
return False
|
||||
resolved = self._resolve_env_credentials()
|
||||
@@ -642,8 +582,7 @@ class ClientLifecycleMixin:
|
||||
route_changed = normalize_route_base_url(self.base_url) != normalize_route_base_url(base_url)
|
||||
prior_api_key, prior_base_url = self.api_key, self.base_url
|
||||
prior_client_kwargs = dict(self._client_kwargs)
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
self.api_key, self.base_url = api_key, base_url
|
||||
self._sync_client_kwargs_credentials()
|
||||
# A base-url change moves the route: recompute TLS material and default headers.
|
||||
self._reapply_route_client_config(route_changed=route_changed)
|
||||
@@ -664,8 +603,7 @@ class ClientLifecycleMixin:
|
||||
return True
|
||||
|
||||
def _try_refresh_vertex_client_credentials(self) -> bool:
|
||||
"""Re-mint the Vertex OAuth2 access token (~1h TTL; long gateway sessions 401 on the expired
|
||||
bearer) and rebuild the OpenAI client."""
|
||||
"""Re-mint the Vertex OAuth2 token (~1h TTL; long sessions 401 on the expired bearer) and rebuild the client."""
|
||||
if self.api_mode != "chat_completions" or self.provider != "vertex":
|
||||
return False
|
||||
try:
|
||||
@@ -674,12 +612,12 @@ class ClientLifecycleMixin:
|
||||
except Exception as exc:
|
||||
logger.debug("Vertex credential refresh failed: %s", exc)
|
||||
return False
|
||||
if not _valid_credential_pair(token, base_url) or not self._adopt_openai_credentials(
|
||||
ok = _valid_credential_pair(token, base_url) and self._adopt_openai_credentials(
|
||||
token, base_url, reason="vertex_credential_refresh",
|
||||
):
|
||||
return False
|
||||
logger.info("Vertex AI OAuth token refreshed")
|
||||
return True
|
||||
)
|
||||
if ok:
|
||||
logger.info("Vertex AI OAuth token refreshed")
|
||||
return ok
|
||||
|
||||
def _apply_copilot_token(self, token: str, enterprise_base_url: Any, *, reason: str) -> bool:
|
||||
self.api_key = token
|
||||
@@ -690,11 +628,10 @@ class ClientLifecycleMixin:
|
||||
return self._replace_primary_openai_client(reason=reason)
|
||||
|
||||
def _try_refresh_copilot_client_credentials(self) -> bool:
|
||||
"""Refresh Copilot credentials and rebuild the shared OpenAI client.
|
||||
"""Refresh Copilot credentials and rebuild the shared OpenAI client (caller enforces the single-shot guard).
|
||||
|
||||
The raw GitHub token is stable, but the short-TTL *exchanged* IDE token is what authenticates and
|
||||
expires mid-turn (``401 IDE token expired``); re-resolving the raw token leaves the same expired
|
||||
JWT on the wire, so force a fresh exchange. Caller enforces the single-shot guard.
|
||||
The raw GitHub token is stable; the short-TTL *exchanged* IDE token is what expires mid-turn (``401 IDE
|
||||
token expired``), so force a fresh exchange rather than re-resolving the raw token.
|
||||
"""
|
||||
if not self._is_copilot_provider():
|
||||
return False
|
||||
@@ -704,9 +641,9 @@ class ClientLifecycleMixin:
|
||||
except Exception as exc:
|
||||
logger.debug("Copilot credential refresh failed: %s", exc)
|
||||
return False
|
||||
if not isinstance(new_token, str) or not new_token.strip():
|
||||
new_token, enterprise_base_url = (new_token.strip() if isinstance(new_token, str) else ""), None
|
||||
if not new_token:
|
||||
return False
|
||||
new_token, enterprise_base_url = new_token.strip(), None
|
||||
# Fall back to the raw token only if the exchange itself is unavailable.
|
||||
try:
|
||||
evict_cached_exchanged_token(new_token)
|
||||
@@ -715,17 +652,16 @@ class ClientLifecycleMixin:
|
||||
new_token, enterprise_base_url = api_token.strip(), exchanged_base_url
|
||||
except Exception as exc:
|
||||
logger.debug("Copilot 401 re-exchange failed, using resolved token: %s", exc)
|
||||
if self._apply_copilot_token(new_token, enterprise_base_url, reason="copilot_credential_refresh"):
|
||||
ok = self._apply_copilot_token(new_token, enterprise_base_url, reason="copilot_credential_refresh")
|
||||
if ok:
|
||||
logger.info("Copilot credentials refreshed from %s", token_source)
|
||||
return True
|
||||
return False
|
||||
return ok
|
||||
|
||||
def _try_recover_stale_copilot_credential(self) -> bool:
|
||||
"""Force a fresh Copilot token exchange + client rebuild after a 400.
|
||||
"""Force a fresh Copilot token exchange + client rebuild after a 400 (single-shot, caller-guarded).
|
||||
|
||||
Copilot surfaces a stale credential as ``400 model_not_available_for_integrator`` /
|
||||
``model_not_supported``, not a 401 — typically a raw ``ghu_`` token cached when the startup
|
||||
exchange degraded, routing to the restricted integrator allowlist. Single-shot (caller-guarded).
|
||||
Copilot surfaces a stale credential as ``400 model_not_available_for_integrator`` / ``model_not_supported``
|
||||
— typically a raw ``ghu_`` token cached when the startup exchange degraded (restricted integrator allowlist).
|
||||
"""
|
||||
if not self._is_copilot_provider():
|
||||
return False
|
||||
@@ -735,7 +671,7 @@ class ClientLifecycleMixin:
|
||||
if not isinstance(raw_token, str) or not raw_token.strip():
|
||||
return False
|
||||
raw_token = raw_token.strip()
|
||||
# Drop any cached (possibly degraded/raw) exchanged token so the next exchange hits the network.
|
||||
# Evict the cached (possibly degraded/raw) exchanged token so the exchange hits the network.
|
||||
evict_cached_exchanged_token(raw_token)
|
||||
api_token, enterprise_base_url = get_copilot_api_token(raw_token)
|
||||
except Exception as exc:
|
||||
@@ -743,23 +679,21 @@ class ClientLifecycleMixin:
|
||||
return False
|
||||
if not isinstance(api_token, str) or not api_token.strip():
|
||||
return False
|
||||
# If the exchange STILL degraded to the raw token, a rebuild won't help — don't burn the
|
||||
# single-shot retry on an identical request.
|
||||
# Exchange STILL degraded to the raw token: a rebuild won't help — don't burn the single-shot retry.
|
||||
if api_token == raw_token and not enterprise_base_url:
|
||||
logger.warning(
|
||||
"Copilot stale-credential recovery: exchange still degraded to "
|
||||
"raw token; skipping retry (network/exchange endpoint unavailable)."
|
||||
"Copilot stale-credential recovery: exchange still degraded to raw token; skipping retry "
|
||||
"(network/exchange endpoint unavailable)."
|
||||
)
|
||||
return False
|
||||
if self._apply_copilot_token(api_token.strip(), enterprise_base_url, reason="copilot_stale_credential_recovery"):
|
||||
ok = self._apply_copilot_token(api_token.strip(), enterprise_base_url, reason="copilot_stale_credential_recovery")
|
||||
if ok:
|
||||
logger.info("Copilot credentials re-exchanged after stale-credential 400 (source=%s)", token_source)
|
||||
return True
|
||||
return False
|
||||
return ok
|
||||
|
||||
def _try_refresh_anthropic_client_credentials(self) -> bool:
|
||||
# Only the native Anthropic provider rotates OAuth tokens; other anthropic_messages providers
|
||||
# (MiniMax, Alibaba, ...) use their own keys, and Azure endpoints use static API keys (a refresh
|
||||
# would pick up the ~/.claude OAuth token and break auth).
|
||||
# Only native Anthropic rotates OAuth tokens; other anthropic_messages providers (MiniMax, Alibaba, ...)
|
||||
# and Azure use static keys — a refresh would pick up the ~/.claude OAuth token and break auth.
|
||||
if (
|
||||
self.api_mode != "anthropic_messages" or not hasattr(self, "_anthropic_api_key")
|
||||
or self.provider != "anthropic"
|
||||
@@ -772,47 +706,39 @@ class ClientLifecycleMixin:
|
||||
except Exception as exc:
|
||||
logger.debug("Anthropic credential refresh failed: %s", exc)
|
||||
return False
|
||||
if not isinstance(new_token, str) or not new_token.strip():
|
||||
return False
|
||||
new_token = new_token.strip()
|
||||
if new_token == self._anthropic_api_key:
|
||||
new_token = new_token.strip() if isinstance(new_token, str) else ""
|
||||
if not new_token or new_token == self._anthropic_api_key:
|
||||
return False
|
||||
with suppress(Exception):
|
||||
self._anthropic_client.close()
|
||||
try:
|
||||
self._anthropic_client = self._build_direct_anthropic_client(new_token, getattr(self, "_anthropic_base_url", None))
|
||||
base_url = getattr(self, "_anthropic_base_url", None)
|
||||
self._anthropic_client = self._build_direct_anthropic_client(new_token, base_url)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to rebuild Anthropic client after credential refresh: %s", exc)
|
||||
return False
|
||||
self._anthropic_api_key = new_token
|
||||
self._is_anthropic_oauth = self._anthropic_oauth_flag(new_token)
|
||||
self._anthropic_api_key, self._is_anthropic_oauth = new_token, self._anthropic_oauth_flag(new_token)
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------ route-derived client config
|
||||
|
||||
def _apply_client_headers_for_base_url(self, base_url: str, *, apply_user_headers: bool = True) -> None:
|
||||
for host, build in _ROUTE_DEFAULT_HEADERS:
|
||||
if base_url_host_matches(base_url, host):
|
||||
self._client_kwargs["default_headers"] = build(self, base_url)
|
||||
break
|
||||
else:
|
||||
# No URL-specific headers — fall back to profile.default_headers before clearing.
|
||||
profile_headers = None
|
||||
# No URL-specific headers — fall back to profile.default_headers, else clear.
|
||||
self._client_kwargs.pop("default_headers", None)
|
||||
with suppress(Exception):
|
||||
from providers import get_provider_profile
|
||||
profile = get_provider_profile(self.provider)
|
||||
if profile and profile.default_headers:
|
||||
profile_headers = dict(profile.default_headers)
|
||||
if profile_headers:
|
||||
self._client_kwargs["default_headers"] = profile_headers
|
||||
else:
|
||||
self._client_kwargs.pop("default_headers", None)
|
||||
# User-configured overrides win over URL/profile defaults for the same route; a credential swap
|
||||
# to another endpoint must not inherit them.
|
||||
if profile and profile.default_headers and (profile_headers := dict(profile.default_headers)):
|
||||
self._client_kwargs["default_headers"] = profile_headers
|
||||
# User overrides win over URL/profile defaults for the same route; a swap to another endpoint must not
|
||||
# inherit them.
|
||||
if apply_user_headers:
|
||||
self._apply_user_default_headers()
|
||||
# Per-provider extra_headers applied last so they survive credential swaps and rebuilds.
|
||||
# SECURITY: values may carry credentials — never log them.
|
||||
# Per-provider extra_headers last so they survive swaps/rebuilds. SECURITY: may carry credentials; never log.
|
||||
if self.api_mode not in ("anthropic_messages", "bedrock_converse"):
|
||||
try:
|
||||
from hermes_cli.config import apply_custom_provider_extra_headers_to_client_kwargs
|
||||
@@ -821,9 +747,8 @@ class ClientLifecycleMixin:
|
||||
logger.debug("custom-provider extra_headers skipped", exc_info=True)
|
||||
|
||||
def _apply_user_default_headers(self) -> None:
|
||||
"""Merge ``model.default_headers`` from config onto the OpenAI client (user values win), so custom
|
||||
endpoints behind a WAF that rejects the SDK's identifying headers work. Delegates to
|
||||
``agent.auxiliary_client`` so main and auxiliary clients cannot drift. No-op for Anthropic/Bedrock."""
|
||||
"""Merge config ``model.default_headers`` onto the OpenAI client (user wins; WAFs rejecting SDK headers).
|
||||
Delegates to ``agent.auxiliary_client`` so main and aux clients cannot drift. No-op for Anthropic/Bedrock."""
|
||||
if self.api_mode in ("anthropic_messages", "bedrock_converse"):
|
||||
return
|
||||
from agent.auxiliary_client import _apply_user_default_headers as _merge_user_headers
|
||||
@@ -841,15 +766,12 @@ class ClientLifecycleMixin:
|
||||
if self.api_mode == "anthropic_messages":
|
||||
with suppress(Exception):
|
||||
self._anthropic_client.close()
|
||||
self._anthropic_api_key = runtime_key
|
||||
self._anthropic_base_url = stripped_base
|
||||
self._anthropic_api_key, self._anthropic_base_url = runtime_key, stripped_base
|
||||
self._anthropic_client = self._build_direct_anthropic_client(runtime_key, self._anthropic_base_url)
|
||||
self._is_anthropic_oauth = self._anthropic_oauth_flag(runtime_key)
|
||||
self.api_key = runtime_key
|
||||
self.base_url = stripped_base
|
||||
self.api_key, self.base_url = runtime_key, stripped_base
|
||||
return
|
||||
self.api_key = runtime_key
|
||||
self.base_url = stripped_base
|
||||
self.api_key, self.base_url = runtime_key, stripped_base
|
||||
# Inlined (not _sync_client_kwargs_credentials): tests call this unbound on a SimpleNamespace agent.
|
||||
self._client_kwargs["api_key"] = self.api_key
|
||||
self._client_kwargs["base_url"] = self.base_url
|
||||
@@ -859,8 +781,7 @@ class ClientLifecycleMixin:
|
||||
def _reapply_route_client_config(self, *, route_changed: bool) -> None:
|
||||
"""Recompute route-derived client kwargs (TLS material, default headers) for ``self.base_url``.
|
||||
|
||||
Any rebuild that may have moved ``base_url`` must call this or the new endpoint inherits the old
|
||||
one's config. Shared by pool rotation and the per-turn env refresh so they cannot drift.
|
||||
Any rebuild that may have moved ``base_url`` must call this or the new endpoint inherits the old config.
|
||||
"""
|
||||
self._client_kwargs.pop("ssl_verify", None)
|
||||
self._client_kwargs.pop("ssl_ca_cert", None)
|
||||
@@ -889,6 +810,5 @@ class ClientLifecycleMixin:
|
||||
)
|
||||
|
||||
def _rebuild_anthropic_client(self) -> None:
|
||||
"""Rebuild the Anthropic client after an interrupt or stale call (Bedrock SDK when provider is
|
||||
bedrock; honors ``_oauth_1m_beta_disabled`` so the rebuilt client carries the reduced beta set)."""
|
||||
"""Rebuild the Anthropic client after an interrupt/stale call (Bedrock SDK for bedrock; honors 1M-beta flag)."""
|
||||
self._anthropic_client = self._build_anthropic_client_for_key(self._request_anthropic_client_key())
|
||||
|
||||
Reference in New Issue
Block a user