From 4b09903de5b93a92853a6c3ec398b3b077949b0c Mon Sep 17 00:00:00 2001 From: Shannon Sands Date: Thu, 18 Jun 2026 10:32:49 +1000 Subject: [PATCH] fix Nous auth refresh for idle agents --- agent/auxiliary_client.py | 62 ++++++ gateway/run.py | 14 ++ hermes_cli/nous_auth_keepalive.py | 189 ++++++++++++++++++ hermes_cli/runtime_provider.py | 30 ++- hermes_cli/web_server.py | 7 + tests/agent/test_auxiliary_client.py | 83 ++++++++ tests/hermes_cli/test_nous_auth_keepalive.py | 60 ++++++ .../test_runtime_provider_resolution.py | 60 ++++++ tests/run_agent/test_provider_parity.py | 9 + 9 files changed, 508 insertions(+), 6 deletions(-) create mode 100644 hermes_cli/nous_auth_keepalive.py create mode 100644 tests/hermes_cli/test_nous_auth_keepalive.py diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 4bc9440df3..0afb0add20 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -665,6 +665,13 @@ def _pool_runtime_base_url(entry: Any, fallback: str = "") -> str: return str(url or "").strip().rstrip("/") +def _nous_min_key_ttl_seconds() -> int: + try: + return max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800"))) + except (TypeError, ValueError): + return 1800 + + # ── Codex Responses → chat.completions adapter ───────────────────────────── # All auxiliary consumers call client.chat.completions.create(**kwargs) and # read response.choices[0].message.content. This adapter translates those @@ -1338,6 +1345,57 @@ def _nous_base_url() -> str: return os.getenv("NOUS_INFERENCE_BASE_URL", _NOUS_DEFAULT_BASE_URL) +def _resolve_nous_pool_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]: + """Resolve Nous auxiliary credentials from the selected pool entry.""" + try: + from hermes_cli.auth import _agent_key_is_usable + + pool = load_pool("nous") + except Exception as exc: + logger.debug("Auxiliary Nous pool credential resolution failed: %s", exc) + return None + + if not pool or not pool.has_credentials(): + return None + + try: + entry = pool.select() + except Exception as exc: + logger.debug("Auxiliary Nous pool selection failed: %s", exc) + return None + + if entry is None: + return None + + state = { + "agent_key": getattr(entry, "agent_key", None), + "agent_key_expires_at": getattr(entry, "agent_key_expires_at", None), + "scope": getattr(entry, "scope", None), + } + if force_refresh or not _agent_key_is_usable(state, _nous_min_key_ttl_seconds()): + try: + refreshed = pool.try_refresh_current() + except Exception as exc: + logger.debug("Auxiliary Nous pool refresh failed: %s", exc) + refreshed = None + if refreshed is None: + return None + entry = refreshed + + provider = { + "agent_key": getattr(entry, "agent_key", None), + "agent_key_expires_at": getattr(entry, "agent_key_expires_at", None), + "access_token": getattr(entry, "access_token", None), + "expires_at": getattr(entry, "expires_at", None), + "scope": getattr(entry, "scope", None), + } + api_key = _nous_api_key(provider) + base_url = _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL) + if not api_key or not base_url: + return None + return api_key, base_url + + def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]: """Return fresh Nous runtime credentials when available. @@ -1346,6 +1404,10 @@ def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[ relying only on whatever raw tokens happen to be sitting in auth.json or the credential pool. """ + pooled = _resolve_nous_pool_runtime_api(force_refresh=force_refresh) + if pooled is not None: + return pooled + try: from hermes_cli.auth import resolve_nous_runtime_credentials diff --git a/gateway/run.py b/gateway/run.py index 5b7c63a42f..a388f184ad 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -17642,6 +17642,13 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = atexit.register(remove_pid_file) atexit.register(release_gateway_runtime_lock) + try: + from hermes_cli.nous_auth_keepalive import start_nous_auth_keepalive + + start_nous_auth_keepalive() + except Exception as exc: + logger.debug("Nous auth keepalive did not start: %s", exc) + _ensure_windows_gateway_venv_imports() # MCP tool discovery — run in an executor so the asyncio event loop @@ -17698,6 +17705,13 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # Wait for shutdown await runner.wait_for_shutdown() + try: + from hermes_cli.nous_auth_keepalive import stop_nous_auth_keepalive + + stop_nous_auth_keepalive() + except Exception: + pass + if runner.should_exit_with_failure: if runner.exit_reason: logger.error("Gateway exiting with failure: %s", runner.exit_reason) diff --git a/hermes_cli/nous_auth_keepalive.py b/hermes_cli/nous_auth_keepalive.py new file mode 100644 index 0000000000..947bbd1787 --- /dev/null +++ b/hermes_cli/nous_auth_keepalive.py @@ -0,0 +1,189 @@ +"""Background keepalive for long-lived Nous Portal sessions.""" + +from __future__ import annotations + +import logging +import os +import threading +from typing import Optional + +from hermes_cli.auth import ( + ACCESS_TOKEN_REFRESH_SKEW_SECONDS, + NOUS_INVOKE_JWT_MIN_TTL_SECONDS, + AuthError, + _agent_key_is_usable, + _is_expiring, + get_provider_auth_state, + resolve_nous_runtime_credentials, +) + +logger = logging.getLogger(__name__) + +NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS = 6 * 60 * 60 +NOUS_AUTH_KEEPALIVE_INITIAL_DELAY_SECONDS = 60 + +_keepalive_lock = threading.Lock() +_keepalive_stop = threading.Event() +_keepalive_thread: Optional[threading.Thread] = None + + +def _timeout_seconds(value: Optional[float]) -> float: + if value is not None: + return float(value) + try: + return float(os.getenv("HERMES_NOUS_TIMEOUT_SECONDS", "15")) + except (TypeError, ValueError): + return 15.0 + + +def _entry_state(entry: object) -> dict: + return { + "agent_key": getattr(entry, "agent_key", None), + "agent_key_expires_at": getattr(entry, "agent_key_expires_at", None), + "scope": getattr(entry, "scope", None), + } + + +def _refresh_selected_pool_entry( + *, + min_key_ttl_seconds: int, +) -> Optional[bool]: + """Refresh the current Nous credential pool entry when it is stale. + + Returns True when a pool entry exists and is usable/refreshed, False when a + pool exists but no entry can be used, and None when no Nous pool exists. + """ + try: + from agent.credential_pool import load_pool + + pool = load_pool("nous") + except Exception as exc: + logger.debug("Nous auth keepalive: credential pool unavailable: %s", exc) + return None + + if not pool or not pool.has_credentials(): + return None + + try: + entry = pool.select() + except Exception as exc: + logger.debug("Nous auth keepalive: credential pool selection failed: %s", exc) + return False + + if entry is None: + return False + + access_expiring = _is_expiring( + getattr(entry, "expires_at", None), + ACCESS_TOKEN_REFRESH_SKEW_SECONDS, + ) + key_usable = _agent_key_is_usable(_entry_state(entry), min_key_ttl_seconds) + if access_expiring or not key_usable: + refreshed = pool.try_refresh_current() + if refreshed is None: + return False + logger.debug("Nous auth keepalive: refreshed credential pool entry") + return True + + return True + + +def refresh_nous_auth_keepalive_once( + *, + min_key_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, + timeout_seconds: Optional[float] = None, +) -> bool: + """Refresh Nous auth once if credentials are configured.""" + min_key_ttl_seconds = max(60, int(min_key_ttl_seconds)) + + pool_result = _refresh_selected_pool_entry( + min_key_ttl_seconds=min_key_ttl_seconds, + ) + if pool_result is not None: + return pool_result + + state = get_provider_auth_state("nous") + if not state: + return False + + try: + resolve_nous_runtime_credentials( + timeout_seconds=_timeout_seconds(timeout_seconds), + ) + logger.debug("Nous auth keepalive: refreshed singleton auth state") + return True + except AuthError as exc: + if exc.relogin_required: + logger.info("Nous auth keepalive requires re-login: %s", exc) + else: + logger.debug("Nous auth keepalive failed: %s", exc) + return False + except Exception as exc: + logger.debug("Nous auth keepalive failed: %s", exc) + return False + + +def _keepalive_loop( + stop_event: threading.Event, + *, + interval_seconds: int, + initial_delay_seconds: int, + min_key_ttl_seconds: int, + timeout_seconds: Optional[float], +) -> None: + if initial_delay_seconds > 0 and stop_event.wait(initial_delay_seconds): + return + + while not stop_event.is_set(): + refresh_nous_auth_keepalive_once( + min_key_ttl_seconds=min_key_ttl_seconds, + timeout_seconds=timeout_seconds, + ) + stop_event.wait(interval_seconds) + + +def start_nous_auth_keepalive( + *, + interval_seconds: int = NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS, + initial_delay_seconds: int = NOUS_AUTH_KEEPALIVE_INITIAL_DELAY_SECONDS, + min_key_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, + timeout_seconds: Optional[float] = None, +) -> Optional[threading.Thread]: + """Start the process-wide Nous auth keepalive thread.""" + if interval_seconds <= 0: + return None + + global _keepalive_thread + with _keepalive_lock: + if _keepalive_thread is not None and _keepalive_thread.is_alive(): + return _keepalive_thread + + _keepalive_stop.clear() + _keepalive_thread = threading.Thread( + target=_keepalive_loop, + args=(_keepalive_stop,), + kwargs={ + "interval_seconds": int(interval_seconds), + "initial_delay_seconds": max(0, int(initial_delay_seconds)), + "min_key_ttl_seconds": max(60, int(min_key_ttl_seconds)), + "timeout_seconds": timeout_seconds, + }, + daemon=True, + name="nous-auth-keepalive", + ) + _keepalive_thread.start() + logger.debug("Nous auth keepalive started") + return _keepalive_thread + + +def stop_nous_auth_keepalive(timeout: float = 5.0) -> None: + """Stop the keepalive thread. Intended for graceful shutdown/tests.""" + global _keepalive_thread + with _keepalive_lock: + thread = _keepalive_thread + _keepalive_stop.set() + if thread is not None and thread.is_alive(): + thread.join(timeout=timeout) + with _keepalive_lock: + if _keepalive_thread is thread: + _keepalive_thread = None diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index 2c5dd0a7fd..f15de5ba75 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -1495,10 +1495,10 @@ def resolve_runtime_provider( # For Nous, the pool entry's runtime_api_key is the agent_key # compatibility field. It must be an invoke JWT. The pool doesn't # refresh it during selection (that would trigger network calls in - # non-runtime contexts like `hermes auth list`). If the key is - # expired, clear pool_api_key so we fall through to - # resolve_nous_runtime_credentials() which handles refresh. - if provider == "nous" and entry is not None and pool_api_key: + # non-runtime contexts like `hermes auth list`). If the key is + # expired/missing, refresh the selected pool entry before falling back + # to singleton auth resolution. + if provider == "nous" and entry is not None: min_ttl = max(60, env_int("HERMES_NOUS_MIN_KEY_TTL_SECONDS", 1800)) nous_state = { "agent_key": getattr(entry, "agent_key", None), @@ -1506,8 +1506,26 @@ def resolve_runtime_provider( "scope": getattr(entry, "scope", None), } if not _agent_key_is_usable(nous_state, min_ttl): - logger.debug("Nous pool entry agent_key expired/missing, falling through to runtime resolution") - pool_api_key = "" + logger.debug("Nous pool entry agent_key expired/missing, refreshing selected pool entry") + try: + refreshed = pool.try_refresh_current() + except Exception as exc: + logger.debug("Nous pool entry refresh failed: %s", exc) + refreshed = None + if refreshed is not None: + entry = refreshed + pool_api_key = ( + getattr(entry, "runtime_api_key", None) + or getattr(entry, "access_token", "") + ) + nous_state = { + "agent_key": getattr(entry, "agent_key", None), + "agent_key_expires_at": getattr(entry, "agent_key_expires_at", None), + "scope": getattr(entry, "scope", None), + } + if not pool_api_key or not _agent_key_is_usable(nous_state, min_ttl): + logger.debug("Nous pool entry agent_key still unavailable, falling through to runtime resolution") + pool_api_key = "" if entry is not None and pool_api_key: return _resolve_runtime_from_pool_entry( provider=provider, diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index ade50c6005..4227e62111 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -12823,6 +12823,13 @@ def start_server( """ import uvicorn + try: + from hermes_cli.nous_auth_keepalive import start_nous_auth_keepalive + + start_nous_auth_keepalive() + except Exception as exc: + _log.debug("Nous auth keepalive did not start: %s", exc) + # Phase 0: stash the auth-gate flag on app.state so middleware / SPA-token # injection / WS-auth paths can branch on it consistently. Phase 3.5 # uses this to decide whether to refuse the bind, log the gate-on diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 8ec6102f2e..dac9956b49 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -1071,6 +1071,89 @@ class TestAuxiliaryPoolAwareness: assert mock_openai.call_args.kwargs["api_key"] == pooled_token assert mock_openai.call_args.kwargs["base_url"] == "https://inference.pool.example/v1" + def test_try_nous_refreshes_stale_pool_entry(self): + stale_token = _jwt_with_claims({ + "scope": "inference:invoke", + "exp": int(time.time() - 60), + }) + fresh_token = _jwt_with_claims({ + "scope": "inference:invoke", + "exp": int(time.time() + 3600), + }) + + class _Entry: + def __init__(self, token): + self.access_token = "pooled-access-token" + self.agent_key = token + self.agent_key_expires_at = "2099-01-01T00:00:00+00:00" + self.scope = "inference:invoke" + self.inference_base_url = "https://inference.pool.example/v1" + + class _Pool: + refreshed = False + + def has_credentials(self): + return True + + def select(self): + return _Entry(stale_token) + + def try_refresh_current(self): + self.refreshed = True + return _Entry(fresh_token) + + pool = _Pool() + with ( + patch("agent.auxiliary_client.load_pool", return_value=pool), + patch("agent.auxiliary_client.OpenAI") as mock_openai, + patch("hermes_cli.models.get_nous_recommended_aux_model", return_value=None), + ): + from agent.auxiliary_client import _try_nous + + client, model = _try_nous() + + assert pool.refreshed is True + assert client is not None + assert model == "google/gemini-3-flash-preview" + assert mock_openai.call_args.kwargs["api_key"] == fresh_token + assert mock_openai.call_args.kwargs["base_url"] == "https://inference.pool.example/v1" + + def test_resolve_nous_runtime_api_rejects_stale_pool_entry_when_refresh_fails(self): + stale_token = _jwt_with_claims({ + "scope": "inference:invoke", + "exp": int(time.time() - 60), + }) + + class _Entry: + access_token = "pooled-access-token" + agent_key = stale_token + agent_key_expires_at = "2099-01-01T00:00:00+00:00" + scope = "inference:invoke" + inference_base_url = "https://inference.pool.example/v1" + + class _Pool: + def has_credentials(self): + return True + + def select(self): + return _Entry() + + def try_refresh_current(self): + return None + + with ( + patch("agent.auxiliary_client.load_pool", return_value=_Pool()), + patch( + "hermes_cli.auth.resolve_nous_runtime_credentials", + side_effect=RuntimeError("no singleton auth"), + ), + ): + from agent.auxiliary_client import _resolve_nous_runtime_api + + runtime = _resolve_nous_runtime_api() + + assert runtime is None + def test_try_nous_uses_portal_recommendation_for_text(self): """When the Portal recommends a compaction model, _try_nous honors it.""" fresh_base = "https://inference-api.nousresearch.com/v1" diff --git a/tests/hermes_cli/test_nous_auth_keepalive.py b/tests/hermes_cli/test_nous_auth_keepalive.py new file mode 100644 index 0000000000..9e633a1417 --- /dev/null +++ b/tests/hermes_cli/test_nous_auth_keepalive.py @@ -0,0 +1,60 @@ +from hermes_cli import nous_auth_keepalive as keepalive + + +def test_keepalive_refreshes_stale_pool_entry(monkeypatch): + class _Entry: + access_token = "pooled-access-token" + expires_at = "2000-01-01T00:00:00+00:00" + agent_key = "" + agent_key_expires_at = None + scope = "inference:invoke" + + class _Pool: + refreshed = False + + def has_credentials(self): + return True + + def select(self): + return _Entry() + + def try_refresh_current(self): + self.refreshed = True + return _Entry() + + pool = _Pool() + monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: pool) + + assert keepalive.refresh_nous_auth_keepalive_once() is True + assert pool.refreshed is True + + +def test_keepalive_falls_back_to_singleton_state(monkeypatch): + calls = [] + + class _Pool: + def has_credentials(self): + return False + + def _resolve_nous_runtime_credentials(**kwargs): + calls.append(kwargs) + return { + "provider": "nous", + "api_key": "fresh-agent-key", + "base_url": "https://inference-api.nousresearch.com/v1", + } + + monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: _Pool()) + monkeypatch.setattr( + keepalive, + "get_provider_auth_state", + lambda provider: {"access_token": "stored-access-token"}, + ) + monkeypatch.setattr( + keepalive, + "resolve_nous_runtime_credentials", + _resolve_nous_runtime_credentials, + ) + + assert keepalive.refresh_nous_auth_keepalive_once(timeout_seconds=15.0) is True + assert calls == [{"timeout_seconds": 15.0}] diff --git a/tests/hermes_cli/test_runtime_provider_resolution.py b/tests/hermes_cli/test_runtime_provider_resolution.py index 3e788fe3d5..8df00200d7 100644 --- a/tests/hermes_cli/test_runtime_provider_resolution.py +++ b/tests/hermes_cli/test_runtime_provider_resolution.py @@ -1,8 +1,25 @@ +import base64 +import json +import time + import pytest from hermes_cli import runtime_provider as rp +def _fake_invoke_jwt(ttl_seconds=3600): + header = base64.urlsafe_b64encode(b'{"alg":"none","typ":"JWT"}').decode().rstrip("=") + payload = base64.urlsafe_b64encode( + json.dumps( + { + "scope": "inference:invoke", + "exp": int(time.time() + ttl_seconds), + } + ).encode() + ).decode().rstrip("=") + return f"{header}.{payload}.sig" + + def test_resolve_runtime_provider_uses_credential_pool(monkeypatch): class _Entry: access_token = "pool-token" @@ -977,6 +994,49 @@ def test_named_custom_provider_does_not_shadow_builtin_provider(monkeypatch): assert resolved["requested_provider"] == "nous" +def test_nous_pool_entry_refreshes_expired_agent_key(monkeypatch): + stale_token = _fake_invoke_jwt(ttl_seconds=-60) + fresh_token = _fake_invoke_jwt(ttl_seconds=3600) + + class _Entry: + def __init__(self, token): + self.access_token = "pool-access-token" + self.agent_key = token + self.agent_key_expires_at = "2099-01-01T00:00:00+00:00" + self.scope = "inference:invoke" + self.base_url = "https://inference.pool.example/v1" + self.source = "manual:nous" + + @property + def runtime_api_key(self): + return self.agent_key + + class _Pool: + refreshed = False + + def has_credentials(self): + return True + + def select(self): + return _Entry(stale_token) + + def try_refresh_current(self): + self.refreshed = True + return _Entry(fresh_token) + + pool = _Pool() + monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "nous") + monkeypatch.setattr(rp, "load_pool", lambda provider: pool) + monkeypatch.setattr(rp, "_get_model_config", lambda: {"provider": "nous"}) + + resolved = rp.resolve_runtime_provider(requested="nous") + + assert pool.refreshed is True + assert resolved["provider"] == "nous" + assert resolved["api_key"] == fresh_token + assert resolved["base_url"] == "https://inference.pool.example/v1" + + def test_named_custom_provider_wins_over_builtin_alias(monkeypatch): """A custom_providers entry named after a built-in *alias* (not a canonical provider name) must win over the built-in. Regression guard for #15743: diff --git a/tests/run_agent/test_provider_parity.py b/tests/run_agent/test_provider_parity.py index c99ab433d4..8229b0f020 100644 --- a/tests/run_agent/test_provider_parity.py +++ b/tests/run_agent/test_provider_parity.py @@ -56,6 +56,15 @@ class _FakeOpenAI: pass +@pytest.fixture(autouse=True) +def _reset_auxiliary_provider_state(): + from agent.auxiliary_client import _reset_aux_unhealthy_cache + + _reset_aux_unhealthy_cache() + yield + _reset_aux_unhealthy_cache() + + def _make_agent(monkeypatch, provider, api_mode="chat_completions", base_url="https://openrouter.ai/api/v1", model=None): monkeypatch.setattr("run_agent.get_tool_definitions", lambda **kw: _tool_defs("web_search", "terminal")) monkeypatch.setattr("run_agent.check_toolset_requirements", lambda: {})