Teknium
ba030bc0db
Tests did monkeypatch.setattr(<facade module>, name) where name is now defined in a sibling module and the production path reads the sibling's binding. Where production reads through BOTH bindings the setattr is duplicated onto the defining module (import added next to the existing alias import); where only the sibling reads it the target is repointed. Seams whose production readers go through the facade are left alone.
135 lines
5.1 KiB
Python
135 lines
5.1 KiB
Python
"""Concurrent Nous 401 recovery must not stampede the shared OAuth grant.
|
|
|
|
Sep 2 2026 incident: ~120 subagent processes shared one Nous OAuth pool entry
|
|
whose access token hit its hourly expiry. Every process got a 401, every
|
|
process force-refreshed, and each rotation invalidated the token a sibling had
|
|
just adopted — 81 refreshes and ~540 401s in eight minutes. Processes that
|
|
lost the auth-store flock race had their only entry benched ("matched no nous
|
|
entry ... pool size 0") and surfaced the 401 to the user as "out of funds".
|
|
|
|
Two invariants pinned here:
|
|
|
|
1. ``resolve_nous_runtime_credentials(force_refresh=True, stale_access_token=X)``
|
|
does NOT POST a refresh when the store already holds a usable token that
|
|
is not X — a peer already rotated; adopt it.
|
|
2. A lock-timeout during a pool-level Nous refresh leaves the entry
|
|
untouched instead of marking it exhausted.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
|
|
import hermes_cli.auth as auth_mod
|
|
import hermes_cli.auth_nous as auth_nous
|
|
from agent.credential_pool import CredentialPool, PooledCredential
|
|
|
|
from tests.hermes_cli.test_auth_nous_provider import _invoke_jwt, _setup_nous_auth
|
|
|
|
|
|
def test_forced_refresh_adopts_peer_rotation_instead_of_reposting(tmp_path, monkeypatch):
|
|
hermes_home = tmp_path / "hermes"
|
|
peer_token = _invoke_jwt(seconds=3600)
|
|
failed_token = _invoke_jwt(seconds=3000) # what THIS process still holds
|
|
_setup_nous_auth(
|
|
hermes_home,
|
|
access_token=peer_token,
|
|
refresh_token="rt-after-peer-rotation",
|
|
scope=auth_mod.DEFAULT_NOUS_SCOPE,
|
|
expires_at=auth_mod.datetime.fromtimestamp(
|
|
auth_mod.time.time() + 3600, tz=auth_mod.timezone.utc
|
|
).isoformat(),
|
|
expires_in=3600,
|
|
)
|
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
|
|
|
posts = []
|
|
|
|
def _fake_refresh_access_token(*, client, portal_base_url, client_id, refresh_token):
|
|
posts.append(refresh_token)
|
|
return {
|
|
"access_token": _invoke_jwt(seconds=7200),
|
|
"refresh_token": "rt-should-not-happen",
|
|
"expires_in": 7200,
|
|
"token_type": "Bearer",
|
|
"scope": auth_mod.DEFAULT_NOUS_SCOPE,
|
|
}
|
|
|
|
monkeypatch.setattr(auth_mod, "_refresh_access_token", _fake_refresh_access_token)
|
|
monkeypatch.setattr(auth_nous, "_refresh_access_token", _fake_refresh_access_token)
|
|
|
|
creds = auth_mod.resolve_nous_runtime_credentials(
|
|
force_refresh=True, stale_access_token=failed_token
|
|
)
|
|
|
|
assert posts == [], "peer already rotated — must not consume the refresh token again"
|
|
assert creds["api_key"] == peer_token
|
|
|
|
# Same call WITHOUT the hint keeps the pre-existing force semantics.
|
|
auth_mod.resolve_nous_runtime_credentials(force_refresh=True)
|
|
assert posts == ["rt-after-peer-rotation"]
|
|
|
|
|
|
def test_lock_timeout_during_nous_refresh_does_not_bench_entry(monkeypatch, caplog):
|
|
entry = PooledCredential(
|
|
id="267aed",
|
|
provider="nous",
|
|
auth_type="oauth",
|
|
access_token=_invoke_jwt(seconds=3600),
|
|
refresh_token="rt",
|
|
label="test@nous",
|
|
source="device_code",
|
|
priority=0,
|
|
)
|
|
pool = CredentialPool.__new__(CredentialPool)
|
|
pool._lock = __import__("threading").RLock()
|
|
pool._entries = [entry]
|
|
pool._active_leases = {}
|
|
pool._current_id = None
|
|
pool._max_concurrent = 2
|
|
pool._unmatched_rotation_streak = 0
|
|
pool.provider = "nous"
|
|
|
|
monkeypatch.setattr(pool, "_sync_nous_entry_from_auth_store", lambda e: e)
|
|
monkeypatch.setattr(pool, "_persist", lambda *a, **k: None)
|
|
|
|
def _busy(*a, **k):
|
|
raise TimeoutError("Timed out waiting for auth store lock")
|
|
|
|
monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _busy)
|
|
monkeypatch.setattr(auth_nous, "resolve_nous_runtime_credentials", _busy)
|
|
|
|
result = pool._refresh_entry_impl(entry, force=True)
|
|
|
|
assert result is entry
|
|
assert pool._entries[0].last_status is None, "lock contention is not a credential failure"
|
|
|
|
|
|
def test_agent_401_refresh_passes_failed_bearer_as_stale_hint(monkeypatch):
|
|
"""Every 401-recovery caller must hand the auth store the bearer that
|
|
failed — without it ``_already_rotated_by_peer`` can never fire and each
|
|
subagent rotates the shared grant again (the "no crash, N refreshes"
|
|
variant of the Sep 2 stampede).
|
|
"""
|
|
from run_agent import AIAgent
|
|
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.provider = "nous"
|
|
agent.api_mode = "chat_completions"
|
|
agent.api_key = "jwt-that-just-401d"
|
|
agent.base_url = "https://inference-api.nousresearch.com/v1"
|
|
agent._client_kwargs = {}
|
|
monkeypatch.setattr(agent, "_replace_primary_openai_client", lambda **k: True)
|
|
|
|
seen = {}
|
|
|
|
def _fake_resolve(**kwargs):
|
|
seen.update(kwargs)
|
|
return {"api_key": "fresh", "base_url": agent.base_url}
|
|
|
|
monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _fake_resolve)
|
|
monkeypatch.setattr(auth_nous, "resolve_nous_runtime_credentials", _fake_resolve)
|
|
|
|
assert agent._try_refresh_nous_client_credentials(force=True) is True
|
|
assert seen["force_refresh"] is True
|
|
assert seen["stale_access_token"] == "jwt-that-just-401d"
|