diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 135bc64f09..d96754483d 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -1022,6 +1022,29 @@ def build_anthropic_bedrock_client(region: str): ) +class CredentialPersistError(RuntimeError): + """A rotated single-use credential could not be durably committed. + + Anthropic OAuth refresh tokens are single-use: a successful refresh POST + consumes the old refresh token server-side and returns a replacement. The + replacement exists only in memory until it reaches its authoritative + on-disk store (``~/.claude/.credentials.json`` for ``claude_code``, + ``~/.hermes/.anthropic_oauth.json`` for ``hermes_pkce``). + + If that write fails and the caller reports success anyway, the on-disk + (already consumed) pair survives and is re-seeded on the next + ``load_pool()``, so the following refresh replays a spent token and fails + with ``invalid_grant`` / ``refresh_token_reused``. Callers must therefore + treat this as a failed refresh, not a successful one, and fail closed. + """ + + def __init__(self, path: Any, cause: BaseException) -> None: + super().__init__( + f"failed to durably persist rotated Anthropic credentials to {path}: {cause}" + ) + self.path = path + + def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]: """Read Claude Code OAuth credentials from the macOS Keychain. @@ -1300,16 +1323,35 @@ def _refresh_oauth_token(creds: Dict[str, Any]) -> Optional[str]: try: refreshed = refresh_anthropic_oauth_pure(refresh_token, use_json=False) + except Exception as e: + logger.debug("Failed to refresh Claude Code token: %s", e) + return None + + # The POST above already consumed ``refresh_token`` server-side. + # Writing the replacement pair is the commit step of that + # transaction, not a cache update: if it fails, the rotation is + # unrecoverable and the pair still on disk is spent. Fail closed + # rather than handing back an access token whose refresh half was + # lost — reporting success here is what lets a later load replay + # the consumed token and produce ``invalid_grant``. + try: _write_claude_code_credentials( refreshed["access_token"], refreshed["refresh_token"], refreshed["expires_at_ms"], ) - logger.debug("Successfully refreshed Claude Code OAuth token") - return refreshed["access_token"] except Exception as e: - logger.debug("Failed to refresh Claude Code token: %s", e) + logger.error( + "Anthropic OAuth refresh rotated the single-use token but could not " + "commit it to %s (%s) — treating the refresh as failed; " + "re-run 'claude setup-token' to reauthenticate", + claude_code_credentials_path(), + e, + ) return None + + logger.debug("Successfully refreshed Claude Code OAuth token") + return refreshed["access_token"] except Exception as e: # Lock acquisition/read failures should preserve the resolver's # existing fail-soft contract rather than taking down agent startup. @@ -1330,6 +1372,12 @@ def _write_claude_code_credentials( is persisted so that Claude Code's own auth check recognises the credential as valid. Claude Code >=2.1.81 gates on the presence of ``"user:inference"`` in the stored scopes before it will use the token. + + Raises ``CredentialPersistError`` when the rotated pair does not reach the + file. This write is the commit step of the refresh transaction, not a + best-effort cache update: a swallowed failure leaves the consumed + pre-rotation pair on disk to be re-seeded and replayed (see + ``CredentialPersistError``). """ cred_path = claude_code_credentials_path() try: @@ -1381,8 +1429,12 @@ def _write_claude_code_credentials( except OSError: pass raise - except (OSError, IOError) as e: - logger.debug("Failed to write refreshed credentials: %s", e) + except (OSError, IOError, ValueError) as e: + # ValueError covers a corrupt existing file (JSONDecodeError): the + # merge-read is part of the commit, so failing it means the rotated + # pair never landed either. + logger.error("Failed to write refreshed credentials to %s: %s", cred_path, e) + raise CredentialPersistError(cred_path, e) from e def _resolve_claude_code_token_from_credentials(creds: Optional[Dict[str, Any]] = None) -> Optional[str]: @@ -1761,6 +1813,10 @@ def _write_hermes_oauth_credentials( runs ``_seed_from_singletons()``, which reads the stale file and overwrites the freshly-rotated pool entry with the pre-refresh (and, for single-use Anthropic refresh tokens, already-consumed) token pair. + + Raises ``CredentialPersistError`` when the rotated pair does not reach the + file, for the same reason ``_write_claude_code_credentials`` does: this is + the commit step of the refresh transaction. """ oauth_file = _get_hermes_oauth_file() try: @@ -1788,8 +1844,11 @@ def _write_hermes_oauth_credentials( except OSError: pass raise - except (OSError, IOError) as e: - logger.debug("Failed to write refreshed Hermes OAuth credentials: %s", e) + except (OSError, IOError, ValueError) as e: + logger.error( + "Failed to write refreshed Hermes OAuth credentials to %s: %s", oauth_file, e + ) + raise CredentialPersistError(oauth_file, e) from e # --------------------------------------------------------------------------- diff --git a/agent/credential_persistence.py b/agent/credential_persistence.py index 9217f9535e..287fff3ccc 100644 --- a/agent/credential_persistence.py +++ b/agent/credential_persistence.py @@ -130,6 +130,15 @@ def _fingerprint_value(value: Any) -> str | None: return f"sha256:{digest[:16]}" +def fingerprint_secret_value(value: Any) -> str | None: + """Public, non-reversible fingerprint for a single secret value. + + Callers that compare a live secret against the ``secret_fingerprint`` left + on a sanitized (borrowed) pool row need the same digest this module writes. + """ + return _fingerprint_value(value) + + def _credential_secret_fingerprint(payload: Mapping[str, Any]) -> str | None: for key in ("agent_key", "access_token", "refresh_token", "api_key", "token", "secret"): fingerprint = _fingerprint_value(payload.get(key)) diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 67c64182ea..40bd24adac 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -18,6 +18,7 @@ from hermes_constants import OPENROUTER_BASE_URL from hermes_cli.config import load_env from agent.secret_scope import get_secret as _get_secret from agent.credential_persistence import ( + fingerprint_secret_value, is_borrowed_credential_source, sanitize_borrowed_credential_payload, ) @@ -87,6 +88,14 @@ _TERMINAL_AUTH_REASONS = frozenset({ "refresh_token_reused", # Single-use refresh token consumed by another process }) +# Locally generated terminal reason (no HTTP status involved): a refresh POST +# rotated a single-use pair but the replacement never reached its +# authoritative store, so the pre-rotation token still on disk is already +# spent and no retry can recover it. Kept out of _TERMINAL_AUTH_REASONS — +# that set classifies upstream-reported 401 reasons — and handled explicitly +# in _is_terminal_auth_failure(). +CREDENTIAL_PERSIST_FAILED_REASON = "credential_persist_failed" + # How long a DEAD manual credential is preserved before being pruned. # Manual entries (``manual:*``) are independent credentials with no singleton # to re-seed from, so pruning them after a quiet window cleans up dead state @@ -862,13 +871,18 @@ class CredentialPool: Returns False for non-401 status codes — 429 rate limits and 402 billing failures are transient by nature and should keep TTL semantics. + The one status-independent case is + ``CREDENTIAL_PERSIST_FAILED_REASON``: no upstream response is involved + at all, the rotated pair simply never became durable and only a + re-auth can recover it. """ + raw_reason = normalized_error.get("reason") + reason = raw_reason.strip().lower() if isinstance(raw_reason, str) else "" + if reason == CREDENTIAL_PERSIST_FAILED_REASON: + return True if status_code != 401: return False - reason = normalized_error.get("reason") - if not isinstance(reason, str): - return False - return reason.strip().lower() in _TERMINAL_AUTH_REASONS + return reason in _TERMINAL_AUTH_REASONS def _mark_exhausted( self, @@ -1497,6 +1511,50 @@ class CredentialPool: target_path=claude_code_credentials_path(), ) + def _fail_closed_unpersisted_rotation( + self, + entry: PooledCredential, + exc: BaseException, + *, + store: str, + ) -> None: + """Quarantine an entry whose rotated pair never reached its store. + + Anthropic refresh tokens are single-use, and for ``claude_code`` / + ``hermes_pkce`` sources the singleton file — not ``auth.json`` — is the + authoritative copy: ``_seed_from_singletons()`` re-reads it on every + ``load_pool()`` and overwrites the pool entry with whatever it finds. + + So when the refresh POST succeeded but the singleton write failed, the + rotation is not durable: the replacement pair exists only in memory, + while the consumed pre-rotation pair survives on disk and would be + re-seeded over any pool row we persisted. Persisting or returning the + rotated entry here would report a success that a restart silently + undoes, and the next refresh would replay the spent token + (``invalid_grant`` / ``refresh_token_reused``). + + Fail closed instead: never expose or persist the rotated pair, and mark + the entry terminally so it leaves rotation and surfaces as an explicit + re-auth requirement rather than a silent fallback to another provider. + """ + logger.error( + "Anthropic %s refresh rotated the single-use token but could not commit it " + "to %s (%s) — failing closed and quarantining the credential; " + "re-authenticate to recover", + entry.source, + store, + exc, + ) + self._mark_exhausted( + entry, + None, + { + "reason": CREDENTIAL_PERSIST_FAILED_REASON, + "message": f"rotated credential was not durably written to {store}: {exc}", + }, + ) + return None + def _single_use_refresh_lock_timeout(self) -> float: """Lock timeout for single-use-refresh-token providers. @@ -1547,7 +1605,14 @@ class CredentialPool: refreshed["expires_at_ms"], ) except Exception as wexc: - logger.debug("Failed to write refreshed token to credentials file: %s", wexc) + # Authoritative commit failed: do not mark, persist or + # return the rotation as successful. Returning from + # inside this ``try`` deliberately bypasses the + # ``except Exception`` recovery below — that path + # re-POSTs, and there is nothing left to retry with. + return self._fail_closed_unpersisted_rotation( + entry, wexc, store="~/.claude/.credentials.json" + ) # Same rationale for the singleton source hermes_pkce: # _seed_from_singletons() reads ~/.hermes/.anthropic_oauth.json # on every load_pool() and will re-seed the pre-refresh (and @@ -1565,7 +1630,10 @@ class CredentialPool: refreshed["expires_at_ms"], ) except Exception as wexc: - logger.debug("Failed to write refreshed token to Hermes OAuth file: %s", wexc) + # Same transaction rule as claude_code above. + return self._fail_closed_unpersisted_rotation( + entry, wexc, store="~/.hermes/.anthropic_oauth.json" + ) elif self.provider == "openai-codex": # Adopt fresher tokens from auth.json before spending the # refresh_token — single-use tokens consumed by another Hermes @@ -1628,6 +1696,22 @@ class CredentialPool: synced.refresh_token, use_json=synced.source.endswith("hermes_pkce"), ) + # Commit to the authoritative singleton BEFORE marking + # or persisting the pool row. The previous order + # persisted an "ok" entry that a failed write left + # unbacked, and the next load_pool() re-seeded the + # consumed pair straight over it. + try: + from agent.anthropic_adapter import _write_claude_code_credentials + _write_claude_code_credentials( + refreshed["access_token"], + refreshed["refresh_token"], + refreshed["expires_at_ms"], + ) + except Exception as wexc: + return self._fail_closed_unpersisted_rotation( + synced, wexc, store="~/.claude/.credentials.json" + ) updated = replace( synced, access_token=refreshed["access_token"], @@ -1639,15 +1723,6 @@ class CredentialPool: ) self._replace_entry(synced, updated) self._persist() - try: - from agent.anthropic_adapter import _write_claude_code_credentials - _write_claude_code_credentials( - refreshed["access_token"], - refreshed["refresh_token"], - refreshed["expires_at_ms"], - ) - except Exception as wexc: - logger.debug("Failed to write refreshed token to credentials file (retry path): %s", wexc) return updated except Exception as retry_exc: logger.debug("Retry refresh also failed: %s", retry_exc) @@ -2626,11 +2701,23 @@ def _upsert_entry(entries: List[PooledCredential], provider: str, source: str, p field_updates = {} extra_updates = {} _field_names = {f.name for f in fields(existing)} + incoming_token = payload.get("access_token") token_changed = ( - "access_token" in payload - and payload["access_token"] is not None - and payload["access_token"] != existing.access_token + incoming_token is not None + and incoming_token != existing.access_token ) + if token_changed and not existing.access_token: + # Borrowed sources (``claude_code``, env-backed rows, ...) are written + # to auth.json without their secret: a reloaded entry carries only a + # ``secret_fingerprint``. Comparing the freshly re-seeded token against + # that empty string reports a rotation on *every* load, which silently + # cleared the DEAD/exhausted state the previous process had just + # persisted — resurrecting a quarantined credential on restart. + # Compare fingerprints instead, so only a genuinely different secret + # counts as a rotation. + known_fingerprint = existing.extra.get("secret_fingerprint") + if isinstance(known_fingerprint, str) and known_fingerprint: + token_changed = fingerprint_secret_value(incoming_token) != known_fingerprint for key, value in payload.items(): if key in {"id", "priority"} or value is None: continue diff --git a/tests/agent/test_anthropic_credential_persist_failure.py b/tests/agent/test_anthropic_credential_persist_failure.py new file mode 100644 index 0000000000..532ab75c6f --- /dev/null +++ b/tests/agent/test_anthropic_credential_persist_failure.py @@ -0,0 +1,434 @@ +"""Failure-injection coverage for the Anthropic refresh *commit* step. + +Anthropic OAuth refresh tokens are single-use: the POST that returns a new +pair also invalidates the one that was sent. The replacement therefore exists +only in memory until it reaches its authoritative on-disk store — +``~/.claude/.credentials.json`` for ``claude_code`` entries, +``~/.hermes/.anthropic_oauth.json`` for ``hermes_pkce`` ones. Those singletons +are authoritative in the strict sense: ``_seed_from_singletons()`` re-reads +them on every ``load_pool()`` and writes what it finds over the pool row. + +Before this coverage existed, both writers caught ``OSError``/``IOError``, +logged at debug level, and returned nothing, so no caller could tell a durable +commit from a failed one. A refresh could therefore spend the only refresh +token, report success, and leave the consumed pre-rotation pair on disk to be +re-seeded — with the next refresh replaying a spent token and failing with +``invalid_grant`` / ``refresh_token_reused``. + +Every test here forces the writer to fail and asserts the same invariant from +a different entry point: the rotation is never reported, marked, or persisted +as successful, and a subsequent ``load_pool()`` cannot bring the pre-refresh +pair back as a usable credential. + +Companions: ``test_credential_pool_oauth_writethrough.py`` covers the +successful write-through, ``test_credential_pool_anthropic_refresh_race.py`` +the contention between two refreshers. +""" + +from __future__ import annotations + +import json +import os +import time + +import pytest + +from agent import anthropic_adapter as AA +from agent.anthropic_adapter import CredentialPersistError +from agent.credential_pool import ( + AUTH_TYPE_OAUTH, + CREDENTIAL_PERSIST_FAILED_REASON, + STATUS_DEAD, + CredentialPool, + PooledCredential, + load_pool, +) + +# Far enough in the past that every ``_entry_needs_refresh`` check fires. +_EXPIRED_MS = 1_000 + +# Synthetic, non-functional token material. +_STALE_ACCESS = "sk-ant-oat01-stale" +_STALE_REFRESH = "sk-ant-ort01-stale" +_ROTATED_ACCESS = "sk-ant-oat01-rotated" +_ROTATED_REFRESH = "sk-ant-ort01-rotated" + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + """Real on-disk HERMES_HOME so ``load_pool()`` re-reads what we persisted.""" + home = tmp_path / "hermes" + home.mkdir(parents=True, exist_ok=True) + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_TOKEN", raising=False) + (home / "auth.json").write_text( + json.dumps({"version": 1, "providers": {}}), encoding="utf-8" + ) + monkeypatch.setattr( + "hermes_cli.auth.is_provider_explicitly_configured", lambda pid: True + ) + return home + + +@pytest.fixture +def claude_credentials(tmp_path, monkeypatch): + """Point the ``claude_code`` singleton at a tmp file holding a stale pair.""" + cred_path = tmp_path / "claude" / ".credentials.json" + cred_path.parent.mkdir(parents=True, exist_ok=True) + cred_path.write_text( + json.dumps( + { + "claudeAiOauth": { + "accessToken": _STALE_ACCESS, + "refreshToken": _STALE_REFRESH, + "expiresAt": _EXPIRED_MS, + "scopes": ["user:inference", "user:profile"], + } + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(AA, "claude_code_credentials_path", lambda: cred_path) + # The Keychain reader shadows the file on macOS; keep the file the only + # source so this suite behaves identically on every platform. + monkeypatch.setattr(AA, "_read_claude_code_credentials_from_keychain", lambda: None) + return cred_path + + +def _rotating_refresh(*_a, **_kw): + """Stand-in for the token endpoint: always rotates the pair.""" + return { + "access_token": _ROTATED_ACCESS, + "refresh_token": _ROTATED_REFRESH, + "expires_at_ms": int(time.time() * 1000) + 3_600_000, + } + + +# Only the two authoritative Anthropic singletons are made unwritable. The +# pool's own ``auth.json`` commit must keep working, otherwise the quarantine +# these tests assert on could never be persisted and the injection would be +# proving the wrong failure. +_SINGLETON_FILENAMES = frozenset({".credentials.json", ".anthropic_oauth.json"}) + + +def _break_durable_write(monkeypatch): + """Make the singleton atomic rename fail, i.e. the commit never lands.""" + real_replace = os.replace + + def _failing_replace(src, dst): + if os.path.basename(os.fspath(dst)) in _SINGLETON_FILENAMES: + raise OSError(13, "Permission denied") + return real_replace(src, dst) + + monkeypatch.setattr(AA.os, "replace", _failing_replace) + + +def _entry(source: str) -> PooledCredential: + return PooledCredential( + provider="anthropic", + id="anthropic-1", + label="anthropic oauth", + auth_type=AUTH_TYPE_OAUTH, + priority=0, + source=source, + access_token=_STALE_ACCESS, + refresh_token=_STALE_REFRESH, + expires_at_ms=_EXPIRED_MS, + ) + + +def _read_claude_pair(cred_path): + data = json.loads(cred_path.read_text(encoding="utf-8"))["claudeAiOauth"] + return data["accessToken"], data["refreshToken"] + + +# --------------------------------------------------------------------------- +# The writers themselves +# --------------------------------------------------------------------------- + + +def test_claude_code_writer_raises_instead_of_swallowing( + claude_credentials, monkeypatch +): + """A failed durable write must be reported, not logged and dropped.""" + _break_durable_write(monkeypatch) + + with pytest.raises(CredentialPersistError): + AA._write_claude_code_credentials( + _ROTATED_ACCESS, _ROTATED_REFRESH, _EXPIRED_MS + 3_600_000 + ) + + assert _read_claude_pair(claude_credentials) == (_STALE_ACCESS, _STALE_REFRESH), ( + "the failed commit must leave the previous file contents intact" + ) + + +def test_hermes_oauth_writer_raises_instead_of_swallowing(hermes_home, monkeypatch): + oauth_file = hermes_home / ".anthropic_oauth.json" + oauth_file.write_text( + json.dumps( + { + "accessToken": _STALE_ACCESS, + "refreshToken": _STALE_REFRESH, + "expiresAt": _EXPIRED_MS, + } + ), + encoding="utf-8", + ) + _break_durable_write(monkeypatch) + + with pytest.raises(CredentialPersistError): + AA._write_hermes_oauth_credentials( + _ROTATED_ACCESS, _ROTATED_REFRESH, _EXPIRED_MS + 3_600_000 + ) + + on_disk = json.loads(oauth_file.read_text(encoding="utf-8")) + assert on_disk["refreshToken"] == _STALE_REFRESH + + +def test_failed_write_leaves_no_temp_file_behind(claude_credentials, monkeypatch): + """The 0600 temp file must not survive a failed commit.""" + _break_durable_write(monkeypatch) + + with pytest.raises(CredentialPersistError): + AA._write_claude_code_credentials(_ROTATED_ACCESS, _ROTATED_REFRESH, 0) + + leftovers = [ + p.name for p in claude_credentials.parent.iterdir() if ".tmp." in p.name + ] + assert leftovers == [], f"temp credential files left on disk: {leftovers}" + + +# --------------------------------------------------------------------------- +# Direct resolver path (resolve_anthropic_token -> _refresh_oauth_token) +# --------------------------------------------------------------------------- + + +def test_direct_resolver_fails_closed_when_rotation_cannot_commit( + claude_credentials, monkeypatch +): + """``_refresh_oauth_token`` must not hand back a token it could not persist. + + The refresh POST has already spent ``_STALE_REFRESH``; returning the new + access token here would report a rotation that no restart can reproduce, + because the refresh half of the pair was lost with the failed write. + """ + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + _break_durable_write(monkeypatch) + + creds = AA.read_claude_code_credentials() + assert creds is not None + + assert AA._refresh_oauth_token(creds) is None, ( + "a refresh whose authoritative write failed must be reported as a " + "failed refresh, not as a usable access token" + ) + assert _read_claude_pair(claude_credentials) == (_STALE_ACCESS, _STALE_REFRESH) + + +def test_resolve_from_credentials_returns_none_on_failed_commit( + claude_credentials, monkeypatch +): + """The resolver wrapper propagates the fail-closed verdict.""" + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + _break_durable_write(monkeypatch) + + assert AA._resolve_claude_code_token_from_credentials() is None + + +# --------------------------------------------------------------------------- +# Pool path: claude_code +# --------------------------------------------------------------------------- + + +def test_pool_claude_code_fails_closed_and_reload_cannot_resurrect( + hermes_home, claude_credentials, monkeypatch +): + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + _break_durable_write(monkeypatch) + + entry = _entry("claude_code") + pool = CredentialPool("anthropic", [entry]) + + assert pool._refresh_entry(entry, force=True) is None, ( + "an uncommitted rotation must not be returned as a refreshed credential" + ) + + quarantined = pool.entries()[0] + assert quarantined.last_status == STATUS_DEAD + assert quarantined.last_error_reason == CREDENTIAL_PERSIST_FAILED_REASON + assert quarantined.access_token == _STALE_ACCESS, ( + "the rotated pair must never be adopted onto the entry: it is not " + "backed by the authoritative store" + ) + assert quarantined.refresh_token == _STALE_REFRESH + assert _read_claude_pair(claude_credentials) == (_STALE_ACCESS, _STALE_REFRESH) + + reloaded = [ + e for e in load_pool("anthropic").entries() if e.source == "claude_code" + ] + assert reloaded, "the entry should still exist after reload" + assert reloaded[0].refresh_token == _STALE_REFRESH + assert reloaded[0].last_status == STATUS_DEAD, ( + "a reload must not resurrect the pre-refresh pair as a usable " + "credential — that token was already consumed by the refresh POST" + ) + + +def test_reauthentication_clears_the_persist_failure_quarantine( + hermes_home, claude_credentials, monkeypatch +): + """The quarantine is terminal for the spent pair, not for the account. + + Re-running ``claude setup-token`` rewrites the singleton with a genuinely + new access token; ``_upsert_entry`` sees the token change and clears the + terminal status, so the user recovers without hand-editing auth.json. + """ + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + _break_durable_write(monkeypatch) + + entry = _entry("claude_code") + pool = CredentialPool("anthropic", [entry]) + assert pool._refresh_entry(entry, force=True) is None + assert pool.entries()[0].last_status == STATUS_DEAD + + # Restore a working filesystem, then simulate the re-login rewriting the + # authoritative file with a genuinely new pair. + monkeypatch.undo() + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_TOKEN", raising=False) + monkeypatch.setattr( + "hermes_cli.auth.is_provider_explicitly_configured", lambda pid: True + ) + monkeypatch.setattr(AA, "claude_code_credentials_path", lambda: claude_credentials) + monkeypatch.setattr(AA, "_read_claude_code_credentials_from_keychain", lambda: None) + claude_credentials.write_text( + json.dumps( + { + "claudeAiOauth": { + "accessToken": "sk-ant-oat01-relogin", + "refreshToken": "sk-ant-ort01-relogin", + "expiresAt": int(time.time() * 1000) + 3_600_000, + "scopes": ["user:inference", "user:profile"], + } + } + ), + encoding="utf-8", + ) + + reloaded = [ + e for e in load_pool("anthropic").entries() if e.source == "claude_code" + ] + assert reloaded + assert reloaded[0].refresh_token == "sk-ant-ort01-relogin" + assert reloaded[0].last_status != STATUS_DEAD + + +# --------------------------------------------------------------------------- +# Pool path: hermes_pkce +# --------------------------------------------------------------------------- + + +def test_pool_hermes_pkce_fails_closed_and_reload_cannot_resurrect( + hermes_home, monkeypatch +): + oauth_file = hermes_home / ".anthropic_oauth.json" + oauth_file.write_text( + json.dumps( + { + "accessToken": _STALE_ACCESS, + "refreshToken": _STALE_REFRESH, + "expiresAt": _EXPIRED_MS, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + monkeypatch.setattr(AA, "read_claude_code_credentials", lambda: None) + _break_durable_write(monkeypatch) + + entry = _entry("hermes_pkce") + pool = CredentialPool("anthropic", [entry]) + + assert pool._refresh_entry(entry, force=True) is None + + quarantined = pool.entries()[0] + assert quarantined.last_status == STATUS_DEAD + assert quarantined.last_error_reason == CREDENTIAL_PERSIST_FAILED_REASON + assert quarantined.refresh_token == _STALE_REFRESH + + on_disk = json.loads(oauth_file.read_text(encoding="utf-8")) + assert on_disk["refreshToken"] == _STALE_REFRESH + + reloaded = [ + e for e in load_pool("anthropic").entries() if e.source == "hermes_pkce" + ] + assert reloaded + assert reloaded[0].refresh_token == _STALE_REFRESH + assert reloaded[0].last_status == STATUS_DEAD + + +# --------------------------------------------------------------------------- +# Pool path: the sync-and-retry-once recovery branch +# --------------------------------------------------------------------------- + + +def test_retry_path_fails_closed_when_rotation_cannot_commit( + hermes_home, claude_credentials, monkeypatch +): + """The retry branch used to persist an "ok" row *before* committing. + + That ordering meant a failed write left an entry marked healthy in + ``auth.json`` while the authoritative file still held the consumed pair — + exactly the state ``_seed_from_singletons()`` reverses on the next load. + The commit now runs first, and a failure quarantines instead. + + ``_refresh_entry_impl`` is driven directly here: the pre-POST sync in + ``_refresh_entry`` would adopt the newer file pair and return before ever + reaching this branch. + """ + posts: list[str] = [] + + def _refresh(refresh_token, use_json=False): + posts.append(refresh_token) + if refresh_token == _STALE_REFRESH: + # The pair we hold was already spent by another process. + raise RuntimeError("invalid_grant") + return _rotating_refresh() + + # The winner's rotated pair, as seen by our re-read of the shared file. + claude_credentials.write_text( + json.dumps( + { + "claudeAiOauth": { + "accessToken": "sk-ant-oat01-winner", + "refreshToken": "sk-ant-ort01-winner", + "expiresAt": _EXPIRED_MS, + } + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _refresh) + _break_durable_write(monkeypatch) + + entry = _entry("claude_code") + pool = CredentialPool("anthropic", [entry]) + + assert pool._refresh_entry_impl(entry, force=True) is None + assert posts == [_STALE_REFRESH, "sk-ant-ort01-winner"], ( + "the retry branch should re-POST with the synced token exactly once" + ) + + quarantined = pool.entries()[0] + assert quarantined.last_status == STATUS_DEAD + assert quarantined.last_error_reason == CREDENTIAL_PERSIST_FAILED_REASON + assert quarantined.access_token != _ROTATED_ACCESS, ( + "the retry path must not mark the uncommitted rotation as healthy" + ) + assert _read_claude_pair(claude_credentials) == ( + "sk-ant-oat01-winner", + "sk-ant-ort01-winner", + )