diff --git a/agent/anthropic_credentials.py b/agent/anthropic_credentials.py index d187665ae4..a40cdcb298 100644 --- a/agent/anthropic_credentials.py +++ b/agent/anthropic_credentials.py @@ -112,24 +112,116 @@ class CredentialPersistError(RuntimeError): # explicit verdict the resolver happily hands that already-consumed credential # back from a later source and the caller reads a silent success. # -# Kept as non-reversible digests, process-local, and bounded: a spent secret is -# spent forever, so entries never need clearing (a re-auth mints new tokens -# with new fingerprints). +# Kept as non-reversible digests and bounded: a spent secret is spent forever, +# so entries never need clearing (a re-auth mints new tokens with new +# fingerprints). +# +# The registry has TWO scopes, because the credential it protects does: +# * process-local (this OrderedDict) — fast path, always recorded; +# * durable sidecar file next to the shared credential source — the +# authority boundary of ``claude_code``/``hermes_pkce`` is the shared +# singleton file, which other Hermes processes/profiles read with fresh +# interpreters. A process-local verdict only stops the process that +# lost the commit from lying to itself; the sidecar stops every OTHER +# process from leasing the stale pair or re-POSTing the spent refresh +# token. The sidecar stores only one-way fingerprints (never secrets) +# and is written under the same path-keyed cross-process lock that +# serializes refreshes of that source. _SPENT_ROTATION_LOCK = threading.Lock() _SPENT_ROTATION_FINGERPRINTS: "OrderedDict[str, None]" = OrderedDict() _SPENT_ROTATION_MAX_TRACKED = 64 +_SPENT_ROTATION_SIDECAR_VERSION = 1 -def mark_rotation_consumed_uncommitted(*secrets: Any) -> None: +def _spent_rotation_sidecar_path(source_path: Path) -> Path: + """Sidecar registry path for a shared credential source file.""" + return source_path.with_name(source_path.name + ".hermes-spent-rotations.json") + + +def spent_rotation_source_path(source: Any) -> Optional[Path]: + """Map a pool-entry source to the shared singleton file it borrows from. + + Only singleton-backed sources have a cross-process authority boundary; + profile-owned rows are already protected by the process-local registry + plus the pool quarantine. + """ + if source == "claude_code": + return claude_code_credentials_path() + if source == "hermes_pkce": + return _get_hermes_oauth_file() + return None + + +def _read_spent_rotation_sidecar(source_path: Optional[Path]) -> set: + if source_path is None: + return set() + try: + raw = json.loads( + _spent_rotation_sidecar_path(source_path).read_text(encoding="utf-8") + ) + except (OSError, ValueError): + return set() + fingerprints = raw.get("fingerprints") if isinstance(raw, dict) else None + if not isinstance(fingerprints, list): + return set() + return {fp for fp in fingerprints if isinstance(fp, str) and fp} + + +def _append_spent_rotation_sidecar(source_path: Path, fingerprints: list) -> None: + """Merge fingerprints into the sidecar registry (atomic replace). + + Callers on the refresh path already hold the path-keyed cross-process + lock for ``source_path``, so concurrent merge-writes are serialized. + Fail-soft: a sidecar write failure must never mask the fail-closed + verdict already recorded in the process-local registry. + """ + sidecar = _spent_rotation_sidecar_path(source_path) + try: + merged = _read_spent_rotation_sidecar(source_path) + merged.update(fingerprints) + bounded = sorted(merged)[-_SPENT_ROTATION_MAX_TRACKED * 4 :] + payload = json.dumps( + { + "version": _SPENT_ROTATION_SIDECAR_VERSION, + "comment": ( + "Non-secret one-way fingerprints of Anthropic OAuth " + "credentials whose rotation was consumed server-side but " + "never durably committed. Written by Hermes so sibling " + "processes sharing this credential source fail closed " + "instead of replaying a spent single-use refresh token." + ), + "fingerprints": bounded, + }, + indent=2, + ) + sidecar.parent.mkdir(parents=True, exist_ok=True) + tmp = sidecar.with_name(sidecar.name + ".tmp") + tmp.write_text(payload, encoding="utf-8") + os.replace(tmp, sidecar) + except Exception: + logger.debug( + "Failed to persist spent-rotation fingerprints to %s", sidecar, + exc_info=True, + ) + + +def mark_rotation_consumed_uncommitted( + *secrets: Any, source_path: Optional[Path] = None +) -> None: """Record secrets consumed by a refresh whose replacement never committed. Called from every commit-failure path (the direct resolver here and ``CredentialPool._fail_closed_unpersisted_rotation``). Recording the *pre-rotation* pair is what lets later resolution steps recognise the stale copy they read back off disk as unusable rather than as a working token. + + When ``source_path`` names the shared singleton file the credential was + borrowed from, the verdict is additionally persisted to that source's + sidecar registry so other processes/profiles sharing the file adopt it too. """ from agent.credential_persistence import fingerprint_secret_value + recorded: list = [] with _SPENT_ROTATION_LOCK: for secret in secrets: value = str(secret or "").strip() @@ -138,14 +230,24 @@ def mark_rotation_consumed_uncommitted(*secrets: Any) -> None: fingerprint = fingerprint_secret_value(value) if not fingerprint: continue + recorded.append(fingerprint) _SPENT_ROTATION_FINGERPRINTS.pop(fingerprint, None) _SPENT_ROTATION_FINGERPRINTS[fingerprint] = None while len(_SPENT_ROTATION_FINGERPRINTS) > _SPENT_ROTATION_MAX_TRACKED: _SPENT_ROTATION_FINGERPRINTS.popitem(last=False) + if recorded and source_path is not None: + _append_spent_rotation_sidecar(source_path, recorded) -def is_rotation_consumed_uncommitted(secret: Any) -> bool: - """True when *secret* belongs to a rotation that was spent but not committed.""" +def is_rotation_consumed_uncommitted( + secret: Any, *, source_path: Optional[Path] = None +) -> bool: + """True when *secret* belongs to a rotation that was spent but not committed. + + Checks the process-local registry first, then (when ``source_path`` is + given) the durable sidecar registry of the shared credential source, so a + fresh interpreter in another process still sees the terminal verdict. + """ from agent.credential_persistence import fingerprint_secret_value value = str(secret or "").strip() @@ -155,7 +257,9 @@ def is_rotation_consumed_uncommitted(secret: Any) -> bool: if not fingerprint: return False with _SPENT_ROTATION_LOCK: - return fingerprint in _SPENT_ROTATION_FINGERPRINTS + if fingerprint in _SPENT_ROTATION_FINGERPRINTS: + return True + return fingerprint in _read_spent_rotation_sidecar(source_path) def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]: @@ -434,6 +538,19 @@ def _refresh_oauth_token(creds: Dict[str, Any]) -> Optional[str]: logger.debug("No refresh token available — cannot refresh") return None + # Another process may have spent this refresh token and lost the + # commit; its durable sidecar verdict is authoritative for the + # shared source. POSTing it again would just burn the family into + # ``invalid_grant``. + if is_rotation_consumed_uncommitted( + refresh_token, source_path=claude_code_credentials_path() + ): + logger.debug( + "Refresh token was already consumed by an uncommitted rotation " + "- refusing to replay it; re-run 'claude setup-token'" + ) + return None + try: refreshed = refresh_anthropic_oauth_pure(refresh_token, use_json=False) except Exception as e: @@ -470,6 +587,7 @@ def _refresh_oauth_token(creds: Dict[str, Any]) -> Optional[str]: creds.get("accessToken", ""), (current or {}).get("accessToken", ""), (current or {}).get("refreshToken", ""), + source_path=claude_code_credentials_path(), ) return None @@ -563,7 +681,9 @@ def _write_claude_code_credentials( def _resolve_claude_code_token_from_credentials(creds: Optional[Dict[str, Any]] = None) -> Optional[str]: """Resolve a token from Claude Code credential files, refreshing if needed.""" creds = creds or read_claude_code_credentials() - if creds and is_rotation_consumed_uncommitted(creds.get("accessToken", "")): + if creds and is_rotation_consumed_uncommitted( + creds.get("accessToken", ""), source_path=claude_code_credentials_path() + ): # This process already rotated this pair and failed to commit the # replacement. The file still holds the spent copy; treating it as # usable is exactly the silent success this transaction fails closed @@ -646,9 +766,15 @@ def _resolve_anthropic_pool_token() -> Optional[str]: # rotation that was consumed upstream but never committed comes back # here looking healthy. Enumeration is deliberately read-only # (refresh=False), which means nothing on this path would otherwise - # notice that the credential is spent. - if is_rotation_consumed_uncommitted(token) or is_rotation_consumed_uncommitted( - getattr(entry, "refresh_token", None) + # notice that the credential is spent. Singleton-backed sources also + # consult the durable sidecar registry: the failed commit may have + # happened in a DIFFERENT process, whose process-local verdict this + # interpreter never saw. + entry_source_path = spent_rotation_source_path(getattr(entry, "source", None)) + if is_rotation_consumed_uncommitted( + token, source_path=entry_source_path + ) or is_rotation_consumed_uncommitted( + getattr(entry, "refresh_token", None), source_path=entry_source_path ): logger.debug( "Skipping Anthropic pool entry %s: rotated-but-uncommitted credential", diff --git a/agent/credential_pool.py b/agent/credential_pool.py index d17020cacd..8eca1740a5 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -1572,14 +1572,26 @@ class CredentialPool: exc, ) try: - from agent.anthropic_credentials import mark_rotation_consumed_uncommitted + from agent.anthropic_credentials import ( + mark_rotation_consumed_uncommitted, + spent_rotation_source_path, + ) # Quarantining the row is not enough on its own: the singleton file # still holds the spent pair, ``load_pool()`` re-seeds it, and the # read-only resolver (``_resolve_anthropic_pool_token``) would hand # it back as a working token. Record the fingerprints so every - # resolution step in this process recognises it as consumed. - mark_rotation_consumed_uncommitted(entry.access_token, entry.refresh_token) + # resolution step in this process recognises it as consumed — and, + # for singleton-backed sources, persist them to the shared source's + # sidecar registry (we hold that source's path-keyed lock on this + # path) so OTHER processes/profiles sharing the credential file + # adopt the terminal verdict too instead of leasing the stale pair + # or re-POSTing the spent refresh token from a fresh interpreter. + mark_rotation_consumed_uncommitted( + entry.access_token, + entry.refresh_token, + source_path=spent_rotation_source_path(entry.source), + ) except Exception: # pragma: no cover - never block the quarantine logger.debug("Failed to record consumed rotation fingerprints", exc_info=True) self._mark_exhausted( @@ -1618,7 +1630,32 @@ class CredentialPool: ) -> Optional[PooledCredential]: try: if self.provider == "anthropic": - from agent.anthropic_credentials import refresh_anthropic_oauth_pure + from agent.anthropic_credentials import ( + is_rotation_consumed_uncommitted, + refresh_anthropic_oauth_pure, + spent_rotation_source_path, + ) + + # Never POST a refresh token another process already spent. + # The durable sidecar verdict (written by whichever process + # rotated the pair and lost the commit) is what a fresh + # interpreter sees here; without this check, process B would + # replay the consumed single-use token and burn the family + # into ``invalid_grant``. + _entry_source_path = spent_rotation_source_path(entry.source) + if is_rotation_consumed_uncommitted( + entry.refresh_token, source_path=_entry_source_path + ) or is_rotation_consumed_uncommitted( + entry.access_token, source_path=_entry_source_path + ): + return self._fail_closed_unpersisted_rotation( + entry, + RuntimeError( + "credential pair was rotated by another process but the " + "rotation never committed (spent-rotation sidecar verdict)" + ), + store=str(_entry_source_path or "credential store"), + ) refreshed = refresh_anthropic_oauth_pure( entry.refresh_token, @@ -2896,7 +2933,10 @@ def _seed_from_singletons(provider: str, entries: List[PooledCredential]) -> Tup changed = True return changed, active_sources - from agent.anthropic_credentials import read_claude_code_credentials, read_hermes_oauth_credentials + from agent.anthropic_credentials import ( + read_claude_code_credentials, + read_hermes_oauth_credentials, + ) for source_name, creds in ( ("hermes_pkce", read_hermes_oauth_credentials()), diff --git a/tests/agent/test_anthropic_spent_rotation_verdict.py b/tests/agent/test_anthropic_spent_rotation_verdict.py index f23e46f71c..88f73de771 100644 --- a/tests/agent/test_anthropic_spent_rotation_verdict.py +++ b/tests/agent/test_anthropic_spent_rotation_verdict.py @@ -252,3 +252,140 @@ def test_successful_commit_leaves_the_credential_usable( assert AA.resolve_anthropic_token() == _ROTATED_ACCESS assert AA._SPENT_ROTATION_FINGERPRINTS == {} + + +# --------------------------------------------------------------------------- +# Cross-process durability of the verdict (sidecar registry) +# --------------------------------------------------------------------------- + + +def test_failed_commit_persists_the_verdict_to_the_sidecar( + hermes_home, claude_credentials, monkeypatch +): + """The verdict must outlive this process: it lands in the sidecar file.""" + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + _break_durable_write(monkeypatch) + + assert AA._refresh_oauth_token(AA.read_claude_code_credentials()) is None + + sidecar = AA._spent_rotation_sidecar_path(claude_credentials) + assert sidecar.exists(), "the terminal verdict must be durably persisted" + payload = json.loads(sidecar.read_text(encoding="utf-8")) + fingerprints = set(payload["fingerprints"]) + from agent.credential_persistence import fingerprint_secret_value + + assert fingerprint_secret_value(_STALE_REFRESH) in fingerprints + assert fingerprint_secret_value(_STALE_ACCESS) in fingerprints + # Non-secret invariant: no raw token material may reach the sidecar. + raw = sidecar.read_text(encoding="utf-8") + for secret in (_STALE_ACCESS, _STALE_REFRESH, _ROTATED_ACCESS, _ROTATED_REFRESH): + assert secret not in raw + + +_SECOND_PROCESS_WITNESS = r""" +import json +import sys + +cred_path_str, sidecar_dir = sys.argv[1], sys.argv[2] + +from pathlib import Path + +import agent.anthropic_credentials as AA + +cred_path = Path(cred_path_str) +AA.claude_code_credentials_path = lambda: cred_path +AA._read_claude_code_credentials_from_keychain = lambda *a, **k: None + +posted = [] + + +def _must_not_post(refresh_token, *a, **kw): + posted.append(refresh_token) + raise AssertionError("process B replayed a spent refresh token") + + +AA.refresh_anthropic_oauth_pure = _must_not_post + +creds = AA.read_claude_code_credentials() +result = { + "registry_empty": len(AA._SPENT_ROTATION_FINGERPRINTS) == 0, + "sidecar_verdict_access": AA.is_rotation_consumed_uncommitted( + creds["accessToken"], source_path=cred_path + ), + "sidecar_verdict_refresh": AA.is_rotation_consumed_uncommitted( + creds["refreshToken"], source_path=cred_path + ), + "resolved": AA._resolve_claude_code_token_from_credentials(creds), + "posted": posted, +} +print(json.dumps(result)) +""" + + +def test_second_process_adopts_the_terminal_verdict( + hermes_home, claude_credentials, monkeypatch, tmp_path +): + """Two-process witness: A rotates and loses the commit; B fails closed. + + Process B runs in a fresh interpreter whose process-local registry is + empty, sharing only the credential file (and its sidecar). B must neither + lease the stale access token nor POST the spent refresh token — the exact + cross-process gap the process-local OrderedDict could not cover. + """ + import subprocess + import sys + + # Process A: successful POST, failed durable commit. + monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh) + _break_durable_write(monkeypatch) + assert AA._refresh_oauth_token(AA.read_claude_code_credentials()) is None + assert AA._spent_rotation_sidecar_path(claude_credentials).exists() + + # Process B: fresh interpreter, same shared credential source. + import agent as _agent_pkg + + repo_root = str( + __import__("pathlib").Path(_agent_pkg.__file__).resolve().parents[1] + ) + env = dict(os.environ) + env["HERMES_HOME"] = str(hermes_home) + env["PYTHONPATH"] = repo_root + for var in ("ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN"): + env.pop(var, None) + + proc = subprocess.run( + [sys.executable, "-c", _SECOND_PROCESS_WITNESS, str(claude_credentials), str(tmp_path)], + capture_output=True, + text=True, + timeout=60, + env=env, + stdin=subprocess.DEVNULL, + ) + assert proc.returncode == 0, f"witness failed:\n{proc.stdout}\n{proc.stderr}" + verdict = json.loads(proc.stdout.strip().splitlines()[-1]) + + assert verdict["registry_empty"], "precondition: B must start with no local verdict" + assert verdict["sidecar_verdict_access"], "B must see A's verdict via the sidecar" + assert verdict["sidecar_verdict_refresh"] + assert verdict["resolved"] is None, "B must not lease the stale access token" + assert verdict["posted"] == [], "B must not POST the spent refresh token" + + +def test_control_second_process_without_sidecar_still_resolves( + hermes_home, claude_credentials, monkeypatch +): + """Independent-credential control: no verdict, no quarantine. + + With no failed rotation recorded anywhere, the shared file's (valid) + credential resolves normally in this process — proving the sidecar gate + only fires on a recorded verdict, not on every read. + """ + fresh = dict( + json.loads(claude_credentials.read_text(encoding="utf-8")) + ) + fresh["claudeAiOauth"]["expiresAt"] = int(time.time() * 1000) + 3_600_000 + claude_credentials.write_text(json.dumps(fresh), encoding="utf-8") + + assert not AA._spent_rotation_sidecar_path(claude_credentials).exists() + creds = AA.read_claude_code_credentials() + assert AA._resolve_claude_code_token_from_credentials(creds) == _STALE_ACCESS