fix(auth): carry the spent-rotation verdict across processes via a durable sidecar registry
The consumed-but-uncommitted rotation verdict was process-local (_SPENT_ROTATION_FINGERPRINTS), while the credential it protects is explicitly cross-process: ~/.claude/.credentials.json is shared by every Hermes profile and process. A fresh interpreter could lease the stale access token or re-POST the already-spent single-use refresh token and burn the credential family into invalid_grant. - Persist non-secret one-way fingerprints to a sidecar registry next to the shared singleton source (claude_code / hermes_pkce), written under the same path-keyed cross-process lock that serializes refreshes. - Consult the sidecar in the pool resolver, the pool refresh path, and the direct claude_code resolver/refresh before leasing or POSTing. - Two-process regression: A rotates and loses the commit; B (fresh interpreter, empty local registry) must neither lease the stale pair nor POST the spent refresh token. Plus a no-verdict control. Closes the remaining P1 from the exact-head review of f228439b on PR #87891.
This commit is contained in:
Teknium
+137
-11
@@ -112,24 +112,116 @@ class CredentialPersistError(RuntimeError):
|
||||
# explicit verdict the resolver happily hands that already-consumed credential
|
||||
# back from a later source and the caller reads a silent success.
|
||||
#
|
||||
# Kept as non-reversible digests, process-local, and bounded: a spent secret is
|
||||
# spent forever, so entries never need clearing (a re-auth mints new tokens
|
||||
# with new fingerprints).
|
||||
# Kept as non-reversible digests and bounded: a spent secret is spent forever,
|
||||
# so entries never need clearing (a re-auth mints new tokens with new
|
||||
# fingerprints).
|
||||
#
|
||||
# The registry has TWO scopes, because the credential it protects does:
|
||||
# * process-local (this OrderedDict) — fast path, always recorded;
|
||||
# * durable sidecar file next to the shared credential source — the
|
||||
# authority boundary of ``claude_code``/``hermes_pkce`` is the shared
|
||||
# singleton file, which other Hermes processes/profiles read with fresh
|
||||
# interpreters. A process-local verdict only stops the process that
|
||||
# lost the commit from lying to itself; the sidecar stops every OTHER
|
||||
# process from leasing the stale pair or re-POSTing the spent refresh
|
||||
# token. The sidecar stores only one-way fingerprints (never secrets)
|
||||
# and is written under the same path-keyed cross-process lock that
|
||||
# serializes refreshes of that source.
|
||||
_SPENT_ROTATION_LOCK = threading.Lock()
|
||||
_SPENT_ROTATION_FINGERPRINTS: "OrderedDict[str, None]" = OrderedDict()
|
||||
_SPENT_ROTATION_MAX_TRACKED = 64
|
||||
_SPENT_ROTATION_SIDECAR_VERSION = 1
|
||||
|
||||
|
||||
def mark_rotation_consumed_uncommitted(*secrets: Any) -> None:
|
||||
def _spent_rotation_sidecar_path(source_path: Path) -> Path:
|
||||
"""Sidecar registry path for a shared credential source file."""
|
||||
return source_path.with_name(source_path.name + ".hermes-spent-rotations.json")
|
||||
|
||||
|
||||
def spent_rotation_source_path(source: Any) -> Optional[Path]:
|
||||
"""Map a pool-entry source to the shared singleton file it borrows from.
|
||||
|
||||
Only singleton-backed sources have a cross-process authority boundary;
|
||||
profile-owned rows are already protected by the process-local registry
|
||||
plus the pool quarantine.
|
||||
"""
|
||||
if source == "claude_code":
|
||||
return claude_code_credentials_path()
|
||||
if source == "hermes_pkce":
|
||||
return _get_hermes_oauth_file()
|
||||
return None
|
||||
|
||||
|
||||
def _read_spent_rotation_sidecar(source_path: Optional[Path]) -> set:
|
||||
if source_path is None:
|
||||
return set()
|
||||
try:
|
||||
raw = json.loads(
|
||||
_spent_rotation_sidecar_path(source_path).read_text(encoding="utf-8")
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
return set()
|
||||
fingerprints = raw.get("fingerprints") if isinstance(raw, dict) else None
|
||||
if not isinstance(fingerprints, list):
|
||||
return set()
|
||||
return {fp for fp in fingerprints if isinstance(fp, str) and fp}
|
||||
|
||||
|
||||
def _append_spent_rotation_sidecar(source_path: Path, fingerprints: list) -> None:
|
||||
"""Merge fingerprints into the sidecar registry (atomic replace).
|
||||
|
||||
Callers on the refresh path already hold the path-keyed cross-process
|
||||
lock for ``source_path``, so concurrent merge-writes are serialized.
|
||||
Fail-soft: a sidecar write failure must never mask the fail-closed
|
||||
verdict already recorded in the process-local registry.
|
||||
"""
|
||||
sidecar = _spent_rotation_sidecar_path(source_path)
|
||||
try:
|
||||
merged = _read_spent_rotation_sidecar(source_path)
|
||||
merged.update(fingerprints)
|
||||
bounded = sorted(merged)[-_SPENT_ROTATION_MAX_TRACKED * 4 :]
|
||||
payload = json.dumps(
|
||||
{
|
||||
"version": _SPENT_ROTATION_SIDECAR_VERSION,
|
||||
"comment": (
|
||||
"Non-secret one-way fingerprints of Anthropic OAuth "
|
||||
"credentials whose rotation was consumed server-side but "
|
||||
"never durably committed. Written by Hermes so sibling "
|
||||
"processes sharing this credential source fail closed "
|
||||
"instead of replaying a spent single-use refresh token."
|
||||
),
|
||||
"fingerprints": bounded,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
sidecar.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = sidecar.with_name(sidecar.name + ".tmp")
|
||||
tmp.write_text(payload, encoding="utf-8")
|
||||
os.replace(tmp, sidecar)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to persist spent-rotation fingerprints to %s", sidecar,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
|
||||
def mark_rotation_consumed_uncommitted(
|
||||
*secrets: Any, source_path: Optional[Path] = None
|
||||
) -> None:
|
||||
"""Record secrets consumed by a refresh whose replacement never committed.
|
||||
|
||||
Called from every commit-failure path (the direct resolver here and
|
||||
``CredentialPool._fail_closed_unpersisted_rotation``). Recording the
|
||||
*pre-rotation* pair is what lets later resolution steps recognise the stale
|
||||
copy they read back off disk as unusable rather than as a working token.
|
||||
|
||||
When ``source_path`` names the shared singleton file the credential was
|
||||
borrowed from, the verdict is additionally persisted to that source's
|
||||
sidecar registry so other processes/profiles sharing the file adopt it too.
|
||||
"""
|
||||
from agent.credential_persistence import fingerprint_secret_value
|
||||
|
||||
recorded: list = []
|
||||
with _SPENT_ROTATION_LOCK:
|
||||
for secret in secrets:
|
||||
value = str(secret or "").strip()
|
||||
@@ -138,14 +230,24 @@ def mark_rotation_consumed_uncommitted(*secrets: Any) -> None:
|
||||
fingerprint = fingerprint_secret_value(value)
|
||||
if not fingerprint:
|
||||
continue
|
||||
recorded.append(fingerprint)
|
||||
_SPENT_ROTATION_FINGERPRINTS.pop(fingerprint, None)
|
||||
_SPENT_ROTATION_FINGERPRINTS[fingerprint] = None
|
||||
while len(_SPENT_ROTATION_FINGERPRINTS) > _SPENT_ROTATION_MAX_TRACKED:
|
||||
_SPENT_ROTATION_FINGERPRINTS.popitem(last=False)
|
||||
if recorded and source_path is not None:
|
||||
_append_spent_rotation_sidecar(source_path, recorded)
|
||||
|
||||
|
||||
def is_rotation_consumed_uncommitted(secret: Any) -> bool:
|
||||
"""True when *secret* belongs to a rotation that was spent but not committed."""
|
||||
def is_rotation_consumed_uncommitted(
|
||||
secret: Any, *, source_path: Optional[Path] = None
|
||||
) -> bool:
|
||||
"""True when *secret* belongs to a rotation that was spent but not committed.
|
||||
|
||||
Checks the process-local registry first, then (when ``source_path`` is
|
||||
given) the durable sidecar registry of the shared credential source, so a
|
||||
fresh interpreter in another process still sees the terminal verdict.
|
||||
"""
|
||||
from agent.credential_persistence import fingerprint_secret_value
|
||||
|
||||
value = str(secret or "").strip()
|
||||
@@ -155,7 +257,9 @@ def is_rotation_consumed_uncommitted(secret: Any) -> bool:
|
||||
if not fingerprint:
|
||||
return False
|
||||
with _SPENT_ROTATION_LOCK:
|
||||
return fingerprint in _SPENT_ROTATION_FINGERPRINTS
|
||||
if fingerprint in _SPENT_ROTATION_FINGERPRINTS:
|
||||
return True
|
||||
return fingerprint in _read_spent_rotation_sidecar(source_path)
|
||||
|
||||
|
||||
def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]:
|
||||
@@ -434,6 +538,19 @@ def _refresh_oauth_token(creds: Dict[str, Any]) -> Optional[str]:
|
||||
logger.debug("No refresh token available — cannot refresh")
|
||||
return None
|
||||
|
||||
# Another process may have spent this refresh token and lost the
|
||||
# commit; its durable sidecar verdict is authoritative for the
|
||||
# shared source. POSTing it again would just burn the family into
|
||||
# ``invalid_grant``.
|
||||
if is_rotation_consumed_uncommitted(
|
||||
refresh_token, source_path=claude_code_credentials_path()
|
||||
):
|
||||
logger.debug(
|
||||
"Refresh token was already consumed by an uncommitted rotation "
|
||||
"- refusing to replay it; re-run 'claude setup-token'"
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
refreshed = refresh_anthropic_oauth_pure(refresh_token, use_json=False)
|
||||
except Exception as e:
|
||||
@@ -470,6 +587,7 @@ def _refresh_oauth_token(creds: Dict[str, Any]) -> Optional[str]:
|
||||
creds.get("accessToken", ""),
|
||||
(current or {}).get("accessToken", ""),
|
||||
(current or {}).get("refreshToken", ""),
|
||||
source_path=claude_code_credentials_path(),
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -563,7 +681,9 @@ def _write_claude_code_credentials(
|
||||
def _resolve_claude_code_token_from_credentials(creds: Optional[Dict[str, Any]] = None) -> Optional[str]:
|
||||
"""Resolve a token from Claude Code credential files, refreshing if needed."""
|
||||
creds = creds or read_claude_code_credentials()
|
||||
if creds and is_rotation_consumed_uncommitted(creds.get("accessToken", "")):
|
||||
if creds and is_rotation_consumed_uncommitted(
|
||||
creds.get("accessToken", ""), source_path=claude_code_credentials_path()
|
||||
):
|
||||
# This process already rotated this pair and failed to commit the
|
||||
# replacement. The file still holds the spent copy; treating it as
|
||||
# usable is exactly the silent success this transaction fails closed
|
||||
@@ -646,9 +766,15 @@ def _resolve_anthropic_pool_token() -> Optional[str]:
|
||||
# rotation that was consumed upstream but never committed comes back
|
||||
# here looking healthy. Enumeration is deliberately read-only
|
||||
# (refresh=False), which means nothing on this path would otherwise
|
||||
# notice that the credential is spent.
|
||||
if is_rotation_consumed_uncommitted(token) or is_rotation_consumed_uncommitted(
|
||||
getattr(entry, "refresh_token", None)
|
||||
# notice that the credential is spent. Singleton-backed sources also
|
||||
# consult the durable sidecar registry: the failed commit may have
|
||||
# happened in a DIFFERENT process, whose process-local verdict this
|
||||
# interpreter never saw.
|
||||
entry_source_path = spent_rotation_source_path(getattr(entry, "source", None))
|
||||
if is_rotation_consumed_uncommitted(
|
||||
token, source_path=entry_source_path
|
||||
) or is_rotation_consumed_uncommitted(
|
||||
getattr(entry, "refresh_token", None), source_path=entry_source_path
|
||||
):
|
||||
logger.debug(
|
||||
"Skipping Anthropic pool entry %s: rotated-but-uncommitted credential",
|
||||
|
||||
@@ -1572,14 +1572,26 @@ class CredentialPool:
|
||||
exc,
|
||||
)
|
||||
try:
|
||||
from agent.anthropic_credentials import mark_rotation_consumed_uncommitted
|
||||
from agent.anthropic_credentials import (
|
||||
mark_rotation_consumed_uncommitted,
|
||||
spent_rotation_source_path,
|
||||
)
|
||||
|
||||
# Quarantining the row is not enough on its own: the singleton file
|
||||
# still holds the spent pair, ``load_pool()`` re-seeds it, and the
|
||||
# read-only resolver (``_resolve_anthropic_pool_token``) would hand
|
||||
# it back as a working token. Record the fingerprints so every
|
||||
# resolution step in this process recognises it as consumed.
|
||||
mark_rotation_consumed_uncommitted(entry.access_token, entry.refresh_token)
|
||||
# resolution step in this process recognises it as consumed — and,
|
||||
# for singleton-backed sources, persist them to the shared source's
|
||||
# sidecar registry (we hold that source's path-keyed lock on this
|
||||
# path) so OTHER processes/profiles sharing the credential file
|
||||
# adopt the terminal verdict too instead of leasing the stale pair
|
||||
# or re-POSTing the spent refresh token from a fresh interpreter.
|
||||
mark_rotation_consumed_uncommitted(
|
||||
entry.access_token,
|
||||
entry.refresh_token,
|
||||
source_path=spent_rotation_source_path(entry.source),
|
||||
)
|
||||
except Exception: # pragma: no cover - never block the quarantine
|
||||
logger.debug("Failed to record consumed rotation fingerprints", exc_info=True)
|
||||
self._mark_exhausted(
|
||||
@@ -1618,7 +1630,32 @@ class CredentialPool:
|
||||
) -> Optional[PooledCredential]:
|
||||
try:
|
||||
if self.provider == "anthropic":
|
||||
from agent.anthropic_credentials import refresh_anthropic_oauth_pure
|
||||
from agent.anthropic_credentials import (
|
||||
is_rotation_consumed_uncommitted,
|
||||
refresh_anthropic_oauth_pure,
|
||||
spent_rotation_source_path,
|
||||
)
|
||||
|
||||
# Never POST a refresh token another process already spent.
|
||||
# The durable sidecar verdict (written by whichever process
|
||||
# rotated the pair and lost the commit) is what a fresh
|
||||
# interpreter sees here; without this check, process B would
|
||||
# replay the consumed single-use token and burn the family
|
||||
# into ``invalid_grant``.
|
||||
_entry_source_path = spent_rotation_source_path(entry.source)
|
||||
if is_rotation_consumed_uncommitted(
|
||||
entry.refresh_token, source_path=_entry_source_path
|
||||
) or is_rotation_consumed_uncommitted(
|
||||
entry.access_token, source_path=_entry_source_path
|
||||
):
|
||||
return self._fail_closed_unpersisted_rotation(
|
||||
entry,
|
||||
RuntimeError(
|
||||
"credential pair was rotated by another process but the "
|
||||
"rotation never committed (spent-rotation sidecar verdict)"
|
||||
),
|
||||
store=str(_entry_source_path or "credential store"),
|
||||
)
|
||||
|
||||
refreshed = refresh_anthropic_oauth_pure(
|
||||
entry.refresh_token,
|
||||
@@ -2896,7 +2933,10 @@ def _seed_from_singletons(provider: str, entries: List[PooledCredential]) -> Tup
|
||||
changed = True
|
||||
return changed, active_sources
|
||||
|
||||
from agent.anthropic_credentials import read_claude_code_credentials, read_hermes_oauth_credentials
|
||||
from agent.anthropic_credentials import (
|
||||
read_claude_code_credentials,
|
||||
read_hermes_oauth_credentials,
|
||||
)
|
||||
|
||||
for source_name, creds in (
|
||||
("hermes_pkce", read_hermes_oauth_credentials()),
|
||||
|
||||
@@ -252,3 +252,140 @@ def test_successful_commit_leaves_the_credential_usable(
|
||||
|
||||
assert AA.resolve_anthropic_token() == _ROTATED_ACCESS
|
||||
assert AA._SPENT_ROTATION_FINGERPRINTS == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-process durability of the verdict (sidecar registry)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_failed_commit_persists_the_verdict_to_the_sidecar(
|
||||
hermes_home, claude_credentials, monkeypatch
|
||||
):
|
||||
"""The verdict must outlive this process: it lands in the sidecar file."""
|
||||
monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh)
|
||||
_break_durable_write(monkeypatch)
|
||||
|
||||
assert AA._refresh_oauth_token(AA.read_claude_code_credentials()) is None
|
||||
|
||||
sidecar = AA._spent_rotation_sidecar_path(claude_credentials)
|
||||
assert sidecar.exists(), "the terminal verdict must be durably persisted"
|
||||
payload = json.loads(sidecar.read_text(encoding="utf-8"))
|
||||
fingerprints = set(payload["fingerprints"])
|
||||
from agent.credential_persistence import fingerprint_secret_value
|
||||
|
||||
assert fingerprint_secret_value(_STALE_REFRESH) in fingerprints
|
||||
assert fingerprint_secret_value(_STALE_ACCESS) in fingerprints
|
||||
# Non-secret invariant: no raw token material may reach the sidecar.
|
||||
raw = sidecar.read_text(encoding="utf-8")
|
||||
for secret in (_STALE_ACCESS, _STALE_REFRESH, _ROTATED_ACCESS, _ROTATED_REFRESH):
|
||||
assert secret not in raw
|
||||
|
||||
|
||||
_SECOND_PROCESS_WITNESS = r"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
cred_path_str, sidecar_dir = sys.argv[1], sys.argv[2]
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import agent.anthropic_credentials as AA
|
||||
|
||||
cred_path = Path(cred_path_str)
|
||||
AA.claude_code_credentials_path = lambda: cred_path
|
||||
AA._read_claude_code_credentials_from_keychain = lambda *a, **k: None
|
||||
|
||||
posted = []
|
||||
|
||||
|
||||
def _must_not_post(refresh_token, *a, **kw):
|
||||
posted.append(refresh_token)
|
||||
raise AssertionError("process B replayed a spent refresh token")
|
||||
|
||||
|
||||
AA.refresh_anthropic_oauth_pure = _must_not_post
|
||||
|
||||
creds = AA.read_claude_code_credentials()
|
||||
result = {
|
||||
"registry_empty": len(AA._SPENT_ROTATION_FINGERPRINTS) == 0,
|
||||
"sidecar_verdict_access": AA.is_rotation_consumed_uncommitted(
|
||||
creds["accessToken"], source_path=cred_path
|
||||
),
|
||||
"sidecar_verdict_refresh": AA.is_rotation_consumed_uncommitted(
|
||||
creds["refreshToken"], source_path=cred_path
|
||||
),
|
||||
"resolved": AA._resolve_claude_code_token_from_credentials(creds),
|
||||
"posted": posted,
|
||||
}
|
||||
print(json.dumps(result))
|
||||
"""
|
||||
|
||||
|
||||
def test_second_process_adopts_the_terminal_verdict(
|
||||
hermes_home, claude_credentials, monkeypatch, tmp_path
|
||||
):
|
||||
"""Two-process witness: A rotates and loses the commit; B fails closed.
|
||||
|
||||
Process B runs in a fresh interpreter whose process-local registry is
|
||||
empty, sharing only the credential file (and its sidecar). B must neither
|
||||
lease the stale access token nor POST the spent refresh token — the exact
|
||||
cross-process gap the process-local OrderedDict could not cover.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Process A: successful POST, failed durable commit.
|
||||
monkeypatch.setattr(AA, "refresh_anthropic_oauth_pure", _rotating_refresh)
|
||||
_break_durable_write(monkeypatch)
|
||||
assert AA._refresh_oauth_token(AA.read_claude_code_credentials()) is None
|
||||
assert AA._spent_rotation_sidecar_path(claude_credentials).exists()
|
||||
|
||||
# Process B: fresh interpreter, same shared credential source.
|
||||
import agent as _agent_pkg
|
||||
|
||||
repo_root = str(
|
||||
__import__("pathlib").Path(_agent_pkg.__file__).resolve().parents[1]
|
||||
)
|
||||
env = dict(os.environ)
|
||||
env["HERMES_HOME"] = str(hermes_home)
|
||||
env["PYTHONPATH"] = repo_root
|
||||
for var in ("ANTHROPIC_API_KEY", "ANTHROPIC_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN"):
|
||||
env.pop(var, None)
|
||||
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-c", _SECOND_PROCESS_WITNESS, str(claude_credentials), str(tmp_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
assert proc.returncode == 0, f"witness failed:\n{proc.stdout}\n{proc.stderr}"
|
||||
verdict = json.loads(proc.stdout.strip().splitlines()[-1])
|
||||
|
||||
assert verdict["registry_empty"], "precondition: B must start with no local verdict"
|
||||
assert verdict["sidecar_verdict_access"], "B must see A's verdict via the sidecar"
|
||||
assert verdict["sidecar_verdict_refresh"]
|
||||
assert verdict["resolved"] is None, "B must not lease the stale access token"
|
||||
assert verdict["posted"] == [], "B must not POST the spent refresh token"
|
||||
|
||||
|
||||
def test_control_second_process_without_sidecar_still_resolves(
|
||||
hermes_home, claude_credentials, monkeypatch
|
||||
):
|
||||
"""Independent-credential control: no verdict, no quarantine.
|
||||
|
||||
With no failed rotation recorded anywhere, the shared file's (valid)
|
||||
credential resolves normally in this process — proving the sidecar gate
|
||||
only fires on a recorded verdict, not on every read.
|
||||
"""
|
||||
fresh = dict(
|
||||
json.loads(claude_credentials.read_text(encoding="utf-8"))
|
||||
)
|
||||
fresh["claudeAiOauth"]["expiresAt"] = int(time.time() * 1000) + 3_600_000
|
||||
claude_credentials.write_text(json.dumps(fresh), encoding="utf-8")
|
||||
|
||||
assert not AA._spent_rotation_sidecar_path(claude_credentials).exists()
|
||||
creds = AA.read_claude_code_credentials()
|
||||
assert AA._resolve_claude_code_token_from_credentials(creds) == _STALE_ACCESS
|
||||
|
||||
Reference in New Issue
Block a user