fix(honcho): adopt a sibling's on-disk rotation even inside our exchange cooldown
force_refresh_token gated the adopt-from-disk paths behind the failure cooldown. After one of our exchanges failed transiently, a 401 within the next 30s returned None even when a sibling process had already rotated and written a valid grant, so the operation raised HonchoAuthError with a good token sitting on disk. Adopting is a disk read, not an exchange; the cooldown exists to stop replaying a single-use refresh token, so the two adopt checks now run before the gates. _write_config parsed honcho.json twice under the lock and only wrapped the first read into ConfigWriteRefused; _refuse_unparseable now returns the parsed dict and the branches use it. The getattr/isinstance duck-typing collapses to one guard. cli._read_config reuses oauth's tolerant reader (BOM-tolerant, like the strict reader the write side uses) instead of its own utf-8 copy.
This commit is contained in:
@@ -84,11 +84,12 @@ class ConfigWriteRefused(Exception):
|
||||
"""honcho.json exists on disk but does not parse, so no command may overwrite it."""
|
||||
|
||||
|
||||
def _refuse_unparseable(path: Path) -> None:
|
||||
"""Raise ConfigWriteRefused when ``path`` exists but cannot be parsed; writing back ``{}`` would drop every host."""
|
||||
def _refuse_unparseable(path: Path) -> dict:
|
||||
"""Return ``path``'s parsed content ({} when absent); raise ConfigWriteRefused when it exists but cannot
|
||||
be parsed, since writing back ``{}`` would drop every host."""
|
||||
from plugins.memory.honcho.oauth import _read_config_strict
|
||||
try:
|
||||
_read_config_strict(path)
|
||||
return _read_config_strict(path)
|
||||
except (OSError, ValueError) as e:
|
||||
raise ConfigWriteRefused(f"{path} exists but could not be read as JSON ({e}). Nothing was written. "
|
||||
"Fix or move the file, then re-run.") from e
|
||||
@@ -135,17 +136,18 @@ def _write_config(cfg: dict, path: Path | None = None) -> None:
|
||||
"""Persist ``cfg`` under the token refresh's cross-process lock. The object _read_config() returned
|
||||
has only its edits applied onto a fresh read of disk; a plain dict is written whole. A read that
|
||||
resolved to a seed file (~/.honcho or a profile) is written whole only while ``path`` does not exist."""
|
||||
from plugins.memory.honcho.oauth import _config_refresh_lock, _read_config_strict, _refresh_lock
|
||||
from plugins.memory.honcho.oauth import _config_refresh_lock, _refresh_lock
|
||||
from utils import atomic_json_write
|
||||
path = path or _local_config_path()
|
||||
# The file lock is best-effort; _refresh_lock is what keeps an in-process refresh thread out.
|
||||
with _refresh_lock, _config_refresh_lock(path):
|
||||
_refuse_unparseable(path)
|
||||
disk = _refuse_unparseable(path)
|
||||
out = cfg
|
||||
if getattr(cfg, "path", None) == path:
|
||||
out = _apply_edits(cfg.snapshot, cfg, _read_config_strict(path))
|
||||
elif isinstance(cfg, _ReadConfig) and path.exists():
|
||||
out = _apply_edits(cfg.snapshot, cfg, _overlay_local(cfg.snapshot, _read_config_strict(path)))
|
||||
if isinstance(cfg, _ReadConfig):
|
||||
if cfg.path == path:
|
||||
out = _apply_edits(cfg.snapshot, cfg, disk)
|
||||
elif path.exists():
|
||||
out = _apply_edits(cfg.snapshot, cfg, _overlay_local(cfg.snapshot, disk))
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_json_write(path, out, mode=0o600)
|
||||
if isinstance(cfg, _ReadConfig): # a later write on the same object applies only edits made after this one
|
||||
|
||||
@@ -359,18 +359,17 @@ def force_refresh_token(path: Path, host: str, *, failed_access_token: str | Non
|
||||
if cred is None:
|
||||
_expiry_cache.pop(key, None)
|
||||
return None
|
||||
# Adopting a sibling's rotation is a disk read, so it comes before the exchange gates: disk, not
|
||||
# the expiry cache, is the signal (the cache is empty in sibling processes).
|
||||
cached = _expiry_cache.get(key)
|
||||
moved_off_failed = failed_access_token and cred.access_token != failed_access_token
|
||||
moved_off_cached = cached is not None and cred.access_token != cached[1]
|
||||
if (moved_off_failed or moved_off_cached) and not cred.is_expired(now=now):
|
||||
_expiry_cache[key] = (cred.expires_at, cred.access_token)
|
||||
return cred.access_token
|
||||
# Dead grant, or an exchange just failed transiently: callers fail open.
|
||||
if _grant_is_dead(key, cred) or _in_failure_cooldown(key):
|
||||
return None
|
||||
# Disk, not the expiry cache, is the signal: the cache is empty in sibling processes.
|
||||
if failed_access_token and cred.access_token != failed_access_token and not cred.is_expired(now=now):
|
||||
_expiry_cache[key] = (cred.expires_at, cred.access_token)
|
||||
return cred.access_token
|
||||
cached = _expiry_cache.get(key)
|
||||
# Another thread or process already rotated: adopt the newer on-disk token.
|
||||
if cached is not None and cred.access_token != cached[1] and not cred.is_expired(now=now):
|
||||
_expiry_cache[key] = (cred.expires_at, cred.access_token)
|
||||
return cred.access_token
|
||||
rotated = _rotate_and_persist(path, host, key, cred, now=now, op_label="forced refresh")
|
||||
if rotated is not None:
|
||||
logger.info("Honcho OAuth token force-refreshed for host %s after an auth failure", host)
|
||||
|
||||
@@ -195,10 +195,11 @@ class TestForceRefreshToken:
|
||||
)
|
||||
assert oauth.force_refresh_token(path, "hermes") == "hch-at-2"
|
||||
|
||||
@pytest.mark.parametrize("rotated_by", ["first waiter", "sibling process"])
|
||||
@pytest.mark.parametrize("rotated_by", ["first waiter", "sibling process", "sibling process during our cooldown"])
|
||||
def test_401_on_a_bearer_disk_has_moved_off_adopts_without_exchange(self, tmp_path, monkeypatch, rotated_by):
|
||||
"""The failing bearer disagreeing with disk is enough to adopt: a replayed refresh token can
|
||||
revoke the grant, and the expiry cache is empty in a sibling process."""
|
||||
revoke the grant, and the expiry cache is empty in a sibling process. Adopting is a disk read,
|
||||
so our own recent failed exchange (the cooldown) does not block it."""
|
||||
path = tmp_path / "honcho.json"
|
||||
far = time.time() + 7200
|
||||
_write(path, {"hosts": {"hermes": _host_block(expires_at=far)}})
|
||||
@@ -208,6 +209,8 @@ class TestForceRefreshToken:
|
||||
else:
|
||||
_write(path, {"hosts": {"hermes": {**_host_block(refresh="hch-rt-new2", expires_at=far), "apiKey": "hch-at-new2"}}})
|
||||
oauth._expiry_cache.clear()
|
||||
if rotated_by.endswith("cooldown"):
|
||||
oauth._refresh_failure_at[(str(path), "hermes")] = time.monotonic()
|
||||
monkeypatch.setattr(oauth, "_http_post_form_status", lambda *a, **k: pytest.fail("must adopt the on-disk grant, not exchange"))
|
||||
assert oauth.force_refresh_token(path, "hermes", failed_access_token="hch-at-old") == "hch-at-new2"
|
||||
assert oauth._expiry_cache[(str(path), "hermes")][1] == "hch-at-new2"
|
||||
|
||||
Reference in New Issue
Block a user