"""Auto-resume restart-loop breaker (defense-3). Defenses 1 and 2 (the ``_HERMES_GATEWAY`` guard on ``hermes gateway stop|restart`` + ``terminal_tool``, and the cron-creation lifecycle filter) stop the agent scheduling its own restart via cron/CLI. They do NOT cover every SIGTERM source (raw ``launchctl kickstart``, a bad external monitor, any repeated crash): the supervisor respawns, the gateway auto-resumes the restart-interrupted session, whose next turn re-runs the offending logic. This module is the last-resort circuit breaker. Each boot with restart-interrupted sessions pending is timestamped and persisted (each boot is a fresh process, so in-memory state is useless). Boots CHAIN while consecutive gaps stay within ``max_gap_seconds``, so slow crash cycles (a wedged loop killed by the liveness watchdog every ~150s) trip exactly like the fast ~10s respawn loop. When tripped, the caller SKIPS auto-resume for that boot — the gateway still serves real inbound messages, it just stops replaying the session that keeps killing it. State lives in ``/gateway/restart_loop.json`` (profile-scoped). Best-effort: any read/write failure fails OPEN (no false trip) because a broken breaker must never wedge a healthy gateway. """ from __future__ import annotations import json import logging import time from typing import List, Optional from hermes_constants import get_hermes_home import contextlib logger = logging.getLogger("gateway.run") # A legitimate operator restart (or two) never trips; a ~10s respawn loop does # within a few cycles. DEFAULT_MAX_RESTARTS = 3 DEFAULT_WINDOW_SECONDS = 60 # Longest gap between consecutive restart-interrupted boots that still counts # them as the SAME loop. A fixed-window prune only sees cycles faster than the # window (a slower loop drops its own history every boot and never trips); # chaining on the inter-boot gap makes the breaker period-agnostic, and a # single boot followed by real quiet resets the chain. DEFAULT_MAX_GAP_SECONDS = 300 # Cap the persisted chain; only the newest ``max_restarts`` entries can change # a verdict, the rest are forensics. _MAX_STORED_BOOTS = 50 def _state_path(): return get_hermes_home() / "gateway" / "restart_loop.json" def _load_boots() -> List[float]: try: data = json.loads(_state_path().read_text(encoding="utf-8")) return [float(t) for t in data.get("boots", []) if isinstance(t, (int, float))] except (OSError, ValueError, TypeError): return [] def _save_boots(boots: List[float]) -> None: try: path = _state_path() path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"boots": boots}), encoding="utf-8") except OSError: pass def _chain_gap(window_seconds: int, max_gap_seconds: int) -> float: """Inter-boot gap that still links two boots. Floored by ``window_seconds`` so widening the window never makes the breaker *less* sensitive.""" return float(max(1, window_seconds, max_gap_seconds)) def _chain_ending_at(boots: List[float], ts: float, gap: float) -> List[float]: """Unbroken chain of boots leading up to ``ts`` (oldest first). Walks backwards keeping boots while each successive gap stays within ``gap``; the first wider gap ends the chain (older boots belong to an already-resolved episode). Nothing recent enough -> empty list, which is how a healthy gateway forgets an old loop. """ chain: List[float] = [] prev = ts for t in sorted(boots, reverse=True): if t > ts: # Clock moved backwards (NTP step, restored state file): treat the # future entry as adjacent rather than dropping the whole chain. chain.append(t) continue if prev - t > gap: break chain.append(t) prev = t chain.reverse() return chain def record_restart_interrupted_boot( window_seconds: int = DEFAULT_WINDOW_SECONDS, *, now: Optional[float] = None, max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS, ) -> List[float]: """Record a restart-interrupted boot; return the pruned chain + now (most recent last). Best-effort — a persistence failure returns the in-memory list without raising. """ ts = time.time() if now is None else now boots = _chain_ending_at(_load_boots(), ts, _chain_gap(window_seconds, max_gap_seconds)) boots.append(ts) _save_boots(boots[-_MAX_STORED_BOOTS:]) return boots def clear() -> None: """Remove the persisted boot log (used on clean shutdown / by tests).""" with contextlib.suppress(OSError): _state_path().unlink(missing_ok=True) def check_and_record( max_restarts: int = DEFAULT_MAX_RESTARTS, window_seconds: int = DEFAULT_WINDOW_SECONDS, *, now: Optional[float] = None, max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS, ) -> bool: """Record this boot and return True when auto-resume should be SKIPPED. The single entry point the gateway calls: appends the current boot, then checks whether the updated chain has reached ``max_restarts``. """ boots = record_restart_interrupted_boot( window_seconds, now=now, max_gap_seconds=max_gap_seconds ) tripped = len(boots) >= max_restarts if max_restarts > 0 else False if tripped: logger.warning( "Restart-loop breaker TRIPPED: %d chained restart-interrupted " "gateway boots (no gap wider than %ds; threshold %d). Skipping " "auto-resume to break a suspected SIGTERM-respawn loop (#30719, " "#81642). Restart-interrupted sessions stay resume-pending and " "will continue on the next real user message. If this is a false " "positive, delete %s.", len(boots), int(_chain_gap(window_seconds, max_gap_seconds)), max_restarts, _state_path(), ) return tripped