From 8fdfe04371703e72061dc7cdc74ba52705403722 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 15 Aug 2026 02:09:03 -0700 Subject: [PATCH] fix(update): probe gateway loop liveness before drain; bounded escalation for wedged gateways MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gateway whose asyncio event loop is stalled (e.g. an in-loop compression pass, #72707) cannot process SIGTERM/SIGUSR1 shutdown. The updater's drain wait then burned the full 180s budget, warned "Gateway PID X still running after 180.0s — restart may fail", and `hermes update` could deadlock behind the wedged process — the user cannot update their way out of the stall. Fix: before any drain wait, read the loop-liveness heartbeat file the gateway rewrites every 30s (#66892). Classification: - alive (fresh heartbeat): busy-but-alive loop — take the normal graceful drain, honoring the in-flight cron drain floor (#86684). - wedged (heartbeat for this PID stale >90s = 3 missed beats): the loop is provably dead; drain is pointless. Bounded escalation: SIGTERM + 5s grace, then SIGKILL + 5s wait, then proceed (~10s worst case, far under the 180s drain budget). - unknown (missing/corrupt file, PID mismatch): never escalate on ambiguity — full drain path. Wired into launchd_restart, systemd_restart, and both updater gateway-shutdown sites (systemd unit drain + manual profile gateways). The probe is a local stat + JSON read (well inside the 10s query tier of the subprocess timeout tiering). The cron drain floor from #86684 is bypassed ONLY when the loop is provably dead — a merely busy gateway still refreshes its heartbeat and keeps the full drain budget. Root cause of the loop stall itself (compression blocking the loop) is #72707 territory and deliberately out of scope here. Fixes #81642 --- hermes_cli/gateway.py | 138 ++++++++++ hermes_cli/update_cmd.py | 66 ++++- .../hermes_cli/test_update_wedged_gateway.py | 251 ++++++++++++++++++ 3 files changed, 446 insertions(+), 9 deletions(-) create mode 100644 tests/hermes_cli/test_update_wedged_gateway.py diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index da52c8e1c2..ea58a89e47 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -324,6 +324,113 @@ def _wait_for_pid_exit(pid: int, timeout: float) -> bool: _time.sleep(0.5) +# --- Wedged-gateway detection + bounded escalation (#81642) ----------------- +# +# A gateway whose asyncio loop is stalled (e.g. an in-loop compression pass, +# #72707) cannot process SIGTERM/SIGUSR1 shutdown: the drain wait then burns +# the full drain budget (180s by default), warns "still running after 180.0s +# — restart may fail", and `hermes update` can deadlock behind it. The loop +# publishes a liveness signal precisely for this case: an asyncio task +# rewrites ``state/gateway.heartbeat`` every 30s (#66892), so a frozen loop +# stops refreshing the file while a busy-but-alive loop keeps refreshing it. +# +# ``probe_gateway_loop_liveness`` reads that signal (a local stat + JSON read, +# instant — far inside the 10s query tier of the subprocess timeout doc) and +# classifies the gateway BEFORE any drain wait begins: +# +# - ``alive`` — heartbeat is fresh: the loop is dispatching (possibly busy). +# Callers must take the normal graceful-drain path, which +# honours the in-flight cron drain floor (#86684). +# - ``wedged`` — heartbeat belongs to this PID but has gone stale well past +# several missed beats: the loop is provably dead. Draining +# is pointless (nothing can run the drain), so callers may +# escalate immediately via ``_escalate_wedged_gateway``. +# - ``unknown`` — no heartbeat / unreadable / PID mismatch (older gateway, +# still starting up, stale file from a previous process). +# Treated like ``alive``: never escalate on ambiguity. +# +# The distinction matters: only a *provably dead* loop may bypass the cron +# drain floor. A merely busy gateway still answers the probe (fresh file) +# and keeps its full drain budget. + +GATEWAY_LOOP_ALIVE = "alive" +GATEWAY_LOOP_WEDGED = "wedged" +GATEWAY_LOOP_UNKNOWN = "unknown" + +# Heartbeat cadence is 30s (gateway.shutdown_watchdog.DEFAULT_HEARTBEAT_INTERVAL_S). +# Three missed beats is decisive without false-positiving on one slow write. +DEFAULT_LOOP_LIVENESS_STALE_AFTER_S = 90.0 + + +def probe_gateway_loop_liveness( + pid: int, + *, + stale_after: float = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S, + home: Path | None = None, +) -> str: + """Classify a gateway PID's event loop as alive / wedged / unknown. + + Reads the loop-liveness heartbeat file the gateway rewrites every 30s + while its loop is dispatching. Never raises; any ambiguity (missing + file, unreadable JSON, PID mismatch) returns ``GATEWAY_LOOP_UNKNOWN`` + so callers default to the safe graceful-drain path. + """ + try: + stale_budget = max(float(stale_after), 0.0) + except (TypeError, ValueError): + stale_budget = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S + try: + from gateway.shutdown_watchdog import get_loop_heartbeat_path + + path = get_loop_heartbeat_path(home) + mtime = path.stat().st_mtime + payload = json.loads(path.read_text(encoding="utf-8")) + heartbeat_pid = int(payload.get("pid", 0)) + except Exception: + return GATEWAY_LOOP_UNKNOWN + if heartbeat_pid <= 0 or int(pid) <= 0 or heartbeat_pid != int(pid): + # No heartbeat for THIS process — old gateway version, still starting + # up, or a stale file from a previous PID. Not evidence of a wedge. + return GATEWAY_LOOP_UNKNOWN + age = time.time() - mtime + if age > stale_budget: + return GATEWAY_LOOP_WEDGED + return GATEWAY_LOOP_ALIVE + + +def _escalate_wedged_gateway( + pid: int, + *, + term_grace: float = 5.0, + kill_wait: float = 5.0, +) -> bool: + """Bounded stop for a gateway whose loop is provably dead (#81642). + + SIGTERM first (the process may still have a live signal handler thread + even with a dead loop), a short grace, then SIGKILL. Total worst case is + ``term_grace + kill_wait`` (~10s by default) — never the 180s drain + budget, which only makes sense when a loop exists to run the drain. + + Callers MUST have classified the gateway as ``GATEWAY_LOOP_WEDGED`` + before calling this: escalating a merely busy gateway would bypass the + in-flight cron drain floor (#86684) and SIGKILL live work. + + Returns True once the PID has left the process table. + """ + try: + terminate_pid(pid, force=False) + except (ProcessLookupError, PermissionError, OSError): + return _wait_for_pid_exit(pid, 1.0) + if _wait_for_pid_exit(pid, max(float(term_grace), 0.0)): + return True + try: + terminate_pid(pid, force=True) + print(f"⚠ Gateway PID {pid} unresponsive to SIGTERM; sent SIGKILL") + except (ProcessLookupError, PermissionError, OSError): + pass + return _wait_for_pid_exit(pid, max(float(kill_wait), 0.0)) + + def _get_ancestor_pids() -> set[int]: """Return the set of PIDs in the current process's ancestor chain. @@ -3674,6 +3781,23 @@ def systemd_restart(system: bool = False): from gateway.status import get_running_pid pid = get_running_pid() or _systemd_main_pid(system=system) + if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED: + # Health probe says the event loop is provably dead (#81642): SIGUSR1 + # can never drain it, so the graceful wait below would burn the full + # budget. Bounded escalation (SIGTERM grace → SIGKILL, ~10s) then let + # systemd relaunch the unit. A busy-but-alive gateway (fresh + # heartbeat) never takes this path — its in-flight work, including + # the #86684 cron drain floor, keeps the full graceful budget. + print( + f"⚠ Gateway PID {pid} event loop is unresponsive — " + "skipping graceful drain and forcing a bounded stop..." + ) + _escalate_wedged_gateway(pid) + svc = get_service_name() + _run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30) + _run_systemctl(["restart", svc], system=system, check=False, timeout=90) + _wait_for_systemd_service_restart(system=system, previous_pid=pid) + return if pid is not None: scope_label = _service_scope_label(system).capitalize() svc = get_service_name() @@ -4826,6 +4950,20 @@ def launchd_restart(): print("✓ Service restart requested") _clear_launchd_unsupported_marker() return + if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED: + # Health probe says the event loop is provably dead (#81642): + # the gateway cannot process a graceful shutdown, so waiting the + # full drain budget only stalls the restart (and `hermes update` + # behind it) for 180s. Bounded escalation instead: SIGTERM grace + # → SIGKILL → proceed, ~10s worst case. Never taken for a + # busy-but-alive gateway — a fresh heartbeat keeps the drain path + # (and the #86684 cron drain floor) fully intact. + print( + f"⚠ Gateway PID {pid} event loop is unresponsive — " + "skipping drain and forcing a bounded stop..." + ) + _escalate_wedged_gateway(pid) + pid = None if pid is not None: # Announce the drain BEFORE waiting on it. This wait can run for # the full drain budget (180s by default) while the old gateway diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index a91b8909de..c9df3f94de 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -5856,14 +5856,41 @@ def _cmd_update_impl(args, gateway_mode: bool): _graceful_ok = False if _main_pid > 0: - print( - f" → {svc_name}: draining (up to {int(_drain_budget)}s)..." - ) - _graceful_ok = _graceful_restart_via_sigusr1( - _main_pid, - drain_timeout=_drain_budget, + from hermes_cli.gateway import ( + GATEWAY_LOOP_WEDGED, + _escalate_wedged_gateway, + probe_gateway_loop_liveness, ) + if ( + probe_gateway_loop_liveness(_main_pid) + == GATEWAY_LOOP_WEDGED + ): + # Loop-liveness probe says the gateway's event + # loop is provably dead (#81642): SIGUSR1 can + # never drain it, so waiting the full budget + # (180s default) only wedges the update too. + # Bounded escalation (SIGTERM grace → SIGKILL, + # ~10s) then restart the unit. A busy gateway + # keeps a fresh heartbeat and never takes this + # path — its drain (incl. the #86684 cron + # floor) is untouched. + print( + f" ⚠ {svc_name}: gateway event loop is " + "unresponsive — skipping drain, forcing " + "a bounded stop..." + ) + _escalate_wedged_gateway(_main_pid) + _graceful_ok = True + else: + print( + f" → {svc_name}: draining (up to {int(_drain_budget)}s)..." + ) + _graceful_ok = _graceful_restart_via_sigusr1( + _main_pid, + drain_timeout=_drain_budget, + ) + if _graceful_ok: # Gateway exited after a planned restart. # ``Restart=always`` means systemd WILL respawn @@ -6127,10 +6154,31 @@ def _cmd_update_impl(args, gateway_mode: bool): f" → {proc.profile}: draining gateway PID {pid} " f"(up to {int(_drain_budget)}s)..." ) - drained = _graceful_restart_via_sigusr1( - pid, - drain_timeout=_drain_budget, + from hermes_cli.gateway import ( + GATEWAY_LOOP_WEDGED, + _escalate_wedged_gateway, + probe_gateway_loop_liveness, ) + + if probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED: + # Loop-liveness probe: this gateway's event loop is + # provably dead (#81642) — SIGUSR1/SIGTERM shutdown can + # never run, so the drain wait would burn the full budget + # and stall the update. Bounded stop instead (SIGTERM + # grace → SIGKILL, ~10s). A busy-but-alive gateway keeps + # a fresh heartbeat and never takes this branch, so live + # drains (incl. the #86684 cron floor) are unaffected. + print( + f" ⚠ {proc.profile}: gateway event loop is " + "unresponsive — skipping drain, forcing a bounded stop..." + ) + _escalate_wedged_gateway(pid) + drained = True + else: + drained = _graceful_restart_via_sigusr1( + pid, + drain_timeout=_drain_budget, + ) if not drained: try: os.kill(pid, _signal.SIGTERM) diff --git a/tests/hermes_cli/test_update_wedged_gateway.py b/tests/hermes_cli/test_update_wedged_gateway.py new file mode 100644 index 0000000000..ab7e4733bd --- /dev/null +++ b/tests/hermes_cli/test_update_wedged_gateway.py @@ -0,0 +1,251 @@ +"""Tests for the wedged-gateway health probe + bounded escalation (#81642). + +A gateway whose event loop is stalled cannot process a graceful shutdown, so +the updater's drain wait used to burn the full 180s budget ("Gateway PID X +still running after 180.0s — restart may fail") and could deadlock `hermes +update`. The fix probes the loop-liveness heartbeat file BEFORE draining and, +only when the loop is provably dead, escalates SIGTERM → SIGKILL bounded to +seconds. A busy-but-alive gateway (fresh heartbeat) must keep the full drain +path — including the in-flight cron drain floor from #86684. +""" + +import json +import os +import time + +import pytest + +import hermes_cli.gateway as gateway_cli +from gateway.shutdown_watchdog import get_loop_heartbeat_path, write_loop_heartbeat + + +def _write_heartbeat(home, pid, age_s=0.0): + """Write a heartbeat file for ``pid`` whose mtime is ``age_s`` old.""" + path = get_loop_heartbeat_path(home) + write_loop_heartbeat(pid=pid, home=home) + if age_s: + stamp = time.time() - age_s + os.utime(path, (stamp, stamp)) + return path + + +class TestProbeGatewayLoopLiveness: + def test_fresh_heartbeat_is_alive(self, tmp_path): + """A gateway that refreshed its heartbeat recently is busy, not wedged.""" + _write_heartbeat(tmp_path, pid=4242, age_s=5.0) + assert ( + gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_ALIVE + ) + + def test_stale_heartbeat_is_wedged(self, tmp_path): + """A heartbeat several missed beats old proves the loop is dead.""" + _write_heartbeat(tmp_path, pid=4242, age_s=600.0) + assert ( + gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_WEDGED + ) + + def test_heartbeat_just_inside_budget_is_alive(self, tmp_path): + """Boundary: age below the stale budget must NOT classify as wedged.""" + _write_heartbeat(tmp_path, pid=4242, age_s=60.0) + assert ( + gateway_cli.probe_gateway_loop_liveness( + 4242, stale_after=90.0, home=tmp_path + ) + == gateway_cli.GATEWAY_LOOP_ALIVE + ) + + def test_missing_heartbeat_is_unknown(self, tmp_path): + """No heartbeat file (older gateway, fresh start) is not evidence.""" + assert ( + gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + + def test_pid_mismatch_is_unknown_even_when_stale(self, tmp_path): + """A stale file from a PREVIOUS process must not condemn the new PID.""" + _write_heartbeat(tmp_path, pid=1111, age_s=600.0) + assert ( + gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + + def test_corrupt_heartbeat_is_unknown(self, tmp_path): + path = get_loop_heartbeat_path(tmp_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{not json", encoding="utf-8") + stamp = time.time() - 600.0 + os.utime(path, (stamp, stamp)) + assert ( + gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + + def test_nonpositive_pid_is_unknown(self, tmp_path): + _write_heartbeat(tmp_path, pid=4242, age_s=600.0) + assert ( + gateway_cli.probe_gateway_loop_liveness(0, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + + def test_invalid_stale_after_falls_back_to_default(self, tmp_path): + _write_heartbeat(tmp_path, pid=4242, age_s=600.0) + assert ( + gateway_cli.probe_gateway_loop_liveness( + 4242, stale_after="bogus", home=tmp_path + ) + == gateway_cli.GATEWAY_LOOP_WEDGED + ) + + def test_probe_never_raises_on_unreadable_path(self, monkeypatch): + monkeypatch.setattr( + "gateway.shutdown_watchdog.get_loop_heartbeat_path", + lambda home=None: (_ for _ in ()).throw(OSError("boom")), + ) + assert ( + gateway_cli.probe_gateway_loop_liveness(4242) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + + +class TestEscalateWedgedGateway: + def test_sigterm_grace_suffices_without_sigkill(self, monkeypatch): + """If SIGTERM lands (signal-handler thread alive), no SIGKILL is sent.""" + signals = [] + monkeypatch.setattr( + gateway_cli, + "terminate_pid", + lambda pid, force=False: signals.append(("kill" if force else "term", pid)), + ) + monkeypatch.setattr( + gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: True + ) + + assert gateway_cli._escalate_wedged_gateway(4242) is True + assert signals == [("term", 4242)] + + def test_escalates_to_sigkill_when_sigterm_ignored(self, monkeypatch): + signals = [] + waits = [] + + def fake_wait(pid, timeout): + waits.append(timeout) + # First wait (SIGTERM grace) times out; second (post-SIGKILL) succeeds. + return len(waits) > 1 + + monkeypatch.setattr( + gateway_cli, + "terminate_pid", + lambda pid, force=False: signals.append(("kill" if force else "term", pid)), + ) + monkeypatch.setattr(gateway_cli, "_wait_for_pid_exit", fake_wait) + + assert gateway_cli._escalate_wedged_gateway(4242) is True + assert signals == [("term", 4242), ("kill", 4242)] + + def test_total_wait_budget_is_bounded_well_under_drain(self, monkeypatch): + """Worst case must be seconds, never the 180s drain budget.""" + waits = [] + monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None) + monkeypatch.setattr( + gateway_cli, + "_wait_for_pid_exit", + lambda pid, timeout: waits.append(timeout) or False, + ) + + assert gateway_cli._escalate_wedged_gateway(4242) is False + assert sum(waits) < 30.0 + + def test_process_already_gone_is_success(self, monkeypatch): + def raise_gone(pid, force=False): + raise ProcessLookupError + + monkeypatch.setattr(gateway_cli, "terminate_pid", raise_gone) + monkeypatch.setattr( + gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: True + ) + + assert gateway_cli._escalate_wedged_gateway(4242) is True + + def test_sigkill_permission_error_does_not_raise(self, monkeypatch): + calls = [] + + def term(pid, force=False): + calls.append(force) + if force: + raise PermissionError + + monkeypatch.setattr(gateway_cli, "terminate_pid", term) + monkeypatch.setattr( + gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: False + ) + + assert gateway_cli._escalate_wedged_gateway(4242) is False + assert calls == [False, True] + + +class TestLaunchdRestartWedgedIntegration: + """launchd_restart must skip the 180s drain only for a wedged loop.""" + + def _setup(self, monkeypatch, liveness): + events = [] + monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway") + monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501") + monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0) + monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: 4242) + monkeypatch.setattr( + gateway_cli, "_request_gateway_self_restart", lambda pid: False + ) + monkeypatch.setattr( + gateway_cli, + "probe_gateway_loop_liveness", + lambda pid, **kw: events.append("probe") or liveness, + ) + monkeypatch.setattr( + gateway_cli, + "_escalate_wedged_gateway", + lambda pid, **kw: events.append("escalate") or True, + ) + monkeypatch.setattr( + gateway_cli, + "terminate_pid", + lambda pid, force=False: events.append("sigterm"), + ) + monkeypatch.setattr( + gateway_cli, + "_wait_for_gateway_exit", + lambda timeout, force_after=None: events.append(("drain", timeout)) or True, + ) + monkeypatch.setattr( + gateway_cli.subprocess, + "run", + lambda *a, **k: events.append("kickstart") + or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""), + ) + monkeypatch.setattr( + gateway_cli, "_clear_launchd_unsupported_marker", lambda: None + ) + return events + + def test_wedged_gateway_skips_drain_and_escalates(self, monkeypatch): + events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_WEDGED) + gateway_cli.launchd_restart() + assert "escalate" in events + # The 180s drain wait must never run for a wedged loop. + assert not any(isinstance(e, tuple) and e[0] == "drain" for e in events) + + def test_busy_gateway_keeps_full_drain_budget(self, monkeypatch): + """A busy-but-alive gateway (fresh heartbeat) must NOT be escalated — + that would bypass the in-flight cron drain floor (#86684).""" + events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_ALIVE) + gateway_cli.launchd_restart() + assert "escalate" not in events + assert ("drain", 180.0) in events + + def test_unknown_liveness_keeps_full_drain_budget(self, monkeypatch): + """Ambiguity (no heartbeat) must never trigger escalation.""" + events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_UNKNOWN) + gateway_cli.launchd_restart() + assert "escalate" not in events + assert ("drain", 180.0) in events