fix(update): probe gateway loop liveness before drain; bounded escalation for wedged gateways

A gateway whose asyncio event loop is stalled (e.g. an in-loop
compression pass, #72707) cannot process SIGTERM/SIGUSR1 shutdown.
The updater's drain wait then burned the full 180s budget, warned
"Gateway PID X still running after 180.0s — restart may fail", and
`hermes update` could deadlock behind the wedged process — the user
cannot update their way out of the stall.

Fix: before any drain wait, read the loop-liveness heartbeat file the
gateway rewrites every 30s (#66892). Classification:

- alive (fresh heartbeat): busy-but-alive loop — take the normal
  graceful drain, honoring the in-flight cron drain floor (#86684).
- wedged (heartbeat for this PID stale >90s = 3 missed beats): the
  loop is provably dead; drain is pointless. Bounded escalation:
  SIGTERM + 5s grace, then SIGKILL + 5s wait, then proceed (~10s
  worst case, far under the 180s drain budget).
- unknown (missing/corrupt file, PID mismatch): never escalate on
  ambiguity — full drain path.

Wired into launchd_restart, systemd_restart, and both updater
gateway-shutdown sites (systemd unit drain + manual profile
gateways). The probe is a local stat + JSON read (well inside the
10s query tier of the subprocess timeout tiering).

The cron drain floor from #86684 is bypassed ONLY when the loop is
provably dead — a merely busy gateway still refreshes its heartbeat
and keeps the full drain budget.

Root cause of the loop stall itself (compression blocking the loop)
is #72707 territory and deliberately out of scope here.

Fixes #81642
This commit is contained in:
Teknium
2026-08-15 02:09:03 -07:00
parent be083358b2
commit 8fdfe04371
3 changed files with 446 additions and 9 deletions
+138
View File
@@ -324,6 +324,113 @@ def _wait_for_pid_exit(pid: int, timeout: float) -> bool:
_time.sleep(0.5)
# --- Wedged-gateway detection + bounded escalation (#81642) -----------------
#
# A gateway whose asyncio loop is stalled (e.g. an in-loop compression pass,
# #72707) cannot process SIGTERM/SIGUSR1 shutdown: the drain wait then burns
# the full drain budget (180s by default), warns "still running after 180.0s
# — restart may fail", and `hermes update` can deadlock behind it. The loop
# publishes a liveness signal precisely for this case: an asyncio task
# rewrites ``state/gateway.heartbeat`` every 30s (#66892), so a frozen loop
# stops refreshing the file while a busy-but-alive loop keeps refreshing it.
#
# ``probe_gateway_loop_liveness`` reads that signal (a local stat + JSON read,
# instant — far inside the 10s query tier of the subprocess timeout doc) and
# classifies the gateway BEFORE any drain wait begins:
#
# - ``alive`` — heartbeat is fresh: the loop is dispatching (possibly busy).
# Callers must take the normal graceful-drain path, which
# honours the in-flight cron drain floor (#86684).
# - ``wedged`` — heartbeat belongs to this PID but has gone stale well past
# several missed beats: the loop is provably dead. Draining
# is pointless (nothing can run the drain), so callers may
# escalate immediately via ``_escalate_wedged_gateway``.
# - ``unknown`` — no heartbeat / unreadable / PID mismatch (older gateway,
# still starting up, stale file from a previous process).
# Treated like ``alive``: never escalate on ambiguity.
#
# The distinction matters: only a *provably dead* loop may bypass the cron
# drain floor. A merely busy gateway still answers the probe (fresh file)
# and keeps its full drain budget.
GATEWAY_LOOP_ALIVE = "alive"
GATEWAY_LOOP_WEDGED = "wedged"
GATEWAY_LOOP_UNKNOWN = "unknown"
# Heartbeat cadence is 30s (gateway.shutdown_watchdog.DEFAULT_HEARTBEAT_INTERVAL_S).
# Three missed beats is decisive without false-positiving on one slow write.
DEFAULT_LOOP_LIVENESS_STALE_AFTER_S = 90.0
def probe_gateway_loop_liveness(
pid: int,
*,
stale_after: float = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S,
home: Path | None = None,
) -> str:
"""Classify a gateway PID's event loop as alive / wedged / unknown.
Reads the loop-liveness heartbeat file the gateway rewrites every 30s
while its loop is dispatching. Never raises; any ambiguity (missing
file, unreadable JSON, PID mismatch) returns ``GATEWAY_LOOP_UNKNOWN``
so callers default to the safe graceful-drain path.
"""
try:
stale_budget = max(float(stale_after), 0.0)
except (TypeError, ValueError):
stale_budget = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S
try:
from gateway.shutdown_watchdog import get_loop_heartbeat_path
path = get_loop_heartbeat_path(home)
mtime = path.stat().st_mtime
payload = json.loads(path.read_text(encoding="utf-8"))
heartbeat_pid = int(payload.get("pid", 0))
except Exception:
return GATEWAY_LOOP_UNKNOWN
if heartbeat_pid <= 0 or int(pid) <= 0 or heartbeat_pid != int(pid):
# No heartbeat for THIS process — old gateway version, still starting
# up, or a stale file from a previous PID. Not evidence of a wedge.
return GATEWAY_LOOP_UNKNOWN
age = time.time() - mtime
if age > stale_budget:
return GATEWAY_LOOP_WEDGED
return GATEWAY_LOOP_ALIVE
def _escalate_wedged_gateway(
pid: int,
*,
term_grace: float = 5.0,
kill_wait: float = 5.0,
) -> bool:
"""Bounded stop for a gateway whose loop is provably dead (#81642).
SIGTERM first (the process may still have a live signal handler thread
even with a dead loop), a short grace, then SIGKILL. Total worst case is
``term_grace + kill_wait`` (~10s by default) — never the 180s drain
budget, which only makes sense when a loop exists to run the drain.
Callers MUST have classified the gateway as ``GATEWAY_LOOP_WEDGED``
before calling this: escalating a merely busy gateway would bypass the
in-flight cron drain floor (#86684) and SIGKILL live work.
Returns True once the PID has left the process table.
"""
try:
terminate_pid(pid, force=False)
except (ProcessLookupError, PermissionError, OSError):
return _wait_for_pid_exit(pid, 1.0)
if _wait_for_pid_exit(pid, max(float(term_grace), 0.0)):
return True
try:
terminate_pid(pid, force=True)
print(f"⚠ Gateway PID {pid} unresponsive to SIGTERM; sent SIGKILL")
except (ProcessLookupError, PermissionError, OSError):
pass
return _wait_for_pid_exit(pid, max(float(kill_wait), 0.0))
def _get_ancestor_pids() -> set[int]:
"""Return the set of PIDs in the current process's ancestor chain.
@@ -3674,6 +3781,23 @@ def systemd_restart(system: bool = False):
from gateway.status import get_running_pid
pid = get_running_pid() or _systemd_main_pid(system=system)
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
# Health probe says the event loop is provably dead (#81642): SIGUSR1
# can never drain it, so the graceful wait below would burn the full
# budget. Bounded escalation (SIGTERM grace → SIGKILL, ~10s) then let
# systemd relaunch the unit. A busy-but-alive gateway (fresh
# heartbeat) never takes this path — its in-flight work, including
# the #86684 cron drain floor, keeps the full graceful budget.
print(
f"⚠ Gateway PID {pid} event loop is unresponsive — "
"skipping graceful drain and forcing a bounded stop..."
)
_escalate_wedged_gateway(pid)
svc = get_service_name()
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
_run_systemctl(["restart", svc], system=system, check=False, timeout=90)
_wait_for_systemd_service_restart(system=system, previous_pid=pid)
return
if pid is not None:
scope_label = _service_scope_label(system).capitalize()
svc = get_service_name()
@@ -4826,6 +4950,20 @@ def launchd_restart():
print("✓ Service restart requested")
_clear_launchd_unsupported_marker()
return
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
# Health probe says the event loop is provably dead (#81642):
# the gateway cannot process a graceful shutdown, so waiting the
# full drain budget only stalls the restart (and `hermes update`
# behind it) for 180s. Bounded escalation instead: SIGTERM grace
# → SIGKILL → proceed, ~10s worst case. Never taken for a
# busy-but-alive gateway — a fresh heartbeat keeps the drain path
# (and the #86684 cron drain floor) fully intact.
print(
f"⚠ Gateway PID {pid} event loop is unresponsive — "
"skipping drain and forcing a bounded stop..."
)
_escalate_wedged_gateway(pid)
pid = None
if pid is not None:
# Announce the drain BEFORE waiting on it. This wait can run for
# the full drain budget (180s by default) while the old gateway
+57 -9
View File
@@ -5856,14 +5856,41 @@ def _cmd_update_impl(args, gateway_mode: bool):
_graceful_ok = False
if _main_pid > 0:
print(
f" → {svc_name}: draining (up to {int(_drain_budget)}s)..."
)
_graceful_ok = _graceful_restart_via_sigusr1(
_main_pid,
drain_timeout=_drain_budget,
from hermes_cli.gateway import (
GATEWAY_LOOP_WEDGED,
_escalate_wedged_gateway,
probe_gateway_loop_liveness,
)
if (
probe_gateway_loop_liveness(_main_pid)
== GATEWAY_LOOP_WEDGED
):
# Loop-liveness probe says the gateway's event
# loop is provably dead (#81642): SIGUSR1 can
# never drain it, so waiting the full budget
# (180s default) only wedges the update too.
# Bounded escalation (SIGTERM grace → SIGKILL,
# ~10s) then restart the unit. A busy gateway
# keeps a fresh heartbeat and never takes this
# path — its drain (incl. the #86684 cron
# floor) is untouched.
print(
f" ⚠ {svc_name}: gateway event loop is "
"unresponsive — skipping drain, forcing "
"a bounded stop..."
)
_escalate_wedged_gateway(_main_pid)
_graceful_ok = True
else:
print(
f" → {svc_name}: draining (up to {int(_drain_budget)}s)..."
)
_graceful_ok = _graceful_restart_via_sigusr1(
_main_pid,
drain_timeout=_drain_budget,
)
if _graceful_ok:
# Gateway exited after a planned restart.
# ``Restart=always`` means systemd WILL respawn
@@ -6127,10 +6154,31 @@ def _cmd_update_impl(args, gateway_mode: bool):
f" → {proc.profile}: draining gateway PID {pid} "
f"(up to {int(_drain_budget)}s)..."
)
drained = _graceful_restart_via_sigusr1(
pid,
drain_timeout=_drain_budget,
from hermes_cli.gateway import (
GATEWAY_LOOP_WEDGED,
_escalate_wedged_gateway,
probe_gateway_loop_liveness,
)
if probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
# Loop-liveness probe: this gateway's event loop is
# provably dead (#81642) — SIGUSR1/SIGTERM shutdown can
# never run, so the drain wait would burn the full budget
# and stall the update. Bounded stop instead (SIGTERM
# grace → SIGKILL, ~10s). A busy-but-alive gateway keeps
# a fresh heartbeat and never takes this branch, so live
# drains (incl. the #86684 cron floor) are unaffected.
print(
f" ⚠ {proc.profile}: gateway event loop is "
"unresponsive — skipping drain, forcing a bounded stop..."
)
_escalate_wedged_gateway(pid)
drained = True
else:
drained = _graceful_restart_via_sigusr1(
pid,
drain_timeout=_drain_budget,
)
if not drained:
try:
os.kill(pid, _signal.SIGTERM)
@@ -0,0 +1,251 @@
"""Tests for the wedged-gateway health probe + bounded escalation (#81642).
A gateway whose event loop is stalled cannot process a graceful shutdown, so
the updater's drain wait used to burn the full 180s budget ("Gateway PID X
still running after 180.0s — restart may fail") and could deadlock `hermes
update`. The fix probes the loop-liveness heartbeat file BEFORE draining and,
only when the loop is provably dead, escalates SIGTERM → SIGKILL bounded to
seconds. A busy-but-alive gateway (fresh heartbeat) must keep the full drain
path — including the in-flight cron drain floor from #86684.
"""
import json
import os
import time
import pytest
import hermes_cli.gateway as gateway_cli
from gateway.shutdown_watchdog import get_loop_heartbeat_path, write_loop_heartbeat
def _write_heartbeat(home, pid, age_s=0.0):
"""Write a heartbeat file for ``pid`` whose mtime is ``age_s`` old."""
path = get_loop_heartbeat_path(home)
write_loop_heartbeat(pid=pid, home=home)
if age_s:
stamp = time.time() - age_s
os.utime(path, (stamp, stamp))
return path
class TestProbeGatewayLoopLiveness:
def test_fresh_heartbeat_is_alive(self, tmp_path):
"""A gateway that refreshed its heartbeat recently is busy, not wedged."""
_write_heartbeat(tmp_path, pid=4242, age_s=5.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_ALIVE
)
def test_stale_heartbeat_is_wedged(self, tmp_path):
"""A heartbeat several missed beats old proves the loop is dead."""
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_WEDGED
)
def test_heartbeat_just_inside_budget_is_alive(self, tmp_path):
"""Boundary: age below the stale budget must NOT classify as wedged."""
_write_heartbeat(tmp_path, pid=4242, age_s=60.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
4242, stale_after=90.0, home=tmp_path
)
== gateway_cli.GATEWAY_LOOP_ALIVE
)
def test_missing_heartbeat_is_unknown(self, tmp_path):
"""No heartbeat file (older gateway, fresh start) is not evidence."""
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_pid_mismatch_is_unknown_even_when_stale(self, tmp_path):
"""A stale file from a PREVIOUS process must not condemn the new PID."""
_write_heartbeat(tmp_path, pid=1111, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_corrupt_heartbeat_is_unknown(self, tmp_path):
path = get_loop_heartbeat_path(tmp_path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("{not json", encoding="utf-8")
stamp = time.time() - 600.0
os.utime(path, (stamp, stamp))
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_nonpositive_pid_is_unknown(self, tmp_path):
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(0, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_invalid_stale_after_falls_back_to_default(self, tmp_path):
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
4242, stale_after="bogus", home=tmp_path
)
== gateway_cli.GATEWAY_LOOP_WEDGED
)
def test_probe_never_raises_on_unreadable_path(self, monkeypatch):
monkeypatch.setattr(
"gateway.shutdown_watchdog.get_loop_heartbeat_path",
lambda home=None: (_ for _ in ()).throw(OSError("boom")),
)
assert (
gateway_cli.probe_gateway_loop_liveness(4242)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
class TestEscalateWedgedGateway:
def test_sigterm_grace_suffices_without_sigkill(self, monkeypatch):
"""If SIGTERM lands (signal-handler thread alive), no SIGKILL is sent."""
signals = []
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False: signals.append(("kill" if force else "term", pid)),
)
monkeypatch.setattr(
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: True
)
assert gateway_cli._escalate_wedged_gateway(4242) is True
assert signals == [("term", 4242)]
def test_escalates_to_sigkill_when_sigterm_ignored(self, monkeypatch):
signals = []
waits = []
def fake_wait(pid, timeout):
waits.append(timeout)
# First wait (SIGTERM grace) times out; second (post-SIGKILL) succeeds.
return len(waits) > 1
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False: signals.append(("kill" if force else "term", pid)),
)
monkeypatch.setattr(gateway_cli, "_wait_for_pid_exit", fake_wait)
assert gateway_cli._escalate_wedged_gateway(4242) is True
assert signals == [("term", 4242), ("kill", 4242)]
def test_total_wait_budget_is_bounded_well_under_drain(self, monkeypatch):
"""Worst case must be seconds, never the 180s drain budget."""
waits = []
monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None)
monkeypatch.setattr(
gateway_cli,
"_wait_for_pid_exit",
lambda pid, timeout: waits.append(timeout) or False,
)
assert gateway_cli._escalate_wedged_gateway(4242) is False
assert sum(waits) < 30.0
def test_process_already_gone_is_success(self, monkeypatch):
def raise_gone(pid, force=False):
raise ProcessLookupError
monkeypatch.setattr(gateway_cli, "terminate_pid", raise_gone)
monkeypatch.setattr(
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: True
)
assert gateway_cli._escalate_wedged_gateway(4242) is True
def test_sigkill_permission_error_does_not_raise(self, monkeypatch):
calls = []
def term(pid, force=False):
calls.append(force)
if force:
raise PermissionError
monkeypatch.setattr(gateway_cli, "terminate_pid", term)
monkeypatch.setattr(
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: False
)
assert gateway_cli._escalate_wedged_gateway(4242) is False
assert calls == [False, True]
class TestLaunchdRestartWedgedIntegration:
"""launchd_restart must skip the 180s drain only for a wedged loop."""
def _setup(self, monkeypatch, liveness):
events = []
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0)
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: 4242)
monkeypatch.setattr(
gateway_cli, "_request_gateway_self_restart", lambda pid: False
)
monkeypatch.setattr(
gateway_cli,
"probe_gateway_loop_liveness",
lambda pid, **kw: events.append("probe") or liveness,
)
monkeypatch.setattr(
gateway_cli,
"_escalate_wedged_gateway",
lambda pid, **kw: events.append("escalate") or True,
)
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False: events.append("sigterm"),
)
monkeypatch.setattr(
gateway_cli,
"_wait_for_gateway_exit",
lambda timeout, force_after=None: events.append(("drain", timeout)) or True,
)
monkeypatch.setattr(
gateway_cli.subprocess,
"run",
lambda *a, **k: events.append("kickstart")
or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""),
)
monkeypatch.setattr(
gateway_cli, "_clear_launchd_unsupported_marker", lambda: None
)
return events
def test_wedged_gateway_skips_drain_and_escalates(self, monkeypatch):
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_WEDGED)
gateway_cli.launchd_restart()
assert "escalate" in events
# The 180s drain wait must never run for a wedged loop.
assert not any(isinstance(e, tuple) and e[0] == "drain" for e in events)
def test_busy_gateway_keeps_full_drain_budget(self, monkeypatch):
"""A busy-but-alive gateway (fresh heartbeat) must NOT be escalated —
that would bypass the in-flight cron drain floor (#86684)."""
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_ALIVE)
gateway_cli.launchd_restart()
assert "escalate" not in events
assert ("drain", 180.0) in events
def test_unknown_liveness_keeps_full_drain_budget(self, monkeypatch):
"""Ambiguity (no heartbeat) must never trigger escalation."""
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_UNKNOWN)
gateway_cli.launchd_restart()
assert "escalate" not in events
assert ("drain", 180.0) in events