fix(update): probe gateway loop liveness before drain; bounded escalation for wedged gateways
A gateway whose asyncio event loop is stalled (e.g. an in-loop compression pass, #72707) cannot process SIGTERM/SIGUSR1 shutdown. The updater's drain wait then burned the full 180s budget, warned "Gateway PID X still running after 180.0s — restart may fail", and `hermes update` could deadlock behind the wedged process — the user cannot update their way out of the stall. Fix: before any drain wait, read the loop-liveness heartbeat file the gateway rewrites every 30s (#66892). Classification: - alive (fresh heartbeat): busy-but-alive loop — take the normal graceful drain, honoring the in-flight cron drain floor (#86684). - wedged (heartbeat for this PID stale >90s = 3 missed beats): the loop is provably dead; drain is pointless. Bounded escalation: SIGTERM + 5s grace, then SIGKILL + 5s wait, then proceed (~10s worst case, far under the 180s drain budget). - unknown (missing/corrupt file, PID mismatch): never escalate on ambiguity — full drain path. Wired into launchd_restart, systemd_restart, and both updater gateway-shutdown sites (systemd unit drain + manual profile gateways). The probe is a local stat + JSON read (well inside the 10s query tier of the subprocess timeout tiering). The cron drain floor from #86684 is bypassed ONLY when the loop is provably dead — a merely busy gateway still refreshes its heartbeat and keeps the full drain budget. Root cause of the loop stall itself (compression blocking the loop) is #72707 territory and deliberately out of scope here. Fixes #81642
This commit is contained in:
@@ -324,6 +324,113 @@ def _wait_for_pid_exit(pid: int, timeout: float) -> bool:
|
||||
_time.sleep(0.5)
|
||||
|
||||
|
||||
# --- Wedged-gateway detection + bounded escalation (#81642) -----------------
|
||||
#
|
||||
# A gateway whose asyncio loop is stalled (e.g. an in-loop compression pass,
|
||||
# #72707) cannot process SIGTERM/SIGUSR1 shutdown: the drain wait then burns
|
||||
# the full drain budget (180s by default), warns "still running after 180.0s
|
||||
# — restart may fail", and `hermes update` can deadlock behind it. The loop
|
||||
# publishes a liveness signal precisely for this case: an asyncio task
|
||||
# rewrites ``state/gateway.heartbeat`` every 30s (#66892), so a frozen loop
|
||||
# stops refreshing the file while a busy-but-alive loop keeps refreshing it.
|
||||
#
|
||||
# ``probe_gateway_loop_liveness`` reads that signal (a local stat + JSON read,
|
||||
# instant — far inside the 10s query tier of the subprocess timeout doc) and
|
||||
# classifies the gateway BEFORE any drain wait begins:
|
||||
#
|
||||
# - ``alive`` — heartbeat is fresh: the loop is dispatching (possibly busy).
|
||||
# Callers must take the normal graceful-drain path, which
|
||||
# honours the in-flight cron drain floor (#86684).
|
||||
# - ``wedged`` — heartbeat belongs to this PID but has gone stale well past
|
||||
# several missed beats: the loop is provably dead. Draining
|
||||
# is pointless (nothing can run the drain), so callers may
|
||||
# escalate immediately via ``_escalate_wedged_gateway``.
|
||||
# - ``unknown`` — no heartbeat / unreadable / PID mismatch (older gateway,
|
||||
# still starting up, stale file from a previous process).
|
||||
# Treated like ``alive``: never escalate on ambiguity.
|
||||
#
|
||||
# The distinction matters: only a *provably dead* loop may bypass the cron
|
||||
# drain floor. A merely busy gateway still answers the probe (fresh file)
|
||||
# and keeps its full drain budget.
|
||||
|
||||
GATEWAY_LOOP_ALIVE = "alive"
|
||||
GATEWAY_LOOP_WEDGED = "wedged"
|
||||
GATEWAY_LOOP_UNKNOWN = "unknown"
|
||||
|
||||
# Heartbeat cadence is 30s (gateway.shutdown_watchdog.DEFAULT_HEARTBEAT_INTERVAL_S).
|
||||
# Three missed beats is decisive without false-positiving on one slow write.
|
||||
DEFAULT_LOOP_LIVENESS_STALE_AFTER_S = 90.0
|
||||
|
||||
|
||||
def probe_gateway_loop_liveness(
|
||||
pid: int,
|
||||
*,
|
||||
stale_after: float = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S,
|
||||
home: Path | None = None,
|
||||
) -> str:
|
||||
"""Classify a gateway PID's event loop as alive / wedged / unknown.
|
||||
|
||||
Reads the loop-liveness heartbeat file the gateway rewrites every 30s
|
||||
while its loop is dispatching. Never raises; any ambiguity (missing
|
||||
file, unreadable JSON, PID mismatch) returns ``GATEWAY_LOOP_UNKNOWN``
|
||||
so callers default to the safe graceful-drain path.
|
||||
"""
|
||||
try:
|
||||
stale_budget = max(float(stale_after), 0.0)
|
||||
except (TypeError, ValueError):
|
||||
stale_budget = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S
|
||||
try:
|
||||
from gateway.shutdown_watchdog import get_loop_heartbeat_path
|
||||
|
||||
path = get_loop_heartbeat_path(home)
|
||||
mtime = path.stat().st_mtime
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
heartbeat_pid = int(payload.get("pid", 0))
|
||||
except Exception:
|
||||
return GATEWAY_LOOP_UNKNOWN
|
||||
if heartbeat_pid <= 0 or int(pid) <= 0 or heartbeat_pid != int(pid):
|
||||
# No heartbeat for THIS process — old gateway version, still starting
|
||||
# up, or a stale file from a previous PID. Not evidence of a wedge.
|
||||
return GATEWAY_LOOP_UNKNOWN
|
||||
age = time.time() - mtime
|
||||
if age > stale_budget:
|
||||
return GATEWAY_LOOP_WEDGED
|
||||
return GATEWAY_LOOP_ALIVE
|
||||
|
||||
|
||||
def _escalate_wedged_gateway(
|
||||
pid: int,
|
||||
*,
|
||||
term_grace: float = 5.0,
|
||||
kill_wait: float = 5.0,
|
||||
) -> bool:
|
||||
"""Bounded stop for a gateway whose loop is provably dead (#81642).
|
||||
|
||||
SIGTERM first (the process may still have a live signal handler thread
|
||||
even with a dead loop), a short grace, then SIGKILL. Total worst case is
|
||||
``term_grace + kill_wait`` (~10s by default) — never the 180s drain
|
||||
budget, which only makes sense when a loop exists to run the drain.
|
||||
|
||||
Callers MUST have classified the gateway as ``GATEWAY_LOOP_WEDGED``
|
||||
before calling this: escalating a merely busy gateway would bypass the
|
||||
in-flight cron drain floor (#86684) and SIGKILL live work.
|
||||
|
||||
Returns True once the PID has left the process table.
|
||||
"""
|
||||
try:
|
||||
terminate_pid(pid, force=False)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
return _wait_for_pid_exit(pid, 1.0)
|
||||
if _wait_for_pid_exit(pid, max(float(term_grace), 0.0)):
|
||||
return True
|
||||
try:
|
||||
terminate_pid(pid, force=True)
|
||||
print(f"⚠ Gateway PID {pid} unresponsive to SIGTERM; sent SIGKILL")
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
return _wait_for_pid_exit(pid, max(float(kill_wait), 0.0))
|
||||
|
||||
|
||||
def _get_ancestor_pids() -> set[int]:
|
||||
"""Return the set of PIDs in the current process's ancestor chain.
|
||||
|
||||
@@ -3674,6 +3781,23 @@ def systemd_restart(system: bool = False):
|
||||
from gateway.status import get_running_pid
|
||||
|
||||
pid = get_running_pid() or _systemd_main_pid(system=system)
|
||||
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
|
||||
# Health probe says the event loop is provably dead (#81642): SIGUSR1
|
||||
# can never drain it, so the graceful wait below would burn the full
|
||||
# budget. Bounded escalation (SIGTERM grace → SIGKILL, ~10s) then let
|
||||
# systemd relaunch the unit. A busy-but-alive gateway (fresh
|
||||
# heartbeat) never takes this path — its in-flight work, including
|
||||
# the #86684 cron drain floor, keeps the full graceful budget.
|
||||
print(
|
||||
f"⚠ Gateway PID {pid} event loop is unresponsive — "
|
||||
"skipping graceful drain and forcing a bounded stop..."
|
||||
)
|
||||
_escalate_wedged_gateway(pid)
|
||||
svc = get_service_name()
|
||||
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
|
||||
_run_systemctl(["restart", svc], system=system, check=False, timeout=90)
|
||||
_wait_for_systemd_service_restart(system=system, previous_pid=pid)
|
||||
return
|
||||
if pid is not None:
|
||||
scope_label = _service_scope_label(system).capitalize()
|
||||
svc = get_service_name()
|
||||
@@ -4826,6 +4950,20 @@ def launchd_restart():
|
||||
print("✓ Service restart requested")
|
||||
_clear_launchd_unsupported_marker()
|
||||
return
|
||||
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
|
||||
# Health probe says the event loop is provably dead (#81642):
|
||||
# the gateway cannot process a graceful shutdown, so waiting the
|
||||
# full drain budget only stalls the restart (and `hermes update`
|
||||
# behind it) for 180s. Bounded escalation instead: SIGTERM grace
|
||||
# → SIGKILL → proceed, ~10s worst case. Never taken for a
|
||||
# busy-but-alive gateway — a fresh heartbeat keeps the drain path
|
||||
# (and the #86684 cron drain floor) fully intact.
|
||||
print(
|
||||
f"⚠ Gateway PID {pid} event loop is unresponsive — "
|
||||
"skipping drain and forcing a bounded stop..."
|
||||
)
|
||||
_escalate_wedged_gateway(pid)
|
||||
pid = None
|
||||
if pid is not None:
|
||||
# Announce the drain BEFORE waiting on it. This wait can run for
|
||||
# the full drain budget (180s by default) while the old gateway
|
||||
|
||||
@@ -5856,14 +5856,41 @@ def _cmd_update_impl(args, gateway_mode: bool):
|
||||
|
||||
_graceful_ok = False
|
||||
if _main_pid > 0:
|
||||
print(
|
||||
f" → {svc_name}: draining (up to {int(_drain_budget)}s)..."
|
||||
)
|
||||
_graceful_ok = _graceful_restart_via_sigusr1(
|
||||
_main_pid,
|
||||
drain_timeout=_drain_budget,
|
||||
from hermes_cli.gateway import (
|
||||
GATEWAY_LOOP_WEDGED,
|
||||
_escalate_wedged_gateway,
|
||||
probe_gateway_loop_liveness,
|
||||
)
|
||||
|
||||
if (
|
||||
probe_gateway_loop_liveness(_main_pid)
|
||||
== GATEWAY_LOOP_WEDGED
|
||||
):
|
||||
# Loop-liveness probe says the gateway's event
|
||||
# loop is provably dead (#81642): SIGUSR1 can
|
||||
# never drain it, so waiting the full budget
|
||||
# (180s default) only wedges the update too.
|
||||
# Bounded escalation (SIGTERM grace → SIGKILL,
|
||||
# ~10s) then restart the unit. A busy gateway
|
||||
# keeps a fresh heartbeat and never takes this
|
||||
# path — its drain (incl. the #86684 cron
|
||||
# floor) is untouched.
|
||||
print(
|
||||
f" ⚠ {svc_name}: gateway event loop is "
|
||||
"unresponsive — skipping drain, forcing "
|
||||
"a bounded stop..."
|
||||
)
|
||||
_escalate_wedged_gateway(_main_pid)
|
||||
_graceful_ok = True
|
||||
else:
|
||||
print(
|
||||
f" → {svc_name}: draining (up to {int(_drain_budget)}s)..."
|
||||
)
|
||||
_graceful_ok = _graceful_restart_via_sigusr1(
|
||||
_main_pid,
|
||||
drain_timeout=_drain_budget,
|
||||
)
|
||||
|
||||
if _graceful_ok:
|
||||
# Gateway exited after a planned restart.
|
||||
# ``Restart=always`` means systemd WILL respawn
|
||||
@@ -6127,10 +6154,31 @@ def _cmd_update_impl(args, gateway_mode: bool):
|
||||
f" → {proc.profile}: draining gateway PID {pid} "
|
||||
f"(up to {int(_drain_budget)}s)..."
|
||||
)
|
||||
drained = _graceful_restart_via_sigusr1(
|
||||
pid,
|
||||
drain_timeout=_drain_budget,
|
||||
from hermes_cli.gateway import (
|
||||
GATEWAY_LOOP_WEDGED,
|
||||
_escalate_wedged_gateway,
|
||||
probe_gateway_loop_liveness,
|
||||
)
|
||||
|
||||
if probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
|
||||
# Loop-liveness probe: this gateway's event loop is
|
||||
# provably dead (#81642) — SIGUSR1/SIGTERM shutdown can
|
||||
# never run, so the drain wait would burn the full budget
|
||||
# and stall the update. Bounded stop instead (SIGTERM
|
||||
# grace → SIGKILL, ~10s). A busy-but-alive gateway keeps
|
||||
# a fresh heartbeat and never takes this branch, so live
|
||||
# drains (incl. the #86684 cron floor) are unaffected.
|
||||
print(
|
||||
f" ⚠ {proc.profile}: gateway event loop is "
|
||||
"unresponsive — skipping drain, forcing a bounded stop..."
|
||||
)
|
||||
_escalate_wedged_gateway(pid)
|
||||
drained = True
|
||||
else:
|
||||
drained = _graceful_restart_via_sigusr1(
|
||||
pid,
|
||||
drain_timeout=_drain_budget,
|
||||
)
|
||||
if not drained:
|
||||
try:
|
||||
os.kill(pid, _signal.SIGTERM)
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Tests for the wedged-gateway health probe + bounded escalation (#81642).
|
||||
|
||||
A gateway whose event loop is stalled cannot process a graceful shutdown, so
|
||||
the updater's drain wait used to burn the full 180s budget ("Gateway PID X
|
||||
still running after 180.0s — restart may fail") and could deadlock `hermes
|
||||
update`. The fix probes the loop-liveness heartbeat file BEFORE draining and,
|
||||
only when the loop is provably dead, escalates SIGTERM → SIGKILL bounded to
|
||||
seconds. A busy-but-alive gateway (fresh heartbeat) must keep the full drain
|
||||
path — including the in-flight cron drain floor from #86684.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import hermes_cli.gateway as gateway_cli
|
||||
from gateway.shutdown_watchdog import get_loop_heartbeat_path, write_loop_heartbeat
|
||||
|
||||
|
||||
def _write_heartbeat(home, pid, age_s=0.0):
|
||||
"""Write a heartbeat file for ``pid`` whose mtime is ``age_s`` old."""
|
||||
path = get_loop_heartbeat_path(home)
|
||||
write_loop_heartbeat(pid=pid, home=home)
|
||||
if age_s:
|
||||
stamp = time.time() - age_s
|
||||
os.utime(path, (stamp, stamp))
|
||||
return path
|
||||
|
||||
|
||||
class TestProbeGatewayLoopLiveness:
|
||||
def test_fresh_heartbeat_is_alive(self, tmp_path):
|
||||
"""A gateway that refreshed its heartbeat recently is busy, not wedged."""
|
||||
_write_heartbeat(tmp_path, pid=4242, age_s=5.0)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
||||
== gateway_cli.GATEWAY_LOOP_ALIVE
|
||||
)
|
||||
|
||||
def test_stale_heartbeat_is_wedged(self, tmp_path):
|
||||
"""A heartbeat several missed beats old proves the loop is dead."""
|
||||
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
||||
== gateway_cli.GATEWAY_LOOP_WEDGED
|
||||
)
|
||||
|
||||
def test_heartbeat_just_inside_budget_is_alive(self, tmp_path):
|
||||
"""Boundary: age below the stale budget must NOT classify as wedged."""
|
||||
_write_heartbeat(tmp_path, pid=4242, age_s=60.0)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(
|
||||
4242, stale_after=90.0, home=tmp_path
|
||||
)
|
||||
== gateway_cli.GATEWAY_LOOP_ALIVE
|
||||
)
|
||||
|
||||
def test_missing_heartbeat_is_unknown(self, tmp_path):
|
||||
"""No heartbeat file (older gateway, fresh start) is not evidence."""
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
||||
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
||||
)
|
||||
|
||||
def test_pid_mismatch_is_unknown_even_when_stale(self, tmp_path):
|
||||
"""A stale file from a PREVIOUS process must not condemn the new PID."""
|
||||
_write_heartbeat(tmp_path, pid=1111, age_s=600.0)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
||||
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
||||
)
|
||||
|
||||
def test_corrupt_heartbeat_is_unknown(self, tmp_path):
|
||||
path = get_loop_heartbeat_path(tmp_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("{not json", encoding="utf-8")
|
||||
stamp = time.time() - 600.0
|
||||
os.utime(path, (stamp, stamp))
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
||||
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
||||
)
|
||||
|
||||
def test_nonpositive_pid_is_unknown(self, tmp_path):
|
||||
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(0, home=tmp_path)
|
||||
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
||||
)
|
||||
|
||||
def test_invalid_stale_after_falls_back_to_default(self, tmp_path):
|
||||
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(
|
||||
4242, stale_after="bogus", home=tmp_path
|
||||
)
|
||||
== gateway_cli.GATEWAY_LOOP_WEDGED
|
||||
)
|
||||
|
||||
def test_probe_never_raises_on_unreadable_path(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"gateway.shutdown_watchdog.get_loop_heartbeat_path",
|
||||
lambda home=None: (_ for _ in ()).throw(OSError("boom")),
|
||||
)
|
||||
assert (
|
||||
gateway_cli.probe_gateway_loop_liveness(4242)
|
||||
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
||||
)
|
||||
|
||||
|
||||
class TestEscalateWedgedGateway:
|
||||
def test_sigterm_grace_suffices_without_sigkill(self, monkeypatch):
|
||||
"""If SIGTERM lands (signal-handler thread alive), no SIGKILL is sent."""
|
||||
signals = []
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"terminate_pid",
|
||||
lambda pid, force=False: signals.append(("kill" if force else "term", pid)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: True
|
||||
)
|
||||
|
||||
assert gateway_cli._escalate_wedged_gateway(4242) is True
|
||||
assert signals == [("term", 4242)]
|
||||
|
||||
def test_escalates_to_sigkill_when_sigterm_ignored(self, monkeypatch):
|
||||
signals = []
|
||||
waits = []
|
||||
|
||||
def fake_wait(pid, timeout):
|
||||
waits.append(timeout)
|
||||
# First wait (SIGTERM grace) times out; second (post-SIGKILL) succeeds.
|
||||
return len(waits) > 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"terminate_pid",
|
||||
lambda pid, force=False: signals.append(("kill" if force else "term", pid)),
|
||||
)
|
||||
monkeypatch.setattr(gateway_cli, "_wait_for_pid_exit", fake_wait)
|
||||
|
||||
assert gateway_cli._escalate_wedged_gateway(4242) is True
|
||||
assert signals == [("term", 4242), ("kill", 4242)]
|
||||
|
||||
def test_total_wait_budget_is_bounded_well_under_drain(self, monkeypatch):
|
||||
"""Worst case must be seconds, never the 180s drain budget."""
|
||||
waits = []
|
||||
monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"_wait_for_pid_exit",
|
||||
lambda pid, timeout: waits.append(timeout) or False,
|
||||
)
|
||||
|
||||
assert gateway_cli._escalate_wedged_gateway(4242) is False
|
||||
assert sum(waits) < 30.0
|
||||
|
||||
def test_process_already_gone_is_success(self, monkeypatch):
|
||||
def raise_gone(pid, force=False):
|
||||
raise ProcessLookupError
|
||||
|
||||
monkeypatch.setattr(gateway_cli, "terminate_pid", raise_gone)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: True
|
||||
)
|
||||
|
||||
assert gateway_cli._escalate_wedged_gateway(4242) is True
|
||||
|
||||
def test_sigkill_permission_error_does_not_raise(self, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def term(pid, force=False):
|
||||
calls.append(force)
|
||||
if force:
|
||||
raise PermissionError
|
||||
|
||||
monkeypatch.setattr(gateway_cli, "terminate_pid", term)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout: False
|
||||
)
|
||||
|
||||
assert gateway_cli._escalate_wedged_gateway(4242) is False
|
||||
assert calls == [False, True]
|
||||
|
||||
|
||||
class TestLaunchdRestartWedgedIntegration:
|
||||
"""launchd_restart must skip the 180s drain only for a wedged loop."""
|
||||
|
||||
def _setup(self, monkeypatch, liveness):
|
||||
events = []
|
||||
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
|
||||
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
|
||||
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0)
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: 4242)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_request_gateway_self_restart", lambda pid: False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"probe_gateway_loop_liveness",
|
||||
lambda pid, **kw: events.append("probe") or liveness,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"_escalate_wedged_gateway",
|
||||
lambda pid, **kw: events.append("escalate") or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"terminate_pid",
|
||||
lambda pid, force=False: events.append("sigterm"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli,
|
||||
"_wait_for_gateway_exit",
|
||||
lambda timeout, force_after=None: events.append(("drain", timeout)) or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli.subprocess,
|
||||
"run",
|
||||
lambda *a, **k: events.append("kickstart")
|
||||
or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_clear_launchd_unsupported_marker", lambda: None
|
||||
)
|
||||
return events
|
||||
|
||||
def test_wedged_gateway_skips_drain_and_escalates(self, monkeypatch):
|
||||
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_WEDGED)
|
||||
gateway_cli.launchd_restart()
|
||||
assert "escalate" in events
|
||||
# The 180s drain wait must never run for a wedged loop.
|
||||
assert not any(isinstance(e, tuple) and e[0] == "drain" for e in events)
|
||||
|
||||
def test_busy_gateway_keeps_full_drain_budget(self, monkeypatch):
|
||||
"""A busy-but-alive gateway (fresh heartbeat) must NOT be escalated —
|
||||
that would bypass the in-flight cron drain floor (#86684)."""
|
||||
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_ALIVE)
|
||||
gateway_cli.launchd_restart()
|
||||
assert "escalate" not in events
|
||||
assert ("drain", 180.0) in events
|
||||
|
||||
def test_unknown_liveness_keeps_full_drain_budget(self, monkeypatch):
|
||||
"""Ambiguity (no heartbeat) must never trigger escalation."""
|
||||
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_UNKNOWN)
|
||||
gateway_cli.launchd_restart()
|
||||
assert "escalate" not in events
|
||||
assert ("drain", 180.0) in events
|
||||
Reference in New Issue
Block a user