fix(desktop): keep the orphan sweep unconditional; guard the probe fix with a mutation-checked test
Follow-up on the salvaged #87158: the reaper-side fix (probe with cleanup_stale=False so the sweep never deletes its own exclusion evidence) fully protects a healthy standalone gateway, so the web_server.py skip-sweep gate is dropped — a stale-but-present registration must not veto the #77276 orphan reap that motivated the sweep in the first place. New regression test drives the exact failure: a registration that fails liveness validation only surfaces its PID when probed non-destructively; with the destructive default the standalone gateway would be hard-killed (TerminateProcess, no drain). Mutation-checked: reverting the probe to get_running_pid() fails the test.
This commit is contained in:
+10
-10
@@ -472,22 +472,22 @@ async def _lifespan(app: "FastAPI"):
|
||||
# WebSocket, and a newly forked gateway then races the same credential,
|
||||
# splitting messages across parallel session trees (#77276).
|
||||
#
|
||||
# Only reap when no live, registered gateway is in the record: a
|
||||
# healthy standalone gateway (launched via `hermes gateway run`, no
|
||||
# service supervisor) is not an orphan, and the sweep would kill it.
|
||||
# On Windows every layer of the launcher chain (stub -> venv python ->
|
||||
# runtime python) carries "gateway run" in its command line, so
|
||||
# The sweep itself still runs unconditionally — a stale-but-present
|
||||
# registration must not veto the #77276 orphan reap. Protection for
|
||||
# a healthy standalone gateway (launched via `hermes gateway run`,
|
||||
# no service supervisor) lives INSIDE the reaper: it probes the
|
||||
# registration with cleanup_stale=False so the recorded PID always
|
||||
# joins the exclusion set, even when liveness validation would have
|
||||
# unlinked the record mid-sweep. That matters most on Windows, where
|
||||
# every layer of the launcher chain (stub -> venv python -> runtime
|
||||
# python) carries "gateway run" in its command line, so
|
||||
# find_gateway_pids() matches processes the pidfile exclusion cannot
|
||||
# see, and os.kill(SIGTERM) is a hard TerminateProcess — the
|
||||
# gateway's planned-stop watcher (0.5s poll) has no time to drain.
|
||||
# cleanup_stale=False keeps the probe from deleting the registration
|
||||
# record it is checking.
|
||||
try:
|
||||
from gateway.status import get_running_pid
|
||||
from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
|
||||
|
||||
if get_running_pid(cleanup_stale=False) is None:
|
||||
_reap_unsupervised_gateway_orphans()
|
||||
_reap_unsupervised_gateway_orphans()
|
||||
except Exception:
|
||||
_log.exception("Desktop startup: orphan gateway reap failed")
|
||||
|
||||
|
||||
@@ -746,6 +746,58 @@ class TestReapUnsupervisedGatewayOrphansWindows:
|
||||
assert result is False # no orphans reaped
|
||||
assert killed_pids == [] # nothing was killed
|
||||
|
||||
def test_windows_probe_never_cleans_stale_registration(self, monkeypatch):
|
||||
"""The reaper's registration probe must pass cleanup_stale=False.
|
||||
|
||||
With the destructive default, a registration that fails liveness
|
||||
validation is unlinked mid-sweep, the recorded PID never joins the
|
||||
exclusion set, and a healthy standalone gateway (`hermes gateway
|
||||
run`, no service supervisor) is hard-killed. On Windows SIGTERM is
|
||||
TerminateProcess — no graceful drain (#87158).
|
||||
"""
|
||||
recorded_pid = 52615
|
||||
|
||||
monkeypatch.setattr(gateway, "is_windows", lambda: True)
|
||||
monkeypatch.setattr(gateway, "is_macos", lambda: False)
|
||||
monkeypatch.setattr(gateway, "supports_systemd_services", lambda: False)
|
||||
|
||||
recorded = SimpleNamespace(pid=recorded_pid, parent=lambda: None)
|
||||
self._install_fake_psutil(monkeypatch, [recorded])
|
||||
|
||||
# Simulate the failing-liveness-validation registration: the probe
|
||||
# only returns the recorded PID when asked NOT to clean up; the
|
||||
# destructive default path would unlink the record and return None.
|
||||
seen_kwargs = []
|
||||
|
||||
def probing_get_running_pid(cleanup_stale=True):
|
||||
seen_kwargs.append(cleanup_stale)
|
||||
return None if cleanup_stale else recorded_pid
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gateway.status.get_running_pid", probing_get_running_pid
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gateway,
|
||||
"find_gateway_pids",
|
||||
lambda exclude_pids=None: [
|
||||
p for p in [recorded_pid] if p not in (exclude_pids or set())
|
||||
],
|
||||
)
|
||||
|
||||
killed_pids = []
|
||||
monkeypatch.setattr(
|
||||
gateway.os, "kill", lambda pid, sig: killed_pids.append((pid, sig))
|
||||
)
|
||||
|
||||
result = gateway._reap_unsupervised_gateway_orphans()
|
||||
|
||||
assert seen_kwargs == [False], (
|
||||
"reaper probe must use cleanup_stale=False so the sweep never "
|
||||
f"deletes its own exclusion evidence, got {seen_kwargs}"
|
||||
)
|
||||
assert result is False
|
||||
assert killed_pids == [] # the standalone gateway survived
|
||||
|
||||
|
||||
class TestReaperCandidateIsSupervisorOwned:
|
||||
"""Regression for the Windows pidfile-less supervisor-owned case (#83683).
|
||||
|
||||
Reference in New Issue
Block a user