fix(cron): widen the lock-first liveness check to 'hermes cron status'
Sibling site of the salvaged #95947 fix (same file): cron_status declared 'Gateway is not running — cron jobs will NOT fire' from a bare find_gateway_pids() miss even while the runtime lock proved the gateway alive. Now the not-running verdict requires both the scan AND the lock to read dead; when only the lock answers, the pid line falls back to the recorded gateway pid (or is omitted). Two regression tests pin the false-alarm suppression and the genuine not-running warning.
This commit is contained in:
+24
-4
@@ -404,7 +404,24 @@ def cron_status():
|
||||
return
|
||||
|
||||
pids = find_gateway_pids()
|
||||
if pids:
|
||||
gateway_alive_via_lock = False
|
||||
if not pids:
|
||||
# Same false-alarm class the cronjob tool fixed (#95947): the pid scan
|
||||
# can transiently miss a live gateway (just after a restart) while the
|
||||
# runtime lock — held for exactly the gateway's lifetime — proves the
|
||||
# ticker's process is alive. Only declare "not running" when both the
|
||||
# scan AND the lock say so.
|
||||
try:
|
||||
from gateway.status import get_running_pid, is_gateway_runtime_lock_active
|
||||
|
||||
if is_gateway_runtime_lock_active():
|
||||
gateway_alive_via_lock = True
|
||||
lock_pid = get_running_pid()
|
||||
if lock_pid:
|
||||
pids = [lock_pid]
|
||||
except Exception:
|
||||
pass
|
||||
if pids or gateway_alive_via_lock:
|
||||
# The gateway PROCESS is alive — but the cron ticker THREAD inside it
|
||||
# can die silently, or stay alive while every tick fails. Check both
|
||||
# the liveness heartbeat and the last-successful-tick marker so we
|
||||
@@ -432,7 +449,8 @@ def cron_status():
|
||||
f"no heartbeat for {int(hb_age)}s (expected every ~60s).",
|
||||
Colors.YELLOW,
|
||||
))
|
||||
print(f" PID: {', '.join(map(str, pids))}")
|
||||
if pids:
|
||||
print(f" PID: {', '.join(map(str, pids))}")
|
||||
print(" Cron jobs may NOT be firing. Restart: hermes gateway restart")
|
||||
elif hb_age is not None and ok_age is not None and ok_age > STALE_AFTER:
|
||||
# Loop is alive (fresh heartbeat) but no tick has SUCCEEDED in a
|
||||
@@ -442,7 +460,8 @@ def cron_status():
|
||||
f"succeeded in {int(ok_age)}s — ticks may be failing.",
|
||||
Colors.YELLOW,
|
||||
))
|
||||
print(f" PID: {', '.join(map(str, pids))}")
|
||||
if pids:
|
||||
print(f" PID: {', '.join(map(str, pids))}")
|
||||
last_error = get_ticker_last_error()
|
||||
if last_error:
|
||||
# Show WHY ticks fail — e.g. a root-rewritten jobs.json
|
||||
@@ -470,7 +489,8 @@ def cron_status():
|
||||
print(" Check the gateway log for 'Cron tick error'.")
|
||||
else:
|
||||
print(color("✓ Gateway is running — cron jobs will fire automatically", Colors.GREEN))
|
||||
print(f" PID: {', '.join(map(str, pids))}")
|
||||
if pids:
|
||||
print(f" PID: {', '.join(map(str, pids))}")
|
||||
if hb_age is not None:
|
||||
print(f" Ticker heartbeat: {int(hb_age)}s ago")
|
||||
else:
|
||||
|
||||
@@ -278,3 +278,43 @@ class TestRuntimeLockFirstLiveness:
|
||||
patch("hermes_cli.gateway.find_gateway_pids", return_value=[424242]),
|
||||
):
|
||||
assert cron_cli._builtin_gateway_liveness() is True
|
||||
|
||||
|
||||
class TestCronStatusLockFirst:
|
||||
"""`hermes cron status` shares the lock-first false-alarm fix (#95947).
|
||||
|
||||
Sibling site of `_builtin_gateway_liveness`: it previously declared
|
||||
"Gateway is not running — cron jobs will NOT fire" from a bare
|
||||
`find_gateway_pids()` miss even while the runtime lock proved the
|
||||
gateway (and its ticker) alive.
|
||||
"""
|
||||
|
||||
def _run_status(self, *, pids, lock_active, lock_pid=None):
|
||||
from unittest.mock import patch
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
import hermes_cli.cron as cron_cli
|
||||
|
||||
out = io.StringIO()
|
||||
with (
|
||||
patch("hermes_cli.cron._active_cron_provider_name", return_value="builtin"),
|
||||
patch("hermes_cli.gateway.find_gateway_pids", return_value=list(pids)),
|
||||
patch(
|
||||
"gateway.status.is_gateway_runtime_lock_active",
|
||||
return_value=lock_active,
|
||||
),
|
||||
patch("gateway.status.get_running_pid", return_value=lock_pid),
|
||||
redirect_stdout(out),
|
||||
):
|
||||
cron_cli.cron_status()
|
||||
return out.getvalue()
|
||||
|
||||
def test_lock_active_suppresses_not_running_false_alarm(self, hermes_env):
|
||||
text = self._run_status(pids=[], lock_active=True, lock_pid=4242)
|
||||
assert "NOT fire" not in text
|
||||
assert "Gateway is running" in text or "running" in text
|
||||
|
||||
def test_no_lock_no_pids_still_warns(self, hermes_env):
|
||||
text = self._run_status(pids=[], lock_active=False)
|
||||
assert "NOT fire" in text
|
||||
|
||||
Reference in New Issue
Block a user