fix(cron): widen the lock-first liveness check to 'hermes cron status'

Sibling site of the salvaged #95947 fix (same file): cron_status
declared 'Gateway is not running — cron jobs will NOT fire' from a bare
find_gateway_pids() miss even while the runtime lock proved the gateway
alive. Now the not-running verdict requires both the scan AND the lock
to read dead; when only the lock answers, the pid line falls back to
the recorded gateway pid (or is omitted).

Two regression tests pin the false-alarm suppression and the genuine
not-running warning.
This commit is contained in:
kshitijk4poor
2026-08-27 14:31:27 +05:30
committed by kshitij
parent f2d043eb08
commit 1e5fb70fb5
2 changed files with 64 additions and 4 deletions
+24 -4
View File
@@ -404,7 +404,24 @@ def cron_status():
return
pids = find_gateway_pids()
if pids:
gateway_alive_via_lock = False
if not pids:
# Same false-alarm class the cronjob tool fixed (#95947): the pid scan
# can transiently miss a live gateway (just after a restart) while the
# runtime lock — held for exactly the gateway's lifetime — proves the
# ticker's process is alive. Only declare "not running" when both the
# scan AND the lock say so.
try:
from gateway.status import get_running_pid, is_gateway_runtime_lock_active
if is_gateway_runtime_lock_active():
gateway_alive_via_lock = True
lock_pid = get_running_pid()
if lock_pid:
pids = [lock_pid]
except Exception:
pass
if pids or gateway_alive_via_lock:
# The gateway PROCESS is alive — but the cron ticker THREAD inside it
# can die silently, or stay alive while every tick fails. Check both
# the liveness heartbeat and the last-successful-tick marker so we
@@ -432,7 +449,8 @@ def cron_status():
f"no heartbeat for {int(hb_age)}s (expected every ~60s).",
Colors.YELLOW,
))
print(f" PID: {', '.join(map(str, pids))}")
if pids:
print(f" PID: {', '.join(map(str, pids))}")
print(" Cron jobs may NOT be firing. Restart: hermes gateway restart")
elif hb_age is not None and ok_age is not None and ok_age > STALE_AFTER:
# Loop is alive (fresh heartbeat) but no tick has SUCCEEDED in a
@@ -442,7 +460,8 @@ def cron_status():
f"succeeded in {int(ok_age)}s — ticks may be failing.",
Colors.YELLOW,
))
print(f" PID: {', '.join(map(str, pids))}")
if pids:
print(f" PID: {', '.join(map(str, pids))}")
last_error = get_ticker_last_error()
if last_error:
# Show WHY ticks fail — e.g. a root-rewritten jobs.json
@@ -470,7 +489,8 @@ def cron_status():
print(" Check the gateway log for 'Cron tick error'.")
else:
print(color("✓ Gateway is running — cron jobs will fire automatically", Colors.GREEN))
print(f" PID: {', '.join(map(str, pids))}")
if pids:
print(f" PID: {', '.join(map(str, pids))}")
if hb_age is not None:
print(f" Ticker heartbeat: {int(hb_age)}s ago")
else:
@@ -278,3 +278,43 @@ class TestRuntimeLockFirstLiveness:
patch("hermes_cli.gateway.find_gateway_pids", return_value=[424242]),
):
assert cron_cli._builtin_gateway_liveness() is True
class TestCronStatusLockFirst:
"""`hermes cron status` shares the lock-first false-alarm fix (#95947).
Sibling site of `_builtin_gateway_liveness`: it previously declared
"Gateway is not running — cron jobs will NOT fire" from a bare
`find_gateway_pids()` miss even while the runtime lock proved the
gateway (and its ticker) alive.
"""
def _run_status(self, *, pids, lock_active, lock_pid=None):
from unittest.mock import patch
import io
from contextlib import redirect_stdout
import hermes_cli.cron as cron_cli
out = io.StringIO()
with (
patch("hermes_cli.cron._active_cron_provider_name", return_value="builtin"),
patch("hermes_cli.gateway.find_gateway_pids", return_value=list(pids)),
patch(
"gateway.status.is_gateway_runtime_lock_active",
return_value=lock_active,
),
patch("gateway.status.get_running_pid", return_value=lock_pid),
redirect_stdout(out),
):
cron_cli.cron_status()
return out.getvalue()
def test_lock_active_suppresses_not_running_false_alarm(self, hermes_env):
text = self._run_status(pids=[], lock_active=True, lock_pid=4242)
assert "NOT fire" not in text
assert "Gateway is running" in text or "running" in text
def test_no_lock_no_pids_still_warns(self, hermes_env):
text = self._run_status(pids=[], lock_active=False)
assert "NOT fire" in text