fix(cron): persisted-state recovery re-arms recurring job stuck in stale last_status=error (t_8b5480b3)

The 2026-08-14 incident (t_20e23f84): 4 recurring no_agent interval jobs
EAGAIN-failed at 12:50 and recorded ZERO executions for ~1h47m, surviving a
gateway restart, cleared only by operator `cron resume` / force-run. The
in-memory stale-claim sweep (t_3778a491, already on origin/main) heals a
leaked `_running_job_ids` claim in-process, but a recurring job whose
PERSISTED state shows last_status=error and whose next_run_at was re-armed
into the future by mark_job_run is invisible to that sweep: it is not in the
running set and not due, so it just sits — the restart-surviving half.

cron/jobs.py::_get_due_jobs_locked now re-arms such a recurring job to
next_run_at=now when all hold: persisted last_status==error, last_run_at older
than cadence+grace (so it is a real wedge, not a normal transient-error retry),
next_run_at in the future, and not running in this process. The scheduler then
re-dispatches it on the next tick without force-run/resume. Logs
cron.persisted_error.recovered, bumps a probe-visible counter, appends a JSONL
row. Within-cadence errors are never force-re-armed.

Tests: tests/cron/test_recurring_persisted_error_recovery.py (clean behavioral
RED on unfixed main / GREEN here; 2 consecutive auto-fires; within-cadence not
re-armed). Full tests/cron/: 713 passed, 1 skipped.
This commit is contained in:
devops
2026-08-15 21:53:54 +01:00
committed by kshitij
parent b52b725f62
commit 122bfad535
3 changed files with 332 additions and 7 deletions
+157
View File
@@ -899,6 +899,132 @@ def _compute_grace_seconds(schedule: dict) -> int:
return MIN_GRACE
# Durable (persisted-state) recovery counter for a recurring job wedged in a
# stale ``last_status == "error"`` state with ``next_run_at`` parked in the
# future. This is the restart-surviving half of the recurring-cron wedge
# (t_20e23f84 / t_8b5480b3): the in-memory stale-claim sweep in scheduler.py
# heals a leaked ``_running_job_ids`` claim in-process, but a job whose
# persisted ``next_run_at`` was re-armed into the future (the normal
# post-error re-arm by ``mark_job_run``) is invisible to that sweep — it is
# not in the running set, so nothing force-releases it, and it is not due, so
# ``get_due_jobs`` never returns it. It just sits. This recovery re-arms
# ``next_run_at`` to now so the next tick re-dispatches it, exactly like the
# operator's force-run / mech_red_guard's ``cron resume`` but built-in.
_persisted_error_recoveries: int = 0
_PERSISTED_ERROR_RECOVERY_HISTORY = 20
_persisted_error_recoveries_recent: list = []
def _job_is_stale_error_recurring(
job: Dict[str, Any],
schedule: Dict[str, Any],
now: datetime,
) -> bool:
"""True when a recurring job is wedged in a stale persisted error state.
Condition (all must hold):
* it is a recurring (cron/interval) job (checked by caller);
* its persisted ``last_status == "error"`` (a prior fire errored and the
job never recovered);
* it has NOT successfully re-fired within its natural cadence — its
``last_run_at`` is older than ``cadence + grace``, so this is not a
normal transient-error retry that will fire on its own soon, it is a
job that has been sitting errored for a full period with no recovery;
* it is not currently running in this process (a live run must never be
re-armed underneath itself, #62002-style).
``last_run_at`` being older than one cadence is the key discriminator: a
job that errors and is retried on its normal schedule keeps ``last_run_at``
fresh (mark_job_run stamps it on every fire, success or failure), so the
stale check does not fire for a job that is merely erroring-and-retrying.
"""
if job.get("last_status") != "error":
return False
if _job_running_in_this_process(str(job.get("id") or "")):
return False
last_run = job.get("last_run_at")
if not last_run:
return False
try:
last_run_dt = _ensure_aware(datetime.fromisoformat(last_run))
except (ValueError, TypeError):
return False
age_seconds = (now - last_run_dt).total_seconds()
if age_seconds < 0:
return False
cadence_seconds = _schedule_cadence_seconds(schedule)
if cadence_seconds is None:
# Unknown cadence (croniter unavailable / malformed expr): fall back to
# the grace window so a badly-parked job is still recovered, but never
# re-arm anything younger than the 2h grace cap.
cadence_seconds = _compute_grace_seconds(schedule)
return age_seconds > (cadence_seconds + _compute_grace_seconds(schedule))
def _schedule_cadence_seconds(schedule: Dict[str, Any]) -> Optional[float]:
"""Approximate the natural period of a schedule, in seconds, or None.
Interval jobs use ``minutes * 60``. Cron jobs measure the gap between the
next two fire times with croniter (falling back to None when croniter is
missing or the expr is malformed).
"""
if not isinstance(schedule, dict):
return None
kind = schedule.get("kind")
if kind == "interval":
minutes = schedule.get("minutes")
try:
return float(minutes) * 60.0 if minutes else None
except (TypeError, ValueError):
return None
if kind == "cron":
if not _ensure_croniter():
return None
expr = schedule.get("expr")
if not expr:
return None
try:
base = _hermes_now()
it = croniter(expr, base)
first = it.get_next(datetime)
second = it.get_next(datetime)
gap = (second - first).total_seconds()
return gap if gap > 0 else None
except Exception:
return None
return None
def _record_persisted_error_recovery(job: Dict[str, Any], previous_next_run: str) -> None:
"""Persist a countable, probe-visible signal for one stale-error re-arm."""
global _persisted_error_recoveries
now = _hermes_now()
entry = {
"job_id": job.get("id"),
"name": job.get("name") or job.get("id"),
"previous_next_run_at": previous_next_run,
"rearmed_at": now.isoformat(),
}
_persisted_error_recoveries += 1
_persisted_error_recoveries_recent.append(entry)
del _persisted_error_recoveries_recent[:-_PERSISTED_ERROR_RECOVERY_HISTORY]
try:
path = _current_cron_store().cron_dir / "persisted_error_recoveries.jsonl"
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(entry) + "\n")
except Exception as exc: # never let telemetry break a tick
logger.debug("Could not append persisted-error-recovery record: %s", exc)
def get_persisted_error_recovery_stats() -> Dict[str, Any]:
"""Probe-visible snapshot of persisted-error recoveries."""
return {
"persisted_error_recoveries": _persisted_error_recoveries,
"recent": list(_persisted_error_recoveries_recent),
}
def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]:
"""
Compute the next run time for a schedule.
@@ -3096,6 +3222,37 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:
break
continue
# Persisted-state stale-error recovery (t_8b5480b3): a recurring
# job whose persisted state shows last_status=error and whose
# last_run_at is older than a full cadence has been sitting wedged
# (errored once, next_run_at re-armed into the future by
# mark_job_run, but never re-dispatched — the restart-surviving
# half of the 2026-08-14 incident, invisible to the in-memory
# in-flight sweep). Re-arm next_run_at to now so the next tick
# returns it as due and re-dispatches it WITHOUT force-run/resume.
if (
kind in ("cron", "interval")
and next_run_dt > now
and _job_is_stale_error_recurring(job, schedule, now)
):
jid = job.get("id")
logger.warning(
"cron.persisted_error.recovered job='%s' id=%s — recurring "
"job wedged in stale last_status=error without re-firing for "
"a full cadence; re-arming next_run_at to now so it "
"re-dispatches without force-run/resume",
job.get("name", jid),
jid,
)
_record_persisted_error_recovery(job, next_run)
job["next_run_at"] = now.isoformat()
next_run_dt = now
for rj in raw_jobs:
if rj["id"] == jid:
rj["next_run_at"] = now.isoformat()
needs_save = True
break
if next_run_dt <= now:
# For recurring jobs, check if the scheduled time is stale
+12 -7
View File
@@ -1,4 +1,4 @@
"""Deterministic reproduction of the recurring-cron EAGAIN wedge (t_8b5480b3).
"""Deterministic reproduction of the recurring-cron EAGAIN in-memory wedge (t_8b5480b3).
Scenario modelled on the 2026-08-14 incident: a recurring no_agent interval job
whose script subprocess raises EAGAIN ([Errno 11] Resource temporarily
@@ -6,12 +6,17 @@ unavailable) during a substrate thread-exhaustion spike. After the failure is
recorded (terminal 'failed' execution row), the job must be re-dispatched on
the NEXT tick once the substrate recovers — with no force-run.
The os-reviewer (t_20e23f84) established the real incident wedge: 4 recurring
no_agent jobs recorded ZERO executions for ~1h47m after EAGAIN while
`next_run_at` kept advancing (they stayed 'due') but `_submit_with_guard`
never dispatched them, and the wedge SURVIVED a gateway restart. That points
at a PERSISTED non-dispatch state, not just the in-memory `_running_job_ids`
leak (which t_3778a491 already bounds).
SCOPE NOTE (honest): this file exercises the in-memory EAGAIN-release class —
the claim taken by ``_submit_with_guard`` is released when the submit/init
path fails, so the NEXT tick re-dispatches. It passes on unfixed base because
a terminal ``failed`` execution row does not itself suppress a recurring job's
next due fire; that is pre-existing behavior, not the fix. The persisted-state
(restart-surviving) half of the incident — a recurring job whose persisted
``last_status=error`` and ``next_run_at`` parked in the future is never
re-dispatched without force-run/resume — is covered by
``test_recurring_persisted_error_recovery.py``, which provides the clean
behavioral RED (unfixed: no execution) / GREEN (fixed: re-dispatched) for the
new recovery path.
This file drives the REAL `tick()` end-to-end against a throwaway HERMES_HOME:
tick 1 -> script EAGAINs (subprocess.run raises OSError 11) -> failed exec row
@@ -0,0 +1,163 @@
"""Persisted-state stale-error recovery for recurring cron jobs (t_8b5480b3).
The 2026-08-14 incident (t_20e23f84): 4 recurring no_agent interval jobs
EAGAIN-failed at 12:50 and then recorded ZERO executions for ~1h47m (~9 missed
cycles) even after the substrate recovered, and the wedge SURVIVED a gateway
restart. The in-memory stale-claim sweep (cron/scheduler.py, t_3778a491)
heals a leaked `_running_job_ids` claim in-process, but that is only the
in-process half. A recurring job whose *persisted* state shows
`last_status == "error"` and whose `next_run_at` was re-armed into the future
by `mark_job_run` is invisible to the in-memory sweep: it is not in the
running set (nothing force-releases it) and it is not due (so `get_due_jobs`
never returns it). It just sits — exactly the restart-surviving symptom
documented in the incident.
This recovery (cron/jobs.py `_get_due_jobs_locked`) re-arms `next_run_at` to
now for a recurring job that:
* is in a persisted `last_status == "error"` state, AND
* has NOT re-fired within a full cadence (`last_run_at` older than
cadence + grace — so it is NOT a normal transient-error retry that will
fire on its own soon), AND
* is not currently running in this process.
The scheduler then re-dispatches it on the next tick WITHOUT force-run/resume
— the built-in equivalent of mech_red_guard's `cron resume`.
Deterministic RED/GREEN: on unfixed code the wedged job is left untouched (no
execution row, no re-dispatch); with the recovery it re-fires and completes.
"""
from __future__ import annotations
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest import mock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
@pytest.fixture
def cron_env(tmp_path, monkeypatch):
"""Isolated cron env + a recurring no_agent interval job."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "cron").mkdir()
(hermes_home / "cron" / "output").mkdir()
(hermes_home / "scripts").mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
import cron.jobs as jobs_mod
monkeypatch.setattr(jobs_mod, "HERMES_DIR", hermes_home)
monkeypatch.setattr(jobs_mod, "CRON_DIR", hermes_home / "cron")
monkeypatch.setattr(jobs_mod, "JOBS_FILE", hermes_home / "cron" / "jobs.json")
monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", hermes_home / "cron" / "output")
job = jobs_mod.create_job(
prompt="probe",
schedule="every 10m",
no_agent=True,
script="probe.py",
)
script = hermes_home / "scripts" / "probe.py"
script.write_text("print('ok')\n")
return {"home": hermes_home, "job_id": job["id"]}
def _setup(cron_env, monkeypatch):
from cron import scheduler as S
from cron import executions as E
import cron.jobs as J
env = cron_env
monkeypatch.setattr(E, "EXECUTIONS_FILE", env["home"] / "cron" / "executions.db")
monkeypatch.setattr(S, "_hermes_home", env["home"])
return S, E, J, env
def _persist_stale_error(J, job_id, *, error_age_minutes=110):
"""Persist the incident wedge state: last_status=error, last_run stale by
more than a full cadence, next_run_at re-armed into the future."""
now = datetime.now(timezone.utc)
J.update_job(
job_id,
{
"next_run_at": (now + timedelta(minutes=5)).isoformat(),
"last_status": "error",
"last_error": "EAGAIN [Errno 11] Resource temporarily unavailable",
"last_run_at": (now - timedelta(minutes=error_age_minutes)).isoformat(),
},
)
class TestPersistedStaleErrorRecovery:
def test_stale_error_recurring_job_redispatches_without_force_run(
self, cron_env, monkeypatch
):
"""GREEN: a recurring job wedged in a persisted stale-error state with
next_run_at parked in the future is re-armed and re-dispatches on the
next tick — no force-run, no resume."""
S, E, J, env = _setup(cron_env, monkeypatch)
job_id = env["job_id"]
_persist_stale_error(J, job_id, error_age_minutes=110)
job = J.get_job(job_id)
with mock.patch("cron.jobs.load_jobs", return_value=[job]):
n = S.tick(verbose=False, sync=True)
latest = E.latest_execution(job_id)
assert latest is not None, (
"wedged stale-error job must be re-dispatched without force-run"
)
assert latest["status"] == "completed"
# The recovery must be countable / probe-visible.
from cron import jobs as Jmod
stats = Jmod.get_persisted_error_recovery_stats()
assert stats["persisted_error_recoveries"] >= 1
def test_two_consecutive_auto_fires_after_persisted_recovery(
self, cron_env, monkeypatch
):
"""GREEN: after the persisted recovery re-arms the job, it fires on
consecutive ticks (no manual intervention)."""
S, E, J, env = _setup(cron_env, monkeypatch)
job_id = env["job_id"]
_persist_stale_error(J, job_id, error_age_minutes=110)
job = J.get_job(job_id)
with mock.patch("cron.jobs.load_jobs", return_value=[job]):
S.tick(verbose=False, sync=True)
latest1 = E.latest_execution(job_id)
assert latest1["status"] == "completed"
# Re-arm due normally and tick again: fire #2.
now = datetime.now(timezone.utc)
J.update_job(job_id, {"next_run_at": (now - timedelta(minutes=1)).isoformat()})
with mock.patch("cron.jobs.load_jobs", return_value=[J.get_job(job_id)]):
S.tick(verbose=False, sync=True)
latest2 = E.latest_execution(job_id)
assert latest2["status"] == "completed"
assert latest2["id"] != latest1["id"], "two distinct executions"
def test_recent_error_within_cadence_not_force_rearmed(self, cron_env, monkeypatch):
"""The discriminator must NOT re-arm a normal transient error whose
last_run is within a full cadence — that job retries on its own
schedule and must be left alone (otherwise we'd over-fire healthy
erroring jobs)."""
S, E, J, env = _setup(cron_env, monkeypatch)
job_id = env["job_id"]
_persist_stale_error(J, job_id, error_age_minutes=5) # within cadence
job = J.get_job(job_id)
with mock.patch("cron.jobs.load_jobs", return_value=[job]):
S.tick(verbose=False, sync=True)
latest = E.latest_execution(job_id)
assert latest is None, (
"a within-cadence error is a normal retry — must not be "
"force-re-armed onto an immediate fire"
)