fix(cron): persisted-state recovery re-arms recurring job stuck in stale last_status=error (t_8b5480b3)
The 2026-08-14 incident (t_20e23f84): 4 recurring no_agent interval jobs EAGAIN-failed at 12:50 and recorded ZERO executions for ~1h47m, surviving a gateway restart, cleared only by operator `cron resume` / force-run. The in-memory stale-claim sweep (t_3778a491, already on origin/main) heals a leaked `_running_job_ids` claim in-process, but a recurring job whose PERSISTED state shows last_status=error and whose next_run_at was re-armed into the future by mark_job_run is invisible to that sweep: it is not in the running set and not due, so it just sits — the restart-surviving half. cron/jobs.py::_get_due_jobs_locked now re-arms such a recurring job to next_run_at=now when all hold: persisted last_status==error, last_run_at older than cadence+grace (so it is a real wedge, not a normal transient-error retry), next_run_at in the future, and not running in this process. The scheduler then re-dispatches it on the next tick without force-run/resume. Logs cron.persisted_error.recovered, bumps a probe-visible counter, appends a JSONL row. Within-cadence errors are never force-re-armed. Tests: tests/cron/test_recurring_persisted_error_recovery.py (clean behavioral RED on unfixed main / GREEN here; 2 consecutive auto-fires; within-cadence not re-armed). Full tests/cron/: 713 passed, 1 skipped.
This commit is contained in:
+157
@@ -899,6 +899,132 @@ def _compute_grace_seconds(schedule: dict) -> int:
|
||||
return MIN_GRACE
|
||||
|
||||
|
||||
# Durable (persisted-state) recovery counter for a recurring job wedged in a
|
||||
# stale ``last_status == "error"`` state with ``next_run_at`` parked in the
|
||||
# future. This is the restart-surviving half of the recurring-cron wedge
|
||||
# (t_20e23f84 / t_8b5480b3): the in-memory stale-claim sweep in scheduler.py
|
||||
# heals a leaked ``_running_job_ids`` claim in-process, but a job whose
|
||||
# persisted ``next_run_at`` was re-armed into the future (the normal
|
||||
# post-error re-arm by ``mark_job_run``) is invisible to that sweep — it is
|
||||
# not in the running set, so nothing force-releases it, and it is not due, so
|
||||
# ``get_due_jobs`` never returns it. It just sits. This recovery re-arms
|
||||
# ``next_run_at`` to now so the next tick re-dispatches it, exactly like the
|
||||
# operator's force-run / mech_red_guard's ``cron resume`` but built-in.
|
||||
_persisted_error_recoveries: int = 0
|
||||
_PERSISTED_ERROR_RECOVERY_HISTORY = 20
|
||||
_persisted_error_recoveries_recent: list = []
|
||||
|
||||
|
||||
def _job_is_stale_error_recurring(
|
||||
job: Dict[str, Any],
|
||||
schedule: Dict[str, Any],
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
"""True when a recurring job is wedged in a stale persisted error state.
|
||||
|
||||
Condition (all must hold):
|
||||
* it is a recurring (cron/interval) job (checked by caller);
|
||||
* its persisted ``last_status == "error"`` (a prior fire errored and the
|
||||
job never recovered);
|
||||
* it has NOT successfully re-fired within its natural cadence — its
|
||||
``last_run_at`` is older than ``cadence + grace``, so this is not a
|
||||
normal transient-error retry that will fire on its own soon, it is a
|
||||
job that has been sitting errored for a full period with no recovery;
|
||||
* it is not currently running in this process (a live run must never be
|
||||
re-armed underneath itself, #62002-style).
|
||||
|
||||
``last_run_at`` being older than one cadence is the key discriminator: a
|
||||
job that errors and is retried on its normal schedule keeps ``last_run_at``
|
||||
fresh (mark_job_run stamps it on every fire, success or failure), so the
|
||||
stale check does not fire for a job that is merely erroring-and-retrying.
|
||||
"""
|
||||
if job.get("last_status") != "error":
|
||||
return False
|
||||
if _job_running_in_this_process(str(job.get("id") or "")):
|
||||
return False
|
||||
last_run = job.get("last_run_at")
|
||||
if not last_run:
|
||||
return False
|
||||
try:
|
||||
last_run_dt = _ensure_aware(datetime.fromisoformat(last_run))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
age_seconds = (now - last_run_dt).total_seconds()
|
||||
if age_seconds < 0:
|
||||
return False
|
||||
cadence_seconds = _schedule_cadence_seconds(schedule)
|
||||
if cadence_seconds is None:
|
||||
# Unknown cadence (croniter unavailable / malformed expr): fall back to
|
||||
# the grace window so a badly-parked job is still recovered, but never
|
||||
# re-arm anything younger than the 2h grace cap.
|
||||
cadence_seconds = _compute_grace_seconds(schedule)
|
||||
return age_seconds > (cadence_seconds + _compute_grace_seconds(schedule))
|
||||
|
||||
|
||||
def _schedule_cadence_seconds(schedule: Dict[str, Any]) -> Optional[float]:
|
||||
"""Approximate the natural period of a schedule, in seconds, or None.
|
||||
|
||||
Interval jobs use ``minutes * 60``. Cron jobs measure the gap between the
|
||||
next two fire times with croniter (falling back to None when croniter is
|
||||
missing or the expr is malformed).
|
||||
"""
|
||||
if not isinstance(schedule, dict):
|
||||
return None
|
||||
kind = schedule.get("kind")
|
||||
if kind == "interval":
|
||||
minutes = schedule.get("minutes")
|
||||
try:
|
||||
return float(minutes) * 60.0 if minutes else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if kind == "cron":
|
||||
if not _ensure_croniter():
|
||||
return None
|
||||
expr = schedule.get("expr")
|
||||
if not expr:
|
||||
return None
|
||||
try:
|
||||
base = _hermes_now()
|
||||
it = croniter(expr, base)
|
||||
first = it.get_next(datetime)
|
||||
second = it.get_next(datetime)
|
||||
gap = (second - first).total_seconds()
|
||||
return gap if gap > 0 else None
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _record_persisted_error_recovery(job: Dict[str, Any], previous_next_run: str) -> None:
|
||||
"""Persist a countable, probe-visible signal for one stale-error re-arm."""
|
||||
global _persisted_error_recoveries
|
||||
now = _hermes_now()
|
||||
entry = {
|
||||
"job_id": job.get("id"),
|
||||
"name": job.get("name") or job.get("id"),
|
||||
"previous_next_run_at": previous_next_run,
|
||||
"rearmed_at": now.isoformat(),
|
||||
}
|
||||
_persisted_error_recoveries += 1
|
||||
_persisted_error_recoveries_recent.append(entry)
|
||||
del _persisted_error_recoveries_recent[:-_PERSISTED_ERROR_RECOVERY_HISTORY]
|
||||
try:
|
||||
path = _current_cron_store().cron_dir / "persisted_error_recoveries.jsonl"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(entry) + "\n")
|
||||
except Exception as exc: # never let telemetry break a tick
|
||||
logger.debug("Could not append persisted-error-recovery record: %s", exc)
|
||||
|
||||
|
||||
def get_persisted_error_recovery_stats() -> Dict[str, Any]:
|
||||
"""Probe-visible snapshot of persisted-error recoveries."""
|
||||
return {
|
||||
"persisted_error_recoveries": _persisted_error_recoveries,
|
||||
"recent": list(_persisted_error_recoveries_recent),
|
||||
}
|
||||
|
||||
|
||||
def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Compute the next run time for a schedule.
|
||||
@@ -3096,6 +3222,37 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:
|
||||
break
|
||||
continue
|
||||
|
||||
# Persisted-state stale-error recovery (t_8b5480b3): a recurring
|
||||
# job whose persisted state shows last_status=error and whose
|
||||
# last_run_at is older than a full cadence has been sitting wedged
|
||||
# (errored once, next_run_at re-armed into the future by
|
||||
# mark_job_run, but never re-dispatched — the restart-surviving
|
||||
# half of the 2026-08-14 incident, invisible to the in-memory
|
||||
# in-flight sweep). Re-arm next_run_at to now so the next tick
|
||||
# returns it as due and re-dispatches it WITHOUT force-run/resume.
|
||||
if (
|
||||
kind in ("cron", "interval")
|
||||
and next_run_dt > now
|
||||
and _job_is_stale_error_recurring(job, schedule, now)
|
||||
):
|
||||
jid = job.get("id")
|
||||
logger.warning(
|
||||
"cron.persisted_error.recovered job='%s' id=%s — recurring "
|
||||
"job wedged in stale last_status=error without re-firing for "
|
||||
"a full cadence; re-arming next_run_at to now so it "
|
||||
"re-dispatches without force-run/resume",
|
||||
job.get("name", jid),
|
||||
jid,
|
||||
)
|
||||
_record_persisted_error_recovery(job, next_run)
|
||||
job["next_run_at"] = now.isoformat()
|
||||
next_run_dt = now
|
||||
for rj in raw_jobs:
|
||||
if rj["id"] == jid:
|
||||
rj["next_run_at"] = now.isoformat()
|
||||
needs_save = True
|
||||
break
|
||||
|
||||
if next_run_dt <= now:
|
||||
|
||||
# For recurring jobs, check if the scheduled time is stale
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Deterministic reproduction of the recurring-cron EAGAIN wedge (t_8b5480b3).
|
||||
"""Deterministic reproduction of the recurring-cron EAGAIN in-memory wedge (t_8b5480b3).
|
||||
|
||||
Scenario modelled on the 2026-08-14 incident: a recurring no_agent interval job
|
||||
whose script subprocess raises EAGAIN ([Errno 11] Resource temporarily
|
||||
@@ -6,12 +6,17 @@ unavailable) during a substrate thread-exhaustion spike. After the failure is
|
||||
recorded (terminal 'failed' execution row), the job must be re-dispatched on
|
||||
the NEXT tick once the substrate recovers — with no force-run.
|
||||
|
||||
The os-reviewer (t_20e23f84) established the real incident wedge: 4 recurring
|
||||
no_agent jobs recorded ZERO executions for ~1h47m after EAGAIN while
|
||||
`next_run_at` kept advancing (they stayed 'due') but `_submit_with_guard`
|
||||
never dispatched them, and the wedge SURVIVED a gateway restart. That points
|
||||
at a PERSISTED non-dispatch state, not just the in-memory `_running_job_ids`
|
||||
leak (which t_3778a491 already bounds).
|
||||
SCOPE NOTE (honest): this file exercises the in-memory EAGAIN-release class —
|
||||
the claim taken by ``_submit_with_guard`` is released when the submit/init
|
||||
path fails, so the NEXT tick re-dispatches. It passes on unfixed base because
|
||||
a terminal ``failed`` execution row does not itself suppress a recurring job's
|
||||
next due fire; that is pre-existing behavior, not the fix. The persisted-state
|
||||
(restart-surviving) half of the incident — a recurring job whose persisted
|
||||
``last_status=error`` and ``next_run_at`` parked in the future is never
|
||||
re-dispatched without force-run/resume — is covered by
|
||||
``test_recurring_persisted_error_recovery.py``, which provides the clean
|
||||
behavioral RED (unfixed: no execution) / GREEN (fixed: re-dispatched) for the
|
||||
new recovery path.
|
||||
|
||||
This file drives the REAL `tick()` end-to-end against a throwaway HERMES_HOME:
|
||||
tick 1 -> script EAGAINs (subprocess.run raises OSError 11) -> failed exec row
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Persisted-state stale-error recovery for recurring cron jobs (t_8b5480b3).
|
||||
|
||||
The 2026-08-14 incident (t_20e23f84): 4 recurring no_agent interval jobs
|
||||
EAGAIN-failed at 12:50 and then recorded ZERO executions for ~1h47m (~9 missed
|
||||
cycles) even after the substrate recovered, and the wedge SURVIVED a gateway
|
||||
restart. The in-memory stale-claim sweep (cron/scheduler.py, t_3778a491)
|
||||
heals a leaked `_running_job_ids` claim in-process, but that is only the
|
||||
in-process half. A recurring job whose *persisted* state shows
|
||||
`last_status == "error"` and whose `next_run_at` was re-armed into the future
|
||||
by `mark_job_run` is invisible to the in-memory sweep: it is not in the
|
||||
running set (nothing force-releases it) and it is not due (so `get_due_jobs`
|
||||
never returns it). It just sits — exactly the restart-surviving symptom
|
||||
documented in the incident.
|
||||
|
||||
This recovery (cron/jobs.py `_get_due_jobs_locked`) re-arms `next_run_at` to
|
||||
now for a recurring job that:
|
||||
* is in a persisted `last_status == "error"` state, AND
|
||||
* has NOT re-fired within a full cadence (`last_run_at` older than
|
||||
cadence + grace — so it is NOT a normal transient-error retry that will
|
||||
fire on its own soon), AND
|
||||
* is not currently running in this process.
|
||||
|
||||
The scheduler then re-dispatches it on the next tick WITHOUT force-run/resume
|
||||
— the built-in equivalent of mech_red_guard's `cron resume`.
|
||||
|
||||
Deterministic RED/GREEN: on unfixed code the wedged job is left untouched (no
|
||||
execution row, no re-dispatch); with the recovery it re-fires and completes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cron_env(tmp_path, monkeypatch):
|
||||
"""Isolated cron env + a recurring no_agent interval job."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "cron").mkdir()
|
||||
(hermes_home / "cron" / "output").mkdir()
|
||||
(hermes_home / "scripts").mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
import cron.jobs as jobs_mod
|
||||
monkeypatch.setattr(jobs_mod, "HERMES_DIR", hermes_home)
|
||||
monkeypatch.setattr(jobs_mod, "CRON_DIR", hermes_home / "cron")
|
||||
monkeypatch.setattr(jobs_mod, "JOBS_FILE", hermes_home / "cron" / "jobs.json")
|
||||
monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", hermes_home / "cron" / "output")
|
||||
|
||||
job = jobs_mod.create_job(
|
||||
prompt="probe",
|
||||
schedule="every 10m",
|
||||
no_agent=True,
|
||||
script="probe.py",
|
||||
)
|
||||
script = hermes_home / "scripts" / "probe.py"
|
||||
script.write_text("print('ok')\n")
|
||||
return {"home": hermes_home, "job_id": job["id"]}
|
||||
|
||||
|
||||
def _setup(cron_env, monkeypatch):
|
||||
from cron import scheduler as S
|
||||
from cron import executions as E
|
||||
import cron.jobs as J
|
||||
|
||||
env = cron_env
|
||||
monkeypatch.setattr(E, "EXECUTIONS_FILE", env["home"] / "cron" / "executions.db")
|
||||
monkeypatch.setattr(S, "_hermes_home", env["home"])
|
||||
return S, E, J, env
|
||||
|
||||
|
||||
def _persist_stale_error(J, job_id, *, error_age_minutes=110):
|
||||
"""Persist the incident wedge state: last_status=error, last_run stale by
|
||||
more than a full cadence, next_run_at re-armed into the future."""
|
||||
now = datetime.now(timezone.utc)
|
||||
J.update_job(
|
||||
job_id,
|
||||
{
|
||||
"next_run_at": (now + timedelta(minutes=5)).isoformat(),
|
||||
"last_status": "error",
|
||||
"last_error": "EAGAIN [Errno 11] Resource temporarily unavailable",
|
||||
"last_run_at": (now - timedelta(minutes=error_age_minutes)).isoformat(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestPersistedStaleErrorRecovery:
|
||||
def test_stale_error_recurring_job_redispatches_without_force_run(
|
||||
self, cron_env, monkeypatch
|
||||
):
|
||||
"""GREEN: a recurring job wedged in a persisted stale-error state with
|
||||
next_run_at parked in the future is re-armed and re-dispatches on the
|
||||
next tick — no force-run, no resume."""
|
||||
S, E, J, env = _setup(cron_env, monkeypatch)
|
||||
job_id = env["job_id"]
|
||||
|
||||
_persist_stale_error(J, job_id, error_age_minutes=110)
|
||||
|
||||
job = J.get_job(job_id)
|
||||
with mock.patch("cron.jobs.load_jobs", return_value=[job]):
|
||||
n = S.tick(verbose=False, sync=True)
|
||||
|
||||
latest = E.latest_execution(job_id)
|
||||
assert latest is not None, (
|
||||
"wedged stale-error job must be re-dispatched without force-run"
|
||||
)
|
||||
assert latest["status"] == "completed"
|
||||
# The recovery must be countable / probe-visible.
|
||||
from cron import jobs as Jmod
|
||||
stats = Jmod.get_persisted_error_recovery_stats()
|
||||
assert stats["persisted_error_recoveries"] >= 1
|
||||
|
||||
def test_two_consecutive_auto_fires_after_persisted_recovery(
|
||||
self, cron_env, monkeypatch
|
||||
):
|
||||
"""GREEN: after the persisted recovery re-arms the job, it fires on
|
||||
consecutive ticks (no manual intervention)."""
|
||||
S, E, J, env = _setup(cron_env, monkeypatch)
|
||||
job_id = env["job_id"]
|
||||
|
||||
_persist_stale_error(J, job_id, error_age_minutes=110)
|
||||
job = J.get_job(job_id)
|
||||
with mock.patch("cron.jobs.load_jobs", return_value=[job]):
|
||||
S.tick(verbose=False, sync=True)
|
||||
latest1 = E.latest_execution(job_id)
|
||||
assert latest1["status"] == "completed"
|
||||
|
||||
# Re-arm due normally and tick again: fire #2.
|
||||
now = datetime.now(timezone.utc)
|
||||
J.update_job(job_id, {"next_run_at": (now - timedelta(minutes=1)).isoformat()})
|
||||
with mock.patch("cron.jobs.load_jobs", return_value=[J.get_job(job_id)]):
|
||||
S.tick(verbose=False, sync=True)
|
||||
latest2 = E.latest_execution(job_id)
|
||||
assert latest2["status"] == "completed"
|
||||
assert latest2["id"] != latest1["id"], "two distinct executions"
|
||||
|
||||
def test_recent_error_within_cadence_not_force_rearmed(self, cron_env, monkeypatch):
|
||||
"""The discriminator must NOT re-arm a normal transient error whose
|
||||
last_run is within a full cadence — that job retries on its own
|
||||
schedule and must be left alone (otherwise we'd over-fire healthy
|
||||
erroring jobs)."""
|
||||
S, E, J, env = _setup(cron_env, monkeypatch)
|
||||
job_id = env["job_id"]
|
||||
|
||||
_persist_stale_error(J, job_id, error_age_minutes=5) # within cadence
|
||||
|
||||
job = J.get_job(job_id)
|
||||
with mock.patch("cron.jobs.load_jobs", return_value=[job]):
|
||||
S.tick(verbose=False, sync=True)
|
||||
|
||||
latest = E.latest_execution(job_id)
|
||||
assert latest is None, (
|
||||
"a within-cadence error is a normal retry — must not be "
|
||||
"force-re-armed onto an immediate fire"
|
||||
)
|
||||
Reference in New Issue
Block a user