diff --git a/cron/jobs.py b/cron/jobs.py index 089299bd8c..a5427d5327 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -899,6 +899,132 @@ def _compute_grace_seconds(schedule: dict) -> int: return MIN_GRACE +# Durable (persisted-state) recovery counter for a recurring job wedged in a +# stale ``last_status == "error"`` state with ``next_run_at`` parked in the +# future. This is the restart-surviving half of the recurring-cron wedge +# (t_20e23f84 / t_8b5480b3): the in-memory stale-claim sweep in scheduler.py +# heals a leaked ``_running_job_ids`` claim in-process, but a job whose +# persisted ``next_run_at`` was re-armed into the future (the normal +# post-error re-arm by ``mark_job_run``) is invisible to that sweep — it is +# not in the running set, so nothing force-releases it, and it is not due, so +# ``get_due_jobs`` never returns it. It just sits. This recovery re-arms +# ``next_run_at`` to now so the next tick re-dispatches it, exactly like the +# operator's force-run / mech_red_guard's ``cron resume`` but built-in. +_persisted_error_recoveries: int = 0 +_PERSISTED_ERROR_RECOVERY_HISTORY = 20 +_persisted_error_recoveries_recent: list = [] + + +def _job_is_stale_error_recurring( + job: Dict[str, Any], + schedule: Dict[str, Any], + now: datetime, +) -> bool: + """True when a recurring job is wedged in a stale persisted error state. + + Condition (all must hold): + * it is a recurring (cron/interval) job (checked by caller); + * its persisted ``last_status == "error"`` (a prior fire errored and the + job never recovered); + * it has NOT successfully re-fired within its natural cadence — its + ``last_run_at`` is older than ``cadence + grace``, so this is not a + normal transient-error retry that will fire on its own soon, it is a + job that has been sitting errored for a full period with no recovery; + * it is not currently running in this process (a live run must never be + re-armed underneath itself, #62002-style). + + ``last_run_at`` being older than one cadence is the key discriminator: a + job that errors and is retried on its normal schedule keeps ``last_run_at`` + fresh (mark_job_run stamps it on every fire, success or failure), so the + stale check does not fire for a job that is merely erroring-and-retrying. + """ + if job.get("last_status") != "error": + return False + if _job_running_in_this_process(str(job.get("id") or "")): + return False + last_run = job.get("last_run_at") + if not last_run: + return False + try: + last_run_dt = _ensure_aware(datetime.fromisoformat(last_run)) + except (ValueError, TypeError): + return False + age_seconds = (now - last_run_dt).total_seconds() + if age_seconds < 0: + return False + cadence_seconds = _schedule_cadence_seconds(schedule) + if cadence_seconds is None: + # Unknown cadence (croniter unavailable / malformed expr): fall back to + # the grace window so a badly-parked job is still recovered, but never + # re-arm anything younger than the 2h grace cap. + cadence_seconds = _compute_grace_seconds(schedule) + return age_seconds > (cadence_seconds + _compute_grace_seconds(schedule)) + + +def _schedule_cadence_seconds(schedule: Dict[str, Any]) -> Optional[float]: + """Approximate the natural period of a schedule, in seconds, or None. + + Interval jobs use ``minutes * 60``. Cron jobs measure the gap between the + next two fire times with croniter (falling back to None when croniter is + missing or the expr is malformed). + """ + if not isinstance(schedule, dict): + return None + kind = schedule.get("kind") + if kind == "interval": + minutes = schedule.get("minutes") + try: + return float(minutes) * 60.0 if minutes else None + except (TypeError, ValueError): + return None + if kind == "cron": + if not _ensure_croniter(): + return None + expr = schedule.get("expr") + if not expr: + return None + try: + base = _hermes_now() + it = croniter(expr, base) + first = it.get_next(datetime) + second = it.get_next(datetime) + gap = (second - first).total_seconds() + return gap if gap > 0 else None + except Exception: + return None + return None + + +def _record_persisted_error_recovery(job: Dict[str, Any], previous_next_run: str) -> None: + """Persist a countable, probe-visible signal for one stale-error re-arm.""" + global _persisted_error_recoveries + now = _hermes_now() + entry = { + "job_id": job.get("id"), + "name": job.get("name") or job.get("id"), + "previous_next_run_at": previous_next_run, + "rearmed_at": now.isoformat(), + } + _persisted_error_recoveries += 1 + _persisted_error_recoveries_recent.append(entry) + del _persisted_error_recoveries_recent[:-_PERSISTED_ERROR_RECOVERY_HISTORY] + try: + path = _current_cron_store().cron_dir / "persisted_error_recoveries.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(entry) + "\n") + except Exception as exc: # never let telemetry break a tick + logger.debug("Could not append persisted-error-recovery record: %s", exc) + + +def get_persisted_error_recovery_stats() -> Dict[str, Any]: + """Probe-visible snapshot of persisted-error recoveries.""" + return { + "persisted_error_recoveries": _persisted_error_recoveries, + "recent": list(_persisted_error_recoveries_recent), + } + + def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]: """ Compute the next run time for a schedule. @@ -3096,6 +3222,37 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: break continue + # Persisted-state stale-error recovery (t_8b5480b3): a recurring + # job whose persisted state shows last_status=error and whose + # last_run_at is older than a full cadence has been sitting wedged + # (errored once, next_run_at re-armed into the future by + # mark_job_run, but never re-dispatched — the restart-surviving + # half of the 2026-08-14 incident, invisible to the in-memory + # in-flight sweep). Re-arm next_run_at to now so the next tick + # returns it as due and re-dispatches it WITHOUT force-run/resume. + if ( + kind in ("cron", "interval") + and next_run_dt > now + and _job_is_stale_error_recurring(job, schedule, now) + ): + jid = job.get("id") + logger.warning( + "cron.persisted_error.recovered job='%s' id=%s — recurring " + "job wedged in stale last_status=error without re-firing for " + "a full cadence; re-arming next_run_at to now so it " + "re-dispatches without force-run/resume", + job.get("name", jid), + jid, + ) + _record_persisted_error_recovery(job, next_run) + job["next_run_at"] = now.isoformat() + next_run_dt = now + for rj in raw_jobs: + if rj["id"] == jid: + rj["next_run_at"] = now.isoformat() + needs_save = True + break + if next_run_dt <= now: # For recurring jobs, check if the scheduled time is stale diff --git a/tests/cron/test_recurring_eagain_redispatch.py b/tests/cron/test_recurring_eagain_redispatch.py index 5e9d499541..4bf56a0ecc 100644 --- a/tests/cron/test_recurring_eagain_redispatch.py +++ b/tests/cron/test_recurring_eagain_redispatch.py @@ -1,4 +1,4 @@ -"""Deterministic reproduction of the recurring-cron EAGAIN wedge (t_8b5480b3). +"""Deterministic reproduction of the recurring-cron EAGAIN in-memory wedge (t_8b5480b3). Scenario modelled on the 2026-08-14 incident: a recurring no_agent interval job whose script subprocess raises EAGAIN ([Errno 11] Resource temporarily @@ -6,12 +6,17 @@ unavailable) during a substrate thread-exhaustion spike. After the failure is recorded (terminal 'failed' execution row), the job must be re-dispatched on the NEXT tick once the substrate recovers — with no force-run. -The os-reviewer (t_20e23f84) established the real incident wedge: 4 recurring -no_agent jobs recorded ZERO executions for ~1h47m after EAGAIN while -`next_run_at` kept advancing (they stayed 'due') but `_submit_with_guard` -never dispatched them, and the wedge SURVIVED a gateway restart. That points -at a PERSISTED non-dispatch state, not just the in-memory `_running_job_ids` -leak (which t_3778a491 already bounds). +SCOPE NOTE (honest): this file exercises the in-memory EAGAIN-release class — +the claim taken by ``_submit_with_guard`` is released when the submit/init +path fails, so the NEXT tick re-dispatches. It passes on unfixed base because +a terminal ``failed`` execution row does not itself suppress a recurring job's +next due fire; that is pre-existing behavior, not the fix. The persisted-state +(restart-surviving) half of the incident — a recurring job whose persisted +``last_status=error`` and ``next_run_at`` parked in the future is never +re-dispatched without force-run/resume — is covered by +``test_recurring_persisted_error_recovery.py``, which provides the clean +behavioral RED (unfixed: no execution) / GREEN (fixed: re-dispatched) for the +new recovery path. This file drives the REAL `tick()` end-to-end against a throwaway HERMES_HOME: tick 1 -> script EAGAINs (subprocess.run raises OSError 11) -> failed exec row diff --git a/tests/cron/test_recurring_persisted_error_recovery.py b/tests/cron/test_recurring_persisted_error_recovery.py new file mode 100644 index 0000000000..50b86673fe --- /dev/null +++ b/tests/cron/test_recurring_persisted_error_recovery.py @@ -0,0 +1,163 @@ +"""Persisted-state stale-error recovery for recurring cron jobs (t_8b5480b3). + +The 2026-08-14 incident (t_20e23f84): 4 recurring no_agent interval jobs +EAGAIN-failed at 12:50 and then recorded ZERO executions for ~1h47m (~9 missed +cycles) even after the substrate recovered, and the wedge SURVIVED a gateway +restart. The in-memory stale-claim sweep (cron/scheduler.py, t_3778a491) +heals a leaked `_running_job_ids` claim in-process, but that is only the +in-process half. A recurring job whose *persisted* state shows +`last_status == "error"` and whose `next_run_at` was re-armed into the future +by `mark_job_run` is invisible to the in-memory sweep: it is not in the +running set (nothing force-releases it) and it is not due (so `get_due_jobs` +never returns it). It just sits — exactly the restart-surviving symptom +documented in the incident. + +This recovery (cron/jobs.py `_get_due_jobs_locked`) re-arms `next_run_at` to +now for a recurring job that: + * is in a persisted `last_status == "error"` state, AND + * has NOT re-fired within a full cadence (`last_run_at` older than + cadence + grace — so it is NOT a normal transient-error retry that will + fire on its own soon), AND + * is not currently running in this process. + +The scheduler then re-dispatches it on the next tick WITHOUT force-run/resume +— the built-in equivalent of mech_red_guard's `cron resume`. + +Deterministic RED/GREEN: on unfixed code the wedged job is left untouched (no +execution row, no re-dispatch); with the recovery it re-fires and completes. +""" +from __future__ import annotations + +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest import mock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + + +@pytest.fixture +def cron_env(tmp_path, monkeypatch): + """Isolated cron env + a recurring no_agent interval job.""" + hermes_home = tmp_path / ".hermes" + hermes_home.mkdir() + (hermes_home / "cron").mkdir() + (hermes_home / "cron" / "output").mkdir() + (hermes_home / "scripts").mkdir() + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + import cron.jobs as jobs_mod + monkeypatch.setattr(jobs_mod, "HERMES_DIR", hermes_home) + monkeypatch.setattr(jobs_mod, "CRON_DIR", hermes_home / "cron") + monkeypatch.setattr(jobs_mod, "JOBS_FILE", hermes_home / "cron" / "jobs.json") + monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", hermes_home / "cron" / "output") + + job = jobs_mod.create_job( + prompt="probe", + schedule="every 10m", + no_agent=True, + script="probe.py", + ) + script = hermes_home / "scripts" / "probe.py" + script.write_text("print('ok')\n") + return {"home": hermes_home, "job_id": job["id"]} + + +def _setup(cron_env, monkeypatch): + from cron import scheduler as S + from cron import executions as E + import cron.jobs as J + + env = cron_env + monkeypatch.setattr(E, "EXECUTIONS_FILE", env["home"] / "cron" / "executions.db") + monkeypatch.setattr(S, "_hermes_home", env["home"]) + return S, E, J, env + + +def _persist_stale_error(J, job_id, *, error_age_minutes=110): + """Persist the incident wedge state: last_status=error, last_run stale by + more than a full cadence, next_run_at re-armed into the future.""" + now = datetime.now(timezone.utc) + J.update_job( + job_id, + { + "next_run_at": (now + timedelta(minutes=5)).isoformat(), + "last_status": "error", + "last_error": "EAGAIN [Errno 11] Resource temporarily unavailable", + "last_run_at": (now - timedelta(minutes=error_age_minutes)).isoformat(), + }, + ) + + +class TestPersistedStaleErrorRecovery: + def test_stale_error_recurring_job_redispatches_without_force_run( + self, cron_env, monkeypatch + ): + """GREEN: a recurring job wedged in a persisted stale-error state with + next_run_at parked in the future is re-armed and re-dispatches on the + next tick — no force-run, no resume.""" + S, E, J, env = _setup(cron_env, monkeypatch) + job_id = env["job_id"] + + _persist_stale_error(J, job_id, error_age_minutes=110) + + job = J.get_job(job_id) + with mock.patch("cron.jobs.load_jobs", return_value=[job]): + n = S.tick(verbose=False, sync=True) + + latest = E.latest_execution(job_id) + assert latest is not None, ( + "wedged stale-error job must be re-dispatched without force-run" + ) + assert latest["status"] == "completed" + # The recovery must be countable / probe-visible. + from cron import jobs as Jmod + stats = Jmod.get_persisted_error_recovery_stats() + assert stats["persisted_error_recoveries"] >= 1 + + def test_two_consecutive_auto_fires_after_persisted_recovery( + self, cron_env, monkeypatch + ): + """GREEN: after the persisted recovery re-arms the job, it fires on + consecutive ticks (no manual intervention).""" + S, E, J, env = _setup(cron_env, monkeypatch) + job_id = env["job_id"] + + _persist_stale_error(J, job_id, error_age_minutes=110) + job = J.get_job(job_id) + with mock.patch("cron.jobs.load_jobs", return_value=[job]): + S.tick(verbose=False, sync=True) + latest1 = E.latest_execution(job_id) + assert latest1["status"] == "completed" + + # Re-arm due normally and tick again: fire #2. + now = datetime.now(timezone.utc) + J.update_job(job_id, {"next_run_at": (now - timedelta(minutes=1)).isoformat()}) + with mock.patch("cron.jobs.load_jobs", return_value=[J.get_job(job_id)]): + S.tick(verbose=False, sync=True) + latest2 = E.latest_execution(job_id) + assert latest2["status"] == "completed" + assert latest2["id"] != latest1["id"], "two distinct executions" + + def test_recent_error_within_cadence_not_force_rearmed(self, cron_env, monkeypatch): + """The discriminator must NOT re-arm a normal transient error whose + last_run is within a full cadence — that job retries on its own + schedule and must be left alone (otherwise we'd over-fire healthy + erroring jobs).""" + S, E, J, env = _setup(cron_env, monkeypatch) + job_id = env["job_id"] + + _persist_stale_error(J, job_id, error_age_minutes=5) # within cadence + + job = J.get_job(job_id) + with mock.patch("cron.jobs.load_jobs", return_value=[job]): + S.tick(verbose=False, sync=True) + + latest = E.latest_execution(job_id) + assert latest is None, ( + "a within-cadence error is a normal retry — must not be " + "force-re-armed onto an immediate fire" + )