fix(cron): re-anchor stale next_run_at after direct jobs.json schedule edits
get_due_jobs() fires purely off the stored next_run_at <= now, with no check that the stored instant is still an occurrence of the schedule's current expression. A direct jobs.json edit that narrows schedule.expr (e.g. daily "0 7 * * *" -> weekdays "0 7 * * 1-5") keeps the stored next_run_at computed under the old expression, so the job fires on days the new expression excludes. The within-grace fire and the catch-up "run once now" path both inherit the wrong instant. Add a best-effort stale-schedule guard on the fire path: when the stored next_run_at is not an occurrence of the current cron expression, re-anchor it via compute_next_run() from the current expression and skip the fire. Non-cron kinds, missing expr, croniter unavailability, and malformed input all report a match so the fire path keeps its existing semantics. Recomputation uses the current expression, so the re-anchor converges and cannot defer a valid job forever. Fixes #93049
This commit is contained in:
@@ -1028,6 +1028,35 @@ def get_persisted_error_recovery_stats() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _cron_next_run_matches_expr(
|
||||
schedule: Dict[str, Any],
|
||||
next_run_dt: datetime,
|
||||
) -> bool:
|
||||
"""Whether ``next_run_dt`` is an occurrence of the schedule's current expr.
|
||||
|
||||
A direct ``jobs.json`` edit can change ``schedule.expr`` while leaving the
|
||||
stored ``next_run_at`` computed under the *old* expression (#93049). The
|
||||
stored instant is stale exactly when it is not an occurrence of the
|
||||
current expression. Validation is best-effort: anything that cannot be
|
||||
checked (non-cron kind, missing expr, croniter unavailable, malformed
|
||||
input) reports a match so the fire path keeps its existing semantics.
|
||||
"""
|
||||
if schedule.get("kind") != "cron":
|
||||
return True
|
||||
expr = schedule.get("expr")
|
||||
if not expr or not _ensure_croniter() or croniter is None:
|
||||
return True
|
||||
try:
|
||||
# The last occurrence at-or-before the stored instant: base croniter
|
||||
# a second past it so an exact occurrence is included, then compare
|
||||
# at second granularity (croniter returns second-precision datetimes).
|
||||
base = next_run_dt + timedelta(seconds=1)
|
||||
prev = croniter(str(expr), base).get_prev(datetime)
|
||||
return abs((prev - next_run_dt).total_seconds()) < 1.0
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Compute the next run time for a schedule.
|
||||
@@ -3406,6 +3435,35 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:
|
||||
|
||||
if next_run_dt <= now:
|
||||
|
||||
# Stale-schedule guard (#93049): a direct jobs.json edit that
|
||||
# changed schedule.expr leaves next_run_at computed under the
|
||||
# old expression, so the job would fire at an instant the
|
||||
# current expression excludes. Both the within-grace fire and
|
||||
# the catch-up "run once now" below inherit that wrong instant,
|
||||
# so re-anchor before either can fire. Recomputation uses the
|
||||
# current expression, so this converges — it cannot defer
|
||||
# forever.
|
||||
if kind == "cron" and not _cron_next_run_matches_expr(
|
||||
schedule, next_run_dt
|
||||
):
|
||||
new_next = compute_next_run(schedule, now.isoformat())
|
||||
logger.info(
|
||||
"Job '%s' next_run_at %s does not match its current "
|
||||
"cron expression %r (direct jobs.json edit?); "
|
||||
"re-anchoring to %s without firing.",
|
||||
job.get("name", job.get("id", "?")),
|
||||
next_run,
|
||||
schedule.get("expr"),
|
||||
new_next,
|
||||
)
|
||||
if new_next:
|
||||
for rj in raw_jobs:
|
||||
if rj["id"] == job["id"]:
|
||||
rj["next_run_at"] = new_next
|
||||
needs_save = True
|
||||
break
|
||||
continue
|
||||
|
||||
# For recurring jobs, check if the scheduled time is stale
|
||||
# (gateway was down and missed the window). Fast-forward to
|
||||
# the next future occurrence instead of firing a stale run.
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Stale-schedule guard on the cron fire path (#93049).
|
||||
|
||||
A direct ``jobs.json`` edit that changes ``schedule.expr`` leaves the stored
|
||||
``next_run_at`` computed under the *old* expression — e.g. editing a daily
|
||||
``0 7 * * *`` job down to ``0 7 * * 1-5`` keeps a Saturday 07:00
|
||||
``next_run_at``. The old due check only compared ``next_run_at <= now``, so
|
||||
the job fired on the day the new expression excludes. The guard re-anchors
|
||||
``next_run_at`` from the current expression and skips the fire.
|
||||
|
||||
These exercise the real store against a temp ``HERMES_HOME`` (no mocks) per
|
||||
the E2EE-over-mocks discipline for file-touching code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_home(tmp_path, monkeypatch):
|
||||
"""Isolated HERMES_HOME so jobs.json doesn't touch the real store."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
yield tmp_path
|
||||
|
||||
|
||||
def _write_cron_job(schedule_expr: str, next_run_at: datetime) -> str:
|
||||
"""Persist a cron job with a pinned next_run_at (the stale-edit shape)."""
|
||||
from cron.jobs import create_job, save_jobs, load_jobs
|
||||
|
||||
job = create_job(prompt="x", schedule="every 5m", name="t")
|
||||
jobs = load_jobs()
|
||||
for j in jobs:
|
||||
if j["id"] == job["id"]:
|
||||
j["schedule"] = {"kind": "cron", "expr": schedule_expr}
|
||||
j["next_run_at"] = next_run_at.isoformat()
|
||||
save_jobs(jobs)
|
||||
return job["id"]
|
||||
|
||||
|
||||
# 2026-08-22 is a Saturday; 2026-08-24 is the following Monday.
|
||||
_SATURDAY_0700 = datetime.fromisoformat("2026-08-22T07:00:00+00:00")
|
||||
_MONDAY_0700 = datetime.fromisoformat("2026-08-24T07:00:00+00:00")
|
||||
|
||||
|
||||
def test_stale_next_run_on_excluded_dow_does_not_fire(temp_home, monkeypatch):
|
||||
"""next_run_at computed under the old daily expr must not fire the new
|
||||
Mon-Fri schedule on a Saturday — the job re-anchors to Monday instead."""
|
||||
from cron.jobs import get_due_jobs, get_job
|
||||
|
||||
monkeypatch.setattr(
|
||||
"cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(seconds=30)
|
||||
)
|
||||
jid = _write_cron_job("0 7 * * 1-5", _SATURDAY_0700)
|
||||
|
||||
due = get_due_jobs()
|
||||
|
||||
assert [j["id"] for j in due if j["id"] == jid] == []
|
||||
stored = get_job(jid)
|
||||
assert stored["next_run_at"].startswith(_MONDAY_0700.isoformat())
|
||||
|
||||
|
||||
def test_matching_next_run_still_fires(temp_home, monkeypatch):
|
||||
"""Control: the same stored instant under an expression it matches (daily)
|
||||
still fires — the guard only blocks schedule-drifted instants."""
|
||||
from cron.jobs import get_due_jobs
|
||||
|
||||
monkeypatch.setattr(
|
||||
"cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(seconds=30)
|
||||
)
|
||||
jid = _write_cron_job("0 7 * * *", _SATURDAY_0700)
|
||||
|
||||
due = get_due_jobs()
|
||||
|
||||
assert jid in [j["id"] for j in due]
|
||||
|
||||
|
||||
def test_stale_next_run_skips_even_inside_catchup_window(temp_home, monkeypatch):
|
||||
"""The catch-up 'run once now' policy must not resurrect a stale instant:
|
||||
hours after the stored time, the drifted job re-anchors without firing."""
|
||||
from cron.jobs import get_due_jobs, get_job
|
||||
|
||||
monkeypatch.setattr(
|
||||
"cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(hours=5)
|
||||
)
|
||||
jid = _write_cron_job("0 7 * * 1-5", _SATURDAY_0700)
|
||||
|
||||
due = get_due_jobs()
|
||||
|
||||
assert [j["id"] for j in due if j["id"] == jid] == []
|
||||
assert get_job(jid)["next_run_at"].startswith(_MONDAY_0700.isoformat())
|
||||
|
||||
|
||||
def test_helper_reports_match_for_non_cron_and_unvalidatable(temp_home):
|
||||
"""Best-effort validation: non-cron kinds, missing expr, and croniter
|
||||
unavailable all report a match so the fire path is unchanged there."""
|
||||
from cron.jobs import _cron_next_run_matches_expr
|
||||
|
||||
assert _cron_next_run_matches_expr({"kind": "interval"}, _SATURDAY_0700) is True
|
||||
assert _cron_next_run_matches_expr({"kind": "cron"}, _SATURDAY_0700) is True
|
||||
# A croniter-matching instant reports a match; an excluded one does not.
|
||||
assert (
|
||||
_cron_next_run_matches_expr(
|
||||
{"kind": "cron", "expr": "0 7 * * *"}, _SATURDAY_0700
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
_cron_next_run_matches_expr(
|
||||
{"kind": "cron", "expr": "0 7 * * 1-5"}, _SATURDAY_0700
|
||||
)
|
||||
is False
|
||||
)
|
||||
Reference in New Issue
Block a user