fix(cron): re-anchor stale next_run_at after direct jobs.json schedule edits

get_due_jobs() fires purely off the stored next_run_at <= now, with no
check that the stored instant is still an occurrence of the schedule's
current expression. A direct jobs.json edit that narrows schedule.expr
(e.g. daily "0 7 * * *" -> weekdays "0 7 * * 1-5") keeps the stored
next_run_at computed under the old expression, so the job fires on days
the new expression excludes. The within-grace fire and the catch-up
"run once now" path both inherit the wrong instant.

Add a best-effort stale-schedule guard on the fire path: when the stored
next_run_at is not an occurrence of the current cron expression,
re-anchor it via compute_next_run() from the current expression and skip
the fire. Non-cron kinds, missing expr, croniter unavailability, and
malformed input all report a match so the fire path keeps its existing
semantics. Recomputation uses the current expression, so the re-anchor
converges and cannot defer a valid job forever.

Fixes #93049
This commit is contained in:
liuhao1024
2026-08-24 01:12:22 +08:00
committed by Teknium
parent 5d8b031514
commit 8f9abc9873
2 changed files with 172 additions and 0 deletions
+58
View File
@@ -1028,6 +1028,35 @@ def get_persisted_error_recovery_stats() -> Dict[str, Any]:
}
def _cron_next_run_matches_expr(
schedule: Dict[str, Any],
next_run_dt: datetime,
) -> bool:
"""Whether ``next_run_dt`` is an occurrence of the schedule's current expr.
A direct ``jobs.json`` edit can change ``schedule.expr`` while leaving the
stored ``next_run_at`` computed under the *old* expression (#93049). The
stored instant is stale exactly when it is not an occurrence of the
current expression. Validation is best-effort: anything that cannot be
checked (non-cron kind, missing expr, croniter unavailable, malformed
input) reports a match so the fire path keeps its existing semantics.
"""
if schedule.get("kind") != "cron":
return True
expr = schedule.get("expr")
if not expr or not _ensure_croniter() or croniter is None:
return True
try:
# The last occurrence at-or-before the stored instant: base croniter
# a second past it so an exact occurrence is included, then compare
# at second granularity (croniter returns second-precision datetimes).
base = next_run_dt + timedelta(seconds=1)
prev = croniter(str(expr), base).get_prev(datetime)
return abs((prev - next_run_dt).total_seconds()) < 1.0
except Exception:
return True
def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]:
"""
Compute the next run time for a schedule.
@@ -3406,6 +3435,35 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:
if next_run_dt <= now:
# Stale-schedule guard (#93049): a direct jobs.json edit that
# changed schedule.expr leaves next_run_at computed under the
# old expression, so the job would fire at an instant the
# current expression excludes. Both the within-grace fire and
# the catch-up "run once now" below inherit that wrong instant,
# so re-anchor before either can fire. Recomputation uses the
# current expression, so this converges — it cannot defer
# forever.
if kind == "cron" and not _cron_next_run_matches_expr(
schedule, next_run_dt
):
new_next = compute_next_run(schedule, now.isoformat())
logger.info(
"Job '%s' next_run_at %s does not match its current "
"cron expression %r (direct jobs.json edit?); "
"re-anchoring to %s without firing.",
job.get("name", job.get("id", "?")),
next_run,
schedule.get("expr"),
new_next,
)
if new_next:
for rj in raw_jobs:
if rj["id"] == job["id"]:
rj["next_run_at"] = new_next
needs_save = True
break
continue
# For recurring jobs, check if the scheduled time is stale
# (gateway was down and missed the window). Fast-forward to
# the next future occurrence instead of firing a stale run.
+114
View File
@@ -0,0 +1,114 @@
"""Stale-schedule guard on the cron fire path (#93049).
A direct ``jobs.json`` edit that changes ``schedule.expr`` leaves the stored
``next_run_at`` computed under the *old* expression — e.g. editing a daily
``0 7 * * *`` job down to ``0 7 * * 1-5`` keeps a Saturday 07:00
``next_run_at``. The old due check only compared ``next_run_at <= now``, so
the job fired on the day the new expression excludes. The guard re-anchors
``next_run_at`` from the current expression and skips the fire.
These exercise the real store against a temp ``HERMES_HOME`` (no mocks) per
the E2EE-over-mocks discipline for file-touching code.
"""
from __future__ import annotations
from datetime import datetime, timedelta
import pytest
@pytest.fixture
def temp_home(tmp_path, monkeypatch):
"""Isolated HERMES_HOME so jobs.json doesn't touch the real store."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
yield tmp_path
def _write_cron_job(schedule_expr: str, next_run_at: datetime) -> str:
"""Persist a cron job with a pinned next_run_at (the stale-edit shape)."""
from cron.jobs import create_job, save_jobs, load_jobs
job = create_job(prompt="x", schedule="every 5m", name="t")
jobs = load_jobs()
for j in jobs:
if j["id"] == job["id"]:
j["schedule"] = {"kind": "cron", "expr": schedule_expr}
j["next_run_at"] = next_run_at.isoformat()
save_jobs(jobs)
return job["id"]
# 2026-08-22 is a Saturday; 2026-08-24 is the following Monday.
_SATURDAY_0700 = datetime.fromisoformat("2026-08-22T07:00:00+00:00")
_MONDAY_0700 = datetime.fromisoformat("2026-08-24T07:00:00+00:00")
def test_stale_next_run_on_excluded_dow_does_not_fire(temp_home, monkeypatch):
"""next_run_at computed under the old daily expr must not fire the new
Mon-Fri schedule on a Saturday — the job re-anchors to Monday instead."""
from cron.jobs import get_due_jobs, get_job
monkeypatch.setattr(
"cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(seconds=30)
)
jid = _write_cron_job("0 7 * * 1-5", _SATURDAY_0700)
due = get_due_jobs()
assert [j["id"] for j in due if j["id"] == jid] == []
stored = get_job(jid)
assert stored["next_run_at"].startswith(_MONDAY_0700.isoformat())
def test_matching_next_run_still_fires(temp_home, monkeypatch):
"""Control: the same stored instant under an expression it matches (daily)
still fires — the guard only blocks schedule-drifted instants."""
from cron.jobs import get_due_jobs
monkeypatch.setattr(
"cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(seconds=30)
)
jid = _write_cron_job("0 7 * * *", _SATURDAY_0700)
due = get_due_jobs()
assert jid in [j["id"] for j in due]
def test_stale_next_run_skips_even_inside_catchup_window(temp_home, monkeypatch):
"""The catch-up 'run once now' policy must not resurrect a stale instant:
hours after the stored time, the drifted job re-anchors without firing."""
from cron.jobs import get_due_jobs, get_job
monkeypatch.setattr(
"cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(hours=5)
)
jid = _write_cron_job("0 7 * * 1-5", _SATURDAY_0700)
due = get_due_jobs()
assert [j["id"] for j in due if j["id"] == jid] == []
assert get_job(jid)["next_run_at"].startswith(_MONDAY_0700.isoformat())
def test_helper_reports_match_for_non_cron_and_unvalidatable(temp_home):
"""Best-effort validation: non-cron kinds, missing expr, and croniter
unavailable all report a match so the fire path is unchanged there."""
from cron.jobs import _cron_next_run_matches_expr
assert _cron_next_run_matches_expr({"kind": "interval"}, _SATURDAY_0700) is True
assert _cron_next_run_matches_expr({"kind": "cron"}, _SATURDAY_0700) is True
# A croniter-matching instant reports a match; an excluded one does not.
assert (
_cron_next_run_matches_expr(
{"kind": "cron", "expr": "0 7 * * *"}, _SATURDAY_0700
)
is True
)
assert (
_cron_next_run_matches_expr(
{"kind": "cron", "expr": "0 7 * * 1-5"}, _SATURDAY_0700
)
is False
)