From 8f9abc98730861cff1606870f2d82d3f253fb862 Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Mon, 24 Aug 2026 01:12:22 +0800 Subject: [PATCH] fix(cron): re-anchor stale next_run_at after direct jobs.json schedule edits get_due_jobs() fires purely off the stored next_run_at <= now, with no check that the stored instant is still an occurrence of the schedule's current expression. A direct jobs.json edit that narrows schedule.expr (e.g. daily "0 7 * * *" -> weekdays "0 7 * * 1-5") keeps the stored next_run_at computed under the old expression, so the job fires on days the new expression excludes. The within-grace fire and the catch-up "run once now" path both inherit the wrong instant. Add a best-effort stale-schedule guard on the fire path: when the stored next_run_at is not an occurrence of the current cron expression, re-anchor it via compute_next_run() from the current expression and skip the fire. Non-cron kinds, missing expr, croniter unavailability, and malformed input all report a match so the fire path keeps its existing semantics. Recomputation uses the current expression, so the re-anchor converges and cannot defer a valid job forever. Fixes #93049 --- cron/jobs.py | 58 +++++++++++++ tests/cron/test_due_stale_cron_edit.py | 114 +++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 tests/cron/test_due_stale_cron_edit.py diff --git a/cron/jobs.py b/cron/jobs.py index b13393f108..c0c103fa56 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -1028,6 +1028,35 @@ def get_persisted_error_recovery_stats() -> Dict[str, Any]: } +def _cron_next_run_matches_expr( + schedule: Dict[str, Any], + next_run_dt: datetime, +) -> bool: + """Whether ``next_run_dt`` is an occurrence of the schedule's current expr. + + A direct ``jobs.json`` edit can change ``schedule.expr`` while leaving the + stored ``next_run_at`` computed under the *old* expression (#93049). The + stored instant is stale exactly when it is not an occurrence of the + current expression. Validation is best-effort: anything that cannot be + checked (non-cron kind, missing expr, croniter unavailable, malformed + input) reports a match so the fire path keeps its existing semantics. + """ + if schedule.get("kind") != "cron": + return True + expr = schedule.get("expr") + if not expr or not _ensure_croniter() or croniter is None: + return True + try: + # The last occurrence at-or-before the stored instant: base croniter + # a second past it so an exact occurrence is included, then compare + # at second granularity (croniter returns second-precision datetimes). + base = next_run_dt + timedelta(seconds=1) + prev = croniter(str(expr), base).get_prev(datetime) + return abs((prev - next_run_dt).total_seconds()) < 1.0 + except Exception: + return True + + def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]: """ Compute the next run time for a schedule. @@ -3406,6 +3435,35 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: if next_run_dt <= now: + # Stale-schedule guard (#93049): a direct jobs.json edit that + # changed schedule.expr leaves next_run_at computed under the + # old expression, so the job would fire at an instant the + # current expression excludes. Both the within-grace fire and + # the catch-up "run once now" below inherit that wrong instant, + # so re-anchor before either can fire. Recomputation uses the + # current expression, so this converges — it cannot defer + # forever. + if kind == "cron" and not _cron_next_run_matches_expr( + schedule, next_run_dt + ): + new_next = compute_next_run(schedule, now.isoformat()) + logger.info( + "Job '%s' next_run_at %s does not match its current " + "cron expression %r (direct jobs.json edit?); " + "re-anchoring to %s without firing.", + job.get("name", job.get("id", "?")), + next_run, + schedule.get("expr"), + new_next, + ) + if new_next: + for rj in raw_jobs: + if rj["id"] == job["id"]: + rj["next_run_at"] = new_next + needs_save = True + break + continue + # For recurring jobs, check if the scheduled time is stale # (gateway was down and missed the window). Fast-forward to # the next future occurrence instead of firing a stale run. diff --git a/tests/cron/test_due_stale_cron_edit.py b/tests/cron/test_due_stale_cron_edit.py new file mode 100644 index 0000000000..98e4fdc84d --- /dev/null +++ b/tests/cron/test_due_stale_cron_edit.py @@ -0,0 +1,114 @@ +"""Stale-schedule guard on the cron fire path (#93049). + +A direct ``jobs.json`` edit that changes ``schedule.expr`` leaves the stored +``next_run_at`` computed under the *old* expression — e.g. editing a daily +``0 7 * * *`` job down to ``0 7 * * 1-5`` keeps a Saturday 07:00 +``next_run_at``. The old due check only compared ``next_run_at <= now``, so +the job fired on the day the new expression excludes. The guard re-anchors +``next_run_at`` from the current expression and skips the fire. + +These exercise the real store against a temp ``HERMES_HOME`` (no mocks) per +the E2EE-over-mocks discipline for file-touching code. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta + +import pytest + + +@pytest.fixture +def temp_home(tmp_path, monkeypatch): + """Isolated HERMES_HOME so jobs.json doesn't touch the real store.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + yield tmp_path + + +def _write_cron_job(schedule_expr: str, next_run_at: datetime) -> str: + """Persist a cron job with a pinned next_run_at (the stale-edit shape).""" + from cron.jobs import create_job, save_jobs, load_jobs + + job = create_job(prompt="x", schedule="every 5m", name="t") + jobs = load_jobs() + for j in jobs: + if j["id"] == job["id"]: + j["schedule"] = {"kind": "cron", "expr": schedule_expr} + j["next_run_at"] = next_run_at.isoformat() + save_jobs(jobs) + return job["id"] + + +# 2026-08-22 is a Saturday; 2026-08-24 is the following Monday. +_SATURDAY_0700 = datetime.fromisoformat("2026-08-22T07:00:00+00:00") +_MONDAY_0700 = datetime.fromisoformat("2026-08-24T07:00:00+00:00") + + +def test_stale_next_run_on_excluded_dow_does_not_fire(temp_home, monkeypatch): + """next_run_at computed under the old daily expr must not fire the new + Mon-Fri schedule on a Saturday — the job re-anchors to Monday instead.""" + from cron.jobs import get_due_jobs, get_job + + monkeypatch.setattr( + "cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(seconds=30) + ) + jid = _write_cron_job("0 7 * * 1-5", _SATURDAY_0700) + + due = get_due_jobs() + + assert [j["id"] for j in due if j["id"] == jid] == [] + stored = get_job(jid) + assert stored["next_run_at"].startswith(_MONDAY_0700.isoformat()) + + +def test_matching_next_run_still_fires(temp_home, monkeypatch): + """Control: the same stored instant under an expression it matches (daily) + still fires — the guard only blocks schedule-drifted instants.""" + from cron.jobs import get_due_jobs + + monkeypatch.setattr( + "cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(seconds=30) + ) + jid = _write_cron_job("0 7 * * *", _SATURDAY_0700) + + due = get_due_jobs() + + assert jid in [j["id"] for j in due] + + +def test_stale_next_run_skips_even_inside_catchup_window(temp_home, monkeypatch): + """The catch-up 'run once now' policy must not resurrect a stale instant: + hours after the stored time, the drifted job re-anchors without firing.""" + from cron.jobs import get_due_jobs, get_job + + monkeypatch.setattr( + "cron.jobs._hermes_now", lambda: _SATURDAY_0700 + timedelta(hours=5) + ) + jid = _write_cron_job("0 7 * * 1-5", _SATURDAY_0700) + + due = get_due_jobs() + + assert [j["id"] for j in due if j["id"] == jid] == [] + assert get_job(jid)["next_run_at"].startswith(_MONDAY_0700.isoformat()) + + +def test_helper_reports_match_for_non_cron_and_unvalidatable(temp_home): + """Best-effort validation: non-cron kinds, missing expr, and croniter + unavailable all report a match so the fire path is unchanged there.""" + from cron.jobs import _cron_next_run_matches_expr + + assert _cron_next_run_matches_expr({"kind": "interval"}, _SATURDAY_0700) is True + assert _cron_next_run_matches_expr({"kind": "cron"}, _SATURDAY_0700) is True + # A croniter-matching instant reports a match; an excluded one does not. + assert ( + _cron_next_run_matches_expr( + {"kind": "cron", "expr": "0 7 * * *"}, _SATURDAY_0700 + ) + is True + ) + assert ( + _cron_next_run_matches_expr( + {"kind": "cron", "expr": "0 7 * * 1-5"}, _SATURDAY_0700 + ) + is False + )