From bcb412cd9d0566648d44b30bc1af2590a42db7f3 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:03:46 +0530 Subject: [PATCH] refactor(cron): share the fire-path telemetry recorder _record_timezone_migration_catchup was a line-for-line clone of _record_persisted_error_recovery (counter bump, bounded recent list, best-effort jsonl append). Extract _append_telemetry_record and route both through it; one shared history cap replaces the two per-counter constants. Also correct the "distinct from catch_up_occurrences" comment: a migrated row that is also past its grace window increments both. No behavior change; both recorders write the same entries to the same files. --- cron/jobs.py | 61 +++++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/cron/jobs.py b/cron/jobs.py index 2d0d44fdf6..85ed2a9848 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -1248,7 +1248,8 @@ def _classify_dispatch_lateness(lateness_seconds: float, grace_seconds: int) -> # ``next_run_at`` to now so the next tick re-dispatches it, exactly like the # operator's force-run / mech_red_guard's ``cron resume`` but built-in. _persisted_error_recoveries: int = 0 -_PERSISTED_ERROR_RECOVERY_HISTORY = 20 +# Bounded in-memory history kept by every probe-visible fire-path counter. +_TELEMETRY_RECENT_HISTORY = 20 _persisted_error_recoveries_recent: list = [] @@ -1351,26 +1352,38 @@ def _schedule_cadence_seconds(schedule: Dict[str, Any]) -> Optional[float]: _cron_cadence_cache: Dict[str, Optional[float]] = {} +def _append_telemetry_record(filename: str, entry: Dict[str, Any], recent: list) -> None: + """Keep ``entry`` in the bounded in-memory ``recent`` list and append it to + ``/`` (best effort — telemetry must never break a tick). + + Shared by the probe-visible fire-path counters (persisted-error recovery, + timezone-migration catch-up); each keeps its own module-level int counter + because tests reset those by name. + """ + recent.append(entry) + del recent[:-_TELEMETRY_RECENT_HISTORY] + try: + path = _current_cron_store().cron_dir / filename + _ensure_cron_dir(path.parent) + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(entry) + "\n") + except Exception as exc: + logger.debug("Could not append %s record: %s", filename, exc) + + def _record_persisted_error_recovery(job: Dict[str, Any], previous_next_run: str) -> None: """Persist a countable, probe-visible signal for one stale-error re-arm.""" global _persisted_error_recoveries - now = _hermes_now() entry = { "job_id": job.get("id"), "name": job.get("name") or job.get("id"), "previous_next_run_at": previous_next_run, - "rearmed_at": now.isoformat(), + "rearmed_at": _hermes_now().isoformat(), } _persisted_error_recoveries += 1 - _persisted_error_recoveries_recent.append(entry) - del _persisted_error_recoveries_recent[:-_PERSISTED_ERROR_RECOVERY_HISTORY] - try: - path = _current_cron_store().cron_dir / "persisted_error_recoveries.jsonl" - _ensure_cron_dir(path.parent) - with open(path, "a", encoding="utf-8") as fh: - fh.write(json.dumps(entry) + "\n") - except Exception as exc: # never let telemetry break a tick - logger.debug("Could not append persisted-error-recovery record: %s", exc) + _append_telemetry_record( + "persisted_error_recoveries.jsonl", entry, _persisted_error_recoveries_recent + ) def get_persisted_error_recovery_stats() -> Dict[str, Any]: @@ -1460,13 +1473,13 @@ def _classify_stale_cron_next_run( # Durable, probe-visible counter for offset-representation migrations caught -# on the fire path. Distinct from `catch_up_occurrences` (which counts runs -# skipped past their grace window) because this one means "an upgrade rewrote -# how next_run_at is represented" — an operator seeing it climb after a deploy -# is seeing the migration drain, and seeing it climb steadily afterwards is -# seeing a timezone that keeps changing under the store. +# on the fire path. Kept separate from `catch_up_occurrences` (runs skipped +# past their grace window — a migrated row that is ALSO past grace increments +# both) because this one means "an upgrade rewrote how next_run_at is +# represented" — an operator seeing it climb after a deploy is seeing the +# migration drain, and seeing it climb steadily afterwards is seeing a +# timezone that keeps changing under the store. _timezone_migration_catchups: int = 0 -_TIMEZONE_MIGRATION_CATCHUP_HISTORY = 20 _timezone_migration_catchups_recent: list = [] @@ -1486,15 +1499,9 @@ def _record_timezone_migration_catchup( "fired_at": _hermes_now().isoformat(), } _timezone_migration_catchups += 1 - _timezone_migration_catchups_recent.append(entry) - del _timezone_migration_catchups_recent[:-_TIMEZONE_MIGRATION_CATCHUP_HISTORY] - try: - path = _current_cron_store().cron_dir / "timezone_migration_catchups.jsonl" - _ensure_cron_dir(path.parent) - with open(path, "a", encoding="utf-8") as fh: - fh.write(json.dumps(entry) + "\n") - except Exception as exc: # never let telemetry break a tick - logger.debug("Could not append timezone-migration-catchup record: %s", exc) + _append_telemetry_record( + "timezone_migration_catchups.jsonl", entry, _timezone_migration_catchups_recent + ) def get_timezone_migration_catchup_stats() -> Dict[str, Any]: