diff --git a/cron/jobs.py b/cron/jobs.py index e2ed85ce1f..e3bfe95eb5 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -1,4 +1,5 @@ -"""Cron job storage: ~/.hermes/cron/jobs.json; output in ~/.hermes/cron/output/{job_id}/{timestamp}.md""" +"""Cron job storage: ~/.hermes/cron/jobs.json; output in +~/.hermes/cron/output/{job_id}/{timestamp}.md""" import contextlib import copy @@ -64,7 +65,7 @@ HERMES_DIR = get_hermes_home().resolve() CRON_DIR = HERMES_DIR / "cron" JOBS_FILE = CRON_DIR / "jobs.json" # Heartbeat: touched every ticker loop so `hermes cron status` can tell the ticker THREAD is alive, -# not just the gateway PROCESS. Success: last tick that completed WITHOUT raising. +# not just the gateway PROCESS; success = last tick that completed WITHOUT raising. TICKER_HEARTBEAT_FILE = CRON_DIR / "ticker_heartbeat" TICKER_SUCCESS_FILE = CRON_DIR / "ticker_last_success" # Single source of truth for the ticker interval (scheduler_provider.py) and the staleness @@ -107,12 +108,10 @@ _IMPORT_STORE = _CronStorePaths(CRON_DIR, JOBS_FILE, OUTPUT_DIR) def _current_cron_store() -> _CronStorePaths: - """Paths pinned to this execution context's profile. - - Precedence: (1) active use_cron_store() override; (2) deliberately re-pointed module constants; - (3) the ACTIVE profile home via get_hermes_home(), so re-pointing HERMES_HOME after import uses - ITS OWN store rather than the user's real jobs.json frozen at import; (4) import-time constants. - """ + """Paths pinned to this execution context's profile. Precedence: (1) active use_cron_store() + override; (2) deliberately re-pointed module constants; (3) the ACTIVE profile home via + get_hermes_home(), so re-pointing HERMES_HOME after import uses ITS OWN store rather than the + user's real jobs.json frozen at import; (4) import-time constants.""" override = _cron_store_override.get() if override is not None: return override @@ -142,11 +141,12 @@ def get_cron_output_dir() -> Path: # Fallback stale-recovery window for a one-shot's running-claim when HERMES_CRON_TIMEOUT=0 -# (unlimited, no bound to derive from); also the floor so a tiny timeout can't expire a claim mid-run. +# (unlimited, no bound to derive from); also the floor so a tiny timeout can't expire a claim +# mid-run. ONESHOT_RUN_CLAIM_TTL_SECONDS = 1800 -# Derived TTL = inactivity timeout × this headroom. The TTL only recovers a claim left by a tick that -# DIED mid-run; the timeout is an *inactivity* limit, not a wall-clock cap, so healthy runs may +# Derived TTL = inactivity timeout × this headroom. The TTL only recovers a claim left by a tick +# that DIED mid-run; the timeout is an *inactivity* limit, not a wall-clock cap, so healthy runs may # legitimately exceed it — hence the headroom. _ONESHOT_RUN_CLAIM_TTL_HEADROOM = 3 @@ -167,10 +167,9 @@ def _oneshot_run_claim_ttl_seconds() -> float: def _job_running_in_this_process(job_id: str) -> bool: - """True when the scheduler in THIS process is still running ``job_id``. - - The run_claim TTL alone cannot distinguish "claiming tick died" from "alive but slow"; the - in-process running set settles the single-gateway case. Lazy import: scheduler imports us.""" + """True when the scheduler in THIS process is still running ``job_id``: the run_claim TTL alone + cannot distinguish "claiming tick died" from "alive but slow". Lazy import: scheduler imports + us.""" try: from cron.scheduler import get_running_job_ids return job_id in get_running_job_ids() @@ -188,11 +187,10 @@ def _jobs_lock_file() -> Path: def _acquire_flock(lock_fd, timeout: float) -> Optional[bool]: - """Bounded exclusive lock: True when acquired, False on timeout, None when no backend exists. - - A blocking flock(LOCK_EX) has NO timeout and is taken while holding the in-process lock, so a - wedged sibling process would freeze EVERY cron function here forever. Poll LOCK_NB against a - deadline instead; the caller decides the degraded mode on timeout.""" + """Bounded exclusive lock: True when acquired, False on timeout, None when no backend exists. A + blocking flock(LOCK_EX) taken under the in-process lock would let a wedged sibling freeze + EVERY cron function forever, so poll LOCK_NB against a deadline; the caller picks the + degraded mode.""" if fcntl is not None: deadline = time.monotonic() + timeout while True: @@ -224,13 +222,12 @@ def _release_flock(lock_fd) -> None: @contextlib.contextmanager def _jobs_lock(): - """Serialize a load_jobs→modify→save_jobs critical section. - - In-process RLock (parallel tick threads) plus a cross-process flock on ``/.jobs.lock`` - (gateway vs. CLI writes — otherwise a `cron pause` could be clobbered and keep firing). Sections - are short (field updates only). Nested calls in one thread reuse the held lock. Without a flock - backend, or on flock timeout (logged loudly), it degrades to in-process-only locking: a briefly - torn cross-process write beats a dead scheduler.""" + """Serialize a load_jobs→modify→save_jobs critical section: in-process RLock (parallel tick + threads) plus a cross-process flock on ``/.jobs.lock`` (gateway vs. CLI writes — + otherwise a `cron pause` could be clobbered and keep firing). Nested calls in one thread + reuse the held lock. Without a flock backend, or on flock timeout (logged loudly), it + degrades to in-process-only locking: a briefly torn cross-process write beats a dead + scheduler.""" depth = getattr(_jobs_lock_state, "depth", 0) if depth: _jobs_lock_state.depth = depth + 1 @@ -278,11 +275,9 @@ def _jobs_lock(): @contextlib.contextmanager def _fire_job_lock(job_id: str): - """Serialize one job's owner mutations and external side effects. - - Unlike the global jobs lock this may be held across network delivery; scoped to one profile + - job so unrelated jobs keep progressing. Fails closed when cross-process locking is unavailable. - """ + """Serialize one job's owner mutations and external side effects. Unlike the global jobs lock + this may be held across network delivery; scoped to one profile + job so unrelated jobs keep + progressing. Fails closed when cross-process locking is unavailable.""" cron_dir = _current_cron_store().cron_dir lock_key = f"{cron_dir.resolve()}::{job_id}" with _fire_fence_locks_guard: @@ -444,10 +439,8 @@ def _normalize_job_record(job: Dict[str, Any]) -> Dict[str, Any]: """Read-safe job shape: legacy/hand-edited records may have nullable ``prompt``, ``name``, ``schedule_display``. Storage is untouched; consumers never crash on formatting.""" normalized = _apply_skill_fields(job) - job_id = _coerce_job_text(normalized.get("id"), "unknown") - prompt = _coerce_job_text(normalized.get("prompt")) - normalized["id"] = job_id - normalized["prompt"] = prompt + job_id = normalized["id"] = _coerce_job_text(normalized.get("id"), "unknown") + prompt = normalized["prompt"] = _coerce_job_text(normalized.get("prompt")) name = _coerce_job_text(normalized.get("name")).strip() if not name: label_source = ( @@ -499,12 +492,10 @@ def is_terminal_job(job: Dict[str, Any]) -> bool: def _is_recoverable_error_job(job: Dict[str, Any]) -> bool: - """True for a recurring job stuck in ``state=error``. - - ``state=error`` is set ONLY when ``compute_next_run()`` fails for a cron/interval job (croniter - missing, malformed schedule); such a job still has future occurrences once the issue resolves, - so treating it as terminal would block due-scan self-heal, pre-advance, dispatch claim and - ``resume_job`` — wedging it forever. ``is_terminal_job()`` alone means "truly done".""" + """True for a recurring job stuck in ``state=error`` (set ONLY when ``compute_next_run()`` fails + for a cron/interval job: croniter missing, malformed schedule). Such a job still has future + occurrences once the issue resolves, so treating it as terminal would block due-scan self-heal, + pre-advance, dispatch claim and ``resume_job`` — wedging it forever.""" return ( job.get("state") == "error" and (job.get("schedule") or {}).get("kind") in {"cron", "interval"} @@ -525,12 +516,9 @@ def _secure_file(path: Path): def _preserve_file_ownership(path: Path, before: Optional[os.stat_result]) -> None: - """Restore a rewritten file's previous owner (POSIX, root writer only). - - Atomic replace makes the file owned by the writer's euid; a root CLI write (e.g. ``docker exec``) - against the unprivileged gateway's store would flip jobs.json to root:root 0600 and lock the - ticker out of every later tick. Root can hand ownership back, so chown when the owner differed. - """ + """Restore a rewritten file's previous owner (POSIX, root writer only): atomic replace makes the + file owned by the writer's euid, so a root CLI write (e.g. ``docker exec``) against the + unprivileged gateway's store would flip jobs.json to root:root 0600 and lock the ticker out.""" if before is None or os.name != "posix": return try: @@ -577,8 +565,9 @@ def ensure_dirs(): # --- Schedule Parsing --- def normalize_repeat_value(repeat: Any) -> Optional[int]: - """Coerce a repeat value (int or user-facing string) into ``Optional[int]``: ``'forever'``-family - -> None, ``'once'``-family -> 1, numeric -> int, 0/negative -> None, else ValueError.""" + """Coerce a repeat value (int or user-facing string) into ``Optional[int]``: + ``'forever'``-family -> None, ``'once'``-family -> 1, numeric -> int, 0/negative -> None, + else ValueError.""" if repeat is None: return None if isinstance(repeat, str): @@ -601,7 +590,7 @@ _DURATION_MULTIPLIERS = {'m': 1, 'h': 60, 'd': 1440} def parse_duration(s: str) -> int: - """Parse a duration string into minutes: "30m" → 30, "2h" → 120, "1d" → 1440, bare "hour" → 60.""" + """Parse a duration into minutes: "30m" → 30, "2h" → 120, "1d" → 1440, bare "hour" → 60.""" s = s.strip().lower() match = re.match(r'^(\d*)\s*(m|min|mins|minute|minutes|h|hr|hrs|hour|hours|d|day|days)$', s) if not match: @@ -693,7 +682,9 @@ def _natural_every_to_cron(rest: str) -> Optional[str]: return f"{minute} {hour} * * {dow}" -def _cron_schedule(expr: str, display: str, missing_croniter: str, invalid_label: str) -> Dict[str, Any]: +def _cron_schedule( + expr: str, display: str, missing_croniter: str, invalid_label: str +) -> Dict[str, Any]: """Validate a cron expression with croniter and build the stored schedule dict.""" if not _ensure_croniter(): raise ValueError(f"{missing_croniter} Install with: pip install croniter") @@ -787,7 +778,8 @@ def _ensure_aware(dt: datetime) -> datetime: def _parse_aware(value: Any) -> Optional[datetime]: - """``_ensure_aware(datetime.fromisoformat(value))``, or None when *value* is not a parseable ISO string.""" + """``_ensure_aware(datetime.fromisoformat(value))``, or None when *value* is not a parseable ISO + string.""" try: return _ensure_aware(datetime.fromisoformat(value)) except Exception: @@ -851,7 +843,8 @@ def _classify_dispatch_lateness(lateness_seconds: float, grace_seconds: int) -> return "on_time" -# Recovery counter for recurring jobs wedged in stale ``last_status == "error"`` with a future next_run_at. +# Recovery counter for recurring jobs wedged in stale ``last_status == "error"`` with a future +# next_run_at. _persisted_error_recoveries: int = 0 # Bounded in-memory history kept by every probe-visible fire-path counter. _TELEMETRY_RECENT_HISTORY = 20 @@ -862,9 +855,9 @@ def _job_is_stale_error_recurring( job: Dict[str, Any], schedule: Dict[str, Any], now: datetime, ) -> bool: """True when a recurring job (caller-checked) is wedged in a stale persisted error state: - ``last_status == "error"``, not running in this process (a live run must never be re-armed - underneath itself), and ``last_run_at`` older than ``cadence + grace``. Age is the key - discriminator: a job merely erroring-and-retrying on schedule stays fresh and is not flagged.""" + ``last_status == "error"``, not running in this process (never re-arm a live run underneath + itself), and ``last_run_at`` older than ``cadence + grace`` (a job merely erroring-and-retrying + on schedule stays fresh and is not flagged).""" if job.get("last_status") != "error": return False if _job_running_in_this_process(str(job.get("id") or "")): @@ -960,9 +953,10 @@ def get_persisted_error_recovery_stats() -> Dict[str, Any]: def _cron_next_run_matches_expr(schedule: Dict[str, Any], next_run_dt: datetime) -> bool: - """Whether ``next_run_dt`` is an occurrence of the schedule's current expr (detects a hand-edited - ``schedule.expr`` whose stored ``next_run_at`` came from the old one). Best-effort: anything - uncheckable (non-cron, no expr, no croniter, malformed) reports a match.""" + """Whether ``next_run_dt`` is an occurrence of the schedule's current expr (detects a + hand-edited ``schedule.expr`` whose stored ``next_run_at`` came from the old one). + Best-effort: anything uncheckable (non-cron, no expr, no croniter, malformed) reports a + match.""" if schedule.get("kind") != "cron": return True expr = schedule.get("expr") @@ -977,7 +971,8 @@ def _cron_next_run_matches_expr(schedule: Dict[str, Any], next_run_dt: datetime) return True -# Classifications for a due cron instant NOT on the current expr (see _classify_stale_cron_next_run). +# Classifications for a due cron instant NOT on the current expr (see +# _classify_stale_cron_next_run). STALE_CRON_MATCH = "match" STALE_CRON_TIMEZONE_MIGRATION = "timezone_migration" STALE_CRON_EXPR_EDIT = "expr_edit" @@ -987,13 +982,12 @@ def _classify_stale_cron_next_run( schedule: Dict[str, Any], raw_next_run_dt: datetime, next_run_dt: datetime, ) -> str: """Explain WHY a stored ``next_run_at`` misses the current cron lattice; the causes need - opposite actions. - - ``expr_edit``: a hand edit changed ``schedule.expr`` — re-anchor WITHOUT firing. - ``timezone_migration``: only the offset representation changed (legacy UTC rows normalized into - the profile tz); treating it as an edit would skip a due, never-fired occurrence. Discriminator: - a stored instant whose OWN wall clock is a legal occurrence, when normalization moved the wall - clock, is a migration. When offsets agree a genuine expr edit can never be misread as one.""" + opposite actions. ``expr_edit``: a hand edit changed ``schedule.expr`` — re-anchor WITHOUT + firing. ``timezone_migration``: only the offset representation changed (legacy UTC rows + normalized into the profile tz); treating it as an edit would skip a due, never-fired + occurrence. Discriminator: normalization moved the wall clock AND the stored instant's OWN + wall clock is a legal occurrence (when offsets agree a genuine expr edit can never be misread + as a migration).""" if _cron_next_run_matches_expr(schedule, next_run_dt): return STALE_CRON_MATCH wall_clock_shifted = raw_next_run_dt.replace(tzinfo=None) != next_run_dt.replace(tzinfo=None) @@ -1002,7 +996,8 @@ def _classify_stale_cron_next_run( return STALE_CRON_EXPR_EDIT -# Offset-migration catch-ups on the fire path: climbing after a deploy = draining; steady = TZ churn. +# Offset-migration catch-ups on the fire path: climbing after a deploy = draining; steady = TZ +# churn. _timezone_migration_catchups: int = 0 _timezone_migration_catchups_recent: list = [] @@ -1066,7 +1061,8 @@ def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None # --- Ticker heartbeat (liveness signal for `hermes cron status`) --- def _write_marker(name: str, text: str, tmp_prefix: str) -> None: - """Atomic (never torn) best-effort marker write; failures swallowed so markers never break the tick.""" + """Atomic (never torn) best-effort marker write; failures swallowed so markers never break the + tick.""" try: ensure_dirs() atomic_write_text(_current_cron_store().cron_dir / name, text, tmp_prefix=tmp_prefix) @@ -1092,7 +1088,8 @@ def _epoch_file_age(name: str) -> Optional[float]: def get_ticker_heartbeat_age() -> Optional[float]: - """Seconds since the ticker loop last iterated; None = missing/unreadable ("cannot determine", not "dead").""" + """Seconds since the ticker loop last iterated; None = missing/unreadable ("cannot determine", + not "dead").""" return _epoch_file_age("ticker_heartbeat") @@ -1116,7 +1113,8 @@ def record_catch_up_occurrence() -> None: def record_ticker_error(message: str) -> None: - """Persist the latest tick failure so `cron status` (another process) can show WHY, not just staleness.""" + """Persist the latest tick failure so `cron status` (another process) can show WHY, not just + staleness.""" _write_marker("ticker_last_error", f"{time.time()}\n{message.strip()}\n", ".terr_") @@ -1142,7 +1140,8 @@ def get_ticker_last_error() -> Optional[str]: def _parse_jobs_file(jobs_file: Path) -> Tuple[Any, bool]: """Tolerant jobs.json parse -> ``(data, used_strict_fallback)``: utf-8-sig absorbs a BOM, strict - failure retries with ``strict=False``. IO/fallback errors propagate (caller decides repair vs bail).""" + failure retries with ``strict=False``. IO/fallback errors propagate (caller decides repair vs + bail).""" with open(jobs_file, "r", encoding="utf-8-sig") as f: raw = f.read() try: @@ -1155,7 +1154,8 @@ def load_jobs() -> List[Dict[str, Any]]: """Load all jobs from storage.""" jobs_file = _current_cron_store().jobs_file ensure_dirs() - # Stamp BEFORE reading (fail-safe, see _record_load_stamp): a racing write then forces the merge. + # Stamp BEFORE reading (fail-safe, see _record_load_stamp): a racing write then forces the + # merge. pre_read_stamp = _jobs_file_stamp(jobs_file) if not jobs_file.exists(): _record_load_stamp(None) @@ -1170,7 +1170,8 @@ def load_jobs() -> List[Dict[str, Any]]: logger.error("Failed to auto-repair jobs.json: %s", e) raise RuntimeError(f"Cron database corrupted and unrepairable: {e}") from e - # Accept the canonical dict, or a bare list (auto-repair); any other top-level shape is corruption. + # Accept the canonical dict, or a bare list (auto-repair); any other top-level shape is + # corruption. repair = "had invalid control characters" if _strict_retry else None if isinstance(data, dict): jobs = data.get("jobs", []) @@ -1216,7 +1217,8 @@ def _peek_jobs_unlocked() -> Optional[List[Dict[str, Any]]]: def _jobs_file_stamp(jobs_file: Path) -> Optional[Tuple[int, int, int]]: """Shrink-merge fast-path stamp ``(mtime_ns, size, ino)``; ``None`` if unstatable. ``st_ino`` is - included because every writer uses mkstemp+rename, so a same-size write in one mtime quantum can't false-match.""" + included because every writer uses mkstemp+rename, so a same-size write in one mtime quantum + can't false-match.""" try: st = jobs_file.stat() return (st.st_mtime_ns, st.st_size, st.st_ino) @@ -1226,8 +1228,9 @@ def _jobs_file_stamp(jobs_file: Path) -> Optional[Tuple[int, int, int]]: def _record_load_stamp(stamp: Optional[Tuple[int, int, int]]) -> None: """Remember jobs.json's stamp for the enclosing _jobs_lock() section (no-op outside one) so the - save path can skip the shrink-merge when disk provably hasn't changed. Capture it BEFORE reading: - a mid-read sibling then mismatches (fail-safe); stamping after would certify an unseen write.""" + save path can skip the shrink-merge when disk provably hasn't changed. Capture it BEFORE + reading: a mid-read sibling then mismatches (fail-safe); stamping after would certify an + unseen write.""" if getattr(_jobs_lock_state, "depth", 0): _jobs_lock_state.load_stamp = stamp @@ -1235,8 +1238,9 @@ def _record_load_stamp(stamp: Optional[Tuple[int, int, int]]) -> None: def _unmerged_disk_jobs( jobs: List[Dict[str, Any]], removed_ids: Optional[Collection[str]] ) -> List[Dict[str, Any]]: - """On-disk jobs missing from *jobs* and not intentionally removed. Stamp match => nothing landed, - return without parsing; unreadable store => ``[]`` (never merge against an unknown baseline).""" + """On-disk jobs missing from *jobs* and not intentionally removed. Stamp match => nothing + landed, return without parsing; unreadable store => ``[]`` (never merge against an unknown + baseline).""" stamp = getattr(_jobs_lock_state, "load_stamp", None) if stamp is not None and _jobs_file_stamp(_current_cron_store().jobs_file) == stamp: return [] @@ -1259,8 +1263,9 @@ def _unmerged_disk_jobs( def _merge_unexpected_disk_jobs( jobs: List[Dict[str, Any]], *, removed_ids: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """*jobs* plus on-disk jobs absent from the payload (under the degraded flock-timeout path a stale - writer would otherwise clobber concurrent creates). Deletes pass ``removed_ids``; never mutates *jobs*.""" + """*jobs* plus on-disk jobs absent from the payload (under the degraded flock-timeout path a + stale writer would otherwise clobber concurrent creates). Deletes pass ``removed_ids``; never + mutates *jobs*.""" recovered = _unmerged_disk_jobs(jobs, removed_ids) if not recovered: return jobs @@ -1302,7 +1307,8 @@ def _save_jobs_unlocked( replace: bool = False, ): """Save all jobs; caller must hold _jobs_lock(). ``removed_ids`` = intentional deletes; - ``replace=True`` skips the shrink-merge guard (wholesale rewrite for tests/disaster recovery).""" + ``replace=True`` skips the shrink-merge guard (wholesale rewrite for tests/disaster + recovery).""" jobs_file = _current_cron_store().jobs_file ensure_dirs() # Owner snapshot BEFORE replace so a root writer can hand the file back to the gateway user. @@ -1353,8 +1359,11 @@ def save_jobs( _MISSING = object() -def _with_job(job_id: Any, fn: Callable[[List[Dict[str, Any]], int, Dict[str, Any]], Any], missing: Any = None) -> Any: - """Run ``fn(jobs, i, job)`` on the first match under ``_jobs_lock()``; ``fn`` saves. *missing* if none.""" +def _with_job( + job_id: Any, fn: Callable[[List[Dict[str, Any]], int, Dict[str, Any]], Any], missing: Any = None +) -> Any: + """Run ``fn(jobs, i, job)`` on the first match under ``_jobs_lock()``; ``fn`` saves. *missing* + if none.""" with _jobs_lock(): jobs = load_jobs() for i, job in enumerate(jobs): @@ -1420,24 +1429,19 @@ def _resolve_default_model_snapshot() -> Optional[str]: if isinstance(cron_model, str) and cron_model.strip(): return cron_model.strip() model_cfg = cfg.get("model") or {} - if isinstance(model_cfg, str): - return model_cfg.strip() or None if isinstance(model_cfg, dict): - default = model_cfg.get("default") or model_cfg.get("model") - if isinstance(default, str): - return default.strip() or None - return None + model_cfg = model_cfg.get("default") or model_cfg.get("model") + return model_cfg.strip() or None if isinstance(model_cfg, str) else None except Exception: return None -def _normalize_job_optional_text(value: Any, *, strip_trailing_slash: bool = False) -> Optional[str]: +def _normalize_job_optional_text( + value: Any, *, strip_trailing_slash: bool = False +) -> Optional[str]: if not isinstance(value, str): return None - text = value.strip() - if strip_trailing_slash: - text = text.rstrip("/") - return text or None + return (value.strip().rstrip("/") if strip_trailing_slash else value.strip()) or None def _normalize_base_url(value: Any) -> Optional[str]: @@ -1452,7 +1456,7 @@ def _normalize_str_list(items: Any) -> Optional[List[str]]: def _normalize_context_from(value: Any) -> Optional[List[str]]: """Accept a job id or a list of ids; anything else is None.""" if isinstance(value, str): - return _normalize_str_list([value]) + value = [value] return _normalize_str_list(value) if isinstance(value, list) else None @@ -1486,7 +1490,8 @@ def _normalize_reasoning_effort(value: Any) -> Optional[str]: return text -# Normalizers for create_job (all fields) / update_job (present fields). Invalid values raise BEFORE storing. +# Normalizers for create_job (all fields) / update_job (present fields). Invalid values raise BEFORE +# storing. _CREATE_FIELD_NORMALIZERS: Dict[str, Callable[[Any], Any]] = { "model": _normalize_job_optional_text, "provider": _normalize_job_optional_text, @@ -1522,7 +1527,7 @@ def _compute_provider_model_snapshots( provider_snapshot: Optional[str] = None model_snapshot: Optional[str] = None if normalized_provider is None: - try: + with contextlib.suppress(Exception): from hermes_cli.runtime_provider import resolve_runtime_provider runtime_kwargs = {"requested": None} @@ -1530,17 +1535,15 @@ def _compute_provider_model_snapshots( runtime_kwargs["explicit_base_url"] = normalized_base_url snap = resolve_runtime_provider(**runtime_kwargs) provider_snapshot = str(snap.get("provider") or "").strip().lower() or None - except Exception: - provider_snapshot = None if normalized_model is None: - try: + with contextlib.suppress(Exception): model_snapshot = _resolve_default_model_snapshot() or None - except Exception: - model_snapshot = None return provider_snapshot, model_snapshot -def _normalized_inference_axes(job: Dict[str, Any]) -> Tuple[Optional[str], Optional[str], Optional[str], bool]: +def _normalized_inference_axes( + job: Dict[str, Any], +) -> Tuple[Optional[str], Optional[str], Optional[str], bool]: """Return the stored inference-routing fields in their semantic form.""" return ( _normalize_job_optional_text(job.get("provider")), @@ -1550,9 +1553,13 @@ def _normalized_inference_axes(job: Dict[str, Any]) -> Tuple[Optional[str], Opti def _validate_job_mode_invariants( - monitor_script: Optional[str], monitor_url: Optional[str], no_agent: bool, script: Optional[str], + monitor_script: Optional[str], + monitor_url: Optional[str], + no_agent: bool, + script: Optional[str], ) -> None: - """Execution-mode invariants shared by create_job and update_job (no bypass via the update door).""" + """Execution-mode invariants shared by create_job and update_job (no bypass via the update + door).""" if monitor_script and monitor_url: raise ValueError( "monitor_script and monitor_url are mutually exclusive — a job " @@ -1575,8 +1582,8 @@ def _oneshot_past_grace_error(run_at: Any) -> ValueError: def _next_run_or_reject_past_oneshot( parsed_schedule: Dict[str, Any], label: str, fallback_run_at: Any, what: str, ) -> Optional[str]: - """``compute_next_run`` that raises (after a warning log) for a one-shot outside the grace window, - so a ghost job with ``next_run_at=None`` can never be stored.""" + """``compute_next_run`` that raises (after a warning log) for a one-shot outside the grace + window, so a ghost job with ``next_run_at=None`` can never be stored.""" next_run_at = compute_next_run(parsed_schedule) if parsed_schedule.get("kind") == "once" and next_run_at is None: run_at = parsed_schedule.get("run_at") or fallback_run_at @@ -1637,7 +1644,7 @@ def create_job( prompt_text = _coerce_job_text(prompt).strip() if not prompt_text and not f["script"] and not normalized_skills: raise ValueError(EMPTY_PAYLOAD_ERROR) - # Reject gateway-lifecycle commands (respawn loops) here, not just the CLI, to cover the cronjob tool. + # Reject gateway-lifecycle commands (respawn loops) here, not just in the CLI: covers the tool. from cron.lifecycle_guard import check_gateway_lifecycle check_gateway_lifecycle(prompt_text, f["script"]) @@ -1690,8 +1697,9 @@ def create_job( "enabled_toolsets": f["enabled_toolsets"], "workdir": f["workdir"], } - # Optional keys are persisted only when explicitly set: an absent key falls back to global config - # (attach/reasoning) or to ``deliver`` (failure_deliver), byte-identical to pre-feature jobs. + # Optional keys are persisted only when explicitly set: an absent key falls back to global + # config (attach/reasoning) or to ``deliver`` (failure_deliver), byte-identical to pre-feature + # jobs. for key, value in ( ("attach_to_session", normalized_attach), ("reasoning_effort", normalized_reasoning_effort), ("failure_deliver", f["failure_deliver"]), @@ -1728,9 +1736,9 @@ def resolve_job_ref(ref: str) -> Optional[Dict[str, Any]]: if not ref: return None jobs = load_jobs() - for job in jobs: - if job["id"] == ref: - return _normalize_job_record(job) + by_id = next((j for j in jobs if j["id"] == ref), None) + if by_id is not None: + return _normalize_job_record(by_id) ref_lower = ref.lower() name_matches = [j for j in jobs if (j.get("name") or "").lower() == ref_lower] if not name_matches: @@ -1804,8 +1812,13 @@ def _apply_schedule_update(updated: Dict[str, Any], updates: Dict[str, Any], job def _fill_missing_next_run(updated: Dict[str, Any]) -> None: - """An enabled, unpaused record must never persist without ``next_run_at`` (it would never fire).""" - if not updated.get("enabled", True) or updated.get("state") == "paused" or updated.get("next_run_at"): + """An enabled, unpaused record must never persist without ``next_run_at`` (it would never fire). + """ + if ( + not updated.get("enabled", True) + or updated.get("state") == "paused" + or updated.get("next_run_at") + ): return next_run = compute_next_run(updated["schedule"]) if next_run is None and updated["schedule"].get("kind") == "once": @@ -1844,11 +1857,12 @@ def update_job(job_id: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]] if "schedule" in updates: _apply_schedule_update(updated, updates, job_id) if inference_fields_changed: - updated["provider_snapshot"], updated["model_snapshot"] = _compute_provider_model_snapshots( + snapshots = _compute_provider_model_snapshots( provider=updated.get("provider"), model=updated.get("model"), base_url=updated.get("base_url"), no_agent=updated.get("no_agent")) + updated["provider_snapshot"], updated["model_snapshot"] = snapshots _fill_missing_next_run(updated) _reject_terminal_activation(job, updated, job_id) jobs[i] = updated @@ -1898,10 +1912,9 @@ def trigger_job(job_id: str, extra_prompt: Optional[str] = None) -> Optional[Dic if not job: return None if is_terminal_job(job): - state = job.get("state") name = job.get("name", job_id) raise ValueError( - f"Cannot run: job '{name}' is {state} (terminal). " + f"Cannot run: job '{name}' is {job.get('state')} (terminal). " f"Create a new occurrence with 'hermes cron resume {name} " "--run-now' or '--at '.") manual_run_at = _hermes_now().isoformat() @@ -1920,7 +1933,9 @@ def trigger_job(job_id: str, extra_prompt: Optional[str] = None) -> Optional[Dic def _claim_is_live(claim: Any, now: datetime, ttl_seconds: float) -> bool: """True for a well-formed claim aged within ``[0, ttl)``: future-dated (clock/TZ skew) or malformed claims count as stale so they can never wedge a job.""" - claimed_at = _parse_aware(claim.get("at")) if isinstance(claim, dict) and claim.get("at") else None + if not isinstance(claim, dict) or not claim.get("at"): + return False + claimed_at = _parse_aware(claim["at"]) return claimed_at is not None and 0 <= (now - claimed_at).total_seconds() < ttl_seconds @@ -1997,7 +2012,8 @@ def remove_job(job_id: str) -> bool: def _set_alert_flag(job_id: str, field: str, value: bool) -> bool: """Set/clear a persisted alert-dedup marker (alert exactly once until the condition heals; - survives restarts) and return the PRIOR value. Fields: ``preflight_alerted``, ``drift_alerted``.""" + survives restarts) and return the PRIOR value. Fields: ``preflight_alerted``, + ``drift_alerted``.""" def apply(jobs, _i, job): prior = bool(job.get(field)) if value: @@ -2028,10 +2044,12 @@ def mark_drift_alerted(job_id: str) -> bool: def note_fire_forward_failure(job_id: str, detail: str) -> bool: """Durably record (as ``last_fire_error``) that a scheduled fire could not be handed to the - runner — written by the dashboard fire webhook when the loopback forward fails. Without it the - miss is invisible (no execution row, last_status only covers started runs); mark_job_run clears it.""" + runner — written by the dashboard fire webhook when the loopback forward fails. Without it + the miss is invisible (no execution row, last_status only covers started runs); mark_job_run + clears it.""" def apply(jobs, _i, job): - job["last_fire_error"] = {"at": _hermes_now().isoformat(), "detail": str(detail or "")[:500]} + job["last_fire_error"] = { + "at": _hermes_now().isoformat(), "detail": str(detail or "")[:500]} save_jobs(jobs) return True @@ -2059,7 +2077,8 @@ def _record_run_outcome( job.pop("last_fire_error", None) job["failure_streak"] = 0 else: - # Consecutive agent-failure streak; delivery failures do NOT count (scheduler._failure_streak_nudge). + # Consecutive agent-failure streak; delivery failures do NOT count + # (scheduler._failure_streak_nudge). job["failure_streak"] = int(job.get("failure_streak") or 0) + 1 job["last_delivery_error"] = delivery_error # Clear both claims: the run is over, so the job is claimable again. @@ -2188,7 +2207,8 @@ def _write_wedged_oneshot_diagnostic(job: Dict[str, Any]) -> None: def _write_missed_oneshot_diagnostic(job: Dict[str, Any], next_run: str) -> None: - """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).""" + """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish). + """ _write_oneshot_diagnostic( job, "# Cron job removed before firing (run time outside grace window)\n\n" @@ -2207,13 +2227,10 @@ def _write_missed_oneshot_diagnostic(job: Dict[str, Any], next_run: str) -> None def claim_dispatch(job_id: str) -> bool: - """Atomically claim a finite one-shot dispatch BEFORE execution. - - Increments ``repeat.completed`` under the jobs lock and persists it, so a tick dying - mid-execution cannot lose the dispatch: finite one-shots become *at-most-times* instead of - *at-least-once*. Returns True if the caller may run the job, False when the limit is already - reached (the stale job is removed). Only ``kind == "once"`` with ``repeat.times > 0`` is claimed. - """ + """Atomically claim a finite one-shot dispatch BEFORE execution: ``repeat.completed`` is bumped + and persisted under the jobs lock so a tick dying mid-execution cannot lose the dispatch + (*at-most-times* instead of *at-least-once*). True if the caller may run the job; False when + the limit is already reached. Only ``kind == "once"`` with ``repeat.times > 0`` is claimed.""" def apply(jobs, i, job): repeat = job.get("repeat") or {} times = repeat.get("times") @@ -2225,17 +2242,21 @@ def claim_dispatch(job_id: str) -> bool: if completed >= times: if job.get("last_run_at") is not None: # A prior run completed normally (mark_job_run raced this tick). Retain the terminal - # record, as mark_job_run's repeat-limit branch does, instead of deleting the status. + # record, as mark_job_run's repeat-limit branch does, instead of deleting the + # status. _complete_job_record(job) save_jobs(jobs) - logger.info("Job '%s': dispatch limit reached (%d/%d) — marking completed", label, completed, times) + logger.info( + "Job '%s': dispatch limit reached (%d/%d) — marking completed", + label, completed, times) return False # A prior tick claimed the dispatch then died — a genuinely wedged claim. Remove it so # it stops appearing due, leaving an operator-visible diagnostic. jobs.pop(i) save_jobs(jobs, removed_ids={job_id}) _write_wedged_oneshot_diagnostic(job) - logger.info("Job '%s': dispatch limit reached (%d/%d) — removing", label, completed, times) + logger.info( + "Job '%s': dispatch limit reached (%d/%d) — removing", label, completed, times) return False # Claim this dispatch before the side effect runs. repeat["completed"] = completed + 1 @@ -2288,11 +2309,9 @@ def clear_run_claim(job_id: str) -> bool: def advance_next_runs(job_ids) -> int: - """Batch form of :func:`advance_next_run`: one load + at most one save for the whole due set. - - One-shot/unknown ids are skipped as in the per-job form. Returns the count advanced. Crash - semantics: persisted once at the end, so a crash mid-batch re-fires the whole set on restart - (at-least-once burst) rather than a prefix — acceptable given the sub-10ms window.""" + """Batch form of :func:`advance_next_run`: one load + at most one save for the whole due set; + one-shot/unknown ids are skipped. Returns the count advanced. Persisted once at the end, so a + crash mid-batch re-fires the whole set on restart rather than a prefix (sub-10ms window).""" ids = set(job_ids) if not ids: return 0 @@ -2342,15 +2361,13 @@ def claim_job_for_fire( job_id: str, *, claim_ttl_seconds: int = 300, force: bool = False, return_job: bool = False, ) -> Union[bool, Dict[str, Any]]: """Atomically claim a job for one external 'fire' (multi-machine at-most-once); True iff THIS - caller won. Used by ``CronScheduler.fire_due`` so exactly one of N replicas runs a job. - - Under the fence + file lock: reject missing/terminal/paused jobs unless ``force`` (an explicit - manual fire, which also enables/resumes the job atomically; external callbacks must leave it - false so a stale callback cannot resurrect a paused job). Lose if a fresh claim (younger than - ``claim_ttl_seconds``) exists. Otherwise stamp ``fire_claim`` and, for recurring jobs, advance - ``next_run_at`` so a stale re-delivery cannot re-fire; one-shots rely on the fresh claim alone. - The TTL lets another fire reclaim a job whose claimant crashed; mark_job_run clears the claim. - """ + caller won (``CronScheduler.fire_due``: exactly one of N replicas runs a job). Under the + fence + file lock: reject missing/terminal/paused jobs unless ``force`` (explicit manual + fire, which also resumes the job atomically; external callbacks must leave it false so a + stale callback cannot resurrect a paused job). Lose if a claim younger than + ``claim_ttl_seconds`` exists (the TTL lets another fire reclaim after a crash; mark_job_run + clears the claim). Otherwise stamp ``fire_claim`` and, for recurring jobs, advance + ``next_run_at`` so a stale re-delivery cannot re-fire.""" def apply(jobs, _i, job): if is_terminal_job(job) and not _is_recoverable_error_job(job): return False @@ -2379,8 +2396,10 @@ def claim_job_for_fire( def heartbeat_fire_claim(job_id: str, *, expected_owner: str) -> bool: """Refresh an active ``fire_claim`` without extending another owner's lease: an execution may outlive the TTL, and the owner check stops a stale runner from refreshing a recovered claim.""" - return _under_fire_fence(job_id, lambda: _with_job( - job_id, lambda jobs, _i, job: _refresh_claim(jobs, job.get("fire_claim"), expected_owner), False)) + def apply(jobs, _i, job): + return _refresh_claim(jobs, job.get("fire_claim"), expected_owner) + + return _under_fire_fence(job_id, lambda: _with_job(job_id, apply, False)) # Completed one-shots are retained in jobs.json (final status stays inspectable) and pruned by @@ -2408,10 +2427,9 @@ def _sweep_completed_oneshots( raw_jobs: List[Dict[str, Any]], now: datetime, *, removed_ids: Optional[Set[str]] = None, ) -> bool: """Prune completed one-shot records past retention (in place; True when anything was removed). - - Removed ids go into *removed_ids* so save_jobs's shrink-merge guard allows the delete. Only - ``kind == "once"`` records in state "completed" are candidates; age is measured from - ``last_run_at``, and a record without a parseable one is kept (never guess into deletion).""" + Removed ids go into *removed_ids* so save_jobs's shrink-merge guard allows the delete. Age is + measured from ``last_run_at``; a record without a parseable one is kept (never guess into + deletion).""" retention_days = _completed_oneshot_retention_days() if retention_days <= 0: return False @@ -2445,12 +2463,10 @@ def _sweep_completed_oneshots( # --- Due scan --- def get_due_jobs() -> List[Dict[str, Any]]: - """Return all jobs due now. - - A recurring job more than one period stale (gateway down, or a previous run overran the - interval) has its backlog collapsed — next_run_at fast-forwards so nothing burst-fires — but - still fires ONCE now, avoiding the perpetual-defer loop for runs longer than interval + grace. - That catch-up fire flows through mark_job_run, so it consumes one ``repeat.times`` run.""" + """Return all jobs due now. A recurring job more than one period stale (gateway down, or a run + overran the interval) has its backlog collapsed — next_run_at fast-forwards so nothing + burst-fires — but still fires ONCE now (via mark_job_run, consuming one ``repeat.times`` run), + avoiding the perpetual-defer loop for runs longer than interval + grace.""" with _jobs_lock(): return _get_due_jobs_locked() @@ -2485,9 +2501,8 @@ class _DueScan: def _normalize_due_scan_records(raw_jobs: List[Dict[str, Any]]) -> bool: """Repair malformed store records in place BEFORE the due scan keys off them: a missing ``id`` - (older writers used ``job_id``), non-dict ``schedule``, or non-ISO timestamp used to raise - mid-tick and abort the whole scan before save_jobs(), freezing the scheduler in a fast-forward - loop. Returns True when anything changed.""" + (older writers used ``job_id``), non-dict ``schedule``, or non-ISO timestamp used to abort the + whole scan before save_jobs(), freezing the scheduler in a fast-forward loop.""" changed = False for rj in raw_jobs: if not rj.get("id"): @@ -2531,7 +2546,8 @@ def _recover_missing_next_run(job: Dict[str, Any], scan: _DueScan) -> Optional[s and would otherwise be silently skipped forever.""" schedule = job.get("schedule", {}) kind = schedule.get("kind") - recovered_next = _recoverable_oneshot_run_at(schedule, scan.now, last_run_at=job.get("last_run_at")) + recovered_next = _recoverable_oneshot_run_at( + schedule, scan.now, last_run_at=job.get("last_run_at")) recovery_kind = "one-shot" if recovered_next else None if not recovered_next and kind in {"cron", "interval"}: recovered_next = compute_next_run(schedule, scan.now.isoformat()) @@ -2549,7 +2565,8 @@ def _recover_missing_next_run(job: Dict[str, Any], scan: _DueScan) -> Optional[s @dataclass class _DueJob: - """One candidate under evaluation: its record, schedule and the stored next_run in raw/aware form.""" + """One candidate under evaluation: its record, schedule and the stored next_run in raw/aware + form.""" job: Dict[str, Any] scan: _DueScan @@ -2599,12 +2616,14 @@ def _repair_timezone_shifted_cron(d: _DueJob) -> bool: def _rearm_stale_error_recurring(d: _DueJob) -> datetime: - """Re-arm a recurring job wedged in persisted last_status=error; returns the effective next_run_dt. - + """Re-arm a recurring job wedged in persisted last_status=error; returns the effective + next_run_dt. + Such a job errored, mark_job_run parked next_run_at in the future, and nothing re-dispatched it (the in-memory stale-claim sweep cannot see it). Interval jobs re-arm to now (always a legal fire); cron jobs re-arm to the next LEGAL occurrence, since re-arming to now would fire at times - the expression excludes. A correctly-parked cron value is left as-is.""" + the expression excludes. A correctly-parked cron value is left as-is. + """ now = d.scan.now if not ( d.kind in ("cron", "interval") @@ -2637,7 +2656,8 @@ def _reanchor_stale_cron(d: _DueJob) -> bool: A direct edit of schedule.expr leaves next_run_at on the old lattice, so re-anchor first (from the current expr, so this converges). An offset-representation migration also moves a legacy - instant off the lattice, and re-anchoring THAT swallowed a due occurrence — so classify, and let + instant off the lattice, and re-anchoring THAT swallowed a due occurrence — so classify, and + let the migration case fall through to fire ONCE (at-most-once holds: nothing re-reads the legacy instant after advance/mark_job_run rewrites it).""" stale_class = _classify_stale_cron_next_run(d.schedule, d.raw_next_run_dt, d.next_run_dt) @@ -2664,10 +2684,12 @@ def _reanchor_stale_cron(d: _DueJob) -> bool: def _fast_forward_missed_recurring(d: _DueJob, grace: int) -> None: """Recurring job past its grace window: skip the accumulated misses, fire once now. - + The fast-forward is persisted immediately — NOT redundant with advance_next_run/mark_job_run: - it protects the crash window before mark_job_run and covers the external fire_due path, which - never calls advance_next_run. mark_job_run re-anchors on completion, so the value is provisional.""" + it + protects the crash window before mark_job_run and covers the external fire_due path, which never + calls advance_next_run. mark_job_run re-anchors on completion, so the value is provisional. + """ if (d.scan.now - d.next_run_dt).total_seconds() <= grace: return new_next = d.recompute_next() @@ -2736,18 +2758,21 @@ def _oneshot_dispatch_limit_reached(job: Dict[str, Any], scan: _DueScan) -> bool def _evaluate_due_job(job: Dict[str, Any], scan: _DueScan, run_claim_ttl: float) -> bool: """Decide whether one enabled, non-terminal job fires this tick, persisting any repairs. Ordering matters: recover missing next_run_at, repair timezone shifts, re-arm stale-error - recurring jobs, then once due: re-anchor stale cron instants, fast-forward missed recurring + recurring jobs; then once due: re-anchor stale cron instants, fast-forward missed recurring runs, retire/guard one-shots, and finally stamp the run claim / dispatch record.""" now = scan.now # Cross-process guard: another process's live one-shot run_claim (younger than TTL) — do NOT - # re-dispatch. Malformed/future-dated claims (clock/TZ skew) count as stale, never eternally fresh. - if job.get("schedule", {}).get("kind") == "once" and _claim_is_live(job.get("run_claim"), now, run_claim_ttl): + # re-dispatch. Malformed/future-dated claims (clock/TZ skew) count as stale, never eternally + # fresh. + if ( + job.get("schedule", {}).get("kind") == "once" + and _claim_is_live(job.get("run_claim"), now, run_claim_ttl) + ): return False next_run = job.get("next_run_at") or _recover_missing_next_run(job, scan) if not next_run: return False - raw_next_run_dt = datetime.fromisoformat(next_run) d = _DueJob(job, scan, next_run, raw_next_run_dt, _ensure_aware(raw_next_run_dt)) kind = d.kind @@ -2778,8 +2803,9 @@ def _evaluate_due_job(job: Dict[str, Any], scan: _DueScan, run_claim_ttl: float) job["run_claim"] = claim scan.persist(job["id"], run_claim=claim) - # Missed-run visibility: persist scheduled-vs-actual timing so separate CLI processes can show - # a late catch-up. Recurring only — expired one-shots were retired above; manual triggers aren't late. + # Missed-run visibility: persist scheduled-vs-actual timing so separate CLI processes can show a + # late catch-up. Recurring only — expired one-shots were retired above; manual triggers aren't + # late. if not manual_run and recurring: lateness = max(0.0, (now - d.next_run_dt).total_seconds()) dispatch_stamp = { @@ -2810,7 +2836,8 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: due = [] for job in jobs: # Per-job containment: one malformed record must never abort the whole scan. Normalization - # above repairs known shapes; this catches FUTURE variants so healthy siblings still run/persist. + # above repairs known shapes; this catches FUTURE variants so healthy siblings still + # run/persist. try: if is_terminal_job(job) and not _is_recoverable_error_job(job): continue @@ -2833,7 +2860,8 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: # --- Run output --- -# Per-run output files (`cron/output//.md`) are capped so a frequent job can't fill the disk. +# Per-run output files (`cron/output//.md`) are capped so a frequent job can't fill +# the disk. _CRON_OUTPUT_DEFAULT_KEEP = 50 @@ -2850,7 +2878,8 @@ def _prune_job_output(job_output_dir: Path, keep: int) -> int: return 0 try: files = sorted( - (f for f in job_output_dir.glob("*.md") if f.is_file()), key=lambda f: f.name, reverse=True) + (f for f in job_output_dir.glob("*.md") if f.is_file()), + key=lambda f: f.name, reverse=True) except OSError: return 0 deleted = 0 @@ -2879,10 +2908,10 @@ def save_job_output(job_id: str, output: str): # --- Skill reference rewriting (curator integration) --- def _canonical_skill_ref(raw: Any) -> str: - """Reduce a job skill reference (possibly an absolute path) to the bare name the curator - matches on, resolving it the same way the scheduler does — otherwise a path-referencing job's - skill looks unreferenced and gets archived. Falls back to plain cleanup if the resolver fails, - so a broken import can never lose a name.""" + """Reduce a job skill reference (possibly an absolute path) to the bare name the curator matches + on, resolving as the scheduler does — otherwise a path-referencing job's skill looks + unreferenced and gets archived. Falls back to plain cleanup so a broken import can never lose + a name.""" value = str(raw or "").strip() if not value: return "" @@ -2895,10 +2924,9 @@ def _canonical_skill_ref(raw: Any) -> str: def referenced_skill_names() -> Set[str]: - """Skill names referenced by ANY cron job, deliberately including paused/disabled ones (a paused - job never bumps its skills, yet resuming it must still find them). The curator uses this to - protect referenced skills from inactivity archival. Names are canonicalized as the scheduler - does, so absolute paths are protected too. A corrupt store yields an empty set, never raises.""" + """Skill names referenced by ANY cron job, deliberately including paused/disabled ones (resuming + must still find them); the curator protects these from inactivity archival. Canonicalized as the + scheduler does, so absolute paths are protected too. A corrupt store yields an empty set.""" try: jobs = load_jobs() except Exception: @@ -2916,13 +2944,12 @@ def referenced_skill_names() -> Set[str]: def rewrite_skill_refs( consolidated: Optional[Dict[str, str]] = None, pruned: Optional[List[str]] = None, ) -> Dict[str, Any]: - """Rewrite cron job skill references after a curator consolidation pass. - - A job listing a consolidated/pruned skill would otherwise run without it (the scheduler logs - and skips missing skills). Consolidated names are replaced by their umbrella target without - duplication, pruned names are dropped, ordering is preserved, and the legacy ``skill`` field is - realigned. Returns ``{"rewrites": [{job_id, job_name, before, after, mapped, dropped}, ...], - "jobs_updated": N, "jobs_scanned": M}``. Load/save exceptions propagate (the curator wraps).""" + """Rewrite cron job skill references after a curator consolidation pass (a job listing a + consolidated/pruned skill would otherwise run without it). Consolidated names map to their + umbrella target without duplication, pruned names are dropped, ordering is preserved, and the + legacy ``skill`` field is realigned. Returns ``{"rewrites": [{job_id, job_name, before, + after, mapped, dropped}, ...], "jobs_updated": N, "jobs_scanned": M}``. Load/save exceptions + propagate.""" consolidated = dict(consolidated or {}) # A skill listed in both wins as "consolidated" — it has a target, the more useful outcome. pruned_set = set(pruned or []) - set(consolidated.keys())