"""Content-free cron service-health and execution telemetry projection.""" from __future__ import annotations import logging import re from dataclasses import dataclass from datetime import datetime from typing import Any, Callable, Optional from agent.monitoring.events import CronExecutionEvent from agent.monitoring.gateway_health import ( GatewayMetric, _contains_any, _safe_instance_id, ) from cron.jobs import ( _compute_grace_seconds, get_catch_up_occurrence_count, get_ticker_heartbeat_age, get_ticker_success_age, load_jobs, ) from cron.scheduler import get_running_job_ids from hermes_time import now as _now logger = logging.getLogger(__name__) _KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} _KNOWN_SOURCES = {"builtin", "direct", "external"} _KNOWN_DELIVERY_OUTCOMES = { "delivered", "failed", "suppressed", "suppressed_acked", "not_configured", } _TERMINAL_STATUSES = {"completed", "failed", "unknown"} @dataclass(frozen=True, slots=True) class CronHealthSnapshot: metrics: list[GatewayMetric] events: list[CronExecutionEvent] # Opaque job key: same sha256:<24 hex> shape as service.instance.id — never the raw id. _job_key = _safe_instance_id _AUTH_RE = re.compile( r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize" r"|unauthorized|forbidden|bearer|401|403)\b|\b(?:access|api|refresh) token\b" ) # Ordered (predicate, class) rules; first match wins. Auth uses word boundaries so # "oauth"/"tokenizer"/"HTTP 4015" do not false-positive. _CRON_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = ( (lambda s: _AUTH_RE.search(s) is not None, "auth_failed"), (_contains_any("rate limit", "429", "quota"), "rate_limited"), (_contains_any("timeout", "timed out"), "timeout"), (_contains_any("network", "connection", "dns", "socket", "unreachable"), "network_error"), (_contains_any("dispatch", "executor"), "dispatch_failed"), (_contains_any("interrupt", "owner exited", "restarted"), "interrupted"), (_contains_any("empty response"), "empty_response"), (_contains_any("config", "missing", "invalid"), "invalid_config"), ) def classify_cron_error(raw: Any) -> str: text = str(raw or "").lower() return next((label for match, label in _CRON_ERROR_RULES if match(text)), "unknown") def _parse_time(raw: Any) -> Optional[datetime]: try: return datetime.fromisoformat(str(raw)) if raw else None except (TypeError, ValueError): return None def _duration_ms(record: dict[str, Any]) -> Optional[int]: start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at")) finish = _parse_time(record.get("finished_at")) if start is None or finish is None: return None try: duration = int((finish - start).total_seconds() * 1000) except (TypeError, ValueError): return None return max(0, duration) def project_execution_event( record: dict[str, Any], *, delivery_outcome: Optional[str] = None ) -> CronExecutionEvent: status = str(record.get("status") or "unknown").lower() source = str(record.get("source") or "unknown").lower() outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None return CronExecutionEvent( status=status if status in _KNOWN_STATUSES else "unknown", job_key=_job_key(record.get("job_id")), # Unknown non-empty sources are bucketed as "external" (not dropped to unknown). source=source if source in _KNOWN_SOURCES or source == "unknown" else "external", duration_ms=_duration_ms(record), delivery_outcome=( outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None ), error_class=( classify_cron_error(record.get("error")) if status in {"failed", "unknown"} else None ), ) def emit_execution_state( record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None ) -> None: """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier.""" if not record: return try: from agent.monitoring import emitter event = project_execution_event(record, delivery_outcome=delivery_outcome) target = emitter.get_emitter() target.emit(event) if event.status in _TERMINAL_STATUSES: target.flush(timeout=1.0) except Exception: logger.debug("cron execution telemetry emit failed", exc_info=True) def _is_overdue(job: dict[str, Any], now: datetime) -> bool: if not job.get("enabled", True): return False next_run = _parse_time(job.get("next_run_at")) schedule = job.get("schedule") if next_run is None or not isinstance(schedule, dict): return False try: if next_run.tzinfo is None and now.tzinfo is not None: next_run = next_run.replace(tzinfo=now.tzinfo) lateness = (now - next_run).total_seconds() return lateness > _compute_grace_seconds(schedule) except (TypeError, ValueError): return False def _job_metrics(metrics: list[GatewayMetric]) -> None: enabled = [job for job in load_jobs() if job.get("enabled", True)] metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {})) metrics.append(GatewayMetric( "hermes.cron.jobs.overdue", sum(1 for job in enabled if _is_overdue(job, _now())), {}, )) def _freshness_metric(name: str, reader: Callable[[], Optional[float]]) -> Callable[[list[GatewayMetric]], None]: def build(metrics: list[GatewayMetric]) -> None: value = reader() if value is not None: metrics.append(GatewayMetric(name, max(0.0, float(value)), {})) return build def _single_metric(name: str, reader: Callable[[], Any]) -> Callable[[list[GatewayMetric]], None]: return lambda metrics: metrics.append(GatewayMetric(name, reader(), {})) # Each group is independently fail-open so one unavailable source never hides the rest. # Readers are wrapped in lambdas so monkeypatching this module's names still takes effect. _METRIC_GROUPS: tuple[tuple[Callable[[list[GatewayMetric]], None], str], ...] = ( (_freshness_metric("hermes.cron.scheduler.heartbeat_age_seconds", lambda: get_ticker_heartbeat_age()), "cron freshness metric unavailable"), (_freshness_metric("hermes.cron.scheduler.last_success_age_seconds", lambda: get_ticker_success_age()), "cron freshness metric unavailable"), (_single_metric("hermes.cron.scheduler.catch_up_occurrences", lambda: get_catch_up_occurrence_count()), "cron catch-up metric unavailable"), (_job_metrics, "cron job metrics unavailable"), (_single_metric("hermes.cron.jobs.running", lambda: len(get_running_job_ids())), "cron running-job metric unavailable"), ) def build_cron_health_snapshot() -> CronHealthSnapshot: metrics: list[GatewayMetric] = [] for build, failure_msg in _METRIC_GROUPS: try: build(metrics) except Exception: logger.debug(failure_msg, exc_info=True) return CronHealthSnapshot(metrics=metrics, events=[]) __all__ = [ "CronHealthSnapshot", "build_cron_health_snapshot", "classify_cron_error", "emit_execution_state", "project_execution_event", ]