Teknium
7eb8ccc315
- otlp_exporter hosts SDK loading (symbol table), header/endpoint/resource helpers shared with gateway_health_export; export module imports them instead of keeping copies (_otlp_config/_resolve_headers/_install_id/_safe_resource_attributes unified). _KEEP_BY_KIND attribute allowlist reused for diagnostic log attrs. - events: _MonitoringEvent base supplies to_dict; field order (wire order) unchanged. - redaction: redact_bounded() replaces 3 inline redact+truncate try/excepts. - EmitterStreamer base owns the shared unsubscribe/flush/shutdown for OTLPStreamer and GatewayDiagnosticLogStreamer. - gateway_health: metric construction helpers, _contains_any shared with cron_health, dead _allowed_logger/redact_gateway_message removed (0 refs). - Comment/docstring compaction keeping every stated invariant.
406 lines
16 KiB
Python
406 lines
16 KiB
Python
"""Gateway health and diagnostics signal producer.
|
|
|
|
Keeps the plane narrow: service health plus redacted operational diagnostics,
|
|
derived from the existing gateway runtime-status contract. No prompts,
|
|
messages, tool args, session history, audit records, or product analytics.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
from dataclasses import dataclass
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
from agent.monitoring import emitter
|
|
from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent
|
|
from agent.monitoring.redaction import redact_bounded
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class GatewayMetric:
|
|
name: str
|
|
value: int | float
|
|
attributes: Dict[str, str]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class GatewayHealthSnapshot:
|
|
metrics: List[GatewayMetric]
|
|
events: List[GatewayHealthEvent | GatewayDiagnosticEvent]
|
|
|
|
|
|
_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
|
|
_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
|
|
_KNOWN_GATEWAY_STATES = {
|
|
"starting", "draining", "stopping", "stopped", "startup_failed", "unknown"
|
|
} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES
|
|
_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
|
|
"connecting", "disconnected", "disabled", "paused", "retrying", "unknown"
|
|
}
|
|
_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
|
|
_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$")
|
|
|
|
|
|
def source_logger_for_export(name: Any) -> Optional[str]:
|
|
"""Return a bounded source-controlled gateway logger name for OTLP scope."""
|
|
value = str(name or "")
|
|
return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None
|
|
|
|
|
|
def _contains_any(*needles: str) -> Callable[[str], bool]:
|
|
return lambda text: any(needle in text for needle in needles)
|
|
|
|
|
|
# Ordered (predicate, class) rules; first match wins, so auth outranks rate-limit etc.
|
|
_GATEWAY_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = (
|
|
(_contains_any("auth", "token", "unauthorized", "forbidden", "401", "403"), "auth_failed"),
|
|
(lambda s: "rate" in s and "limit" in s, "rate_limited"),
|
|
(_contains_any("timeout", "timed out"), "timeout"),
|
|
(_contains_any(
|
|
"network", "connection", "dns", "socket", "connect call failed",
|
|
"failed to connect", "cannot connect", "unreachable", "name resolution",
|
|
), "network_error"),
|
|
(_contains_any("config", "missing", "invalid"), "invalid_config"),
|
|
(_contains_any("startup"), "startup_failed"),
|
|
(_contains_any("fatal"), "platform_fatal"),
|
|
)
|
|
|
|
|
|
def classify_gateway_error(raw: Any) -> str:
|
|
s = str(raw or "").lower()
|
|
return next((label for match, label in _GATEWAY_ERROR_RULES if match(s)), "unknown")
|
|
|
|
|
|
def classify_exit_reason(
|
|
raw: Any, *, state: Any, restart_requested: bool
|
|
) -> Optional[str]:
|
|
"""Reduce free-form shutdown text to a bounded operational class."""
|
|
if restart_requested:
|
|
return "restart_requested"
|
|
state_name = str(state or "").lower()
|
|
if raw is None and state_name != "startup_failed":
|
|
return None
|
|
classified = classify_gateway_error(raw)
|
|
if state_name == "startup_failed":
|
|
return classified if classified != "unknown" else "startup_failed"
|
|
text = str(raw or "").lower()
|
|
if "signal" in text or "sigterm" in text or "sigint" in text:
|
|
return "signal"
|
|
if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")):
|
|
return "planned_stop"
|
|
return classified
|
|
|
|
|
|
def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
|
|
state = str(raw or "unknown").lower()
|
|
return state if state in allowed else "unknown"
|
|
|
|
|
|
def _optional_state(raw: Any, *, allowed: set[str]) -> Optional[str]:
|
|
"""``_bounded_state`` that preserves "absent" (None) instead of coercing to unknown."""
|
|
return None if raw is None else _bounded_state(raw, allowed=allowed)
|
|
|
|
|
|
def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
|
|
return redact_bounded(raw, limit=limit, empty="unknown", unavailable="unknown")
|
|
|
|
|
|
def _safe_instance_id(raw: Any) -> str:
|
|
"""Return a stable opaque instance key without exporting the source ID."""
|
|
value = str(raw or "unknown").encode("utf-8", errors="replace")
|
|
return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
|
|
|
|
|
|
def subsystem_for_logger(logger_name: str) -> str:
|
|
parts = logger_name.split(".")
|
|
if parts[:2] == ["gateway", "relay"]:
|
|
return "platform.relay"
|
|
if parts[:2] == ["gateway", "platforms"] and len(parts) >= 3 and parts[2]:
|
|
return f"platform.{parts[2]}"
|
|
return "platform" if logger_name.startswith("gateway.platforms") else "gateway"
|
|
|
|
|
|
def platform_for_subsystem(subsystem: str) -> Optional[str]:
|
|
return (subsystem.split(".", 1)[1] or None) if subsystem.startswith("platform.") else None
|
|
|
|
|
|
def _parse_active_agents(raw: Any) -> int:
|
|
try:
|
|
from gateway.status import parse_active_agents
|
|
return parse_active_agents(raw)
|
|
except Exception:
|
|
try:
|
|
return max(0, int(raw))
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
|
|
try:
|
|
from gateway.status import derive_gateway_busy
|
|
return derive_gateway_busy(
|
|
gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents
|
|
)
|
|
except Exception:
|
|
return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0)
|
|
|
|
|
|
def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool:
|
|
try:
|
|
from gateway.status import derive_gateway_drainable
|
|
return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state)
|
|
except Exception:
|
|
return bool(gateway_running and gateway_state == "running")
|
|
|
|
|
|
def _base_attrs(*, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]:
|
|
mode = str(supervision_mode or "unknown").lower()
|
|
return {
|
|
"service.instance.id": _safe_instance_id(install_id),
|
|
"service.version": _safe_metric_value(version, limit=64),
|
|
"hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown",
|
|
}
|
|
|
|
|
|
def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric:
|
|
out = dict(attrs)
|
|
for key, val in extra.items():
|
|
if val is not None:
|
|
out[key] = _safe_metric_value(val)
|
|
return GatewayMetric(name=name, value=value, attributes=out)
|
|
|
|
|
|
def _platforms_of(runtime: Optional[dict[str, Any]]) -> dict[str, Any]:
|
|
raw = (runtime or {}).get("platforms")
|
|
return raw if isinstance(raw, dict) else {}
|
|
|
|
|
|
def build_gateway_health_snapshot(
|
|
runtime: Optional[dict[str, Any]],
|
|
*,
|
|
gateway_running: bool,
|
|
profile: str,
|
|
install_id: str,
|
|
version: str,
|
|
supervision_mode: str = "unknown",
|
|
) -> GatewayHealthSnapshot:
|
|
"""Convert gateway_state.json-compatible runtime state into P0 signals."""
|
|
runtime = runtime or {}
|
|
gateway_state = _bounded_state(runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
|
|
active_agents = _parse_active_agents(runtime.get("active_agents", 0))
|
|
busy = _derive_busy(gateway_running, gateway_state, active_agents)
|
|
drainable = _derive_drainable(gateway_running, gateway_state)
|
|
platforms = _platforms_of(runtime)
|
|
base = _base_attrs(install_id=install_id, version=version, supervision_mode=supervision_mode)
|
|
|
|
metrics: list[GatewayMetric] = [
|
|
_metric("hermes.gateway.up", 1 if gateway_running else 0, base),
|
|
_metric("hermes.gateway.active_agents", active_agents, base),
|
|
_metric("hermes.gateway.busy", 1 if busy else 0, base),
|
|
_metric("hermes.gateway.drainable", 1 if drainable else 0, base),
|
|
_metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base),
|
|
_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": gateway_state}),
|
|
]
|
|
|
|
fatal_count = 0
|
|
events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
|
|
for platform, pdata in platforms.items():
|
|
pdata = pdata if isinstance(pdata, dict) else {}
|
|
state = _bounded_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES)
|
|
# classify_* is idempotent on its own labels, so error_class == error_code here.
|
|
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
|
|
is_degraded = state in _FATAL_PLATFORM_STATES
|
|
if is_degraded:
|
|
fatal_count += 1
|
|
pattrs = {"hermes.platform": str(platform), "hermes.platform.state": state}
|
|
metrics.append(_metric("hermes.platform.up", 1 if state in _RUNNING_PLATFORM_STATES else 0, base, **pattrs))
|
|
metrics.append(_metric(
|
|
"hermes.platform.degraded", 1 if is_degraded else 0, base, **pattrs, **{"hermes.error_code": error_code}
|
|
))
|
|
if is_degraded:
|
|
events.append(GatewayDiagnosticEvent(
|
|
name="platform.fatal",
|
|
subsystem=f"platform.{platform}",
|
|
platform=str(platform),
|
|
error_code=error_code,
|
|
error_class=error_code,
|
|
profile=profile,
|
|
version=version,
|
|
severity="error" if state == "fatal" else "warning",
|
|
))
|
|
|
|
events.insert(0, GatewayHealthEvent(
|
|
name="gateway.health_snapshot",
|
|
gateway_state=gateway_state,
|
|
active_agents=active_agents,
|
|
gateway_busy=busy,
|
|
gateway_drainable=drainable,
|
|
platform_count=len(platforms),
|
|
fatal_platform_count=fatal_count,
|
|
profile=profile,
|
|
install_id=install_id,
|
|
version=version,
|
|
supervision_mode=supervision_mode,
|
|
pid=_coerce_pid(runtime.get("pid")),
|
|
))
|
|
return GatewayHealthSnapshot(metrics=metrics, events=events)
|
|
|
|
|
|
def _safe_profile() -> str:
|
|
try:
|
|
from hermes_cli.profiles import get_active_profile_name
|
|
return str(get_active_profile_name() or "default")
|
|
except Exception:
|
|
return "default"
|
|
|
|
|
|
def _safe_version() -> str:
|
|
try:
|
|
from hermes_cli import __version__
|
|
return str(__version__)
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
def _lifecycle_events(
|
|
previous: Optional[dict[str, Any]], current: dict[str, Any], *, profile: str, version: str
|
|
) -> list[GatewayHealthEvent | GatewayDiagnosticEvent]:
|
|
"""Gateway-level transition events: lifecycle, startup_failed diagnostic, exit."""
|
|
old_state = _optional_state((previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
|
|
new_state = _optional_state(current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
|
|
if old_state == new_state or not new_state:
|
|
return []
|
|
restart_requested = bool(current.get("restart_requested"))
|
|
|
|
def health(name: str) -> GatewayHealthEvent:
|
|
return GatewayHealthEvent(
|
|
name=name,
|
|
gateway_state=new_state,
|
|
old_state=old_state,
|
|
new_state=new_state,
|
|
exit_reason=classify_exit_reason(
|
|
current.get("exit_reason"), state=new_state, restart_requested=restart_requested
|
|
),
|
|
restart_requested=restart_requested,
|
|
active_agents=_parse_active_agents(current.get("active_agents", 0)),
|
|
profile=profile,
|
|
version=version,
|
|
pid=_coerce_pid(current.get("pid")),
|
|
)
|
|
|
|
out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [health("gateway.lifecycle")]
|
|
if new_state == "startup_failed":
|
|
error_class = classify_gateway_error(current.get("exit_reason") or "startup_failed")
|
|
out.append(GatewayDiagnosticEvent(
|
|
name="gateway.startup_failed",
|
|
subsystem="gateway",
|
|
error_class=error_class,
|
|
error_code=error_class,
|
|
profile=profile,
|
|
version=version,
|
|
severity="error",
|
|
))
|
|
if new_state == "stopped":
|
|
out.append(health("gateway.exit"))
|
|
return out
|
|
|
|
|
|
def _platform_events(
|
|
previous: Optional[dict[str, Any]], current: dict[str, Any], *, profile: str, version: str
|
|
) -> list[GatewayDiagnosticEvent]:
|
|
"""Per-platform state_change diagnostics, plus platform.fatal when the new state is fatal."""
|
|
old_platforms = _platforms_of(previous)
|
|
out: list[GatewayDiagnosticEvent] = []
|
|
for platform, pdata in _platforms_of(current).items():
|
|
pdata = pdata if isinstance(pdata, dict) else {}
|
|
prev_raw = old_platforms.get(platform, {})
|
|
prev = prev_raw if isinstance(prev_raw, dict) else {}
|
|
old_state = _optional_state(prev.get("state"), allowed=_KNOWN_PLATFORM_STATES)
|
|
new_state = _optional_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES)
|
|
if old_state == new_state or not new_state:
|
|
continue
|
|
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
|
|
common: dict[str, Any] = dict(
|
|
subsystem=f"platform.{platform}",
|
|
platform=str(platform),
|
|
error_code=error_code,
|
|
error_class=error_code,
|
|
profile=profile,
|
|
version=version,
|
|
severity="error" if new_state in {"fatal", "failed", "error"} else "warning",
|
|
)
|
|
out.append(GatewayDiagnosticEvent(
|
|
name="platform.state_change", old_state=old_state, new_state=new_state, **common
|
|
))
|
|
if new_state in _FATAL_PLATFORM_STATES:
|
|
out.append(GatewayDiagnosticEvent(name="platform.fatal", **common))
|
|
return out
|
|
|
|
|
|
def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None:
|
|
"""Emit immediate content-free gateway events for runtime status changes.
|
|
|
|
Called by gateway.status.write_runtime_status after persisting the new status.
|
|
Fully fail-open: failures never affect gateway status writes.
|
|
"""
|
|
try:
|
|
ctx = dict(profile=_safe_profile(), version=_safe_version())
|
|
for ev in _lifecycle_events(previous, current, **ctx) + _platform_events(previous, current, **ctx):
|
|
emitter.emit(ev)
|
|
except Exception:
|
|
logger.debug("gateway runtime status transition emit failed", exc_info=True)
|
|
|
|
|
|
def _coerce_pid(raw: Any) -> Optional[int]:
|
|
try:
|
|
pid = int(raw)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return pid if pid > 0 else None
|
|
|
|
|
|
class GatewayDiagnosticLogHandler(logging.Handler):
|
|
"""Allowlisted warning/error bridge for gateway-owned diagnostics."""
|
|
|
|
def __init__(self, *, profile: str = "default", version: str = "unknown") -> None:
|
|
super().__init__(level=logging.WARNING)
|
|
self.profile = profile
|
|
self.version = version
|
|
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
try:
|
|
if record.levelno < logging.WARNING:
|
|
return
|
|
if not (record.name == "gateway" or record.name.startswith("gateway.")):
|
|
return
|
|
subsystem = subsystem_for_logger(record.name)
|
|
error_class = classify_gateway_error(record.getMessage())
|
|
event = GatewayDiagnosticEvent(
|
|
name=f"gateway.log.{record.levelname.lower()}",
|
|
subsystem=subsystem,
|
|
source_logger=source_logger_for_export(record.name),
|
|
platform=platform_for_subsystem(subsystem),
|
|
error_class=error_class,
|
|
error_code=error_class,
|
|
profile=self.profile,
|
|
version=self.version,
|
|
severity=record.levelname.lower(),
|
|
)
|
|
emitter.get_emitter().emit(event)
|
|
except Exception:
|
|
logger.debug("gateway diagnostic emit failed", exc_info=True)
|
|
|
|
|
|
__all__ = [
|
|
"GatewayMetric",
|
|
"GatewayHealthSnapshot",
|
|
"GatewayDiagnosticLogHandler",
|
|
"build_gateway_health_snapshot",
|
|
"classify_gateway_error",
|
|
"source_logger_for_export",
|
|
]
|