Teknium
7eb8ccc315
- otlp_exporter hosts SDK loading (symbol table), header/endpoint/resource helpers shared with gateway_health_export; export module imports them instead of keeping copies (_otlp_config/_resolve_headers/_install_id/_safe_resource_attributes unified). _KEEP_BY_KIND attribute allowlist reused for diagnostic log attrs. - events: _MonitoringEvent base supplies to_dict; field order (wire order) unchanged. - redaction: redact_bounded() replaces 3 inline redact+truncate try/excepts. - EmitterStreamer base owns the shared unsubscribe/flush/shutdown for OTLPStreamer and GatewayDiagnosticLogStreamer. - gateway_health: metric construction helpers, _contains_any shared with cron_health, dead _allowed_logger/redact_gateway_message removed (0 refs). - Comment/docstring compaction keeping every stated invariant.
430 lines
17 KiB
Python
430 lines
17 KiB
Python
"""Gateway Health & Diagnostics OTLP export runtime.
|
|
|
|
Emits operator-owned gateway service-health metrics plus narrow redacted
|
|
diagnostic events. Deliberately in-process and fail-open so it works under
|
|
systemd, launchd, s6, containers, tmux, nohup, or a plain shell without a
|
|
sidecar/watchdog dependency.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import logging
|
|
import os
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from typing import Any, Callable, Dict, Optional
|
|
|
|
from agent.monitoring import emitter, otlp_exporter
|
|
from agent.monitoring.gateway_health import (
|
|
GatewayDiagnosticLogHandler,
|
|
GatewayMetric,
|
|
_safe_profile as _profile,
|
|
_safe_version as _version,
|
|
build_gateway_health_snapshot,
|
|
source_logger_for_export,
|
|
)
|
|
from agent.monitoring.otlp_exporter import (
|
|
EmitterStreamer,
|
|
_allowlisted_attrs,
|
|
_install_id,
|
|
_monitoring_section,
|
|
_otlp_config,
|
|
_resolve_headers,
|
|
_runtime_resource_attributes,
|
|
_safe_resource_attributes, # noqa: F401 — re-exported for tests
|
|
_signal_endpoint,
|
|
)
|
|
from agent.monitoring.redaction import redact_bounded
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_DEFAULT_DIAGNOSTIC_SCOPE = "hermes.gateway.diagnostics"
|
|
_METRICS_SDK = (
|
|
"OTLPLogExporter", "OTLPMetricExporter", "Observation", "LogRecord", "LoggerProvider",
|
|
"INVALID_SPAN_ID", "INVALID_TRACE_ID", "TraceFlags", "SeverityNumber",
|
|
"BatchLogRecordProcessor", "MeterProvider", "PeriodicExportingMetricReader", "Resource",
|
|
)
|
|
# Every gauge the runtime snapshot can emit MUST be listed here or it is silently dropped.
|
|
_OBSERVABLE_METRIC_NAMES = (
|
|
"hermes.gateway.up",
|
|
"hermes.gateway.state",
|
|
"hermes.gateway.active_agents",
|
|
"hermes.gateway.busy",
|
|
"hermes.gateway.drainable",
|
|
"hermes.gateway.restart_requested",
|
|
"hermes.gateway.background_work",
|
|
"hermes.gateway.background_delegations",
|
|
"hermes.platform.up",
|
|
"hermes.platform.degraded",
|
|
"hermes.cron.scheduler.heartbeat_age_seconds",
|
|
"hermes.cron.scheduler.last_success_age_seconds",
|
|
"hermes.cron.scheduler.catch_up_occurrences",
|
|
"hermes.cron.jobs.enabled",
|
|
"hermes.cron.jobs.running",
|
|
"hermes.cron.jobs.overdue",
|
|
)
|
|
|
|
|
|
def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]:
|
|
# Same allowlist as the span mapping: profile/install_id/ts never egress as attributes.
|
|
return _allowlisted_attrs(event, otlp_exporter._KEEP_BY_KIND["gateway_diagnostic"])
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class GatewayHealthExportRuntime:
|
|
enabled: bool
|
|
reason: str = "disabled"
|
|
streamer: Any = None
|
|
metric_provider: Any = None
|
|
log_handler: Any = None
|
|
log_streamer: Any = None
|
|
thread: Optional[threading.Thread] = None
|
|
stop_event: Optional[threading.Event] = None
|
|
|
|
def shutdown(self) -> None:
|
|
if self.stop_event is not None:
|
|
self.stop_event.set()
|
|
if self.thread is not None:
|
|
self.thread.join(timeout=0.25)
|
|
if self.log_handler is not None:
|
|
try:
|
|
logging.getLogger().removeHandler(self.log_handler)
|
|
except Exception:
|
|
pass
|
|
|
|
# Producers are stopped; drain queued/in-flight events BEFORE detaching subscribers
|
|
# so the terminal lifecycle event cannot race exporter shutdown. Bounded, fail-open.
|
|
subscribers = [item for item in (self.streamer, self.log_streamer) if item is not None]
|
|
try:
|
|
bus = emitter.get_emitter()
|
|
bus.flush(timeout=1.0)
|
|
for sub in subscribers:
|
|
bus.unsubscribe(sub)
|
|
except Exception:
|
|
pass
|
|
|
|
# Network flush/close runs under one bounded daemon-thread deadline so it can
|
|
# never delay gateway teardown indefinitely.
|
|
closeables = subscribers + ([self.metric_provider] if self.metric_provider is not None else [])
|
|
|
|
def _close() -> None:
|
|
for item in closeables:
|
|
try:
|
|
item.shutdown()
|
|
except Exception:
|
|
pass
|
|
|
|
if closeables:
|
|
worker = threading.Thread(
|
|
target=_close, name="hermes-gateway-health-export-shutdown", daemon=True
|
|
)
|
|
worker.start()
|
|
worker.join(timeout=2.0)
|
|
|
|
self.streamer = self.log_streamer = self.metric_provider = self.thread = self.stop_event = None
|
|
|
|
|
|
def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
|
return _monitoring_section(config, "gateway_health_export")
|
|
|
|
|
|
def _enabled(config: Dict[str, Any]) -> bool:
|
|
return bool(_gateway_health_config(config).get("enabled") and otlp_exporter.is_enabled(config))
|
|
|
|
|
|
def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]:
|
|
try:
|
|
return otlp_exporter._require_sdk(_METRICS_SDK, auto_install=auto_install, prompt=prompt)
|
|
except Exception as exc:
|
|
raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc
|
|
|
|
|
|
def _exporter_kwargs(config: Dict[str, Any], signal: str) -> Dict[str, Any]:
|
|
otlp = _otlp_config(config)
|
|
return {
|
|
"endpoint": _signal_endpoint(str(otlp.get("endpoint")), signal),
|
|
"headers": _resolve_headers(otlp.get("headers_env")) or None,
|
|
}
|
|
|
|
|
|
def _resource(config: Dict[str, Any], sdk: Dict[str, Any], telemetry_scope: str) -> Any:
|
|
return sdk["Resource"].create(_runtime_resource_attributes(config, telemetry_scope=telemetry_scope))
|
|
|
|
|
|
# Ordered detection: systemd > s6 > container > launchd > manual (first match wins).
|
|
_SUPERVISION_DETECTORS: tuple[tuple[str, Callable[[], Any]], ...] = (
|
|
("systemd", lambda: os.environ.get("INVOCATION_ID")),
|
|
("s6", lambda: os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION")),
|
|
("container", lambda: os.environ.get("container") or os.path.exists("/.dockerenv")),
|
|
("launchd", lambda: os.environ.get("LAUNCHD_SOCKET")),
|
|
)
|
|
|
|
|
|
def _supervision_mode() -> str:
|
|
return next((mode for mode, detect in _SUPERVISION_DETECTORS if detect()), "manual")
|
|
|
|
|
|
def _read_gateway_snapshot(config: Dict[str, Any]):
|
|
try:
|
|
from gateway.status import read_runtime_status
|
|
runtime = read_runtime_status() or {}
|
|
except Exception:
|
|
runtime = {}
|
|
return build_gateway_health_snapshot(
|
|
runtime,
|
|
gateway_running=True,
|
|
profile=_profile(),
|
|
install_id=_install_id(config),
|
|
version=_version(),
|
|
supervision_mode=_supervision_mode(),
|
|
)
|
|
|
|
|
|
def _read_cron_snapshot():
|
|
from agent.monitoring.cron_health import build_cron_health_snapshot
|
|
|
|
return build_cron_health_snapshot()
|
|
|
|
|
|
def _count(failure_msg: str, module: str, read: Callable[[Any], Any]) -> int:
|
|
"""Best-effort non-negative count read from a lazily imported module; 0 when it can't be imported/read."""
|
|
try:
|
|
return max(0, int(read(importlib.import_module(module))))
|
|
except Exception:
|
|
logger.debug(failure_msg, exc_info=True)
|
|
return 0
|
|
|
|
|
|
def _read_background_work_count() -> int:
|
|
"""Live background/subagent work that ``active_agents`` deliberately does NOT include.
|
|
|
|
``active_agents`` counts foreground turns + in-flight cron + API runs; backgrounded
|
|
``delegate_task`` subagents, ``terminal(background=true)`` processes and kanban workers
|
|
are tracked only by the scale-to-zero guard, so without this a peer churning through
|
|
subagents shows ``active_agents=0``. TASK-granular: a fan-out batch of N contributes N
|
|
(real concurrent load), unlike the pool's one-slot-per-batch accounting. Content-free.
|
|
"""
|
|
return (
|
|
_count("background-work async-delegation count failed", "tools.async_delegation",
|
|
lambda m: m.active_task_count())
|
|
+ _count("background-work process-registry count failed", "tools.process_registry",
|
|
lambda m: m.process_registry.count_running())
|
|
)
|
|
|
|
|
|
def _read_background_delegations_count() -> int:
|
|
"""Live async delegation UNITS (dispatch/pool slots): a batch counts ONE regardless of
|
|
fan-out width, matching the pool's capacity accounting — so operators can see slot
|
|
pressure (alert vs ``max_concurrent_children``) alongside ``background_work``'s real load.
|
|
Delegations only; terminal/kanban work is already folded into ``background_work``."""
|
|
return _count("background-delegations count failed", "tools.async_delegation", lambda m: m.active_count())
|
|
|
|
|
|
def _read_runtime_snapshot(config: Dict[str, Any]):
|
|
gateway_snapshot = _read_gateway_snapshot(config)
|
|
# Background/subagent work is appended to the gateway snapshot so it rides the same base
|
|
# resource attributes (service.instance.id etc.).
|
|
try:
|
|
base = dict(gateway_snapshot.metrics[0].attributes) if gateway_snapshot.metrics else {}
|
|
for name, read in (
|
|
("hermes.gateway.background_work", _read_background_work_count),
|
|
("hermes.gateway.background_delegations", _read_background_delegations_count),
|
|
):
|
|
gateway_snapshot.metrics.append(GatewayMetric(name=name, value=read(), attributes=base))
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"background-work snapshot unavailable; metric not exported (error_type=%s)",
|
|
type(exc).__name__,
|
|
)
|
|
logger.debug("background-work snapshot traceback", exc_info=True)
|
|
try:
|
|
cron_snapshot = _read_cron_snapshot()
|
|
except Exception as exc:
|
|
# Cron telemetry silently dropping out is a release-relevant regression: WARN with only
|
|
# the exception *type* (the message could carry paths); exc_info stays on DEBUG.
|
|
logger.warning(
|
|
"cron health snapshot unavailable; cron telemetry not exported (error_type=%s)",
|
|
type(exc).__name__,
|
|
)
|
|
logger.debug("cron health snapshot traceback", exc_info=True)
|
|
return gateway_snapshot
|
|
gateway_snapshot.metrics.extend(cron_snapshot.metrics)
|
|
return gateway_snapshot
|
|
|
|
|
|
def _emit_snapshot_events(config: Dict[str, Any]) -> None:
|
|
if not _gateway_health_config(config).get("diagnostic_events_enabled", True):
|
|
return
|
|
try:
|
|
for event in _read_runtime_snapshot(config).events:
|
|
emitter.emit(event)
|
|
except Exception:
|
|
logger.debug("gateway health snapshot emit failed", exc_info=True)
|
|
|
|
|
|
def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any:
|
|
gh = _gateway_health_config(config)
|
|
exporter = sdk["OTLPMetricExporter"](**_exporter_kwargs(config, "metrics"))
|
|
interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000
|
|
reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms)
|
|
provider = sdk["MeterProvider"](
|
|
metric_readers=[reader], resource=_resource(config, sdk, "gateway_health")
|
|
)
|
|
meter = provider.get_meter("hermes.gateway.health")
|
|
Observation = sdk["Observation"]
|
|
|
|
def callback(name: str):
|
|
def _cb(_options=None):
|
|
try:
|
|
snapshot = _read_runtime_snapshot(config)
|
|
return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name]
|
|
except Exception:
|
|
logger.debug("gateway metric callback failed", exc_info=True)
|
|
return []
|
|
return _cb
|
|
|
|
for metric_name in _OBSERVABLE_METRIC_NAMES:
|
|
meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)])
|
|
return provider
|
|
|
|
|
|
_SEVERITY_NAMES = {
|
|
"critical": "FATAL",
|
|
"fatal": "FATAL",
|
|
"error": "ERROR",
|
|
"info": "INFO",
|
|
"information": "INFO",
|
|
"debug": "DEBUG",
|
|
}
|
|
|
|
|
|
def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any:
|
|
sev = str(severity or "warning").lower()
|
|
return getattr(sdk["SeverityNumber"], _SEVERITY_NAMES.get(sev, "WARN"))
|
|
|
|
|
|
class GatewayDiagnosticLogStreamer(EmitterStreamer):
|
|
"""Emitter subscriber that sends gateway diagnostic events as OTLP logs."""
|
|
|
|
def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]):
|
|
self._provider = sdk["LoggerProvider"](resource=_resource(config, sdk, "gateway_diagnostics"))
|
|
self._processor = sdk["BatchLogRecordProcessor"](
|
|
sdk["OTLPLogExporter"](**_exporter_kwargs(config, "logs"))
|
|
)
|
|
self._provider.add_log_record_processor(self._processor)
|
|
self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE)
|
|
self._sdk = sdk
|
|
self.exported = 0
|
|
|
|
def __call__(self, batch: list[Dict[str, Any]]) -> None:
|
|
sdk = self._sdk
|
|
for ev in batch:
|
|
if ev.get("event") != "gateway_diagnostic":
|
|
continue
|
|
# The source-controlled Python logger becomes the OTel instrumentation scope:
|
|
# precise code attribution without maintaining a subsystem enum. Rendered
|
|
# messages stay out (they may carry IDs, names, paths, configured strings).
|
|
source_logger = source_logger_for_export(ev.get("source_logger"))
|
|
otel_logger = self._provider.get_logger(source_logger) if source_logger is not None else self._logger
|
|
otel_logger.emit(sdk["LogRecord"](
|
|
timestamp=ev.get("ts_ns"),
|
|
trace_id=sdk["INVALID_TRACE_ID"],
|
|
span_id=sdk["INVALID_SPAN_ID"],
|
|
trace_flags=sdk["TraceFlags"].DEFAULT,
|
|
severity_text=str(ev.get("severity") or "warning").upper(),
|
|
severity_number=_severity_number(sdk, ev.get("severity")),
|
|
body=redact_bounded("gateway diagnostic"),
|
|
attributes=_diagnostic_log_attributes(ev),
|
|
))
|
|
self.exported += 1
|
|
|
|
|
|
def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread:
|
|
interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5)))
|
|
|
|
def _run() -> None:
|
|
while not stop_event.wait(interval):
|
|
_emit_snapshot_events(config)
|
|
|
|
thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True)
|
|
thread.start()
|
|
return thread
|
|
|
|
|
|
def _attach_log_handler(config: Dict[str, Any]) -> Any:
|
|
gh = _gateway_health_config(config)
|
|
if not gh.get("diagnostic_events_enabled", True) or not gh.get("warning_error_events_enabled", True):
|
|
return None
|
|
handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version())
|
|
root = logging.getLogger()
|
|
if handler not in root.handlers:
|
|
root.addHandler(handler)
|
|
return handler
|
|
|
|
|
|
def _gateway_health_event(ev: Dict[str, Any]) -> bool:
|
|
return ev.get("event") in {"gateway_health", "cron_execution"}
|
|
|
|
|
|
def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime:
|
|
"""Start P0 gateway health export if configured. Never raises."""
|
|
if not _enabled(config):
|
|
return GatewayHealthExportRuntime(enabled=False, reason="disabled")
|
|
gh = _gateway_health_config(config)
|
|
metrics_on = gh.get("metrics_enabled", True)
|
|
diagnostics_on = gh.get("diagnostic_events_enabled", True)
|
|
runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled")
|
|
sdk: Optional[Dict[str, Any]] = None
|
|
|
|
if metrics_on or diagnostics_on:
|
|
try:
|
|
sdk = _require_metrics_sdk(prompt=False)
|
|
except Exception:
|
|
logger.warning(
|
|
"monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; "
|
|
"install 'hermes-agent[otlp]'",
|
|
exc_info=True,
|
|
)
|
|
return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable")
|
|
|
|
if metrics_on and sdk is not None:
|
|
try:
|
|
runtime.metric_provider = _start_metric_provider(config, sdk)
|
|
except Exception:
|
|
logger.warning("gateway health OTLP metrics failed to start", exc_info=True)
|
|
runtime.shutdown()
|
|
return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed")
|
|
|
|
if diagnostics_on and sdk is not None:
|
|
try:
|
|
runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event)
|
|
if runtime.streamer is None:
|
|
raise RuntimeError("gateway health span streamer did not start")
|
|
log_streamer = GatewayDiagnosticLogStreamer(config, sdk)
|
|
emitter.get_emitter().subscribe(log_streamer)
|
|
runtime.log_streamer = log_streamer
|
|
except Exception:
|
|
logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True)
|
|
runtime.shutdown()
|
|
return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed")
|
|
|
|
try:
|
|
runtime.log_handler = _attach_log_handler(config)
|
|
except Exception:
|
|
logger.debug("gateway diagnostic log handler failed to attach", exc_info=True)
|
|
if diagnostics_on:
|
|
try:
|
|
_emit_snapshot_events(config)
|
|
runtime.stop_event = threading.Event()
|
|
runtime.thread = _start_snapshot_thread(config, runtime.stop_event)
|
|
except Exception:
|
|
logger.debug("gateway health snapshot thread failed to start", exc_info=True)
|
|
return runtime
|
|
|
|
|
|
__all__ = [
|
|
"GatewayHealthExportRuntime",
|
|
"start_gateway_health_export",
|
|
]
|