"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. Maps gateway monitoring events to OTel spans for the operator-configured ``monitoring.export.otlp`` endpoint (no default destination ships) and hosts the OTLP plumbing shared with ``gateway_health_export``. The OTel SDK is an optional extra (``hermes-agent[otlp]``) imported lazily; ``headers_env`` values are read at export time and never logged or stored. The streaming subscriber runs fail-isolated on the emitter thread; ``event_filter`` keeps other planes off it. """ from __future__ import annotations import importlib import logging import os import re from contextlib import suppress from typing import Any, Callable, Dict, Iterable, List, Optional from agent.monitoring.gateway_health import _safe_instance_id from agent.monitoring.redaction import redact_bounded logger = logging.getLogger(__name__) class OTLPUnavailable(RuntimeError): """Raised when the optional OpenTelemetry SDK isn't installed.""" # ── SDK loading ────────────────────────────────────────────────────────────── _SDK_MODULES: Dict[str, tuple[str, ...]] = { "opentelemetry.sdk.trace": ("TracerProvider",), "opentelemetry.sdk.trace.export": ("BatchSpanProcessor",), "opentelemetry.sdk.resources": ("Resource",), "opentelemetry.exporter.otlp.proto.http.trace_exporter": ("OTLPSpanExporter",), "opentelemetry.exporter.otlp.proto.http._log_exporter": ("OTLPLogExporter",), "opentelemetry.exporter.otlp.proto.http.metric_exporter": ("OTLPMetricExporter",), "opentelemetry.trace": ("SpanKind", "INVALID_SPAN_ID", "INVALID_TRACE_ID", "TraceFlags"), "opentelemetry.metrics": ("Observation",), "opentelemetry._logs": ("LogRecord",), "opentelemetry._logs.severity": ("SeverityNumber",), "opentelemetry.sdk._logs": ("LoggerProvider",), "opentelemetry.sdk._logs.export": ("BatchLogRecordProcessor",), "opentelemetry.sdk.metrics": ("MeterProvider",), "opentelemetry.sdk.metrics.export": ("PeriodicExportingMetricReader",), } _SDK_SYMBOLS: Dict[str, str] = {name: module for module, names in _SDK_MODULES.items() for name in names} _SPAN_SDK = ("TracerProvider", "BatchSpanProcessor", "Resource", "OTLPSpanExporter", "SpanKind") def _require_sdk(names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True, prompt: bool = True) -> Dict[str, Any]: """Import the named OTel SDK symbols, lazily installing the extra on first use. Routes through tools.lazy_deps (feature 'export.otlp') — gated by security.allow_lazy_installs and TTY-prompted (``prompt=False`` from non-interactive contexts). Any lazy-install failure falls through to the import attempt, which raises OTLPUnavailable with a manual install hint. """ if auto_install: with suppress(Exception): from tools.lazy_deps import ensure as _lazy_ensure _lazy_ensure("export.otlp", prompt=prompt) try: return {name: getattr(importlib.import_module(_SDK_SYMBOLS[name]), name) for name in names} except Exception as e: # ImportError or partial install raise OTLPUnavailable( "OTLP export requires the optional dependency. Install with:\n" " pip install 'hermes-agent[otlp]'\n" f"(import error: {e})" ) # ── config + connection plumbing ───────────────────────────────────────────── def _monitoring_section(config: Dict[str, Any], *path: str) -> Dict[str, Any]: node: Any = (config or {}).get("monitoring") or {} for key in path: node = node.get(key) or {} return node def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: return _monitoring_section(config, "export", "otlp") def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value}; missing vars skipped.""" resolved: Dict[str, str] = {} for header_name, env_name in (headers_env or {}).items(): val = os.environ.get(str(env_name)) if val: resolved[str(header_name)] = val else: logger.debug("OTLP header %s: env var %s not set; skipping", header_name, env_name) return resolved _SIGNAL_SUFFIXES = ("/v1/traces", "/v1/metrics") def _signal_endpoint(endpoint: str, signal: str) -> str: """Rewrite a traces/metrics OTLP path to ``/v1/``; other paths pass through.""" target = f"/v1/{signal}" for suffix in _SIGNAL_SUFFIXES: if suffix != target and endpoint.endswith(suffix): return endpoint[: -len(suffix)] + target return endpoint _RESOURCE_ATTRIBUTE_KEYS = frozenset({ "service.name", "service.namespace", "service.version", "service.instance.id", "deployment.environment.name", "cloud.provider", "cloud.platform", "cloud.region", "telemetry.scope", }) _SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$") def _install_id(config: Dict[str, Any]) -> str: try: from agent.monitoring.policy import ensure_install_id return str(ensure_install_id(config)) except Exception: return "unknown" def _safe_resource_attributes(raw: Any) -> Dict[str, str]: """Allowlist bounded resource labels and reject values changed by redaction.""" attrs: Dict[str, str] = {} for key, value in (raw.items() if isinstance(raw, dict) else ()): key = str(key) if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None: continue text = str(value) if key == "service.instance.id": attrs[key] = _safe_instance_id(value) elif _SAFE_RESOURCE_VALUE.fullmatch(text) and redact_bounded(text, limit=128) == text: attrs[key] = text return attrs def _runtime_resource_attributes(config: Dict[str, Any], *, telemetry_scope: str) -> Dict[str, str]: """Build the safe OTLP resource shared by spans, metrics and diagnostic logs.""" attrs = _safe_resource_attributes(_monitoring_section(config, "gateway_health_export").get("resource_attributes")) attrs["service.name"] = "hermes-gateway" attrs["service.instance.id"] = _safe_instance_id(_install_id(config)) attrs["telemetry.scope"] = telemetry_scope return attrs def build_exporter(config: Dict[str, Any]): """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK.""" sdk = _require_sdk() otlp = _otlp_config(config) endpoint = otlp.get("endpoint") if not endpoint: raise ValueError("monitoring.export.otlp.endpoint is not set") return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=_resolve_headers(otlp.get("headers_env")) or None) def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]: return _runtime_resource_attributes(config, telemetry_scope="gateway_monitoring") def _make_provider(config: Dict[str, Any]): sdk = _require_sdk() provider = sdk["TracerProvider"](resource=sdk["Resource"].create(_resource_attributes(config))) processor = sdk["BatchSpanProcessor"](build_exporter(config)) provider.add_span_processor(processor) return provider, processor # ── event -> span attribute mapping ────────────────────────────────────────── # Per-kind attribute allowlists: everything else (profile, install_id, ...) never egresses. _KEEP_BY_KIND: Dict[str, tuple[str, ...]] = { "gateway_health": ( "name", "gateway_state", "old_state", "new_state", "exit_reason", "restart_requested", "active_agents", "gateway_busy", "gateway_drainable", "platform_count", "fatal_platform_count", "version", "supervision_mode", "pid", ), "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", "platform", "old_state", "new_state", "version", "severity"), "cron_execution": ("status", "job_key", "source", "duration_ms", "delivery_outcome", "error_class"), } def _allowlisted_attrs(ev: Dict[str, Any], keys: Iterable[str]) -> Dict[str, Any]: """``hermes.`` attributes for present keys; string values are redacted and bounded.""" attrs: Dict[str, Any] = {} for col in keys: v = ev.get(col) if v is not None: attrs[f"hermes.{col}"] = redact_bounded(v) if isinstance(v, str) else v return attrs def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: """Span attributes for a monitoring event (content-free by construction).""" kind = ev.get("event") attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} attrs.update(_allowlisted_attrs(ev, _KEEP_BY_KIND.get(kind, ()))) # type: ignore[arg-type] return attrs def export_batch(provider, batch: List[Dict[str, Any]]) -> int: """Map a batch of events to OTel spans. Returns spans created.""" tracer = provider.get_tracer("hermes.monitoring") n = 0 for ev in batch: try: tracer.start_span(f"hermes.{ev.get('event', 'event')}", attributes=_span_attrs(ev)).end() n += 1 except Exception: logger.debug("OTLP span map failed", exc_info=True) return n # ── continuous streaming subscribers ───────────────────────────────────────── class EmitterStreamer: """Base for emitter subscribers owning an OTel provider + batch processor. Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.""" _provider: Any _processor: Any exported: int = 0 def shutdown(self) -> None: with suppress(Exception): from agent.monitoring.emitter import get_emitter get_emitter().unsubscribe(self) with suppress(Exception): self._processor.force_flush() self._provider.shutdown() class OTLPStreamer(EmitterStreamer): """A live subscriber that pushes each emitter batch to OTLP as spans.""" def __init__(self, config: Dict[str, Any], *, event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None): self._provider, self._processor = _make_provider(config) self._event_filter = event_filter self.exported = 0 def __call__(self, batch: List[Dict[str, Any]]) -> None: if self._event_filter is not None: batch = [ev for ev in batch if self._event_filter(ev)] if not batch: return self.exported += export_batch(self._provider, batch) def is_available() -> bool: """True when the OTel SDK is already importable (pure check, no auto-install).""" try: _require_sdk(auto_install=False) return True except OTLPUnavailable: return False def is_enabled(config: Dict[str, Any]) -> bool: otlp = _otlp_config(config) return bool(otlp.get("enabled") and otlp.get("endpoint")) def start_streaming( config: Dict[str, Any], *, event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, ) -> Optional[OTLPStreamer]: """If OTLP is enabled, attach a streamer to the singleton emitter. ``event_filter`` scopes the exporter to its plane. Startup is non-interactive: a configured-but-missing SDK is lazily installed once (prompt=False, gated by security.allow_lazy_installs); if it still can't load, log and no-op — never raise into startup. """ if not is_enabled(config): return None try: _require_sdk(prompt=False) except OTLPUnavailable: logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not " "be installed/imported; install 'hermes-agent[otlp]'") return None from agent.monitoring.emitter import get_emitter streamer = OTLPStreamer(config, event_filter=event_filter) get_emitter().subscribe(streamer) return streamer __all__ = [ "OTLPUnavailable", "OTLPStreamer", "build_exporter", "export_batch", "is_available", "is_enabled", "start_streaming", ]