joaomarcos
628372de46
_resource_attributes() in otlp_exporter.py built its own hardcoded resource dict (service.name/instance.id/telemetry.scope only) instead of reusing gateway_health_export.py's _runtime_resource_attributes(), which already applies the resource_attributes allowlist from config. Result: operator-configured attributes like deployment.environment.name reached metrics and diagnostic logs but never spans. Span resource building now delegates to the same _runtime_resource_attributes() helper metrics/logs already use, removing the duplicate implementation instead of patching it in place.
271 lines
10 KiB
Python
271 lines
10 KiB
Python
"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
|
|
|
|
Maps gateway monitoring events to OTel spans and sends them to the endpoint
|
|
configured under ``monitoring.export.otlp``. Lets an operator stream Hermes
|
|
gateway health into their own observability stack (OTEL Collector, DataDog,
|
|
and similar).
|
|
|
|
Notes:
|
|
* The destination is operator-configured; this module only sends to that
|
|
endpoint. No default destination ships.
|
|
* ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an
|
|
optional extra (``pip install hermes-agent[otlp]``), imported lazily so the
|
|
dependency is only required when OTLP export is actually used.
|
|
* ``headers_env`` maps a header name to an environment variable name; values
|
|
are read from the environment at export time and never logged or stored.
|
|
* The continuous subscriber runs in the emitter's dispatcher thread and is
|
|
fail-isolated, so an export error cannot affect the gateway.
|
|
|
|
Only monitoring events (gateway_health / gateway_diagnostic) exist on this
|
|
plane; the ``event_filter`` seam is kept so future planes sharing the emitter
|
|
cannot silently ride along on this exporter.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class OTLPUnavailable(RuntimeError):
|
|
"""Raised when the optional OpenTelemetry SDK isn't installed."""
|
|
|
|
|
|
def _require_sdk(*, auto_install: bool = True, prompt: bool = True):
|
|
"""Import the OTel SDK, lazily installing it on first use if needed.
|
|
|
|
Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK
|
|
triggers the standard venv install flow — same as every other optional
|
|
backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back
|
|
to OTLPUnavailable (with a manual install hint) when the SDK can't be made
|
|
importable (lazy installs disabled, install failed, or auto_install=False).
|
|
|
|
``auto_install``: attempt the lazy install when missing (default True).
|
|
``prompt``: ask before installing when interactive (default True); pass
|
|
False from non-interactive contexts like the continuous streamer.
|
|
"""
|
|
if auto_install:
|
|
try:
|
|
from tools.lazy_deps import ensure as _lazy_ensure
|
|
_lazy_ensure("export.otlp", prompt=prompt)
|
|
except ImportError:
|
|
pass # lazy_deps unavailable — fall through to the import attempt
|
|
except Exception:
|
|
# FeatureUnavailable (lazy installs disabled / declined / failed) —
|
|
# fall through; the import below raises OTLPUnavailable with the hint.
|
|
pass
|
|
try:
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
|
OTLPSpanExporter,
|
|
)
|
|
from opentelemetry.trace import SpanKind
|
|
return {
|
|
"TracerProvider": TracerProvider,
|
|
"BatchSpanProcessor": BatchSpanProcessor,
|
|
"Resource": Resource,
|
|
"OTLPSpanExporter": OTLPSpanExporter,
|
|
"SpanKind": SpanKind,
|
|
}
|
|
except Exception as e: # ImportError or partial install
|
|
raise OTLPUnavailable(
|
|
"OTLP export requires the optional dependency. Install with:\n"
|
|
" pip install 'hermes-agent[otlp]'\n"
|
|
f"(import error: {e})"
|
|
)
|
|
|
|
|
|
def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
|
|
"""Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env.
|
|
|
|
The config stores environment variable names, not secret values; values are
|
|
read from the environment here. Missing variables are skipped (and noted at
|
|
debug level without the value).
|
|
"""
|
|
resolved: Dict[str, str] = {}
|
|
for header_name, env_name in (headers_env or {}).items():
|
|
val = os.environ.get(str(env_name))
|
|
if val:
|
|
resolved[str(header_name)] = val
|
|
else:
|
|
logger.debug("OTLP header %s: env var %s not set; skipping",
|
|
header_name, env_name)
|
|
return resolved
|
|
|
|
|
|
def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
|
mon = (config or {}).get("monitoring") or {}
|
|
export = mon.get("export") or {}
|
|
return export.get("otlp") or {}
|
|
|
|
|
|
def build_exporter(config: Dict[str, Any]):
|
|
"""Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
|
|
sdk = _require_sdk()
|
|
otlp = _otlp_config(config)
|
|
endpoint = otlp.get("endpoint")
|
|
if not endpoint:
|
|
raise ValueError("monitoring.export.otlp.endpoint is not set")
|
|
headers = _resolve_headers(otlp.get("headers_env"))
|
|
return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None)
|
|
|
|
|
|
def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]:
|
|
# Lazy import: gateway_health_export imports this module back (for
|
|
# start_streaming), so the dependency must resolve at call time, not
|
|
# at module load, to avoid a circular import.
|
|
from agent.monitoring.gateway_health_export import _runtime_resource_attributes
|
|
|
|
return _runtime_resource_attributes(config, telemetry_scope="gateway_monitoring")
|
|
|
|
|
|
def _make_provider(config: Dict[str, Any]):
|
|
sdk = _require_sdk()
|
|
resource = sdk["Resource"].create(_resource_attributes(config))
|
|
provider = sdk["TracerProvider"](resource=resource)
|
|
processor = sdk["BatchSpanProcessor"](build_exporter(config))
|
|
provider.add_span_processor(processor)
|
|
return provider, processor
|
|
|
|
|
|
# ── event -> span attribute mapping ──────────────────────────────────────────
|
|
def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Span attributes for a monitoring event (content-free by construction)."""
|
|
kind = ev.get("event")
|
|
attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"}
|
|
keep_by_kind = {
|
|
"gateway_health": ("name", "gateway_state", "old_state", "new_state",
|
|
"exit_reason", "restart_requested", "active_agents",
|
|
"gateway_busy", "gateway_drainable", "platform_count",
|
|
"fatal_platform_count", "version",
|
|
"supervision_mode", "pid"),
|
|
"gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
|
|
"platform", "old_state", "new_state",
|
|
"version", "severity"),
|
|
"cron_execution": ("status", "job_key", "source", "duration_ms",
|
|
"delivery_outcome", "error_class"),
|
|
}
|
|
for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type]
|
|
v = ev.get(col)
|
|
if v is not None:
|
|
if isinstance(v, str):
|
|
try:
|
|
from agent.monitoring.redaction import redact_for_export
|
|
v = (redact_for_export(v) or "[redacted]")[:500]
|
|
except Exception:
|
|
v = "[redaction-unavailable]"
|
|
attrs[f"hermes.{col}"] = v
|
|
return attrs
|
|
|
|
|
|
def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
|
|
"""Map a batch of events to OTel spans. Returns spans created."""
|
|
tracer = provider.get_tracer("hermes.monitoring")
|
|
n = 0
|
|
for ev in batch:
|
|
try:
|
|
name = f"hermes.{ev.get('event', 'event')}"
|
|
span = tracer.start_span(name, attributes=_span_attrs(ev))
|
|
span.end()
|
|
n += 1
|
|
except Exception:
|
|
logger.debug("OTLP span map failed", exc_info=True)
|
|
return n
|
|
|
|
|
|
# ── continuous streaming subscriber ─────────────────────────────────────────
|
|
class OTLPStreamer:
|
|
"""A live subscriber that pushes each emitter batch to OTLP as it lands.
|
|
|
|
Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
config: Dict[str, Any],
|
|
*,
|
|
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
|
):
|
|
self._provider, self._processor = _make_provider(config)
|
|
self._event_filter = event_filter
|
|
self.exported = 0
|
|
|
|
def __call__(self, batch: List[Dict[str, Any]]) -> None:
|
|
if self._event_filter is not None:
|
|
batch = [ev for ev in batch if self._event_filter(ev)]
|
|
if not batch:
|
|
return
|
|
self.exported += export_batch(self._provider, batch)
|
|
|
|
def shutdown(self) -> None:
|
|
try:
|
|
from agent.monitoring.emitter import get_emitter
|
|
get_emitter().unsubscribe(self)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self._processor.force_flush()
|
|
self._provider.shutdown()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def is_available() -> bool:
|
|
"""True when the OTel SDK is already importable. Does NOT auto-install —
|
|
this is a pure check (e.g. for status display)."""
|
|
try:
|
|
_require_sdk(auto_install=False)
|
|
return True
|
|
except OTLPUnavailable:
|
|
return False
|
|
|
|
|
|
def is_enabled(config: Dict[str, Any]) -> bool:
|
|
otlp = _otlp_config(config)
|
|
return bool(otlp.get("enabled") and otlp.get("endpoint"))
|
|
|
|
|
|
def start_streaming(
|
|
config: Dict[str, Any],
|
|
*,
|
|
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
|
) -> Optional[OTLPStreamer]:
|
|
"""If OTLP is enabled, attach a streamer to the singleton emitter.
|
|
|
|
``event_filter`` scopes the exporter to its plane, e.g. gateway-health
|
|
export, so enabling one plane cannot silently export unrelated events.
|
|
|
|
Non-interactive context (startup): attempts a lazy install with prompt=False
|
|
so a configured-but-missing SDK is installed once (gated by
|
|
security.allow_lazy_installs), then streams. If it still can't load, logs and
|
|
no-ops — never blocks or raises into startup.
|
|
"""
|
|
if not is_enabled(config):
|
|
return None
|
|
try:
|
|
_require_sdk(prompt=False)
|
|
except OTLPUnavailable:
|
|
logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
|
|
"be installed/imported; install 'hermes-agent[otlp]'")
|
|
return None
|
|
from agent.monitoring.emitter import get_emitter
|
|
streamer = OTLPStreamer(config, event_filter=event_filter)
|
|
get_emitter().subscribe(streamer)
|
|
return streamer
|
|
|
|
|
|
__all__ = [
|
|
"OTLPUnavailable",
|
|
"OTLPStreamer",
|
|
"build_exporter",
|
|
"export_batch",
|
|
"is_available",
|
|
"is_enabled",
|
|
"start_streaming",
|
|
]
|