Teknium
7eb8ccc315
- otlp_exporter hosts SDK loading (symbol table), header/endpoint/resource helpers shared with gateway_health_export; export module imports them instead of keeping copies (_otlp_config/_resolve_headers/_install_id/_safe_resource_attributes unified). _KEEP_BY_KIND attribute allowlist reused for diagnostic log attrs. - events: _MonitoringEvent base supplies to_dict; field order (wire order) unchanged. - redaction: redact_bounded() replaces 3 inline redact+truncate try/excepts. - EmitterStreamer base owns the shared unsubscribe/flush/shutdown for OTLPStreamer and GatewayDiagnosticLogStreamer. - gateway_health: metric construction helpers, _contains_any shared with cron_health, dead _allowed_logger/redact_gateway_message removed (0 refs). - Comment/docstring compaction keeping every stated invariant.
340 lines
13 KiB
Python
340 lines
13 KiB
Python
"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
|
|
|
|
Maps gateway monitoring events to OTel spans and sends them to the operator
|
|
configured ``monitoring.export.otlp`` endpoint (no default destination ships).
|
|
Also hosts the OTLP plumbing shared with ``gateway_health_export`` (SDK loading,
|
|
header resolution, resource attributes, endpoint mapping).
|
|
|
|
* The OTel SDK is an optional extra (``pip install hermes-agent[otlp]``),
|
|
imported lazily so it is only required when export is actually used.
|
|
* ``headers_env`` maps a header name to an environment variable name; values
|
|
are read from the environment at export time and never logged or stored.
|
|
* The continuous subscriber runs in the emitter's dispatcher thread and is
|
|
fail-isolated, so an export error cannot affect the gateway. The
|
|
``event_filter`` seam keeps future planes sharing the emitter from silently
|
|
riding along on this exporter.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import logging
|
|
import os
|
|
import re
|
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
|
|
|
from agent.monitoring.gateway_health import _safe_instance_id
|
|
from agent.monitoring.redaction import redact_bounded
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class OTLPUnavailable(RuntimeError):
|
|
"""Raised when the optional OpenTelemetry SDK isn't installed."""
|
|
|
|
|
|
# ── SDK loading ──────────────────────────────────────────────────────────────
|
|
_SDK_SYMBOLS: Dict[str, str] = {
|
|
"TracerProvider": "opentelemetry.sdk.trace",
|
|
"BatchSpanProcessor": "opentelemetry.sdk.trace.export",
|
|
"Resource": "opentelemetry.sdk.resources",
|
|
"OTLPSpanExporter": "opentelemetry.exporter.otlp.proto.http.trace_exporter",
|
|
"SpanKind": "opentelemetry.trace",
|
|
"OTLPLogExporter": "opentelemetry.exporter.otlp.proto.http._log_exporter",
|
|
"OTLPMetricExporter": "opentelemetry.exporter.otlp.proto.http.metric_exporter",
|
|
"Observation": "opentelemetry.metrics",
|
|
"INVALID_SPAN_ID": "opentelemetry.trace",
|
|
"INVALID_TRACE_ID": "opentelemetry.trace",
|
|
"TraceFlags": "opentelemetry.trace",
|
|
"LogRecord": "opentelemetry._logs",
|
|
"SeverityNumber": "opentelemetry._logs.severity",
|
|
"LoggerProvider": "opentelemetry.sdk._logs",
|
|
"BatchLogRecordProcessor": "opentelemetry.sdk._logs.export",
|
|
"MeterProvider": "opentelemetry.sdk.metrics",
|
|
"PeriodicExportingMetricReader": "opentelemetry.sdk.metrics.export",
|
|
}
|
|
_SPAN_SDK = ("TracerProvider", "BatchSpanProcessor", "Resource", "OTLPSpanExporter", "SpanKind")
|
|
|
|
|
|
def _require_sdk(
|
|
names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True, prompt: bool = True
|
|
) -> Dict[str, Any]:
|
|
"""Import the named OTel SDK symbols, lazily installing the extra on first use.
|
|
|
|
Routes through tools.lazy_deps (feature 'export.otlp') — gated by
|
|
security.allow_lazy_installs and TTY-prompted (``prompt=False`` from
|
|
non-interactive contexts). Any lazy-install failure falls through to the
|
|
import attempt, which raises OTLPUnavailable with a manual install hint.
|
|
"""
|
|
if auto_install:
|
|
try:
|
|
from tools.lazy_deps import ensure as _lazy_ensure
|
|
_lazy_ensure("export.otlp", prompt=prompt)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
return {
|
|
name: getattr(importlib.import_module(_SDK_SYMBOLS[name]), name)
|
|
for name in names
|
|
}
|
|
except Exception as e: # ImportError or partial install
|
|
raise OTLPUnavailable(
|
|
"OTLP export requires the optional dependency. Install with:\n"
|
|
" pip install 'hermes-agent[otlp]'\n"
|
|
f"(import error: {e})"
|
|
)
|
|
|
|
|
|
# ── config + connection plumbing ─────────────────────────────────────────────
|
|
def _monitoring_section(config: Dict[str, Any], *path: str) -> Dict[str, Any]:
|
|
node: Any = (config or {}).get("monitoring") or {}
|
|
for key in path:
|
|
node = node.get(key) or {}
|
|
return node
|
|
|
|
|
|
def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
|
return _monitoring_section(config, "export", "otlp")
|
|
|
|
|
|
def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
|
|
"""Resolve {header_name: ENV_VAR_NAME} -> {header_name: value}; missing vars skipped."""
|
|
resolved: Dict[str, str] = {}
|
|
for header_name, env_name in (headers_env or {}).items():
|
|
val = os.environ.get(str(env_name))
|
|
if val:
|
|
resolved[str(header_name)] = val
|
|
else:
|
|
logger.debug("OTLP header %s: env var %s not set; skipping", header_name, env_name)
|
|
return resolved
|
|
|
|
|
|
_SIGNAL_SUFFIXES = ("/v1/traces", "/v1/metrics")
|
|
|
|
|
|
def _signal_endpoint(endpoint: str, signal: str) -> str:
|
|
"""Rewrite a traces/metrics OTLP path to ``/v1/<signal>``; other paths pass through."""
|
|
target = f"/v1/{signal}"
|
|
for suffix in _SIGNAL_SUFFIXES:
|
|
if suffix != target and endpoint.endswith(suffix):
|
|
return endpoint[: -len(suffix)] + target
|
|
return endpoint
|
|
|
|
|
|
_RESOURCE_ATTRIBUTE_KEYS = frozenset({
|
|
"service.name",
|
|
"service.namespace",
|
|
"service.version",
|
|
"service.instance.id",
|
|
"deployment.environment.name",
|
|
"cloud.provider",
|
|
"cloud.platform",
|
|
"cloud.region",
|
|
"telemetry.scope",
|
|
})
|
|
_SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$")
|
|
|
|
|
|
def _install_id(config: Dict[str, Any]) -> str:
|
|
try:
|
|
from agent.monitoring.policy import ensure_install_id
|
|
return str(ensure_install_id(config))
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
def _safe_resource_attributes(raw: Any) -> Dict[str, str]:
|
|
"""Allowlist bounded resource labels and reject values changed by redaction."""
|
|
attrs: Dict[str, str] = {}
|
|
for key, value in (raw.items() if isinstance(raw, dict) else ()):
|
|
key = str(key)
|
|
if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None:
|
|
continue
|
|
text = str(value)
|
|
if key == "service.instance.id":
|
|
attrs[key] = _safe_instance_id(value)
|
|
elif _SAFE_RESOURCE_VALUE.fullmatch(text) and redact_bounded(text, limit=128) == text:
|
|
attrs[key] = text
|
|
return attrs
|
|
|
|
|
|
def _runtime_resource_attributes(
|
|
config: Dict[str, Any], *, telemetry_scope: str
|
|
) -> Dict[str, str]:
|
|
"""Build the safe OTLP resource shared by spans, metrics and diagnostic logs."""
|
|
gh = _monitoring_section(config, "gateway_health_export")
|
|
attrs = _safe_resource_attributes(gh.get("resource_attributes"))
|
|
attrs["service.name"] = "hermes-gateway"
|
|
attrs["service.instance.id"] = _safe_instance_id(_install_id(config))
|
|
attrs["telemetry.scope"] = telemetry_scope
|
|
return attrs
|
|
|
|
|
|
def build_exporter(config: Dict[str, Any]):
|
|
"""Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
|
|
sdk = _require_sdk()
|
|
otlp = _otlp_config(config)
|
|
endpoint = otlp.get("endpoint")
|
|
if not endpoint:
|
|
raise ValueError("monitoring.export.otlp.endpoint is not set")
|
|
headers = _resolve_headers(otlp.get("headers_env"))
|
|
return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None)
|
|
|
|
|
|
def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]:
|
|
return _runtime_resource_attributes(config, telemetry_scope="gateway_monitoring")
|
|
|
|
|
|
def _make_provider(config: Dict[str, Any]):
|
|
sdk = _require_sdk()
|
|
resource = sdk["Resource"].create(_resource_attributes(config))
|
|
provider = sdk["TracerProvider"](resource=resource)
|
|
processor = sdk["BatchSpanProcessor"](build_exporter(config))
|
|
provider.add_span_processor(processor)
|
|
return provider, processor
|
|
|
|
|
|
# ── event -> span attribute mapping ──────────────────────────────────────────
|
|
# Per-kind attribute allowlists: everything else (profile, install_id, ...) never egresses.
|
|
_KEEP_BY_KIND: Dict[str, tuple[str, ...]] = {
|
|
"gateway_health": ("name", "gateway_state", "old_state", "new_state",
|
|
"exit_reason", "restart_requested", "active_agents",
|
|
"gateway_busy", "gateway_drainable", "platform_count",
|
|
"fatal_platform_count", "version",
|
|
"supervision_mode", "pid"),
|
|
"gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
|
|
"platform", "old_state", "new_state",
|
|
"version", "severity"),
|
|
"cron_execution": ("status", "job_key", "source", "duration_ms",
|
|
"delivery_outcome", "error_class"),
|
|
}
|
|
|
|
|
|
def _allowlisted_attrs(ev: Dict[str, Any], keys: Iterable[str]) -> Dict[str, Any]:
|
|
"""``hermes.<key>`` attributes for present keys; string values are redacted and bounded."""
|
|
attrs: Dict[str, Any] = {}
|
|
for col in keys:
|
|
v = ev.get(col)
|
|
if v is not None:
|
|
attrs[f"hermes.{col}"] = redact_bounded(v) if isinstance(v, str) else v
|
|
return attrs
|
|
|
|
|
|
def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Span attributes for a monitoring event (content-free by construction)."""
|
|
kind = ev.get("event")
|
|
attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"}
|
|
attrs.update(_allowlisted_attrs(ev, _KEEP_BY_KIND.get(kind, ()))) # type: ignore[arg-type]
|
|
return attrs
|
|
|
|
|
|
def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
|
|
"""Map a batch of events to OTel spans. Returns spans created."""
|
|
tracer = provider.get_tracer("hermes.monitoring")
|
|
n = 0
|
|
for ev in batch:
|
|
try:
|
|
name = f"hermes.{ev.get('event', 'event')}"
|
|
span = tracer.start_span(name, attributes=_span_attrs(ev))
|
|
span.end()
|
|
n += 1
|
|
except Exception:
|
|
logger.debug("OTLP span map failed", exc_info=True)
|
|
return n
|
|
|
|
|
|
# ── continuous streaming subscribers ─────────────────────────────────────────
|
|
class EmitterStreamer:
|
|
"""Base for emitter subscribers owning an OTel provider + batch processor.
|
|
|
|
Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.
|
|
"""
|
|
|
|
_provider: Any
|
|
_processor: Any
|
|
exported: int = 0
|
|
|
|
def shutdown(self) -> None:
|
|
try:
|
|
from agent.monitoring.emitter import get_emitter
|
|
get_emitter().unsubscribe(self)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self._processor.force_flush()
|
|
self._provider.shutdown()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
class OTLPStreamer(EmitterStreamer):
|
|
"""A live subscriber that pushes each emitter batch to OTLP as spans."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: Dict[str, Any],
|
|
*,
|
|
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
|
):
|
|
self._provider, self._processor = _make_provider(config)
|
|
self._event_filter = event_filter
|
|
self.exported = 0
|
|
|
|
def __call__(self, batch: List[Dict[str, Any]]) -> None:
|
|
if self._event_filter is not None:
|
|
batch = [ev for ev in batch if self._event_filter(ev)]
|
|
if not batch:
|
|
return
|
|
self.exported += export_batch(self._provider, batch)
|
|
|
|
|
|
def is_available() -> bool:
|
|
"""True when the OTel SDK is already importable (pure check, no auto-install)."""
|
|
try:
|
|
_require_sdk(auto_install=False)
|
|
return True
|
|
except OTLPUnavailable:
|
|
return False
|
|
|
|
|
|
def is_enabled(config: Dict[str, Any]) -> bool:
|
|
otlp = _otlp_config(config)
|
|
return bool(otlp.get("enabled") and otlp.get("endpoint"))
|
|
|
|
|
|
def start_streaming(
|
|
config: Dict[str, Any],
|
|
*,
|
|
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
|
) -> Optional[OTLPStreamer]:
|
|
"""If OTLP is enabled, attach a streamer to the singleton emitter.
|
|
|
|
``event_filter`` scopes the exporter to its plane. Startup is non-interactive:
|
|
a configured-but-missing SDK is lazily installed once (prompt=False, gated by
|
|
security.allow_lazy_installs); if it still can't load, log and no-op — never
|
|
raise into startup.
|
|
"""
|
|
if not is_enabled(config):
|
|
return None
|
|
try:
|
|
_require_sdk(prompt=False)
|
|
except OTLPUnavailable:
|
|
logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
|
|
"be installed/imported; install 'hermes-agent[otlp]'")
|
|
return None
|
|
from agent.monitoring.emitter import get_emitter
|
|
streamer = OTLPStreamer(config, event_filter=event_filter)
|
|
get_emitter().subscribe(streamer)
|
|
return streamer
|
|
|
|
|
|
__all__ = [
|
|
"OTLPUnavailable",
|
|
"OTLPStreamer",
|
|
"build_exporter",
|
|
"export_batch",
|
|
"is_available",
|
|
"is_enabled",
|
|
"start_streaming",
|
|
]
|