ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix(discord): liveness probe gains a dispatch-side dimension (#109521)

Browse Source 
The Gateway WS health probe sampled only transport state — ready, open,
heartbeat-ACK age, latency. A socket that stays ESTAB and keeps ACKing while
zero gateway frames arrive (the #109521 "connected-but-deaf" incident) read
healthy indefinitely, and the adapter went silent for hours with no log line
and no watchdog firing.

Two defects fixed:

1. Dispatch-side dimension. `on_socket_raw_receive` now stamps
   `_last_gateway_frame_at` for every inbound raw gateway frame — heartbeats
   and ACKs included, so a legitimately quiet server is not flagged. The
   health check gains an `event_silence` reason with its own bound,
   `websocket_event_max_silence_seconds` (default 300s, 0 disables the
   dimension alone). The stamp resets on `on_ready` so a reconnect never
   inherits pre-restart silence. Trip path is unchanged: consecutive
   failures -> retryable `discord_websocket_health_stale` -> the existing
   reconnect watcher builds a fresh adapter.

2. Silent probe disable. `_finite_positive_config_float` / `_config_int`
   mapped anything `float()` rejects ("15s", "nan", "true") to 0.0 with no
   log line, permanently disabling the watchdog invisibly. Unparsable and
   non-positive values now log one WARNING naming the knob and raw value.

The new key rides the existing `_YAML_WEBSOCKET_LIVENESS_KEYS` seeding and is
documented in the Discord guide's liveness section.
This commit is contained in:
salch-cred
2026-09-13 14:30:54 +05:30
committed by kshitij
parent fadcff9227
commit cef499fbb5
3 changed files with 258 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/platforms/discord/adapter.py
+63 -7
View File
@@ -1068,6 +1068,16 @@ class DiscordAdapter(DiscordMediaMixin, BasePlatformAdapter):
self._max_latency_seconds = self._finite_positive_config_float(
"websocket_max_latency_seconds", 30.0,
)
# Dispatch-side dimension (#109521): ready/open/ACK/latency prove the transport, not
# that events are still being DISPATCHED. Last raw gateway frame (any frame —
# heartbeats, ACKs, presence — so a legitimately quiet server is not "dead") gives
# the probe an event-age bound; a socket that stays ESTAB and keeps ACKing while
# zero frames arrive is the connected-but-deaf fingerprint the old probe read healthy.
self._event_max_silence_seconds = self._finite_positive_config_float(
"websocket_event_max_silence_seconds", 300.0,
)
# perf_counter clock (same as _read_websocket_health's ack math); 0 = never saw a frame.
self._last_gateway_frame_at: float = 0.0
self._liveness_task: Optional[asyncio.Task] = None
self._liveness_notification_task: Optional[asyncio.Task] = None
# True while disconnect() intentionally closes discord.py (done callback: shutdown vs crash).
@@ -1102,24 +1112,45 @@ class DiscordAdapter(DiscordMediaMixin, BasePlatformAdapter):
value = _scoped_gate_env(env_key) or None
return default if value is None or value == "" else value
def _warn_liveness_config_disabled(self, key: str, raw: Any) -> None:
"""One-shot warning when a liveness knob resolves to a disabling value (#109521).
Unparsable config (`"15s"`, `nan`, `true`) silently mapped to 0 and turned the whole
watchdog off with no log line — indistinguishable from "the watchdog missed it".
Loud beats silent: the operator's mitigation (cron-restart on no-`[Discord]`-lines)
exists only because nothing in-process ever told them the probe was off.
"""
logger.warning(
"[%s] Discord liveness knob %s=%r is not a usable positive number; "
"the websocket liveness probe may be disabled by this value",
self.name, key, raw,
)
def _finite_positive_config_float(
self, key: str, default: float, *, env_key: Optional[str] = None
) -> float:
"""Resolve a finite positive liveness duration; invalid values disable it."""
"""Resolve a finite positive liveness duration; invalid values disable it (with a warning)."""
raw = self._config_value(key, default, env_key=env_key)
try:
value = float(self._config_value(key, default, env_key=env_key))
value = float(raw)
except (TypeError, ValueError):
self._warn_liveness_config_disabled(key, raw)
return 0.0
return value if math.isfinite(value) and value > 0 else 0.0
if math.isfinite(value) and value > 0:
return value
self._warn_liveness_config_disabled(key, raw)
return 0.0
def _config_int(self, key: str, default: int, *, env_key: Optional[str] = None) -> int:
"""Resolve a positive liveness count; invalid values disable it."""
value = self._config_value(key, default, env_key=env_key)
if isinstance(value, bool):
"""Resolve a positive liveness count; invalid values disable it (with a warning)."""
raw = self._config_value(key, default, env_key=env_key)
if isinstance(raw, bool):
self._warn_liveness_config_disabled(key, raw)
return 0
try:
return int(value)
return int(raw)
except (TypeError, ValueError):
self._warn_liveness_config_disabled(key, raw)
return 0
def _handle_bot_task_done(self, task: asyncio.Task) -> None:
@@ -1227,6 +1258,9 @@ class DiscordAdapter(DiscordMediaMixin, BasePlatformAdapter):
logger.info("[%s] Connected as %s", adapter_self.name, adapter_self._client.user)
await adapter_self._resolve_allowed_usernames()
adapter_self._ready_event.set()
# Fresh connection => no silence history: reset the dispatch-side stamp so a
# reconnect never inherits the pre-restart silence (#109521).
adapter_self._last_gateway_frame_at = time.perf_counter()
if adapter_self._post_connect_task and not adapter_self._post_connect_task.done():
adapter_self._post_connect_task.cancel()
adapter_self._post_connect_task = asyncio.create_task(
@@ -1235,6 +1269,17 @@ class DiscordAdapter(DiscordMediaMixin, BasePlatformAdapter):
if adapter_self._missed_message_backfill_enabled():
adapter_self._ensure_missed_message_backfill_task()
@self._client.event
async def on_socket_raw_receive(msg: str):
"""Stamp every inbound gateway frame for the liveness probe's event-age check.
Fires for ALL frames — heartbeats, ACKs, presence — not just messages, so a
legitimately quiet server is not "dead", but a socket that stays ESTAB while
zero frames arrive (connected-but-deaf, #109521 incident 2) becomes visible.
Intentionally bare: parsing happens in discord.py; this hook only clocks.
"""
adapter_self._last_gateway_frame_at = time.perf_counter()
@self._client.event
async def on_message(message: DiscordMessage):
await adapter_self._dispatch_discord_message(message)
@@ -1590,6 +1635,7 @@ class DiscordAdapter(DiscordMediaMixin, BasePlatformAdapter):
or self._liveness_failure_threshold <= 0
or self._heartbeat_ack_max_age_seconds <= 0
or self._max_latency_seconds <= 0
or self._event_max_silence_seconds <= 0
):
return
if self._liveness_task and not self._liveness_task.done():
@@ -1629,6 +1675,15 @@ class DiscordAdapter(DiscordMediaMixin, BasePlatformAdapter):
return False, "latency_non_finite"
if latency > self._max_latency_seconds:
return False, "latency_exceeded"
# Dispatch-side dimension (#109521): every check above inspects the transport, and a
# socket that is ESTAB and still ACKing can deliver ZERO events for hours while
# reading healthy. The last raw gateway frame (any frame, including heartbeats) is
# the one signal that events are actually arriving. Never-seen-frame counts as
# silent — on_ready fires long before any gap could be legitimate.
if self._event_max_silence_seconds > 0:
frame_age = time.perf_counter() - self._last_gateway_frame_at
if self._last_gateway_frame_at <= 0 or frame_age > self._event_max_silence_seconds:
return False, "event_silence"
return True, "healthy"
async def _liveness_loop(self) -> None:
@@ -6923,6 +6978,7 @@ _YAML_WEBSOCKET_LIVENESS_KEYS = (
("websocket_liveness_failure_threshold", "liveness_failure_threshold", "HERMES_DISCORD_LIVENESS_FAILURE_THRESHOLD"),
("websocket_heartbeat_ack_max_age_seconds", None, None),
("websocket_max_latency_seconds", None, None),
("websocket_event_max_silence_seconds", None, None),
)