fix(gateway): loop guard judges the final authz verdict and reads config.yaml
#91483 added the guard inside the ALLOW_BOTS block. on today's authz layout an admitted bot returns from an earlier rule before that block runs, so the guard never tripped (its own ping-pong test passes turn 21 on this tree). _is_user_authorized now computes the allowlist verdict first and runs every admitted bot-authored message through the guard, so bots admitted by a chat allowlist, allow-all, pairing or delegation are metered too. the budget stays per conversation. settings move from HERMES_BOT_LOOP_* environment variables to config.yaml gateway.bot_loop_guard (defaults added to config_defaults), re-read on every call. an unauthorized bot dm no longer gets a pairing code, so a cooldown produces no outbound traffic. tests rebuilt against real SessionSource objects.
This commit is contained in:
+37
-54
@@ -8,9 +8,12 @@ imports its ``logger`` lazily so records keep the ``"gateway.run"`` name.
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from gateway.bot_loop_guard import BotLoopGuard
|
||||
from gateway.config import Platform
|
||||
from gateway.pairing import _PLATFORM_ALLOWLIST_ENV
|
||||
from gateway.session import SessionSource
|
||||
@@ -22,6 +25,8 @@ from gateway.whatsapp_identity import (
|
||||
_GROUP_CHAT_TYPES = frozenset({"group", "forum", "channel"})
|
||||
_GROUP_FORUM_TYPES = frozenset({"group", "forum"})
|
||||
_TRUTHY = frozenset({"true", "1", "yes"})
|
||||
_BOT_LOOP_GUARD_INIT_LOCK = threading.Lock()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Platform -> ``<PLATFORM>_ALLOWED_USERS`` / ``<PLATFORM>_ALLOW_ALL_USERS``. Shared with the pairing
|
||||
# store's allowlist mirror (single source of truth); plugin platforms are added per-call from the registry.
|
||||
@@ -470,13 +475,44 @@ class GatewayAuthorizationMixin:
|
||||
self._warned_telegram_group_users_legacy = True
|
||||
return source.chat_id in legacy_chat_ids
|
||||
|
||||
def _bot_loop_guard_admits(self, source: SessionSource, adapter_profile: Optional[str]) -> bool:
|
||||
"""Count one admitted bot-authored message; False while its conversation is over budget."""
|
||||
guard = getattr(self, "_bot_loop_guard", None)
|
||||
if guard is None:
|
||||
with _BOT_LOOP_GUARD_INIT_LOCK:
|
||||
guard = getattr(self, "_bot_loop_guard", None)
|
||||
if guard is None:
|
||||
guard = self._bot_loop_guard = BotLoopGuard()
|
||||
platform = source.platform.value if source.platform else ""
|
||||
# One budget per conversation, not per sender pair: inbound only ever shows the other bots,
|
||||
# so a per-pair key would hand N bots N budgets.
|
||||
allowed, state = guard.admit((adapter_profile or "", platform, str(source.chat_id or "")))
|
||||
if state == "tripped":
|
||||
logger.warning(
|
||||
"Bot loop guard is dropping bot messages in %s chat %s: bot %s sent one message too many "
|
||||
"for the window, cooling down (gateway.bot_loop_guard in config.yaml).",
|
||||
platform, source.chat_id, source.user_id,
|
||||
)
|
||||
return allowed
|
||||
|
||||
def _is_user_authorized(self, source: SessionSource, *, allow_adapter_delegation: bool = True) -> bool:
|
||||
"""Whether a user may use the bot.
|
||||
|
||||
Order: trusted-upstream delegation, chat-scoped group allowlists, ``{PLATFORM}_ALLOW_BOTS``,
|
||||
per-platform allow-all, adapter role auth, pairing store, env/config allowlists,
|
||||
``GATEWAY_ALLOW_ALL_USERS``, default deny.
|
||||
``GATEWAY_ALLOW_ALL_USERS``, default deny. A bot-authored message that any of these admits
|
||||
then passes the bot loop guard.
|
||||
"""
|
||||
if not self._principal_authorized(source, allow_adapter_delegation=allow_adapter_delegation):
|
||||
return False
|
||||
if not getattr(source, "is_bot", False):
|
||||
return True
|
||||
# The guard judges the final verdict: the chat-scoped allowlist admits a bot in an allowlisted
|
||||
# chat before the ALLOW_BOTS block runs, so a guard inside that block would never see it.
|
||||
return self._bot_loop_guard_admits(source, self._adapter_profile_for_source(source))
|
||||
|
||||
def _principal_authorized(self, source: SessionSource, *, allow_adapter_delegation: bool) -> bool:
|
||||
"""The allowlist verdict alone, before the bot loop guard."""
|
||||
# HA events are system-generated (HASS_TOKEN); webhook events are HMAC-verified.
|
||||
if source.platform in {Platform.HOMEASSISTANT, Platform.WEBHOOK}:
|
||||
return True
|
||||
@@ -487,59 +523,6 @@ class GatewayAuthorizationMixin:
|
||||
if self._chat_scoped_grant(source, adapter_profile, is_group, allow_adapter_delegation):
|
||||
return True
|
||||
user_id = source.user_id
|
||||
# Pair loop protection for bot-authored traffic (#79077). MUST run
|
||||
# before the group-allowlist shortcut below: TELEGRAM_GROUP_ALLOWED_CHATS
|
||||
# returns True for any sender in an allowlisted chat, bots included, so a
|
||||
# guard placed after it never executes for the exact configuration that
|
||||
# needs it (an allowlisted group with two bots in it).
|
||||
#
|
||||
# Telegram Bot API 10.0 ships no loop guard of its own --
|
||||
# core.telegram.org/api/bots/bot-to-bot requires the BOT to "make
|
||||
# bot-message handling terminate predictably" via dedupe, rate limits and
|
||||
# maximum interaction depth per sender/receiver pair. ALLOW_BOTS is
|
||||
# admission policy only: a bot replying to another bot satisfies the
|
||||
# "mentions" test, so each turn re-arms the peer and the exchange never
|
||||
# terminates (observed 2026-08-21: 132 messages in one group before a
|
||||
# human intervened).
|
||||
#
|
||||
# The budget is per CONVERSATION, not per (sender, receiver): this
|
||||
# process only ever sees inbound messages, so the receiver side is a
|
||||
# constant (our own profile) and keying on the pair would hand every
|
||||
# distinct sender its own budget -- N bots in one group would then need
|
||||
# N x budget messages to trip a guard meant to cap the whole exchange.
|
||||
if getattr(source, "is_bot", False):
|
||||
_allow_var = {
|
||||
Platform.DISCORD: "DISCORD_ALLOW_BOTS",
|
||||
Platform.FEISHU: "FEISHU_ALLOW_BOTS",
|
||||
Platform.TELEGRAM: "TELEGRAM_ALLOW_BOTS",
|
||||
Platform.SLACK: "SLACK_ALLOW_BOTS",
|
||||
}.get(source.platform)
|
||||
if _allow_var and _platform_gate_env(_allow_var, "none").lower().strip() in {"mentions", "all"}:
|
||||
try:
|
||||
from gateway.bot_loop_guard import allow_bot_event
|
||||
|
||||
_ok, _why = allow_bot_event(
|
||||
scope=str(getattr(self, "profile_name", "") or "-"),
|
||||
conversation=str(getattr(source, "chat_id", "") or "-"),
|
||||
sender_bot="bots",
|
||||
receiver_bot="bots",
|
||||
)
|
||||
if not _ok:
|
||||
try:
|
||||
import logging
|
||||
|
||||
logging.getLogger(__name__).warning(
|
||||
"Bot-to-bot loop guard suppressed message from %s in %s (%s)",
|
||||
getattr(source, "user_id", "?"),
|
||||
getattr(source, "chat_id", "?"),
|
||||
_why,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if not user_id:
|
||||
return False
|
||||
|
||||
|
||||
+111
-126
@@ -1,155 +1,140 @@
|
||||
"""Pair loop protection for bot-to-bot messaging.
|
||||
"""Sliding-window budget for bot-authored inbound messages (#91481).
|
||||
|
||||
Telegram Bot API 10.0 (2026-05-08) allows bots to receive messages from other
|
||||
bots. The platform ships NO loop guard: core.telegram.org/api/bots/bot-to-bot
|
||||
states plainly that "Bot-to-bot communication can create infinite reply loops.
|
||||
Bots using this feature must make bot-message handling terminate predictably"
|
||||
and lists the required safeguards:
|
||||
``{PLATFORM}_ALLOW_BOTS`` only decides admission. When two Hermes profiles reply to each
|
||||
other, every reply satisfies the ``mentions`` test again, so nothing ends the exchange.
|
||||
The guard counts admitted bot-authored messages per conversation and, once a conversation
|
||||
exceeds ``max_events`` inside ``window_seconds``, drops further bot messages there for
|
||||
``cooldown_seconds``. Human traffic never enters the guard.
|
||||
|
||||
* Deduplicate repeated messages.
|
||||
* Apply per-chat and per-bot rate limits.
|
||||
* Enforce maximum interaction depth and timeouts, both globally and per
|
||||
sender/receiver pair.
|
||||
|
||||
This module implements a sliding-window budget per (conversation, bot pair).
|
||||
The pair is tracked order-independently -- A->B and B->A count as the SAME
|
||||
pair -- so a two-bot ping-pong burns one shared budget instead of two.
|
||||
|
||||
Observed failure this guards against (2026-08-21, profiles medicina + ytmed):
|
||||
132 messages exchanged in a single Telegram group before a human intervened.
|
||||
TELEGRAM_ALLOW_BOTS=mentions did NOT stop it: when bot A replies to bot B, the
|
||||
reply itself satisfies the "mention" test, so each turn re-armed the other bot.
|
||||
|
||||
Tunables (env, all optional):
|
||||
HERMES_BOT_LOOP_PROTECTION on|off (default on)
|
||||
HERMES_BOT_LOOP_MAX_EVENTS int (default 20)
|
||||
HERMES_BOT_LOOP_WINDOW_SEC int (default 60)
|
||||
HERMES_BOT_LOOP_COOLDOWN_SEC int (default 60)
|
||||
|
||||
Defaults match the OpenClaw reference implementation (20 events / 60 s window /
|
||||
60 s cooldown), which solves the same problem on Discord, Slack, Matrix,
|
||||
Feishu and Google Chat.
|
||||
Settings live in config.yaml under ``gateway.bot_loop_guard``:
|
||||
``enabled`` (true), ``max_events`` (20), ``window_seconds`` (300), ``cooldown_seconds`` (600).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Deque, Dict, Optional, Tuple
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Deque, Dict, Hashable, Tuple
|
||||
|
||||
__all__ = ["allow_bot_event", "loop_guard_state", "reset_loop_guard"]
|
||||
__all__ = ["BotLoopGuard", "BotLoopGuardSettings", "load_settings", "settings_from_config"]
|
||||
|
||||
_TRUTHY = frozenset({"true", "1", "yes", "on"})
|
||||
_FALSY = frozenset({"false", "0", "no", "off"})
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
raw = os.getenv(name)
|
||||
if not raw:
|
||||
@dataclass(frozen=True)
|
||||
class BotLoopGuardSettings:
|
||||
enabled: bool = True
|
||||
max_events: int = 20
|
||||
window_seconds: float = 300.0
|
||||
cooldown_seconds: float = 600.0
|
||||
|
||||
|
||||
def _as_bool(raw, default: bool) -> bool:
|
||||
if isinstance(raw, bool):
|
||||
return raw
|
||||
text = str(raw).strip().lower() if raw is not None else ""
|
||||
if text in _TRUTHY:
|
||||
return True
|
||||
if text in _FALSY:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _as_positive(raw, default: float) -> float:
|
||||
if isinstance(raw, bool):
|
||||
return default
|
||||
try:
|
||||
val = int(str(raw).strip())
|
||||
value = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return val if val > 0 else default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
raw = (os.getenv("HERMES_BOT_LOOP_PROTECTION") or "on").strip().lower()
|
||||
return raw not in {"off", "false", "0", "no", "disabled"}
|
||||
def settings_from_config(cfg) -> BotLoopGuardSettings:
|
||||
"""Read ``gateway.bot_loop_guard`` from a loaded config dict; unusable values keep the default."""
|
||||
from hermes_cli.config import cfg_get
|
||||
|
||||
block = cfg_get(cfg, "gateway", "bot_loop_guard", default=None)
|
||||
if not isinstance(block, dict):
|
||||
return BotLoopGuardSettings()
|
||||
defaults = BotLoopGuardSettings()
|
||||
return BotLoopGuardSettings(
|
||||
enabled=_as_bool(block.get("enabled"), defaults.enabled),
|
||||
max_events=int(_as_positive(block.get("max_events"), defaults.max_events)),
|
||||
window_seconds=_as_positive(block.get("window_seconds"), defaults.window_seconds),
|
||||
cooldown_seconds=_as_positive(block.get("cooldown_seconds"), defaults.cooldown_seconds),
|
||||
)
|
||||
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
# key -> (deque of event timestamps, cooldown_until)
|
||||
_EVENTS: Dict[Tuple[str, str, str], Deque[float]] = {}
|
||||
_COOLDOWN: Dict[Tuple[str, str, str], float] = {}
|
||||
_LAST_SWEEP = 0.0
|
||||
def load_settings() -> BotLoopGuardSettings:
|
||||
"""Settings from the live config.yaml; defaults when the config cannot be read."""
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
return settings_from_config(load_config_readonly())
|
||||
except Exception:
|
||||
return BotLoopGuardSettings()
|
||||
|
||||
|
||||
def _key(scope: str, conversation: str, a: str, b: str) -> Tuple[str, str, str]:
|
||||
"""Order-independent pair key: A->B and B->A collapse to one bucket."""
|
||||
lo, hi = sorted([str(a or "?"), str(b or "?")])
|
||||
return (str(scope or "-"), str(conversation or "-"), f"{lo}|{hi}")
|
||||
class BotLoopGuard:
|
||||
"""Per-conversation sliding window with a cooldown once the budget trips. Thread-safe.
|
||||
|
||||
|
||||
def _sweep(now: float, window: float) -> None:
|
||||
"""Drop buckets untouched for 10 windows so memory stays bounded."""
|
||||
global _LAST_SWEEP
|
||||
if now - _LAST_SWEEP < window:
|
||||
return
|
||||
_LAST_SWEEP = now
|
||||
horizon = now - (window * 10)
|
||||
for k in [k for k, dq in _EVENTS.items() if not dq or dq[-1] < horizon]:
|
||||
_EVENTS.pop(k, None)
|
||||
if _COOLDOWN.get(k, 0.0) < now:
|
||||
_COOLDOWN.pop(k, None)
|
||||
|
||||
|
||||
def allow_bot_event(
|
||||
scope: str,
|
||||
conversation: str,
|
||||
sender_bot: str,
|
||||
receiver_bot: str,
|
||||
*,
|
||||
now: Optional[float] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Return (allowed, reason) for one inbound bot-authored message.
|
||||
|
||||
Call ONLY for messages authored by another bot, at the moment the message
|
||||
is admitted. Human traffic must never reach this function.
|
||||
Settings are re-read on every call so a config.yaml edit takes effect without a restart;
|
||||
``load_config_readonly`` caches on the file signature, so the read is cheap.
|
||||
"""
|
||||
if not _enabled():
|
||||
return True, "guard-disabled"
|
||||
|
||||
max_events = _env_int("HERMES_BOT_LOOP_MAX_EVENTS", 20)
|
||||
window = float(_env_int("HERMES_BOT_LOOP_WINDOW_SEC", 60))
|
||||
cooldown = float(_env_int("HERMES_BOT_LOOP_COOLDOWN_SEC", 60))
|
||||
def __init__(
|
||||
self,
|
||||
settings: Callable[[], BotLoopGuardSettings] = load_settings,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
self._clock = clock
|
||||
self._lock = threading.Lock()
|
||||
self._events: Dict[Hashable, Deque[float]] = {}
|
||||
self._cooldown_until: Dict[Hashable, float] = {}
|
||||
self._last_sweep = 0.0
|
||||
|
||||
t = time.monotonic() if now is None else now
|
||||
k = _key(scope, conversation, sender_bot, receiver_bot)
|
||||
@property
|
||||
def tracked_conversations(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._events)
|
||||
|
||||
with _LOCK:
|
||||
_sweep(t, window)
|
||||
def admit(self, conversation: Hashable) -> Tuple[bool, str]:
|
||||
"""Count one admitted bot-authored message for ``conversation``.
|
||||
|
||||
until = _COOLDOWN.get(k, 0.0)
|
||||
if until > t:
|
||||
return False, f"cooldown:{until - t:.0f}s-left"
|
||||
Returns ``(allowed, state)`` with state one of ``disabled``, ``ok``, ``tripped``
|
||||
(this message exceeded the budget and started the cooldown) or ``cooldown``.
|
||||
"""
|
||||
settings = self._settings()
|
||||
if not settings.enabled:
|
||||
return True, "disabled"
|
||||
now = self._clock()
|
||||
with self._lock:
|
||||
self._sweep(now, settings)
|
||||
if self._cooldown_until.get(conversation, 0.0) > now:
|
||||
return False, "cooldown"
|
||||
events = self._events.setdefault(conversation, deque())
|
||||
cutoff = now - settings.window_seconds
|
||||
while events and events[0] <= cutoff:
|
||||
events.popleft()
|
||||
if len(events) >= settings.max_events:
|
||||
self._cooldown_until[conversation] = now + settings.cooldown_seconds
|
||||
events.clear()
|
||||
return False, "tripped"
|
||||
events.append(now)
|
||||
return True, "ok"
|
||||
|
||||
dq = _EVENTS.setdefault(k, deque())
|
||||
cutoff = t - window
|
||||
while dq and dq[0] < cutoff:
|
||||
dq.popleft()
|
||||
|
||||
if len(dq) >= max_events:
|
||||
_COOLDOWN[k] = t + cooldown
|
||||
dq.clear()
|
||||
return False, f"budget-exceeded:{max_events}/{window:.0f}s"
|
||||
|
||||
dq.append(t)
|
||||
return True, f"ok:{len(dq)}/{max_events}"
|
||||
|
||||
|
||||
def loop_guard_state() -> dict:
|
||||
"""Snapshot for diagnostics."""
|
||||
t = time.monotonic()
|
||||
with _LOCK:
|
||||
return {
|
||||
"enabled": _enabled(),
|
||||
"max_events": _env_int("HERMES_BOT_LOOP_MAX_EVENTS", 20),
|
||||
"window_seconds": _env_int("HERMES_BOT_LOOP_WINDOW_SEC", 60),
|
||||
"cooldown_seconds": _env_int("HERMES_BOT_LOOP_COOLDOWN_SEC", 60),
|
||||
"tracked_pairs": len(_EVENTS),
|
||||
"pairs": {
|
||||
"|".join(k): len(dq) for k, dq in list(_EVENTS.items())[:20]
|
||||
},
|
||||
"cooling_down": {
|
||||
"|".join(k): round(v - t, 1)
|
||||
for k, v in _COOLDOWN.items()
|
||||
if v > t
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def reset_loop_guard() -> None:
|
||||
with _LOCK:
|
||||
_EVENTS.clear()
|
||||
_COOLDOWN.clear()
|
||||
def _sweep(self, now: float, settings: BotLoopGuardSettings) -> None:
|
||||
"""Drop idle conversations at most once per window so memory stays bounded."""
|
||||
if now - self._last_sweep < settings.window_seconds:
|
||||
return
|
||||
self._last_sweep = now
|
||||
idle_cutoff = now - settings.window_seconds
|
||||
for key in [k for k, dq in self._events.items() if not dq or dq[-1] <= idle_cutoff]:
|
||||
del self._events[key]
|
||||
for key in [k for k, until in self._cooldown_until.items() if until <= now]:
|
||||
del self._cooldown_until[key]
|
||||
|
||||
@@ -191,8 +191,10 @@ class GatewayInboundMixin:
|
||||
return None
|
||||
logger.warning("Unauthorized user: %s (%s) on %s", source.user_id, source.user_name, source.platform.value)
|
||||
# In DMs: offer pairing code. In groups: silently ignore.
|
||||
# A bot cannot pair, and answering one during a loop-guard cooldown would be outbound traffic.
|
||||
if (
|
||||
source.chat_type == "dm"
|
||||
and not getattr(source, "is_bot", False)
|
||||
and self._get_unauthorized_dm_behavior(source.platform, profile=source.profile) == "pair"
|
||||
):
|
||||
await self._hm_offer_pairing_code(source)
|
||||
|
||||
@@ -1916,6 +1916,8 @@ DEFAULT_CONFIG = {
|
||||
"loop_watchdog_probe_interval_s": 30.0,
|
||||
"loop_watchdog_probe_timeout_s": 10.0,
|
||||
"loop_watchdog_max_strikes": 3,
|
||||
# Bot-to-bot loop guard: admitted bot messages per conversation before a cooldown (#91481).
|
||||
"bot_loop_guard": {"enabled": True, "max_events": 20, "window_seconds": 300, "cooldown_seconds": 600},
|
||||
# Startup-liveness watchdog: stdlib-only daemon thread armed at process entry that
|
||||
# hard-exits 75 if the loop isn't live within the deadline. Armed before config loads, so
|
||||
# run_gateway() bridges these to HERMES_STARTUP_WATCHDOG / HERMES_STARTUP_WATCHDOG_TIMEOUT_S
|
||||
|
||||
@@ -1,31 +1,29 @@
|
||||
"""Bot-to-bot pair loop protection (Telegram Bot API 10.0).
|
||||
"""Bot-to-bot loop guard in ``_is_user_authorized`` (#91481).
|
||||
|
||||
Telegram ships no loop guard of its own: core.telegram.org/api/bots/bot-to-bot
|
||||
requires the BOT to "make bot-message handling terminate predictably" via
|
||||
dedupe, rate limits and maximum interaction depth per sender/receiver pair.
|
||||
|
||||
Every test here was verified FAILING against the tree without the guard
|
||||
(mutation-checked), which matters more than the count: exercising the real
|
||||
configuration is what makes them meaningful. In particular they run with
|
||||
TELEGRAM_GROUP_ALLOWED_CHATS set, because that env var short-circuits
|
||||
_is_user_authorized with `return True` ~32 lines before the is_bot block --
|
||||
a guard placed in the is_bot block would never run for the one configuration
|
||||
that needs it, and a test without the allowlist would go green through a
|
||||
different code path.
|
||||
The scenarios run with ``TELEGRAM_GROUP_ALLOWED_CHATS`` set: that allowlist admits every sender in
|
||||
the chat, bots included, before the ``ALLOW_BOTS`` block runs. A guard hooked only into the
|
||||
``ALLOW_BOTS`` block never sees a bot in an allowlisted group, which is the configuration that
|
||||
produced the incident.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.bot_loop_guard import BotLoopGuard, BotLoopGuardSettings, load_settings, settings_from_config
|
||||
from gateway.session import Platform, SessionSource
|
||||
|
||||
GROUP_CHAT = "-1001234567890"
|
||||
OTHER_GROUP = "-1009876543210"
|
||||
BOT_A = "111111111"
|
||||
BOT_B = "222222222"
|
||||
HUMAN = "100200300"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(monkeypatch):
|
||||
def _isolate_telegram_env(monkeypatch):
|
||||
for var in (
|
||||
"TELEGRAM_ALLOW_BOTS",
|
||||
"TELEGRAM_ALLOWED_USERS",
|
||||
@@ -34,205 +32,272 @@ def _isolate_env(monkeypatch):
|
||||
"TELEGRAM_GROUP_ALLOWED_CHATS",
|
||||
"GATEWAY_ALLOW_ALL_USERS",
|
||||
"GATEWAY_ALLOWED_USERS",
|
||||
"HERMES_BOT_LOOP_PROTECTION",
|
||||
"HERMES_BOT_LOOP_MAX_EVENTS",
|
||||
"HERMES_BOT_LOOP_WINDOW_SEC",
|
||||
"HERMES_BOT_LOOP_COOLDOWN_SEC",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
from gateway.bot_loop_guard import reset_loop_guard
|
||||
|
||||
reset_loop_guard()
|
||||
yield
|
||||
reset_loop_guard()
|
||||
@pytest.fixture
|
||||
def clock():
|
||||
state = {"t": 1_000_000.0}
|
||||
return SimpleNamespace(now=lambda: state["t"], advance=lambda secs: state.__setitem__("t", state["t"] + secs))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_clock(monkeypatch):
|
||||
"""Drive the guard's sliding window deterministically."""
|
||||
from gateway import bot_loop_guard
|
||||
|
||||
state = {"t": 1_000_000.0}
|
||||
monkeypatch.setattr(bot_loop_guard.time, "monotonic", lambda: state["t"])
|
||||
return SimpleNamespace(
|
||||
advance=lambda secs: state.__setitem__("t", state["t"] + secs),
|
||||
now=lambda: state["t"],
|
||||
)
|
||||
def settings():
|
||||
"""Mutable holder so a test can flip settings on a live guard."""
|
||||
return {"value": BotLoopGuardSettings(max_events=20, window_seconds=60, cooldown_seconds=60)}
|
||||
|
||||
|
||||
def _runner():
|
||||
@pytest.fixture
|
||||
def runner(clock, settings):
|
||||
from gateway.run import GatewayRunner
|
||||
|
||||
runner = object.__new__(GatewayRunner)
|
||||
runner.pairing_store = SimpleNamespace(is_approved=lambda *_a, **_kw: False)
|
||||
runner._bot_loop_guard = BotLoopGuard(settings=lambda: settings["value"], clock=clock.now)
|
||||
return runner
|
||||
|
||||
|
||||
def _bot(user_id: str, chat_id: str = GROUP_CHAT) -> SessionSource:
|
||||
"""A fresh inbound bot message.
|
||||
|
||||
A new SessionSource per turn, mirroring production (each update builds its
|
||||
own). A hand-rolled stub does not work here: _is_user_authorized touches
|
||||
delivered_via_upstream_relay and other real fields and blows up with
|
||||
AttributeError.
|
||||
"""
|
||||
source = SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id=chat_id,
|
||||
chat_type="group",
|
||||
user_id=user_id,
|
||||
user_name=f"Bot{user_id}",
|
||||
def _bot(user_id: str, chat_id: str = GROUP_CHAT, chat_type: str = "group") -> SessionSource:
|
||||
return SessionSource(
|
||||
platform=Platform.TELEGRAM, chat_id=chat_id, chat_type=chat_type,
|
||||
user_id=user_id, user_name=f"Bot{user_id}", is_bot=True,
|
||||
)
|
||||
source.is_bot = True
|
||||
return source
|
||||
|
||||
|
||||
def _human(user_id: str = "100200300", chat_id: str = GROUP_CHAT, chat_type: str = "group") -> SessionSource:
|
||||
source = SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id=chat_id,
|
||||
chat_type=chat_type,
|
||||
user_id=user_id,
|
||||
user_name="Alice",
|
||||
def _human(user_id: str = HUMAN, chat_id: str = GROUP_CHAT, chat_type: str = "group") -> SessionSource:
|
||||
return SessionSource(
|
||||
platform=Platform.TELEGRAM, chat_id=chat_id, chat_type=chat_type,
|
||||
user_id=user_id, user_name="Alice", is_bot=False,
|
||||
)
|
||||
source.is_bot = False
|
||||
return source
|
||||
|
||||
|
||||
def _real_config(monkeypatch, *, max_events="20"):
|
||||
"""The configuration that actually reproduces the incident."""
|
||||
def _incident_config(monkeypatch):
|
||||
"""ALLOW_BOTS on, a human allowlist, and the group-chat allowlist that short-circuits authz."""
|
||||
monkeypatch.setenv("TELEGRAM_ALLOW_BOTS", "mentions")
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "100200300")
|
||||
# The short-circuit that makes guard placement matter.
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", HUMAN)
|
||||
monkeypatch.setenv("TELEGRAM_GROUP_ALLOWED_CHATS", f"{GROUP_CHAT},{OTHER_GROUP}")
|
||||
monkeypatch.setenv("HERMES_BOT_LOOP_MAX_EVENTS", max_events)
|
||||
monkeypatch.setenv("HERMES_BOT_LOOP_WINDOW_SEC", "60")
|
||||
monkeypatch.setenv("HERMES_BOT_LOOP_COOLDOWN_SEC", "60")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 1
|
||||
def _ping_pong(runner, turns: int, chat_id: str = GROUP_CHAT) -> list:
|
||||
return [runner._is_user_authorized(_bot(BOT_A if t % 2 == 0 else BOT_B, chat_id)) for t in range(turns)]
|
||||
|
||||
|
||||
def test_ping_pong_of_40_turns_is_cut_at_turn_21(monkeypatch, fake_clock):
|
||||
"""The incident, reduced: two bots replying to each other must terminate.
|
||||
|
||||
Observed 2026-08-21 in production: 132 messages between two Hermes profiles
|
||||
in one group before a human intervened. With a budget of 20 the 21st
|
||||
inbound bot message must be suppressed.
|
||||
"""
|
||||
_real_config(monkeypatch, max_events="20")
|
||||
runner = _runner()
|
||||
|
||||
verdicts = []
|
||||
for turn in range(40):
|
||||
sender = "111111111" if turn % 2 == 0 else "222222222"
|
||||
verdicts.append(runner._is_user_authorized(_bot(sender)))
|
||||
|
||||
assert verdicts[:20] == [True] * 20, "first 20 turns must pass"
|
||||
assert verdicts[20] is False, "turn 21 must be suppressed"
|
||||
assert not any(verdicts[20:]), "the exchange must not resume inside cooldown"
|
||||
# --- authz integration -----------------------------------------------------
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 2
|
||||
def test_ping_pong_of_40_turns_is_cut_at_the_budget(monkeypatch, runner):
|
||||
_incident_config(monkeypatch)
|
||||
|
||||
verdicts = _ping_pong(runner, 40)
|
||||
|
||||
assert verdicts[:20] == [True] * 20
|
||||
assert verdicts[20] is False
|
||||
assert not any(verdicts[20:])
|
||||
|
||||
|
||||
def test_human_in_same_group_still_authorized_during_cooldown(monkeypatch, fake_clock):
|
||||
"""The guard must never take the group down for people."""
|
||||
_real_config(monkeypatch, max_events="20")
|
||||
runner = _runner()
|
||||
def test_human_in_same_group_stays_authorized_during_cooldown(monkeypatch, runner, clock):
|
||||
_incident_config(monkeypatch)
|
||||
_ping_pong(runner, 25)
|
||||
|
||||
for turn in range(25):
|
||||
runner._is_user_authorized(_bot("111111111" if turn % 2 == 0 else "222222222"))
|
||||
|
||||
assert runner._is_user_authorized(_bot("111111111")) is False, "precondition: bots are cooling down"
|
||||
assert runner._is_user_authorized(_bot(BOT_A)) is False
|
||||
assert runner._is_user_authorized(_human()) is True
|
||||
fake_clock.advance(5)
|
||||
clock.advance(5)
|
||||
assert runner._is_user_authorized(_human()) is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 3
|
||||
|
||||
|
||||
def test_human_dm_unaffected(monkeypatch, fake_clock):
|
||||
"""Human DMs never enter the guard at all."""
|
||||
def test_human_traffic_is_never_metered(monkeypatch, runner):
|
||||
monkeypatch.setenv("TELEGRAM_ALLOW_BOTS", "mentions")
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "100200300")
|
||||
monkeypatch.setenv("HERMES_BOT_LOOP_MAX_EVENTS", "20")
|
||||
runner = _runner()
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", HUMAN)
|
||||
|
||||
for _ in range(50):
|
||||
assert runner._is_user_authorized(_human(chat_id="123", chat_type="dm")) is True
|
||||
|
||||
from gateway.bot_loop_guard import loop_guard_state
|
||||
|
||||
assert loop_guard_state()["tracked_pairs"] == 0, "human traffic must not be metered"
|
||||
assert runner._bot_loop_guard.tracked_conversations == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 4
|
||||
|
||||
|
||||
def test_three_bots_in_one_group_share_a_single_budget(monkeypatch, fake_clock):
|
||||
"""Budget is keyed per CONVERSATION, not per (sender, receiver).
|
||||
|
||||
This process only sees inbound messages, so the receiver is a constant.
|
||||
Keying on the pair would hand each sender its own budget, and N bots would
|
||||
need N x budget messages to trip a guard meant to cap the whole exchange.
|
||||
"""
|
||||
_real_config(monkeypatch, max_events="20")
|
||||
runner = _runner()
|
||||
senders = ["111111111", "222222222", "333333333"]
|
||||
def test_three_bots_in_one_group_share_one_budget(monkeypatch, runner):
|
||||
_incident_config(monkeypatch)
|
||||
senders = [BOT_A, BOT_B, "333333333"]
|
||||
|
||||
verdicts = [runner._is_user_authorized(_bot(senders[i % 3])) for i in range(30)]
|
||||
|
||||
assert verdicts[:20] == [True] * 20
|
||||
assert not any(verdicts[20:]), "three bots must not get 3 x budget"
|
||||
assert not any(verdicts[20:])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 5
|
||||
def test_other_group_has_its_own_budget(monkeypatch, runner):
|
||||
_incident_config(monkeypatch)
|
||||
_ping_pong(runner, 25)
|
||||
assert runner._is_user_authorized(_bot(BOT_A)) is False
|
||||
|
||||
assert runner._is_user_authorized(_bot(BOT_A, chat_id=OTHER_GROUP)) is True
|
||||
assert runner._is_user_authorized(_bot(BOT_B, chat_id=OTHER_GROUP)) is True
|
||||
|
||||
|
||||
def test_other_group_has_its_own_budget(monkeypatch, fake_clock):
|
||||
"""One noisy group must not silence bots elsewhere."""
|
||||
_real_config(monkeypatch, max_events="20")
|
||||
runner = _runner()
|
||||
|
||||
for turn in range(25):
|
||||
runner._is_user_authorized(_bot("111111111" if turn % 2 == 0 else "222222222"))
|
||||
assert runner._is_user_authorized(_bot("111111111")) is False
|
||||
|
||||
assert runner._is_user_authorized(_bot("111111111", chat_id=OTHER_GROUP)) is True
|
||||
assert runner._is_user_authorized(_bot("222222222", chat_id=OTHER_GROUP)) is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 6
|
||||
|
||||
|
||||
def test_slow_traffic_never_blocks(monkeypatch, fake_clock):
|
||||
"""1 message / 10 s for 10 minutes: legitimate pace, zero false positives."""
|
||||
_real_config(monkeypatch, max_events="20")
|
||||
runner = _runner()
|
||||
def test_slow_traffic_never_trips(monkeypatch, runner, clock):
|
||||
_incident_config(monkeypatch)
|
||||
|
||||
verdicts = []
|
||||
for turn in range(60):
|
||||
verdicts.append(runner._is_user_authorized(_bot("111111111" if turn % 2 == 0 else "222222222")))
|
||||
fake_clock.advance(10)
|
||||
verdicts.append(runner._is_user_authorized(_bot(BOT_A if turn % 2 == 0 else BOT_B)))
|
||||
clock.advance(10)
|
||||
|
||||
assert all(verdicts), f"slow traffic blocked at index {verdicts.index(False) if not all(verdicts) else -1}"
|
||||
assert all(verdicts)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- 7
|
||||
def test_window_expiry_readmits_without_tripping(monkeypatch, runner, clock):
|
||||
_incident_config(monkeypatch)
|
||||
assert all(_ping_pong(runner, 20))
|
||||
|
||||
clock.advance(61)
|
||||
assert runner._is_user_authorized(_bot(BOT_A)) is True
|
||||
|
||||
|
||||
def test_kill_switch_disables_the_guard(monkeypatch, fake_clock):
|
||||
"""HERMES_BOT_LOOP_PROTECTION=off restores the previous behaviour exactly."""
|
||||
_real_config(monkeypatch, max_events="20")
|
||||
monkeypatch.setenv("HERMES_BOT_LOOP_PROTECTION", "off")
|
||||
runner = _runner()
|
||||
def test_cooldown_expiry_readmits_with_a_fresh_budget(monkeypatch, runner, clock):
|
||||
_incident_config(monkeypatch)
|
||||
_ping_pong(runner, 21)
|
||||
assert runner._is_user_authorized(_bot(BOT_A)) is False
|
||||
|
||||
verdicts = [
|
||||
runner._is_user_authorized(_bot("111111111" if turn % 2 == 0 else "222222222"))
|
||||
for turn in range(40)
|
||||
]
|
||||
clock.advance(61)
|
||||
assert all(_ping_pong(runner, 20))
|
||||
assert runner._is_user_authorized(_bot(BOT_A)) is False
|
||||
|
||||
assert all(verdicts), "kill switch must suppress the guard entirely"
|
||||
|
||||
def test_disabled_via_config_admits_everything(monkeypatch, runner, settings):
|
||||
_incident_config(monkeypatch)
|
||||
settings["value"] = settings_from_config({"gateway": {"bot_loop_guard": {"enabled": False}}})
|
||||
|
||||
assert all(_ping_pong(runner, 40))
|
||||
|
||||
|
||||
def test_bot_admitted_by_allow_bots_in_a_dm_is_metered(monkeypatch, runner):
|
||||
"""The plain ALLOW_BOTS path (no chat allowlist) is covered too."""
|
||||
monkeypatch.setenv("TELEGRAM_ALLOW_BOTS", "mentions")
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", HUMAN)
|
||||
|
||||
verdicts = [runner._is_user_authorized(_bot(BOT_A, chat_id="123", chat_type="dm")) for _ in range(25)]
|
||||
|
||||
assert verdicts[:20] == [True] * 20
|
||||
assert not any(verdicts[20:])
|
||||
|
||||
|
||||
def test_rejected_bot_messages_do_not_consume_budget(monkeypatch, runner):
|
||||
"""Only admitted bot traffic counts, so an unauthorized bot cannot silence an authorized one."""
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", HUMAN)
|
||||
|
||||
for _ in range(30):
|
||||
assert runner._is_user_authorized(_bot(BOT_A, chat_id="123", chat_type="dm")) is False
|
||||
|
||||
assert runner._bot_loop_guard.tracked_conversations == 0
|
||||
|
||||
|
||||
def test_platform_scopes_the_budget_key(monkeypatch, runner):
|
||||
_incident_config(monkeypatch)
|
||||
_ping_pong(runner, 21)
|
||||
assert runner._is_user_authorized(_bot(BOT_A)) is False
|
||||
|
||||
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
|
||||
discord_bot = SessionSource(
|
||||
platform=Platform.DISCORD, chat_id=GROUP_CHAT, chat_type="group", user_id=BOT_A, is_bot=True,
|
||||
)
|
||||
assert runner._is_user_authorized(discord_bot) is True
|
||||
|
||||
|
||||
def test_tripping_logs_one_operator_warning(monkeypatch, runner, caplog):
|
||||
_incident_config(monkeypatch)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="gateway.authz_mixin"):
|
||||
_ping_pong(runner, 25)
|
||||
|
||||
trips = [r for r in caplog.records if r.name == "gateway.authz_mixin" and "Bot loop guard" in r.getMessage()]
|
||||
assert len(trips) == 1
|
||||
assert GROUP_CHAT in trips[0].getMessage()
|
||||
|
||||
|
||||
def test_guard_is_created_lazily_on_a_bare_runner(monkeypatch):
|
||||
from gateway.run import GatewayRunner
|
||||
|
||||
monkeypatch.setenv("TELEGRAM_ALLOW_BOTS", "mentions")
|
||||
runner = object.__new__(GatewayRunner)
|
||||
runner.pairing_store = SimpleNamespace(is_approved=lambda *_a, **_kw: False)
|
||||
|
||||
assert runner._is_user_authorized(_bot(BOT_A, chat_id="123", chat_type="dm")) is True
|
||||
assert isinstance(runner._bot_loop_guard, BotLoopGuard)
|
||||
|
||||
|
||||
# --- BotLoopGuard unit ------------------------------------------------------
|
||||
|
||||
|
||||
def test_admit_states(clock):
|
||||
guard = BotLoopGuard(settings=lambda: BotLoopGuardSettings(max_events=2, window_seconds=10, cooldown_seconds=30), clock=clock.now)
|
||||
|
||||
assert guard.admit("c") == (True, "ok")
|
||||
assert guard.admit("c") == (True, "ok")
|
||||
assert guard.admit("c") == (False, "tripped")
|
||||
assert guard.admit("c") == (False, "cooldown")
|
||||
clock.advance(31)
|
||||
assert guard.admit("c") == (True, "ok")
|
||||
|
||||
|
||||
def test_concurrent_admits_respect_the_budget(clock):
|
||||
guard = BotLoopGuard(settings=lambda: BotLoopGuardSettings(max_events=20, window_seconds=60, cooldown_seconds=60), clock=clock.now)
|
||||
allowed = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker():
|
||||
for _ in range(10):
|
||||
ok, _state = guard.admit("c")
|
||||
with lock:
|
||||
allowed.append(ok)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(8)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert sum(allowed) == 20
|
||||
|
||||
|
||||
def test_sweep_drops_idle_conversations(clock):
|
||||
guard = BotLoopGuard(settings=lambda: BotLoopGuardSettings(max_events=5, window_seconds=10, cooldown_seconds=10), clock=clock.now)
|
||||
for i in range(50):
|
||||
guard.admit(f"chat-{i}")
|
||||
assert guard.tracked_conversations == 50
|
||||
|
||||
clock.advance(11)
|
||||
guard.admit("fresh")
|
||||
assert guard.tracked_conversations == 1
|
||||
|
||||
|
||||
# --- settings ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_settings_defaults_when_block_missing_or_malformed():
|
||||
defaults = BotLoopGuardSettings()
|
||||
assert settings_from_config({}) == defaults
|
||||
assert settings_from_config(None) == defaults
|
||||
assert settings_from_config({"gateway": {"bot_loop_guard": "yes"}}) == defaults
|
||||
assert settings_from_config({"gateway": {"bot_loop_guard": {"max_events": -1, "window_seconds": "abc"}}}) == defaults
|
||||
assert settings_from_config({"gateway": {"bot_loop_guard": {"enabled": "maybe", "max_events": True}}}) == defaults
|
||||
|
||||
|
||||
def test_settings_parse_configured_values():
|
||||
parsed = settings_from_config({"gateway": {"bot_loop_guard": {
|
||||
"enabled": "false", "max_events": "7", "window_seconds": 30, "cooldown_seconds": 45.5,
|
||||
}}})
|
||||
assert parsed == BotLoopGuardSettings(enabled=False, max_events=7, window_seconds=30.0, cooldown_seconds=45.5)
|
||||
|
||||
|
||||
def test_load_settings_reads_config_yaml(monkeypatch):
|
||||
import hermes_cli.config as hermes_config
|
||||
|
||||
monkeypatch.setattr(hermes_config, "load_config_readonly", lambda: {"gateway": {"bot_loop_guard": {"max_events": 3}}})
|
||||
assert load_settings().max_events == 3
|
||||
|
||||
def boom():
|
||||
raise RuntimeError("no config")
|
||||
|
||||
monkeypatch.setattr(hermes_config, "load_config_readonly", boom)
|
||||
assert load_settings() == BotLoopGuardSettings()
|
||||
|
||||
@@ -47,6 +47,7 @@ def _make_event(
|
||||
chat_id: str,
|
||||
*,
|
||||
profile: str | None = None,
|
||||
is_bot: bool = False,
|
||||
) -> MessageEvent:
|
||||
return MessageEvent(
|
||||
text="hello",
|
||||
@@ -56,6 +57,7 @@ def _make_event(
|
||||
user_id=user_id,
|
||||
chat_id=chat_id,
|
||||
user_name="tester",
|
||||
is_bot=is_bot,
|
||||
chat_type="dm",
|
||||
profile=profile,
|
||||
),
|
||||
@@ -239,6 +241,21 @@ async def test_unauthorized_dm_pairs_by_default(monkeypatch):
|
||||
assert "ABC12DEF" in adapter.send.await_args.args[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unauthorized_bot_dm_is_never_offered_a_pairing_code(monkeypatch):
|
||||
"""A bot cannot pair, and a reply during a loop-guard cooldown would be outbound traffic."""
|
||||
_clear_auth_env(monkeypatch)
|
||||
config = GatewayConfig(platforms={Platform.WHATSAPP: PlatformConfig(enabled=True)})
|
||||
runner, adapter = _make_runner(Platform.WHATSAPP, config)
|
||||
jid = "15551234567@s.whatsapp.net"
|
||||
|
||||
result = await runner._handle_message(_make_event(Platform.WHATSAPP, jid, jid, is_bot=True))
|
||||
|
||||
assert result is None
|
||||
runner.pairing_store.generate_code.assert_not_called()
|
||||
adapter.send.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unauthorized_whatsapp_dm_can_be_ignored(monkeypatch):
|
||||
_clear_auth_env(monkeypatch)
|
||||
|
||||
Reference in New Issue
Block a user