diff --git a/gateway/authz_mixin.py b/gateway/authz_mixin.py index b4b42f7176..04094779a0 100644 --- a/gateway/authz_mixin.py +++ b/gateway/authz_mixin.py @@ -1,18 +1,11 @@ """User-authorization methods for ``GatewayRunner``. -Extracted from ``gateway/run.py`` as part of the god-file decomposition campaign -(``~/.hermes/plans/god-file-decomposition.md``, Phase 3 mechanical mixin lifts). -This mixin holds the inbound-message authorization cluster: whether a user/chat -is allowed to talk to the agent, the per-adapter DM policy, and the -unauthorized-DM behavior. +Inbound-message authorization cluster: whether a user/chat may talk to the +agent, the per-adapter DM policy, and the unauthorized-DM behavior. -Behavior-neutral: every method is lifted verbatim from ``GatewayRunner``. -``self.*`` calls resolve unchanged via the MRO. Neutral dependencies import at -module top; the module-level ``logger`` is imported lazily inside the one method -that uses it (``from gateway.run import logger`` resolves at call time, when -``gateway.run`` is fully loaded) so this module never imports ``gateway.run`` at -import time -> no import cycle. The lazy import preserves the exact logger name -(``"gateway.run"``) so log records are unchanged. +``self.*`` calls resolve via the MRO. ``gateway.run`` is never imported at +module import time (cycle); the one method that logs imports its ``logger`` +lazily so records keep the ``"gateway.run"`` logger name. """ from __future__ import annotations @@ -21,24 +14,44 @@ import os from typing import Optional from gateway.config import Platform +from gateway.pairing import _PLATFORM_ALLOWLIST_ENV from gateway.session import SessionSource from gateway.whatsapp_identity import ( expand_whatsapp_aliases as _expand_whatsapp_auth_aliases, normalize_whatsapp_identifier as _normalize_whatsapp_identifier, ) +_GROUP_CHAT_TYPES = frozenset({"group", "forum", "channel"}) +_TRUTHY = frozenset({"true", "1", "yes"}) + +# Platform -> ``_ALLOWED_USERS`` / ``_ALLOW_ALL_USERS``. +# Shared with the pairing store's allowlist mirror (single source of truth); +# plugin platforms are added per-call from the platform registry. +_ALLOWED_USERS_ENV = {Platform(k): v for k, v in _PLATFORM_ALLOWLIST_ENV.items()} +_ALLOW_ALL_ENV = { + p: v.replace("_ALLOWED_USERS", "_ALLOW_ALL_USERS") for p, v in _ALLOWED_USERS_ENV.items() +} +_GROUP_USER_ENV = {Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_USERS"} +_GROUP_CHAT_ENV = { + Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_CHATS", + Platform.QQBOT: "QQ_GROUP_ALLOWED_USERS", +} +_ALLOW_BOTS_ENV = { + Platform.DISCORD: "DISCORD_ALLOW_BOTS", + Platform.FEISHU: "FEISHU_ALLOW_BOTS", + Platform.TELEGRAM: "TELEGRAM_ALLOW_BOTS", + Platform.SLACK: "SLACK_ALLOW_BOTS", +} + def _platform_gate_env(name: str, default: str = "") -> str: - """Read a platform allow/deny gate env var with per-profile isolation. + """Read an allow/deny gate env var with per-profile isolation. When a profile secret scope is installed AND multiplexing is active, a - key absent from the scope returns ``default`` instead of falling through - to ``os.environ``. Under multiplex the process env may hold ANOTHER - profile's first-writer-bridged value (the YAML→env bridges in the - Discord/Telegram adapters' ``_apply_yaml_config`` are first-writer-wins), - so falling through would leak profile A's allowlist into profile B - (issue #72348). Single-profile deployments — no scope installed, or - multiplex off — behave exactly like the legacy ``os.getenv`` read. + scoped miss returns ``default`` instead of falling through to + ``os.environ``: the process env may hold ANOTHER profile's first-writer + bridged value, so a fallthrough would leak allowlists across profiles. + Single-profile deployments behave exactly like ``os.getenv``. """ if not name: return default @@ -48,55 +61,41 @@ def _platform_gate_env(name: str, default: str = "") -> str: scope = current_secret_scope() if scope is not None and is_multiplex_active(): val = scope.get(name) - if val is None: - return default - return str(val).strip() + return default if val is None else str(val).strip() except Exception: pass return (os.getenv(name) or default).strip() -def _auth_env(name: str, default: str = "") -> str: - """Read allowlist/auth env with per-profile isolation under multiplex. +_auth_env = _platform_gate_env - Same rules as ``_platform_gate_env``: a scoped miss under multiplex - returns ``default`` and does not fall through to ``os.environ``. The - process env may hold another profile's first-writer-bridged value, so - a fallthrough would leak allowlists and allow-all flags across profiles - (issue #72348). Single-profile deployments keep the legacy - ``os.getenv`` read. - """ - return _platform_gate_env(name, default) + +def _registry_entry(platform): + """Platform-registry entry for a (plugin) platform, or None.""" + if platform is None: + return None + try: + from gateway.platform_registry import platform_registry + + return platform_registry.get(platform.value) + except Exception: + return None def _platform_declares_allowed_users_env(platform) -> bool: """Whether a plugin platform's registry entry declares ``allowed_users_env``. - Such platforms (Buzz, DingTalk, …) document ``PlatformConfig.extra - .allowed_users`` as the config-file spelling of that env allowlist, so - the live adapter's extra is a valid authorization source when the env - var is absent (#98738 / #82871). Built-in platforms and unknown entries - return False. + Such platforms (Buzz, DingTalk, ...) document ``PlatformConfig.extra + .allowed_users`` as the config-file spelling of that env allowlist, so the + live adapter's extra is a valid authorization source when the env var is + absent. Built-in platforms and unknown entries return False. """ - if platform is None: - return False - try: - from gateway.platform_registry import platform_registry - - entry = platform_registry.get(platform.value) - return bool(entry and entry.allowed_users_env) - except Exception: - return False + entry = _registry_entry(platform) + return bool(entry and entry.allowed_users_env) def _coerce_allow_set(raw) -> set[str]: - """Parse allowlist values from config or env var into a set of strings. - - Handles both list inputs (YAML sequences) and comma-separated string - inputs (env vars or scalar YAML values). A scalar string is split on - commas so ``allow_from: "123,456"`` yields ``{"123", "456"}``, not - ``{"1", "2", "3", ",", ...}``. - """ + """Parse an allowlist (YAML list or comma-separated scalar) into a set of strings.""" if raw is None: return set() if isinstance(raw, list): @@ -104,15 +103,14 @@ def _coerce_allow_set(raw) -> set[str]: return {part.strip() for part in str(raw).split(",") if part.strip()} +def _allows(allowed: set[str], candidate: Optional[str]) -> bool: + return "*" in allowed or candidate in allowed + + # --------------------------------------------------------------------------- -# Nostr npub → hex normalization (Buzz and future Nostr-based platforms). -# -# ``BUZZ_ALLOWED_USERS`` accepts either a 64-char hex pubkey or an ``npub1…`` -# bech32 string, but inbound event pubkeys are always hex. Without decoding, -# the central allowlist comparison string-matches the raw npub against the -# hex pubkey and an operator who listed only their npub sees every message -# rejected ("Unauthorized user: ", #78428). Pure stdlib; mirrors -# the decoder in plugins/platforms/buzz/adapter.py. +# Nostr npub -> hex (Buzz). ``BUZZ_ALLOWED_USERS`` accepts hex or ``npub1…`` +# but inbound pubkeys are always hex, so npub entries must be decoded. +# Pure stdlib; mirrors plugins/platforms/buzz/adapter.py. # --------------------------------------------------------------------------- _BECH32_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" @@ -159,9 +157,8 @@ def _npub_to_hex(npub: str) -> Optional[str]: npub = npub.strip().lower() if not npub.startswith("npub1"): return None - data_part = npub[len("npub1"):] try: - data = [_BECH32_CHARSET.index(c) for c in data_part] + data = [_BECH32_CHARSET.index(c) for c in npub[len("npub1"):]] except ValueError: return None if _bech32_polymod(_bech32_hrp_expand("npub") + data) != 1: @@ -173,13 +170,7 @@ def _npub_to_hex(npub: str) -> Optional[str]: def _normalize_nostr_allow_entries(entries: set) -> set: - """Expand npub entries in an allowlist set to their hex pubkey form. - - Hex entries pass through unchanged; each valid ``npub1…`` entry is decoded - and its 64-char hex form added, so either form authorizes the same - identity (#78428). Invalid entries are kept as-is (they simply never - match an inbound hex pubkey). - """ + """Add the hex form of every valid ``npub1…`` entry; invalid entries are kept as-is.""" expanded = set(entries) for entry in entries: if entry.lower().startswith("npub1"): @@ -192,6 +183,9 @@ def _normalize_nostr_allow_entries(entries: set) -> set: class GatewayAuthorizationMixin: """User/chat authorization methods for ``GatewayRunner``.""" + # ``getattr(self, ...)`` throughout: test helpers build bare runners via + # ``object.__new__`` without ``adapters`` / ``config`` (pitfalls.md #17). + def _authorization_adapter( self, platform: Optional[Platform], @@ -199,19 +193,12 @@ class GatewayAuthorizationMixin: ): """Resolve the live adapter whose intake policy should gate authorization. - In multiplex mode, secondary-profile adapters live in - ``_profile_adapters[profile]`` while the default/active profile uses - ``self.adapters``. ``SessionSource.profile`` selects which map to consult. - When a stamped profile has its own adapter registry entry, the default - profile's same-platform adapter must not be consulted as a fallback. - - Consult ``_profile_adapters`` *before* comparing against - ``_active_profile_name()``. Multiplex turns wrap authz in - ``_profile_runtime_scope``, which overrides ``HERMES_HOME`` so - ``get_active_profile_name()`` returns the secondary profile for the - duration of the turn. Treating that scoped name as "primary" would - look up ``self.adapters`` (empty for secondary-only platforms like - A2A) and default-deny an already-authenticated peer. + Secondary-profile adapters live in ``_profile_adapters[profile]``; the + launched (primary) profile owns ``self.adapters``. ``_profile_adapters`` + is consulted BEFORE comparing against the active profile name: multiplex + turns override ``HERMES_HOME`` so ``_active_profile_name()`` reports the + secondary profile mid-turn, and treating it as primary would hand it the + default bot (or default-deny secondary-only platforms like A2A). """ if not platform: return None @@ -220,12 +207,8 @@ class GatewayAuthorizationMixin: profile_adapters = getattr(self, "_profile_adapters", None) or {} if profile_name in profile_adapters: return profile_adapters[profile_name].get(platform) - # Adapter ownership is process-wide: only the profile the gateway - # was LAUNCHED as owns ``self.adapters``. ``_active_profile_name()`` - # reads the per-turn HERMES_HOME override, so inside a secondary - # profile's ``_profile_runtime_scope`` it reports that secondary - # and would hand it the default bot. Compare against the identity - # captured at construction instead. + # Compare against the identity captured at construction, not the + # per-turn HERMES_HOME-derived name. primary_profile = getattr(self, "_primary_profile_name", None) if not primary_profile: active_profile_fn = getattr(self, "_active_profile_name", None) @@ -235,14 +218,12 @@ class GatewayAuthorizationMixin: except Exception: primary_profile = None if profile_name == primary_profile: - adapters = getattr(self, "adapters", None) or {} - return adapters.get(platform) + return (getattr(self, "adapters", None) or {}).get(platform) # Fail closed: a stamped secondary profile with no registry entry - # (e.g. its adapter failed to connect) must NOT fall back to the - # default profile's adapter — that sends replies out the wrong bot. + # (adapter failed to connect) must NOT fall back to the default + # profile's adapter -- that sends replies out the wrong bot. return None - adapters = getattr(self, "adapters", None) or {} - return adapters.get(platform) + return (getattr(self, "adapters", None) or {}).get(platform) def _adapter_for_source(self, source: Optional[SessionSource]): """Resolve the live adapter for an inbound ``SessionSource``.""" @@ -251,86 +232,75 @@ class GatewayAuthorizationMixin: transport_adapter = self._registered_transport_adapter(source) if transport_adapter is not None: return transport_adapter - # Relay ingress deliberately keeps the underlying platform on the - # source so session keys and display policy remain Slack/Discord/etc. - # Delivery still has to use the one live RelayAdapter that owns the - # authenticated connector socket. Looking up the underlying platform - # here silently disables streaming, typing, and tool progress when a - # managed gateway does not also run that platform's native adapter. + # Relay ingress keeps the underlying platform on the source (session + # keys / display policy), but delivery must use the one process-level + # RelayAdapter that owns the connector socket -- secondary profiles do + # not register their own, so a profile-aware lookup would silently + # disable streaming/typing/tool progress. if getattr(source, "delivered_via_upstream_relay", False) is True: - # One process-level RelayAdapter owns the connector socket for all - # multiplexed profiles. Secondary profiles intentionally do not - # register their own relay adapters, so profile-aware lookup would - # fail and suppress streamed delivery for those profiles. - adapters = getattr(self, "adapters", None) or {} - return adapters.get(Platform.RELAY) - # ``getattr`` guards test fixtures that build a bare source via - # SimpleNamespace and omit ``profile`` (see AGENTS.md pitfall #17). + return (getattr(self, "adapters", None) or {}).get(Platform.RELAY) + # ``getattr``: test fixtures build bare SimpleNamespace sources without ``profile``. return self._authorization_adapter( getattr(source, "platform", None), getattr(source, "profile", None), ) + def _owning_profile(self, adapter, platform): + """Return (registered, profile) for a live adapter: profile is None for primary.""" + if adapter is (getattr(self, "adapters", None) or {}).get(platform): + return True, None + for profile, profile_adapters in (getattr(self, "_profile_adapters", None) or {}).items(): + if adapter is profile_adapters.get(platform): + return True, profile + return False, None + def _registered_transport_adapter(self, source: SessionSource): """Return the registered adapter that created *source*, if retained. - ``source.profile`` is the runtime/session namespace. A chat-based - profile route can therefore differ from the adapter profile when one - shared credential serves several routed runtimes. ``build_source`` - keeps the receiving adapter as in-process provenance so replies and - intake-policy checks stay on that transport without weakening the - fail-closed fallback for restored or hand-built sources. + ``source.profile`` is the runtime/session namespace and may differ from + the adapter profile when one shared credential serves several routed + runtimes; ``build_source`` keeps the receiving adapter as in-process + provenance so replies and intake-policy checks stay on that transport. + Restored or hand-built sources fall back (fail-closed) to profile lookup. """ adapter_ref = getattr(source, "_transport_adapter_ref", None) adapter = adapter_ref() if callable(adapter_ref) else None platform = getattr(source, "platform", None) if adapter is None or platform is None: return None - if adapter is (getattr(self, "adapters", None) or {}).get(platform): - return adapter - profile_maps = getattr(self, "_profile_adapters", None) or {} - for profile_adapters in profile_maps.values(): - if adapter is profile_adapters.get(platform): - return adapter - return None + registered, _ = self._owning_profile(adapter, platform) + return adapter if registered else None def _adapter_profile_for_source(self, source: SessionSource) -> Optional[str]: """Resolve the transport-owning profile for adapter policy lookups.""" adapter = self._registered_transport_adapter(source) - platform = getattr(source, "platform", None) if adapter is not None: - if adapter is (getattr(self, "adapters", None) or {}).get(platform): - return None - for profile, profile_adapters in ( - getattr(self, "_profile_adapters", None) or {} - ).items(): - if adapter is profile_adapters.get(platform): - return profile + registered, profile = self._owning_profile(adapter, getattr(source, "platform", None)) + if registered: + return profile return getattr(source, "profile", None) + def _adapter_flag(self, platform, name: str, profile) -> bool: + if not platform: + return False + adapter = self._authorization_adapter(platform, profile) + return adapter is not None and bool(getattr(adapter, name, False)) + def _adapter_authorization_is_upstream( self, platform: Optional[Platform], *, profile: Optional[str] = None, ) -> bool: - """Whether the adapter for *platform* delegates authz to a trusted upstream. + """Whether the adapter delegates authz to a trusted authenticated upstream. - Mirrors ``BasePlatformAdapter.authorization_is_upstream``. The relay - adapter sets this True: the Team Gateway connector authenticates the - gateway's WS and resolves owner-only author bindings before delivering, - so an inbound relay event is already authorized as this instance's bound - user. Unlike ``_adapter_enforces_own_access_policy`` (a LOCAL config - policy the gateway mirrors only when it's an allowlist), this is an - UPSTREAM decision the gateway honors directly. Defaults to ``False`` when - the adapter is unknown or doesn't expose the flag. + Mirrors ``BasePlatformAdapter.authorization_is_upstream`` (True for the + relay adapter: the connector authenticates the WS and resolves owner + bindings before delivering). Unlike ``_adapter_enforces_own_access_policy`` + (a LOCAL policy only trusted when it is an allowlist) this UPSTREAM + decision is honored directly. False when the adapter is unknown. """ - if not platform: - return False - adapter = self._authorization_adapter(platform, profile) - if adapter is None: - return False - return bool(getattr(adapter, "authorization_is_upstream", False)) + return self._adapter_flag(platform, "authorization_is_upstream", profile) def _adapter_enforces_own_access_policy( self, @@ -338,27 +308,38 @@ class GatewayAuthorizationMixin: *, profile: Optional[str] = None, ) -> bool: - """Whether the adapter for *platform* gates access at intake itself. + """Whether the adapter gates access at intake itself. - Mirrors ``BasePlatformAdapter.enforces_own_access_policy``. Adapters - such as WeCom, Weixin, Yuanbao, QQBot, and WhatsApp evaluate their - documented ``dm_policy`` / ``group_policy`` / ``allow_from`` config before a - message is dispatched to the gateway. The flag alone is NOT "already - authorized": these adapters default to ``open``, which forwards every - sender, so ``_is_user_authorized`` only trusts the adapter when its - effective policy for the chat type is an actual ``allowlist`` restriction - (see that method). Defaults to ``False`` when the adapter is unknown or - doesn't expose the flag. + Mirrors ``BasePlatformAdapter.enforces_own_access_policy`` (WeCom, Weixin, + Yuanbao, QQBot, WhatsApp). The flag alone is NOT "already authorized": + these adapters default to ``open``, so ``_is_user_authorized`` only trusts + them when the effective policy is an actual ``allowlist``. + """ + return self._adapter_flag(platform, "enforces_own_access_policy", profile) + + def _config_extra(self, platform) -> dict: + """``config.platforms[platform].extra`` as a dict ({} when absent).""" + config = getattr(self, "config", None) + platform_cfg = ( + config.platforms.get(platform) + if config is not None and hasattr(config, "platforms") + else None + ) + extra = getattr(platform_cfg, "extra", None) if platform_cfg else None + return extra if isinstance(extra, dict) else {} + + def _adapter_setting(self, platform, attr: str, extra_key: str, profile): + """Live adapter's resolved ``attr``, else ``config.extra[extra_key]``. + + The adapter value is preferred because it already folds in the + ``_*`` env override, which is not always bridged back into + ``config.extra``; the extra fallback serves bare runners with no adapter. """ - if not platform: - return False - # Some test helpers build a bare GatewayRunner via object.__new__ and - # never set ``adapters``; treat a missing/empty map as "no adapter" - # rather than raising (see pitfalls.md #17). adapter = self._authorization_adapter(platform, profile) - if adapter is None: - return False - return bool(getattr(adapter, "enforces_own_access_policy", False)) + value = getattr(adapter, attr, None) if adapter is not None else None + if value is None: + value = self._config_extra(platform).get(extra_key) + return value def _adapter_dm_policy( self, @@ -366,35 +347,15 @@ class GatewayAuthorizationMixin: *, profile: Optional[str] = None, ) -> str: - """Best-effort read of an own-policy adapter's effective DM policy. + """Lowercased effective ``dm_policy`` (open/allowlist/disabled/pairing), ``""`` if unknown. - Returns the lowercased ``dm_policy`` (``"open"`` / ``"allowlist"`` / - ``"disabled"`` / ``"pairing"``) for *platform*, or ``""`` when unknown. - Prefers the live adapter's resolved ``_dm_policy`` — which already folds - in both ``config.extra`` and the ``_DM_POLICY`` env var (the - env var is not always bridged back into ``config.extra``) — and falls - back to ``config.extra`` for bare runners built without a live adapter. - - Used by ``_is_user_authorized`` to decide whether an own-policy adapter - actually restricted DM senders to a configured allowlist (trustworthy) - or merely forwarded everyone under ``dm_policy: open`` / for a pairing - handshake (not authorization). "Reached the gateway" only carries an - authorization signal in the ``allowlist`` case. + "Reached the gateway" only carries an authorization signal in the + ``allowlist`` case; ``open`` forwards everyone and ``pairing`` forwards + unpaired DMs for the handshake. """ if not platform: return "" - adapter = self._authorization_adapter(platform, profile) - policy = getattr(adapter, "_dm_policy", None) if adapter is not None else None - if policy is None: - config = getattr(self, "config", None) - platform_cfg = ( - config.platforms.get(platform) - if config is not None and hasattr(config, "platforms") - else None - ) - extra = getattr(platform_cfg, "extra", None) if platform_cfg else None - if isinstance(extra, dict): - policy = extra.get("dm_policy") + policy = self._adapter_setting(platform, "_dm_policy", "dm_policy", profile) return str(policy or "").strip().lower() def _adapter_group_policy( @@ -403,33 +364,10 @@ class GatewayAuthorizationMixin: *, profile: Optional[str] = None, ) -> str: - """Best-effort read of an own-policy adapter's effective group policy. - - Mirror of ``_adapter_dm_policy`` for group / forum / channel traffic: - returns the lowercased ``group_policy`` (``"open"`` / ``"allowlist"`` / - ``"disabled"``) for *platform*, or ``""`` when unknown. Prefers the live - adapter's resolved ``_group_policy`` and falls back to ``config.extra`` - for bare runners built without a live adapter. - - Used by ``_is_user_authorized`` to decide whether an own-policy adapter - restricted group senders to a configured allowlist (trustworthy) or - forwarded the whole channel under ``group_policy: open`` (not - authorization). - """ + """Lowercased effective ``group_policy`` (open/allowlist/disabled), ``""`` if unknown.""" if not platform: return "" - adapter = self._authorization_adapter(platform, profile) - policy = getattr(adapter, "_group_policy", None) if adapter is not None else None - if policy is None: - config = getattr(self, "config", None) - platform_cfg = ( - config.platforms.get(platform) - if config is not None and hasattr(config, "platforms") - else None - ) - extra = getattr(platform_cfg, "extra", None) if platform_cfg else None - if isinstance(extra, dict): - policy = extra.get("group_policy") + policy = self._adapter_setting(platform, "_group_policy", "group_policy", profile) return str(policy or "").strip().lower() def _adapter_group_has_sender_allowlist( @@ -439,29 +377,15 @@ class GatewayAuthorizationMixin: *, profile: Optional[str] = None, ) -> bool: - """Whether a per-group sender allowlist gated this group message. + """Whether a per-group sender allowlist (WeCom ``groups..allow_from``) gated this message. - WeCom supports ``groups..allow_from`` on top of the top-level - ``group_policy``. A group may be open at the chat level while still - restricting which senders inside that group can invoke Hermes. If such a - message reached the gateway, the adapter already checked that sender - allowlist, so it is a trustworthy intake decision rather than the - fail-open ``group_policy: open`` case. + A group may be open at the chat level while restricting senders; if such + a message reached the gateway the adapter already checked that sender + allowlist, so it is a trustworthy intake decision. """ if not platform or not chat_id: return False - adapter = self._authorization_adapter(platform, profile) - groups = getattr(adapter, "_groups", None) if adapter is not None else None - if groups is None: - config = getattr(self, "config", None) - platform_cfg = ( - config.platforms.get(platform) - if config is not None and hasattr(config, "platforms") - else None - ) - extra = getattr(platform_cfg, "extra", None) if platform_cfg else None - if isinstance(extra, dict): - groups = extra.get("groups") + groups = self._adapter_setting(platform, "_groups", "groups", profile) if not isinstance(groups, dict): return False @@ -486,203 +410,100 @@ class GatewayAuthorizationMixin: return False def _pairing_store_for(self, source: "SessionSource"): - """Pick the per-profile PairingStore for a source, falling back to global. - - In a multiplexing gateway, each profile owns its own pairing whitelist - so isolation is preserved. When the source has no profile (single- - profile gateway, or a path that hasn't stamped profile yet) or the - profile isn't registered, fall back to ``self.pairing_store`` (the - global default) so existing behavior is preserved. - """ + """Per-profile PairingStore for a source, else the global ``self.pairing_store``.""" per_profile = getattr(self, "pairing_stores", None) or {} profile = getattr(source, "profile", None) if profile and profile in per_profile: return per_profile[profile] return getattr(self, "pairing_store", None) + def _adapter_extra_for_source(self, source) -> dict: + adapter = self._adapter_for_source(source) + if adapter is None: + return {} + return getattr(getattr(adapter, "config", None), "extra", None) or {} + def _is_user_authorized( self, source: SessionSource, *, allow_adapter_delegation: bool = True, ) -> bool: - """ - Check if a user is authorized to use the bot. - - Checks in order: - 1. Per-platform allow-all flag (e.g., DISCORD_ALLOW_ALL_USERS=true) - 2. Environment variable allowlists (TELEGRAM_ALLOWED_USERS, etc.) - 3. DM pairing approved list - 4. Global allow-all (GATEWAY_ALLOW_ALL_USERS=true) - 5. Default: deny + """Check if a user is authorized to use the bot. + + Order: trusted-upstream delegation, chat-scoped group allowlists, + ``{PLATFORM}_ALLOW_BOTS``, per-platform allow-all, adapter role auth, + pairing store, env/config allowlists, ``GATEWAY_ALLOW_ALL_USERS``, + default deny. """ from gateway.run import logger - # Home Assistant events are system-generated (state changes), not - # user-initiated messages. The HASS_TOKEN already authenticates the - # connection, so HA events are always authorized. - # Webhook events are authenticated via HMAC signature validation in - # the adapter itself — no user allowlist applies. + # HA events are system-generated (HASS_TOKEN authenticates the + # connection); webhook events are HMAC-verified in the adapter. if source.platform in {Platform.HOMEASSISTANT, Platform.WEBHOOK}: return True adapter_profile = self._adapter_profile_for_source(source) + is_group = source.chat_type in _GROUP_CHAT_TYPES - # Relay (and any adapter whose authorization is enforced by a trusted - # authenticated upstream): the Team Gateway connector authenticates this - # gateway's WS with a per-instance secret and resolves owner-only author - # bindings BEFORE delivering, so an inbound relay event was already - # authorized as this instance's bound user (the author id is the one the - # connector observed, never gateway-asserted). There is no local - # RELAY_ALLOWED_USERS env allowlist to consult, and default-denying for - # its absence is the bug this branch fixes. This is delegation to a - # trusted upstream, NOT a fail-open: it fires only for an event that was - # actually delivered over the authenticated relay WS (the transport - # stamps ``delivered_via_upstream_relay``), or whose platform's adapter - # explicitly declares ``authorization_is_upstream=True``; every direct - # network-exposed adapter leaves the flag False and its events unmarked, - # so the env-allowlist default-deny below still applies unchanged. - # - # The delivery marker is the PRIMARY signal: a relay *message* inbound - # carries the UNDERLYING platform (``source.platform`` == discord/…), - # NOT ``Platform.RELAY``, because that's what session-keying and egress - # need — so keying authz off ``source.platform`` would miss (the relay - # adapter is registered under ``Platform.RELAY``) and default-deny the - # user ("Unauthorized user on discord"). The adapter-flag check is - # retained for events whose ``source.platform`` IS ``Platform.RELAY`` - # (e.g. the interaction-passthrough path). - # ``is True`` (not just truthiness): the marker is a real bool on a - # SessionSource, and an explicit identity check refuses to authorize a - # non-bool stand-in (e.g. a MagicMock attribute auto-vivifies truthy in - # tests) — defensive against accidental fail-open. + # Trusted-upstream delegation (relay): the Team Gateway connector + # authenticates this gateway's WS and resolves owner-only author + # bindings BEFORE delivering, so the event is already authorized and + # there is no local RELAY_ALLOWED_USERS to consult. Not a fail-open: it + # fires only for events actually delivered over the authenticated relay + # WS (transport stamps ``delivered_via_upstream_relay``) or whose + # adapter declares ``authorization_is_upstream=True``. The delivery + # marker is PRIMARY because a relayed message carries the UNDERLYING + # platform (discord/...), not ``Platform.RELAY``; the adapter-flag check + # covers events whose platform IS RELAY (interaction passthrough). + # ``is True``: a MagicMock stand-in must not auto-truthy into authz. if allow_adapter_delegation and ( source.delivered_via_upstream_relay is True - or self._adapter_authorization_is_upstream( - source.platform, - profile=adapter_profile, - ) + or self._adapter_authorization_is_upstream(source.platform, profile=adapter_profile) ): return True user_id = source.user_id - # Telegram (and similar) authorize entire group/forum/channel chats - # by chat ID via TELEGRAM_GROUP_ALLOWED_CHATS / QQ_GROUP_ALLOWED_USERS. - # That allowlist is chat-scoped, so it must work even when - # source.user_id is None — Telegram emits anonymous-admin posts, - # sender_chat traffic, and channel broadcasts with no `from_user`, - # and an operator who explicitly listed the chat expects those to - # be honored. Run this check before the no-user-id guard below so - # documented behavior matches reality - # (website/docs/reference/environment-variables.md, - # website/docs/user-guide/messaging/telegram.md). - if source.chat_type in {"group", "forum", "channel"} and source.chat_id: - chat_allowlist_env = { - Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_CHATS", - Platform.QQBOT: "QQ_GROUP_ALLOWED_USERS", - }.get(source.platform, "") - if chat_allowlist_env: - raw_chat_allowlist = _platform_gate_env(chat_allowlist_env) - if raw_chat_allowlist: - allowed_group_ids = { - cid.strip() - for cid in raw_chat_allowlist.split(",") - if cid.strip() - } - if "*" in allowed_group_ids or source.chat_id in allowed_group_ids: - return True - - # Fallback: also check adapter-level config (config.yaml) - # for platforms..extra.group_allowed_chats. - # The Telegram observe-unmentioned mode strips user_id from - # triggered group messages (_apply_telegram_group_observe_attribution), - # so the env-var-only check above misses config.yaml-configured - # allowlists. Read the live adapter's config.extra as a fallback. + # Chat-scoped group allowlists (TELEGRAM_GROUP_ALLOWED_CHATS / + # QQ_GROUP_ALLOWED_USERS) must work with ``user_id is None`` (anonymous + # admins, sender_chat posts, channel broadcasts), so they run before + # the no-user-id guard. + if is_group and source.chat_id: + chat_allowlist_env = _GROUP_CHAT_ENV.get(source.platform, "") + if chat_allowlist_env and _allows( + _coerce_allow_set(_platform_gate_env(chat_allowlist_env)), source.chat_id + ): + return True + # config.yaml fallback (``extra.group_allowed_chats``): Telegram + # observe-unmentioned mode strips user_id from triggered group + # messages, so the env-only check above misses config allowlists. try: - adapter = self._adapter_for_source(source) - if adapter is not None: - extra = getattr(getattr(adapter, "config", None), "extra", None) or {} - adapter_group_allowed = extra.get("group_allowed_chats") - if adapter_group_allowed: - allowed = _coerce_allow_set(adapter_group_allowed) - if "*" in allowed or source.chat_id in allowed: - return True + adapter_group_allowed = self._adapter_extra_for_source(source).get( + "group_allowed_chats" + ) + if adapter_group_allowed and _allows( + _coerce_allow_set(adapter_group_allowed), source.chat_id + ): + return True except Exception: pass - # Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist (#4466). - # Checked before the no-user-id guard below: some platforms deliver - # bot/automation traffic with no user_id at all -- e.g. Slack Workflow - # Builder posts arrive as subtype=bot_message with user=None -- so - # deferring past the guard would reject them outright (the same reason - # the chat-scoped allowlist above runs early). - platform_allow_bots_map = { - Platform.DISCORD: "DISCORD_ALLOW_BOTS", - Platform.FEISHU: "FEISHU_ALLOW_BOTS", - Platform.TELEGRAM: "TELEGRAM_ALLOW_BOTS", - Platform.SLACK: "SLACK_ALLOW_BOTS", - } + # Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist. + # Also before the no-user-id guard: Slack Workflow Builder posts arrive + # as bot_message with user=None. if getattr(source, "is_bot", False): - allow_bots_var = platform_allow_bots_map.get(source.platform) + allow_bots_var = _ALLOW_BOTS_ENV.get(source.platform) if allow_bots_var and _platform_gate_env(allow_bots_var, "none").lower().strip() in {"mentions", "all"}: return True if not user_id: return False - platform_env_map = { - Platform.TELEGRAM: "TELEGRAM_ALLOWED_USERS", - Platform.DISCORD: "DISCORD_ALLOWED_USERS", - Platform.WHATSAPP: "WHATSAPP_ALLOWED_USERS", - Platform.WHATSAPP_CLOUD: "WHATSAPP_CLOUD_ALLOWED_USERS", - Platform.SLACK: "SLACK_ALLOWED_USERS", - Platform.SIGNAL: "SIGNAL_ALLOWED_USERS", - Platform.EMAIL: "EMAIL_ALLOWED_USERS", - Platform.SMS: "SMS_ALLOWED_USERS", - Platform.MATTERMOST: "MATTERMOST_ALLOWED_USERS", - Platform.MATRIX: "MATRIX_ALLOWED_USERS", - Platform.DINGTALK: "DINGTALK_ALLOWED_USERS", - Platform.FEISHU: "FEISHU_ALLOWED_USERS", - Platform.WECOM: "WECOM_ALLOWED_USERS", - Platform.WECOM_CALLBACK: "WECOM_CALLBACK_ALLOWED_USERS", - Platform.WEIXIN: "WEIXIN_ALLOWED_USERS", - Platform.BLUEBUBBLES: "BLUEBUBBLES_ALLOWED_USERS", - Platform.QQBOT: "QQ_ALLOWED_USERS", - Platform.YUANBAO: "YUANBAO_ALLOWED_USERS", - } - platform_group_user_env_map = { - Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_USERS", - } - platform_group_chat_env_map = { - Platform.TELEGRAM: "TELEGRAM_GROUP_ALLOWED_CHATS", - Platform.QQBOT: "QQ_GROUP_ALLOWED_USERS", - } - platform_allow_all_map = { - Platform.TELEGRAM: "TELEGRAM_ALLOW_ALL_USERS", - Platform.DISCORD: "DISCORD_ALLOW_ALL_USERS", - Platform.WHATSAPP: "WHATSAPP_ALLOW_ALL_USERS", - Platform.WHATSAPP_CLOUD: "WHATSAPP_CLOUD_ALLOW_ALL_USERS", - Platform.SLACK: "SLACK_ALLOW_ALL_USERS", - Platform.SIGNAL: "SIGNAL_ALLOW_ALL_USERS", - Platform.EMAIL: "EMAIL_ALLOW_ALL_USERS", - Platform.SMS: "SMS_ALLOW_ALL_USERS", - Platform.MATTERMOST: "MATTERMOST_ALLOW_ALL_USERS", - Platform.MATRIX: "MATRIX_ALLOW_ALL_USERS", - Platform.DINGTALK: "DINGTALK_ALLOW_ALL_USERS", - Platform.FEISHU: "FEISHU_ALLOW_ALL_USERS", - Platform.WECOM: "WECOM_ALLOW_ALL_USERS", - Platform.WECOM_CALLBACK: "WECOM_CALLBACK_ALLOW_ALL_USERS", - Platform.WEIXIN: "WEIXIN_ALLOW_ALL_USERS", - Platform.BLUEBUBBLES: "BLUEBUBBLES_ALLOW_ALL_USERS", - Platform.QQBOT: "QQ_ALLOW_ALL_USERS", - Platform.YUANBAO: "YUANBAO_ALLOW_ALL_USERS", - } - - # Plugin platforms: check the registry for auth env var names + platform_env_map = dict(_ALLOWED_USERS_ENV) + platform_allow_all_map = dict(_ALLOW_ALL_ENV) if source.platform not in platform_env_map: try: - from gateway.platform_registry import platform_registry - - entry = platform_registry.get(source.platform.value) + entry = _registry_entry(source.platform) if entry: if entry.allowed_users_env: platform_env_map[source.platform] = entry.allowed_users_env @@ -691,168 +512,98 @@ class GatewayAuthorizationMixin: except Exception: pass - # Per-platform allow-all flag (e.g., DISCORD_ALLOW_ALL_USERS=true) platform_allow_all_var = platform_allow_all_map.get(source.platform, "") - if platform_allow_all_var and _auth_env(platform_allow_all_var).lower() in {"true", "1", "yes"}: + if platform_allow_all_var and _auth_env(platform_allow_all_var).lower() in _TRUTHY: return True - # Adapter-verified role auth: the Discord adapter already confirmed the - # user holds a role in DISCORD_ALLOWED_ROLES before dispatching the message. - # Compare with ``is True`` so the real bool field authorizes while a - # MagicMock source (test fixtures using ``object.__new__`` runners with - # mock sources) does not auto-truthy through this gate (see pitfall #13). - if ( - allow_adapter_delegation - and getattr(source, "role_authorized", False) is True - ): + # Adapter-verified role auth (Discord confirmed DISCORD_ALLOWED_ROLES + # before dispatch). ``is True`` so a MagicMock source cannot pass. + if allow_adapter_delegation and getattr(source, "role_authorized", False) is True: return True - # Check pairing store. A pairing entry is a first-class authorization - # grant, created only by a trusted operator approving a pairing code - # (hermes gateway pairing approve / the authenticated dashboard) — an - # inbound sender can never reach approve_code, so this is not an - # attacker-controlled path. Honored as a UNION with the allowlist: a - # paired user is authorized regardless of the allowlist, and when an - # allowlist IS configured, operator approval also writes the user into - # that allowlist (see PairingStore._approve_user), keeping a single - # operator-visible source of truth. (#23778: the original bypass was the - # inbound message/approval-button gate, not this gate; that gate is - # fixed separately.) - # In multiplex gateways, route to the per-profile PairingStore so each - # profile's whitelist is isolated; falls back to the global store when - # the source has no profile or the profile isn't registered. + # Pairing store: a first-class grant created only by a trusted operator + # approving a code (inbound senders can never reach approve_code). + # Honored as a UNION with the allowlist; approval also mirrors the user + # into a configured allowlist (PairingStore._approve_user). platform_name = source.platform.value if source.platform else "" pairing_store = self._pairing_store_for(source) if pairing_store is not None and pairing_store.is_approved(platform_name, user_id): return True - # Check platform-specific and global allowlists platform_allowlist = _auth_env(platform_env_map.get(source.platform, "")) group_user_allowlist = "" group_chat_allowlist = "" if source.chat_type in {"group", "forum"}: - group_user_allowlist = _auth_env(platform_group_user_env_map.get(source.platform, "")) - group_chat_allowlist = _auth_env(platform_group_chat_env_map.get(source.platform, "")) + group_user_allowlist = _auth_env(_GROUP_USER_ENV.get(source.platform, "")) + group_chat_allowlist = _auth_env(_GROUP_CHAT_ENV.get(source.platform, "")) global_allowlist = _auth_env("GATEWAY_ALLOWED_USERS") if not platform_allowlist and not group_user_allowlist and not group_chat_allowlist and not global_allowlist: - # No env allowlist configured. Adapters that own their own - # config-driven access policy (dm_policy / group_policy / - # allow_from / group_allow_from) gate access at intake, so for those - # platforms we can honor the adapter's decision instead of the - # env-only default-deny below -- but ONLY when that decision was an - # actual allowlist restriction. - # - # The adapters default dm_policy / group_policy to "open", which - # forwards EVERY sender. Reading "reached the gateway" as - # authorization in that case would admit the whole external network - # with no operator-configured allowlist -- the fail-open SECURITY.md - # §2.6 forbids ("an allowlist is required for every enabled - # network-exposed adapter ... code paths that fail open when no - # allowlist is configured are code bugs"). "disabled" never - # forwards, and "pairing" forwards unpaired DMs only so the gateway - # can run its pairing handshake (the pairing-store check above - # already denied this sender). So trust the adapter only when its - # effective policy for THIS chat type is "allowlist"; for "open" / - # "pairing" / anything else, fall through to default-deny, where - # GATEWAY_ALLOW_ALL_USERS, the per-platform {PLATFORM}_ALLOW_ALL_USERS - # flag (checked above), and the pairing flow remain the explicit - # opt-ins to broader access. (#34515 follow-up: trusting "open" was a - # fail-open.) + # No env allowlist. Own-policy adapters gate at intake, but their + # decision is only trusted when the effective policy for THIS chat + # type is ``allowlist``: ``open`` forwards EVERY sender (trusting it + # is the fail-open SECURITY.md §2.6 forbids), ``disabled`` never + # forwards, ``pairing`` forwards unpaired DMs for the handshake + # (already denied by the pairing-store check above). Anything else + # falls through to default-deny; GATEWAY_ALLOW_ALL_USERS, the + # per-platform ALLOW_ALL flag and pairing stay the explicit opt-ins. if allow_adapter_delegation and self._adapter_enforces_own_access_policy( - source.platform, - profile=adapter_profile, + source.platform, profile=adapter_profile ): - if source.chat_type in {"group", "forum", "channel"}: - effective_policy = self._adapter_group_policy( - source.platform, - profile=adapter_profile, - ) + if is_group: + effective_policy = self._adapter_group_policy(source.platform, profile=adapter_profile) if self._adapter_group_has_sender_allowlist( - source.platform, - source.chat_id, - profile=adapter_profile, + source.platform, source.chat_id, profile=adapter_profile ): return True else: - effective_policy = self._adapter_dm_policy( - source.platform, - profile=adapter_profile, - ) + effective_policy = self._adapter_dm_policy(source.platform, profile=adapter_profile) if effective_policy == "allowlist": - # Trust allowlist intake only when the live adapter still - # allowlists this sender. Pairing revoke can clear + # Re-check DMs against the live adapter when it exposes + # ``_is_dm_allowed``: pairing revoke can clear # WHATSAPP_ALLOWED_USERS while a construction-time - # ``_allow_from`` snapshot would otherwise keep authorizing - # until restart; re-check when the adapter exposes a DM - # allowlist helper. Adapters without that helper keep the - # historical "reached the gateway under allowlist policy" - # rubber-stamp (#34515). - if source.chat_type not in {"group", "forum", "channel"}: - adapter = self._authorization_adapter( - source.platform, - profile=adapter_profile, - ) - dm_check = ( - getattr(adapter, "_is_dm_allowed", None) - if adapter is not None - else None - ) + # ``_allow_from`` snapshot would keep authorizing until + # restart. Adapters without the helper keep the historical + # "reached the gateway under allowlist policy" rubber-stamp. + if not is_group: + adapter = self._authorization_adapter(source.platform, profile=adapter_profile) + dm_check = getattr(adapter, "_is_dm_allowed", None) if adapter is not None else None if callable(dm_check): return bool(dm_check(user_id)) return True - # Some adapters (e.g. Telegram) gate access via config.extra.allow_from / - # group_allow_from at intake but do not override enforces_own_access_policy. - # Check their allowlist here so config.yaml-configured allow_from works - # without requiring a separate {PLATFORM}_ALLOWED_USERS env var. + # Adapters (e.g. Telegram) that gate via config.extra.allow_from / + # group_allow_from without setting enforces_own_access_policy. adapter = self._adapter_for_source(source) if adapter is not None: extra = getattr(getattr(adapter, "config", None), "extra", None) or {} - if source.chat_type in {"group", "forum", "channel"}: - adapter_allow = extra.get("group_allow_from") - else: - adapter_allow = extra.get("allow_from") + adapter_allow = extra.get("group_allow_from" if is_group else "allow_from") if not adapter_allow and _platform_declares_allowed_users_env(source.platform): - # Plugin platforms whose registry entry declares - # ``allowed_users_env`` (e.g. Buzz) carry the same - # operator-configured allowlist in - # ``PlatformConfig.extra.allowed_users``. Under multiplex - # the YAML→env bridge is first-writer-wins, so only the - # default profile's list ever reaches the env var read - # above; consult the live (profile-routed) adapter's own - # config so a secondary profile's allowlist authorizes its - # users (#98738 / #82871). An absent/empty entry changes - # nothing here — the default-deny below still applies. + # Plugin platforms (Buzz) carry the env allowlist as + # ``extra.allowed_users``. Under multiplex the YAML->env + # bridge is first-writer-wins, so only the default profile's + # list reaches the env read above; consult the live + # profile-routed adapter's config instead. adapter_allow = extra.get("allowed_users") if adapter_allow: allowed = _coerce_allow_set(adapter_allow) normalize = getattr(adapter, "normalize_user_id", None) if callable(normalize): - # Ids and allowlist entries may use different - # spellings of the same principal (e.g. Buzz hex - # pubkeys vs npubs) — normalize the entries. + # Ids and entries may spell the same principal + # differently (Buzz hex vs npub). allowed = {normalize(entry) or entry for entry in allowed} - if user_id in allowed or "*" in allowed: + if _allows(allowed, user_id): return True - # No allowlists configured -- check global allow-all flag - return _auth_env("GATEWAY_ALLOW_ALL_USERS").lower() in {"true", "1", "yes"} + return _auth_env("GATEWAY_ALLOW_ALL_USERS").lower() in _TRUTHY - # Telegram can optionally authorize group traffic by chat ID. - # Keep this separate from TELEGRAM_GROUP_ALLOWED_USERS, which gates - # the sender user ID for group/forum messages. + # Telegram group traffic authorized by chat ID (separate from + # TELEGRAM_GROUP_ALLOWED_USERS, which gates the sender). if group_chat_allowlist and source.chat_type in {"group", "forum"} and source.chat_id: - allowed_group_ids = { - chat_id.strip() for chat_id in group_chat_allowlist.split(",") if chat_id.strip() - } - if "*" in allowed_group_ids or source.chat_id in allowed_group_ids: + if _allows(_coerce_allow_set(group_chat_allowlist), source.chat_id): return True - # Backward-compat shim for #15027: prior to PR #17686, - # TELEGRAM_GROUP_ALLOWED_USERS was (mis)used as a chat-ID allowlist. - # Values starting with "-" are Telegram chat IDs, not user IDs, so if - # users still have those in TELEGRAM_GROUP_ALLOWED_USERS we honor them - # as chat IDs and warn once. The correct var is now - # TELEGRAM_GROUP_ALLOWED_CHATS. + # Backward-compat: TELEGRAM_GROUP_ALLOWED_USERS was once (mis)used as a + # chat-ID allowlist. "-"-prefixed values are chat IDs, so honor them as + # such and warn once; the correct var is TELEGRAM_GROUP_ALLOWED_CHATS. if ( source.platform == Platform.TELEGRAM and group_user_allowlist @@ -860,9 +611,7 @@ class GatewayAuthorizationMixin: and source.chat_id ): legacy_chat_ids = { - v.strip() - for v in group_user_allowlist.split(",") - if v.strip().startswith("-") + v.strip() for v in group_user_allowlist.split(",") if v.strip().startswith("-") } if legacy_chat_ids: if not getattr(self, "_warned_telegram_group_users_legacy", False): @@ -877,37 +626,24 @@ class GatewayAuthorizationMixin: if source.chat_id in legacy_chat_ids: return True - # Check if user is in any allowlist. In group/forum chats, - # TELEGRAM_GROUP_ALLOWED_USERS is the scoped allowlist and should not - # imply DM access; TELEGRAM_ALLOWED_USERS remains the platform-wide - # allowlist and still works everywhere for backward compatibility. - allowed_ids = set() - if platform_allowlist: - allowed_ids.update(uid.strip() for uid in platform_allowlist.split(",") if uid.strip()) - if group_user_allowlist: - allowed_ids.update(uid.strip() for uid in group_user_allowlist.split(",") if uid.strip()) - if global_allowlist: - allowed_ids.update(uid.strip() for uid in global_allowlist.split(",") if uid.strip()) + # In group/forum chats TELEGRAM_GROUP_ALLOWED_USERS is the scoped + # allowlist and does not imply DM access; TELEGRAM_ALLOWED_USERS is + # platform-wide and works everywhere. + allowed_ids = ( + _coerce_allow_set(platform_allowlist) + | _coerce_allow_set(group_user_allowlist) + | _coerce_allow_set(global_allowlist) + ) - # Adapters that resolve username-shaped allowlist entries to numeric - # IDs at connect time (Discord's ``_resolve_allowed_usernames``) keep - # the authoritative resolved set in adapter memory and mirror it into - # the process env. The gateway's per-turn .env hot-reload - # (``load_hermes_dotenv(override=True)`` in - # ``_reload_runtime_env_preserving_config_authority``) restores the - # RAW username strings from the .env file into the env, so from the - # second agent turn onward ``platform_allowlist`` holds usernames - # while ``source.user_id`` is numeric — the operator is admitted by - # the adapter but dropped here as "Unauthorized user" (Aug 2026: - # responded once, then silence). Union in the adapter's resolved IDs - # so runtime resolution survives env reloads. This is a UNION of the - # resolution of entries already present in the configured allowlist — - # never a widening: the empty-allowlist fail-closed branch above has - # already returned, and adapters only resolve entries the operator - # wrote. Guarded on ``platform_allowlist`` so group/global-only - # configurations never consult adapter memory, and duck-typed + - # type-checked so bare-runner test fixtures with mock adapters - # (pitfall #17) cannot auto-truthy their way into an authorization. + # Adapters that resolve username-shaped entries to numeric IDs at + # connect time (Discord) keep the resolved set in memory; the per-turn + # .env hot-reload restores the RAW usernames into the env, so from the + # second turn on ``platform_allowlist`` holds usernames while user_id is + # numeric. Union in the adapter's resolved IDs. Never a widening: the + # empty-allowlist branch already returned and adapters only resolve + # operator-written entries. Guarded on ``platform_allowlist`` so + # group/global-only configs never consult adapter memory; duck-typed + + # type-checked so mock adapters cannot auto-truthy in. if platform_allowlist: try: adapter = self._adapter_for_source(source) @@ -926,8 +662,6 @@ class GatewayAuthorizationMixin: if isinstance(entry, (str, int)) and str(entry).strip() ) - # "*" in any allowlist means allow everyone (consistent with - # SIGNAL_GROUP_ALLOWED_USERS precedent) if "*" in allowed_ids: return True @@ -935,8 +669,8 @@ class GatewayAuthorizationMixin: if "@" in user_id: check_ids.add(user_id.split("@")[0]) - # WhatsApp (Baileys + Cloud): resolve phone↔LID / JID aliases so - # device-suffix and bare-phone allowlist entries match the same principal. + # WhatsApp (Baileys + Cloud): resolve phone<->LID / JID aliases so + # device-suffix and bare-phone entries match the same principal. if source.platform in {Platform.WHATSAPP, Platform.WHATSAPP_CLOUD}: normalized_allowed_ids = set() for allowed_id in allowed_ids: @@ -949,27 +683,16 @@ class GatewayAuthorizationMixin: if normalized_user_id: check_ids.add(normalized_user_id) - # SimpleX: SIMPLEX_ALLOWED_USERS accepts either the numeric contactId - # or the contact's display name. The adapter sets user_id=contactId for - # stability across renames, but the SimpleX UI never surfaces the - # numeric id — operators only see display names, so that's what they - # naturally put in the env var. Match both so the allowlist works - # regardless of which form was chosen. - # Plugin platform: compare by value since Platform.SIMPLEX is not a - # hardcoded enum member (it's a dynamic plugin platform). - if ( - source.platform is not None - and source.platform.value == "simplex" - and source.user_name - ): + platform_value = source.platform.value if source.platform is not None else None + + # SimpleX: user_id is the numeric contactId (stable across renames) but + # the UI only surfaces display names, so match both. Plugin platform: + # compare by value. + if platform_value == "simplex" and source.user_name: check_ids.add(source.user_name) - # Buzz (Nostr-based): BUZZ_ALLOWED_USERS accepts npub or hex, but - # inbound event pubkeys are always 64-char hex. Decode npub entries - # to hex so an operator who listed only their npub authorizes the - # same identity as the hex form (#78428). Hex entries pass through - # unchanged, so existing hex-only allowlists keep working. - if source.platform is not None and source.platform.value == "buzz": + # Buzz: allowlist may hold npub or hex; inbound pubkeys are hex. + if platform_value == "buzz": allowed_ids = _normalize_nostr_allow_entries(allowed_ids) if user_id.startswith("npub"): hex_user = _npub_to_hex(user_id) @@ -987,95 +710,43 @@ class GatewayAuthorizationMixin: """Return how unauthorized DMs should be handled for a platform. Resolution order: - 1. Explicit per-platform ``unauthorized_dm_behavior`` in config — always wins. - 2. Email defaults to ``"ignore"`` unless explicitly opted into - pairing. Inboxes may contain arbitrary unread human messages, so - replying with pairing codes is not a safe platform default. - 3. Explicit global ``unauthorized_dm_behavior`` in config — wins for - chat-shaped platforms when no per-platform override is set. - 4. When an adapter-level DM policy opts into pairing or silent drop, honor it. - 5. When an allowlist (``PLATFORM_ALLOWED_USERS``, - ``PLATFORM_GROUP_ALLOWED_USERS`` / ``PLATFORM_GROUP_ALLOWED_CHATS``, - or ``GATEWAY_ALLOWED_USERS``) is configured, default to ``"ignore"`` — - the allowlist signals that the owner has deliberately restricted - access; spamming unknown contacts with pairing codes is both noisy - and a potential info-leak. (#9337) - 6. No allowlist and no explicit config → ``"pair"`` (open-gateway default). + 1. Explicit per-platform ``unauthorized_dm_behavior`` in config. + 2. Email -> ``"ignore"`` (inboxes hold arbitrary human mail; pairing + codes are not a safe default; matches GatewayConfig). + 3. Explicit (non-default) global ``unauthorized_dm_behavior``. + 4. Adapter dm_policy: ``pairing`` -> ``"pair"``; ``allowlist`` / + ``disabled`` -> ``"ignore"``. + 5. Any configured allowlist -> ``"ignore"`` (the operator restricted + access; spamming unknown contacts with codes is noisy and leaks). + 6. Otherwise ``"pair"`` (open-gateway default). """ config = getattr(self, "config", None) - # Check for an explicit per-platform override first. if config and hasattr(config, "get_unauthorized_dm_behavior") and platform: - platform_cfg = config.platforms.get(platform) if hasattr(config, "platforms") else None - if platform_cfg and "unauthorized_dm_behavior" in getattr(platform_cfg, "extra", {}): - # Operator explicitly configured behavior for this platform — respect it. + if "unauthorized_dm_behavior" in self._config_extra(platform): return config.get_unauthorized_dm_behavior(platform) - # Email is inbox-shaped, not chat-shaped: an agent mailbox may contain - # unrelated unread human email. Require an explicit per-platform - # ``unauthorized_dm_behavior: pair`` opt-in before replying to unknown - # senders with pairing codes. Keep this before the global fallback to - # match GatewayConfig.get_unauthorized_dm_behavior(). if platform == Platform.EMAIL: return "ignore" - # Check for an explicit global config override. if config and hasattr(config, "unauthorized_dm_behavior"): - if config.unauthorized_dm_behavior != "pair": # non-default → explicit override + if config.unauthorized_dm_behavior != "pair": return config.unauthorized_dm_behavior - # Config-driven dm_policy (WeCom / Weixin / Yuanbao / QQBot). An - # allowlist or disabled DM policy means the operator restricted access, - # so unauthorized DMs should be dropped silently rather than answered - # with a pairing code. An explicit pairing policy opts back into codes. - # Prefer the profile-scoped live adapter's resolved policy in multiplex - # mode; fall back to the default profile's config.extra. if platform: dm_policy = self._adapter_dm_policy(platform, profile=profile) - if not dm_policy and config and hasattr(config, "platforms"): - platform_cfg = config.platforms.get(platform) - extra = getattr(platform_cfg, "extra", None) if platform_cfg else None - if isinstance(extra, dict): - dm_policy = str(extra.get("dm_policy") or "").strip().lower() + if not dm_policy: + dm_policy = str(self._config_extra(platform).get("dm_policy") or "").strip().lower() if dm_policy == "pairing": return "pair" if dm_policy in {"allowlist", "disabled"}: return "ignore" - # No explicit override. Fall back to allowlist-aware default: - # if any allowlist is configured for this platform, silently drop - # unauthorized messages instead of sending pairing codes. - if platform: - platform_env_map = { - Platform.TELEGRAM: "TELEGRAM_ALLOWED_USERS", - Platform.DISCORD: "DISCORD_ALLOWED_USERS", - Platform.WHATSAPP: "WHATSAPP_ALLOWED_USERS", - Platform.WHATSAPP_CLOUD: "WHATSAPP_CLOUD_ALLOWED_USERS", - Platform.SLACK: "SLACK_ALLOWED_USERS", - Platform.SIGNAL: "SIGNAL_ALLOWED_USERS", - Platform.EMAIL: "EMAIL_ALLOWED_USERS", - Platform.SMS: "SMS_ALLOWED_USERS", - Platform.MATTERMOST: "MATTERMOST_ALLOWED_USERS", - Platform.MATRIX: "MATRIX_ALLOWED_USERS", - Platform.DINGTALK: "DINGTALK_ALLOWED_USERS", - Platform.FEISHU: "FEISHU_ALLOWED_USERS", - Platform.WECOM: "WECOM_ALLOWED_USERS", - Platform.WECOM_CALLBACK: "WECOM_CALLBACK_ALLOWED_USERS", - Platform.WEIXIN: "WEIXIN_ALLOWED_USERS", - Platform.BLUEBUBBLES: "BLUEBUBBLES_ALLOWED_USERS", - Platform.QQBOT: "QQ_ALLOWED_USERS", - } - platform_group_env_map = { - Platform.TELEGRAM: ( - "TELEGRAM_GROUP_ALLOWED_USERS", - "TELEGRAM_GROUP_ALLOWED_CHATS", - ), - Platform.QQBOT: ("QQ_GROUP_ALLOWED_USERS",), - } - if _platform_gate_env(platform_env_map.get(platform, "")).strip(): - return "ignore" - for env_key in platform_group_env_map.get(platform, ()): - if _platform_gate_env(env_key).strip(): + # Historical: Yuanbao is absent from this allowlist-aware default. + env_key = "" if platform == Platform.YUANBAO else _ALLOWED_USERS_ENV.get(platform, "") + group_keys = (_GROUP_USER_ENV.get(platform), _GROUP_CHAT_ENV.get(platform)) + for key in (env_key, *group_keys): + if key and _platform_gate_env(key).strip(): return "ignore" if _platform_gate_env("GATEWAY_ALLOWED_USERS").strip(): diff --git a/gateway/browser_control_broker.py b/gateway/browser_control_broker.py index d794f0999d..5319873a75 100644 --- a/gateway/browser_control_broker.py +++ b/gateway/browser_control_broker.py @@ -1,66 +1,33 @@ """Transport-neutral browser-control broker core. -This module is the in-process heart of the browser-control feature: it binds -an *identity-scoped controller* (the party that physically drives a browser) -to *callers* (agents talking to that browser over any transport) without the -broker itself knowing anything about HTTP, WebSocket, or any wire format. The -transport layers built in later phases wrap this core; nothing here routes -traffic. +Binds an identity-scoped *controller* (the party driving a browser) to +*callers* (agents talking to it over any transport) without knowing HTTP, +WebSocket, or any wire format. The browser is a stateful single-owner +resource while the agent side is multi-tenant and multi-transport, so every +rule here makes a class of violation structurally impossible: -Why a broker at all: the browser is a stateful, single-owner resource and the -agent side is multi-tenant (many principals, profiles, sessions) and -multi-transport (local API, remote API, …). A controller must never be -addressable by a caller that merely resembles the right identity, and a -command must never be completable twice, cancellable by a stranger, or -observable after its owner has gone away. Every rule below exists to make one -of those violations structurally impossible rather than merely discouraged. - -Contract (each rule is exercised by tests/gateway/test_browser_control_broker.py): - -- **Registration tickets are short-lived, single-use, identity-bound, and - cryptographically random.** ``mint_ticket`` returns an opaque value - (``secrets``-derived, >= 32 chars) plus an expiry derived from the injected - clock; ``consume_ticket`` exchanges it exactly once for the - :class:`ControllerScope` it was minted for, raising - :class:`ControllerTicketInvalid` for unknown, already-consumed, or expired values. - The ticket is the only cross-transport credential minted here; transports +- Registration tickets are short-lived, single-use, identity-bound, and + ``secrets``-random; ``consume_ticket`` exchanges one exactly once. The + ticket is the only cross-transport credential minted here; transports decide how to carry it. +- ``select`` matches on *every* stable identity field (principal, profile, + session, controller id, browser profile id, transport family) plus the + controller's currently negotiated capability set; partial matches yield + ``None``. A same-identity reconnect renegotiates capabilities in place. +- One pending command per command id; ``complete`` is single-shot (late + completion after cancel/detach returns ``False``). +- ``cancel`` aborts only the command matching scope + tool_call_id. +- ``detach`` fails closed: pending work of that scope raises + :class:`ControllerCancelled` instead of racing a late ``complete``. +- ``disconnect`` marks a transport offline without cancelling running work; + re-attaching the same identity refreshes the callback and may still + deliver the original result. Detach, cancel, identity replacement, and + timeout remain terminal. -- **Exact identity and capability selection.** ``attach`` registers a send - callback under a :class:`ControllerScope`; ``select`` returns a controller - only when the caller's scope matches on *every stable identity field* — - principal, profile, session, controller id, browser profile id, and - transport family — and the requested capability is present in the - controller's current negotiated capability set. Partial matches return - ``None``. A same-identity reconnect may renegotiate capabilities without - creating an ambiguous second controller. - -- **One pending command per command id; single-shot completion.** Each - ``dispatch`` mints a fresh command id, emits one - ``browser.controller.command`` frame, and parks a waiter keyed by that id. - ``complete`` resolves a command exactly once and returns ``False`` for any - later attempt (late completion after cancellation or detach is ignored). - -- **Scoped cancellation.** ``cancel`` aborts only the pending command whose - scope and tool_call_id match, emits a ``browser.controller.cancel`` frame - for it, and returns ``False`` when nothing matched. - -- **Detach fails closed.** ``detach`` removes the controller and cancels every - pending command of that scope; waiting dispatchers observe - :class:`ControllerCancelled` rather than hanging or racing a detached - controller's late ``complete``. - -- **Unexpected disconnects are recoverable.** ``disconnect`` marks an exact - transport owner offline without accepting new dispatches or cancelling - already-running work. Re-attaching the same stable identity refreshes the - transport callback and can deliver a terminal result for the original - command. Explicit detach, cancel, identity replacement, and timeout remain - terminal boundaries. - -Thread-safety: all public state transitions happen under a single reentrant -lock; the send callback is invoked *outside* the lock so a controller may -synchronously ``complete`` from inside its own send (the no-op round trip), -and waiters are parked on per-command events, not on the broker lock. +Thread-safety: all state transitions happen under one reentrant lock; the +send callback is invoked *outside* it so a controller may synchronously +``complete`` from inside its own send, and waiters park on per-command +events, never on the broker lock. """ from __future__ import annotations @@ -83,12 +50,11 @@ DEFAULT_COMMAND_TIMEOUT = 30.0 MAX_DEFERRED_CANCELS = 512 #: Current wire protocol version. Registration requires this exact integer; -#: booleans are rejected even though ``bool`` subclasses ``int`` in Python. +#: booleans are rejected even though ``bool`` subclasses ``int``. BROWSER_CONTROL_PROTOCOL_VERSION = 1 -#: Exact controller capability contract shared by every transport. The broker -#: never accepts arbitrary browser methods: raw CDP, script evaluation, console -#: access, uploads, and other privileged surfaces remain outside this allowlist. +#: Exact controller capability allowlist shared by every transport. Raw CDP, +#: script evaluation, console access, and other privileged surfaces stay out. BROWSER_CONTROL_CAPABILITIES = frozenset( { "controller.noop", @@ -105,39 +71,20 @@ BROWSER_CONTROL_CAPABILITIES = frozenset( } ) -#: Privileged capabilities (Phase 8 Task 30) that are never negotiable through -#: the base allowlist. ``browser_evaluate`` executes JavaScript in the page -#: context; ``browser_cdp`` is raw CDP. Both are fail-closed unless the broker -#: runs in Developer Mode (``browser.extension_control.developer_mode``) AND -#: the controller explicitly negotiated the capability. -BROWSER_CONTROL_DEVELOPER_CAPABILITIES = frozenset( - { - "browser_cdp", - "browser_evaluate", - } -) +#: Privileged capabilities (JS eval, raw CDP): fail-closed unless Developer +#: Mode (``browser.extension_control.developer_mode``) is on AND the +#: controller explicitly negotiated them. +BROWSER_CONTROL_DEVELOPER_CAPABILITIES = frozenset({"browser_cdp", "browser_evaluate"}) -#: Artifact-transport capabilities (Phase 8 Task 29). These are regular -#: (non-developer) capabilities because upload/download of bounded, validated -#: artifacts is a safe surface; the payloads are never carried in controller -#: frames. Artifact actions are dispatched only after the broker validates the -#: referenced artifact id against the attached store ("approved artifact id -#: only"). +#: Artifact transport capabilities. Non-developer because payloads never ride +#: in controller frames: only a store-validated ``artifact_id`` is dispatched. BROWSER_CONTROL_ARTIFACT_CAPABILITIES = frozenset( - { - "browser_artifact_download", - "browser_artifact_upload", - } + {"browser_artifact_download", "browser_artifact_upload"} ) -#: The complete set a controller may negotiate: base + artifact. Developer -#: capabilities are admitted by :func:`filter_browser_control_capabilities` -#: only when Developer Mode is enabled. -BROWSER_CONTROL_ALL_CAPABILITIES = frozenset( - BROWSER_CONTROL_CAPABILITIES - | BROWSER_CONTROL_ARTIFACT_CAPABILITIES - | BROWSER_CONTROL_DEVELOPER_CAPABILITIES -) +#: Wire method names for controller frames; transports carry them verbatim. +FRAME_COMMAND = "browser.controller.command" +FRAME_CANCEL = "browser.controller.cancel" def browser_control_protocol_supported(value: Any) -> bool: @@ -145,18 +92,12 @@ def browser_control_protocol_supported(value: Any) -> bool: return type(value) is int and value == BROWSER_CONTROL_PROTOCOL_VERSION -def browser_control_developer_mode(config: Optional[dict] = None) -> bool: - """Return the explicit Developer Mode flag (disabled by default). - - Reads ``browser.extension_control.developer_mode`` from the global - config. Developer Mode is the *additional* gate for ``browser_evaluate`` - and raw CDP; it never widens the base action allowlist on its own. - """ +def _extension_control_flag(config: Optional[dict], key: str) -> bool: + """Read ``browser.extension_control.`` as a literal ``True`` (default off).""" if config is None: try: - # Read-only flag probe on every browser tool call / check_fn - # evaluation: skip load_config()'s defensive deepcopy (~135us); - # this function only reads nested dicts and never mutates. + # Hot path (every browser tool call / check_fn): the read-only + # loader skips load_config()'s deepcopy; we never mutate. from hermes_cli.config import load_config_readonly config = load_config_readonly() @@ -170,7 +111,17 @@ def browser_control_developer_mode(config: Optional[dict] = None) -> bool: extension_control = browser.get("extension_control") if not isinstance(extension_control, dict): return False - return extension_control.get("developer_mode", False) is True + return extension_control.get(key, False) is True + + +def browser_control_developer_mode(config: Optional[dict] = None) -> bool: + """Explicit Developer Mode flag; gates ``browser_evaluate``/raw CDP only.""" + return _extension_control_flag(config, "developer_mode") + + +def browser_control_enabled(config: Optional[dict] = None) -> bool: + """Return the explicit browser-control feature flag (disabled by default).""" + return _extension_control_flag(config, "enabled") def filter_browser_control_capabilities( @@ -180,30 +131,19 @@ def filter_browser_control_capabilities( ) -> frozenset: """Return the permitted subset of a JSON/RPC capability list. - A malformed non-list value has no capabilities. Unknown or non-string - entries are ignored; registration rejects an empty returned set. - - Base and artifact capabilities always pass. Developer capabilities - (``browser_evaluate``, ``browser_cdp``) pass only when Developer Mode - is explicitly enabled — either passed in or read from the live config. + Non-list input has no capabilities; unknown/non-string entries are + dropped (registration rejects an empty returned set). Developer + capabilities pass only when Developer Mode is on (passed in, else read + from live config). """ if not isinstance(value, list): return frozenset() - allowed = frozenset(BROWSER_CONTROL_CAPABILITIES | BROWSER_CONTROL_ARTIFACT_CAPABILITIES) + allowed = BROWSER_CONTROL_CAPABILITIES | BROWSER_CONTROL_ARTIFACT_CAPABILITIES if developer_mode is None: developer_mode = browser_control_developer_mode() if developer_mode is True: - allowed = frozenset(allowed | BROWSER_CONTROL_DEVELOPER_CAPABILITIES) - return frozenset( - capability - for capability in value - if isinstance(capability, str) and capability in allowed - ) - -#: Wire method names for controller frames. Transport-neutral by contract: -#: transports carry these envelopes verbatim. -FRAME_COMMAND = "browser.controller.command" -FRAME_CANCEL = "browser.controller.cancel" + allowed = allowed | BROWSER_CONTROL_DEVELOPER_CAPABILITIES + return frozenset(c for c in value if isinstance(c, str) and c in allowed) class BrowserControlError(Exception): @@ -232,12 +172,7 @@ class ControllerRejected(BrowserControlError): @dataclass(frozen=True) class ControllerScope: - """Exact identity of a browser controller plus its capability set. - - Equality is structural over *all* fields, so two scopes differing in any - single field (including ``transport_family``) never match — this is the - "exact identity" contract. - """ + """Exact controller identity plus capability set; equality is over all fields.""" principal_id: Optional[str] = None profile_id: Optional[str] = None @@ -249,7 +184,7 @@ class ControllerScope: def _scope_identity(scope: ControllerScope) -> tuple: - """Return stable controller identity, excluding negotiated capabilities.""" + """Stable identity fields, excluding negotiated capabilities.""" return ( scope.principal_id, scope.profile_id, @@ -286,9 +221,9 @@ class _Controller: owner: Any = None connected: bool = True deferred_cancels: list[dict] = field(default_factory=list) - # Serialize command/cancel writes with detach or replacement. Broker state - # is never held while waiting for this lock, so a transport callback may - # synchronously call complete() without deadlocking the broker. + # Serializes command/cancel writes with detach or replacement. Broker + # state is never held while waiting on it, so a transport callback may + # synchronously complete() without deadlocking. send_lock: threading.Lock = field(default_factory=threading.Lock) @@ -307,16 +242,7 @@ class _PendingCommand: class BrowserControlBroker: """Thread-safe broker core binding controllers to callers. - Parameters - ---------- - ticket_ttl: - Lifetime of minted tickets in clock seconds. - command_timeout: - Seconds a ``dispatch`` waits for completion before raising - :class:`ControllerTimeout`. - clock: - Injectable time source (defaults to ``time.monotonic``); tests pin it - to make expiry deterministic. + ``clock`` is injectable (default ``time.monotonic``) so tests pin expiry. """ def __init__( @@ -334,18 +260,15 @@ class BrowserControlBroker: self._tickets: Dict[str, _TicketRecord] = {} self._controllers: Dict[ControllerScope, _Controller] = {} self._pending: Dict[str, _PendingCommand] = {} - # Developer Mode gates privileged capabilities (browser_evaluate, - # browser_cdp). None defers to the live config on every selection so - # a mid-process config change is honored without restart — including - # REVOKING raw CDP/eval from already-attached controllers; an - # explicit bool pins the gate for tests and multi-tenant hosts. + # None defers to live config on every selection so flipping + # developer_mode off REVOKES raw CDP/eval from attached controllers + # without restart; an explicit bool pins the gate (tests, multi-tenant). self._developer_mode_pinned: Optional[bool] = ( None if developer_mode is None else developer_mode is True ) - # Artifact stores keyed by resolved profile id; ``None`` is the - # default/unscoped store (tests, single-profile hosts). A multiplex - # listener attaches one store per profile so profile A touching the - # artifact route first can never pin profile B to A's physical root. + # Artifact stores keyed by profile id; ``None`` is the default slot. + # Per-profile stores keep multiplex profile A from pinning profile B + # to A's physical root. self._artifact_stores: Dict[Optional[str], Any] = {} def _developer_mode_now(self) -> bool: @@ -357,17 +280,20 @@ class BrowserControlBroker: except Exception: return False + @property + def developer_mode(self) -> bool: + """Whether privileged capabilities may be selected/dispatched.""" + return self._developer_mode_now() + def attach_artifact_store( self, store: Any, *, profile_id: Optional[str] = None ) -> None: - """Attach an artifact store for "approved artifact id only". + """Attach a store exposing ``validate(artifact_id, *, scope) -> receipt`` + (raising the artifacts module's :class:`ArtifactError` subclasses). - ``store`` must expose ``validate(artifact_id, *, scope) -> receipt`` - raising the artifacts module's :class:`ArtifactError` subclasses. ``profile_id`` scopes the store to one profile on multiplex hosts; - ``None`` registers the default store. ``store=None`` clears that - slot; dispatching an artifact action without a resolvable store - fails closed. + ``None`` registers the default store. ``store=None`` clears the slot. + Artifact actions without a resolvable store fail closed. """ if store is None: self._artifact_stores.pop(profile_id, None) @@ -375,22 +301,11 @@ class BrowserControlBroker: self._artifact_stores[profile_id] = store def _artifact_store_for_scope(self, scope: "ControllerScope") -> Any: - """Select the artifact store for one controller scope. - - Prefers the exact profile-scoped store, falling back to the default - (``None``) slot so single-profile hosts and existing tests keep the - historical one-store behaviour. - """ + """Exact profile-scoped store, else the default (``None``) slot so + single-profile hosts keep the historical one-store behaviour.""" profile = getattr(scope, "profile_id", None) or None store = self._artifact_stores.get(profile) - if store is not None: - return store - return self._artifact_stores.get(None) - - @property - def developer_mode(self) -> bool: - """Whether privileged capabilities may be selected/dispatched.""" - return self._developer_mode_now() + return store if store is not None else self._artifact_stores.get(None) # ------------------------------------------------------------------ # Registration tickets @@ -400,18 +315,19 @@ class BrowserControlBroker: """Mint a short-lived, single-use ticket bound to ``scope``.""" now = self._clock() with self._lock: - self._prune_tickets(now) + for stale in [v for v, rec in self._tickets.items() if rec.expires_at <= now]: + del self._tickets[stale] value = secrets.token_urlsafe(32) record = _TicketRecord(scope=scope, expires_at=now + self._ticket_ttl) self._tickets[value] = record return Ticket(value=value, expires_at=record.expires_at) def consume_ticket(self, value: str) -> ControllerScope: - """Exchange a ticket for its scope, exactly once. + """Exchange a ticket for its scope exactly once. - Raises :class:`ControllerTicketInvalid` for unknown, already-consumed, or - expired tickets. The expiry check happens against the live clock at - consume time, so a ticket that outlived its TTL can never be used. + Raises :class:`ControllerTicketInvalid` for unknown, already-consumed, + or expired tickets; expiry is checked against the live clock at + consume time. """ now = self._clock() with self._lock: @@ -425,16 +341,27 @@ class BrowserControlBroker: record.consumed = True return record.scope - def _prune_tickets(self, now: float) -> None: - """Drop expired tickets (caller must hold the lock).""" - expired = [value for value, rec in self._tickets.items() if rec.expires_at <= now] - for value in expired: - del self._tickets[value] - # ------------------------------------------------------------------ # Controller registration / selection # ------------------------------------------------------------------ + def _controller_for_identity_locked(self, scope: ControllerScope) -> Optional[_Controller]: + """Attached controller sharing ``scope``'s stable identity (any capabilities).""" + return next( + (c for c in self._controllers.values() if _same_scope_identity(c.scope, scope)), + None, + ) + + def _lane_scopes_locked(self, session_id: str, principal_id: str, family: str) -> list[ControllerScope]: + """Attached scopes bound to one session lane (session + principal + transport).""" + return [ + scope + for scope in self._controllers + if scope.session_id == session_id + and scope.principal_id == principal_id + and scope.transport_family == family + ] + def attach( self, scope: ControllerScope, @@ -444,55 +371,39 @@ class BrowserControlBroker: ) -> None: """Attach or refresh the controller for one stable identity. - A reconnect with the same principal/profile/session/controller/browser - profile/transport identity refreshes the send callback and negotiated - capabilities without cancelling pending work. Capabilities are not an - identity field. A different controller or browser profile in the same - authenticated session lane hard-replaces the previous identity. + A same-identity reconnect refreshes the send callback and negotiated + capabilities without cancelling pending work. A different controller + or browser profile in the same authenticated session lane + hard-replaces the previous identity. """ while True: with self._lock: - existing_entry = next( - ( - (candidate_scope, controller) - for candidate_scope, controller in self._controllers.items() - if _same_scope_identity(candidate_scope, scope) - ), - None, - ) + existing = self._controller_for_identity_locked(scope) lane_scopes = [ - candidate_scope - for candidate_scope in self._controllers - if candidate_scope.principal_id == scope.principal_id - and candidate_scope.profile_id == scope.profile_id - and candidate_scope.session_id == scope.session_id - and candidate_scope.transport_family == scope.transport_family - and not _same_scope_identity(candidate_scope, scope) + candidate + for candidate in self._controllers + if candidate.principal_id == scope.principal_id + and candidate.profile_id == scope.profile_id + and candidate.session_id == scope.session_id + and candidate.transport_family == scope.transport_family + and not _same_scope_identity(candidate, scope) ] - if existing_entry is None and not lane_scopes: - self._controllers[scope] = _Controller( - scope=scope, - send=send, - owner=owner, - ) + if existing is None and not lane_scopes: + self._controllers[scope] = _Controller(scope=scope, send=send, owner=owner) return - # A different identity in the same authenticated session lane is a - # hard replacement, not a recoverable reconnect. Terminalize it - # before inserting the successor so session lookup stays unique. + # Hard replacement, not a recoverable reconnect: terminalize the + # lane siblings before inserting so session lookup stays unique. if lane_scopes: for lane_scope in lane_scopes: self.detach(lane_scope, notify_controller=False) continue - if existing_entry is not None: - existing_scope, existing = existing_entry - with existing.send_lock: with self._lock: - if self._controllers.get(existing_scope) is not existing: + if self._controllers.get(existing.scope) is not existing: continue - self._controllers.pop(existing_scope, None) + self._controllers.pop(existing.scope, None) existing.scope = scope existing.send = send existing.owner = owner @@ -509,17 +420,13 @@ class BrowserControlBroker: try: send(frame) except Exception: - logger.exception( - "failed to flush deferred browser-controller cancel" - ) + logger.exception("failed to flush deferred browser-controller cancel") unsent = deferred[index:] break if unsent: with self._lock: if self._controllers.get(scope) is existing: - existing.deferred_cancels = unsent[ - -MAX_DEFERRED_CANCELS: - ] + existing.deferred_cancels = unsent[-MAX_DEFERRED_CANCELS:] raise ConnectionError( "browser controller reconnect could not flush deferred cancels" ) @@ -531,49 +438,28 @@ class BrowserControlBroker: def select(self, scope: ControllerScope, capability: str) -> Optional[_Controller]: """Return the connected controller matching identity and capability. - The caller's capabilities are not authoritative on reconnect. Selection - matches the stable identity fields, then checks the attached - controller's current negotiated capability set. Offline controllers - preserve old pending work but never accept new dispatches. - - Privileged capabilities (``browser_evaluate``, ``browser_cdp``) are - additionally gated on Developer Mode: with the gate off they are - never selectable, even when a controller somehow negotiated them. - The gate consults the LIVE flag on every selection (unless pinned at - construction), so flipping ``developer_mode`` off in config revokes - raw CDP/eval from already-attached controllers without a restart. + The caller's capabilities are not authoritative: identity is matched, + then the controller's *current* negotiated set is checked. Offline + controllers never accept new dispatches. Developer capabilities are + additionally gated on the LIVE Developer Mode flag (unless pinned). """ - if ( - capability in BROWSER_CONTROL_DEVELOPER_CAPABILITIES - and not self._developer_mode_now() - ): + if capability in BROWSER_CONTROL_DEVELOPER_CAPABILITIES and not self._developer_mode_now(): return None with self._lock: - matches = [ - controller - for controller in self._controllers.values() - if _same_scope_identity(controller.scope, scope) - and controller.connected - and capability in controller.scope.capabilities - ] - return matches[0] if len(matches) == 1 else None + controller = self._controller_for_identity_locked(scope) + if controller is None or not controller.connected: + return None + return controller if capability in controller.scope.capabilities else None def is_owner(self, scope: ControllerScope, owner: Any) -> bool: - """Return whether ``owner`` is the exact live transport for ``scope``. + """Whether ``owner`` is the exact live transport for ``scope``. - Ownership is independent of capabilities. Transport handlers use this - for heartbeat and result admission so a least-privilege controller does - not need to request ``controller.noop`` merely to complete a real action. + Independent of capabilities, so a least-privilege controller need not + request ``controller.noop`` merely to heartbeat or complete an action. """ with self._lock: - matches = [ - controller - for controller in self._controllers.values() - if _same_scope_identity(controller.scope, scope) - and controller.connected - and controller.owner is owner - ] - return len(matches) == 1 + controller = self._controller_for_identity_locked(scope) + return controller is not None and controller.connected and controller.owner is owner def disconnect( self, @@ -583,20 +469,12 @@ class BrowserControlBroker: ) -> bool: """Mark one exact controller transport offline without cancelling work.""" with self._lock: - entry = next( - ( - (candidate_scope, controller) - for candidate_scope, controller in self._controllers.items() - if _same_scope_identity(candidate_scope, scope) - ), - None, - ) - if entry is None: + controller = self._controller_for_identity_locked(scope) + if controller is None: return False - candidate_scope, controller = entry with controller.send_lock: with self._lock: - if self._controllers.get(candidate_scope) is not controller: + if self._controllers.get(controller.scope) is not controller: return False if owner is not _OWNER_UNSET and controller.owner is not owner: return False @@ -613,10 +491,8 @@ class BrowserControlBroker: ) -> None: """Remove the controller for ``scope`` and fail its pending work closed. - Every pending command of the scope is marked cancelled and resolved, - so waiting dispatchers raise :class:`ControllerCancelled`; a late - ``complete`` for any of them returns ``False`` (the command id is no - longer pending). + Pending commands resolve cancelled (dispatchers raise + :class:`ControllerCancelled`); a late ``complete`` returns ``False``. """ with self._lock: controller = self._controllers.get(scope) @@ -634,7 +510,7 @@ class BrowserControlBroker: pendings = self._pending_for_scope_locked(scope) for pending in pendings: self._resolve_pending(pending, cancelled=True) - # Keep the old generation's send lock through cancellation so a + # Hold the old generation's send lock through cancellation so a # command frame can never overtake its terminal cancel frame. if notify_controller: self._emit_cancel_frames(controller, pendings) @@ -653,25 +529,11 @@ class BrowserControlBroker: ) -> Any: """Send one controller command and block for its completion. - Emits a ``browser.controller.command`` frame carrying a fresh command - id, then waits up to ``command_timeout`` seconds. Returns the - controller's completion result, or raises: - - - :class:`ControllerUnavailable` — no exact scope/capability match; - - :class:`ControllerCancelled` — cancelled via ``cancel``/``detach``; - - :class:`ControllerTimeout` — no completion within the timeout; - - :class:`ControllerRejected` — completed with ``ok=False``. - - Exactly one pending command exists per command id; ``complete`` is - single-shot, so a command can never resolve twice. - - Artifact actions (``browser_artifact_upload`` / - ``browser_artifact_download``) additionally require an attached - artifact store and a live, scope-bound artifact reference: the - ``arguments`` mapping must carry an ``artifact_id`` whose validation - passes against the store ("approved artifact id only"). The payload - is never carried in the frame — only the id travels to the - controller. + Raises :class:`ControllerUnavailable` (no exact match), + :class:`ControllerCancelled`, :class:`ControllerTimeout`, or + :class:`ControllerRejected` (``ok=False``). Artifact actions also + require an attached store and an approved ``artifact_id`` argument; + only the id travels in the frame, never the payload. """ controller = self.select(scope, action) if controller is None: @@ -702,17 +564,9 @@ class BrowserControlBroker: ) with controller.send_lock: with self._lock: - # select() intentionally runs outside the send lock. Revalidate - # the exact live controller after acquiring it so disconnect or - # identity replacement cannot leave a stale command waiting. - attached = next( - ( - candidate - for candidate in self._controllers.values() - if _same_scope_identity(candidate.scope, scope) - ), - None, - ) + # select() ran outside the send lock; revalidate the live + # controller so disconnect/replacement can't strand a command. + attached = self._controller_for_identity_locked(scope) if attached is not controller or not controller.connected: raise ControllerUnavailable( f"controller for scope {scope!r} detached before dispatch" @@ -723,8 +577,7 @@ class BrowserControlBroker: try: controller.send(frame) except Exception: - # The command never left the building; unreserve the id and - # surface the transport failure to the caller. + # Never left the building: unreserve the id, surface the error. with self._lock: self._pending.pop(command_id, None) raise @@ -741,16 +594,7 @@ class BrowserControlBroker: if timed_out: with controller.send_lock: with self._lock: - active = next( - ( - candidate - for candidate in self._controllers.values() - if _same_scope_identity(candidate.scope, scope) - ), - None, - ) - if active is None: - active = controller + active = self._controller_for_identity_locked(scope) or controller if not active.connected: self._defer_cancel_locked(active, pending) active = None @@ -779,10 +623,8 @@ class BrowserControlBroker: ) -> bool: """Resolve a pending command by id; ``False`` when none is pending. - Safe to call from inside the controller's own ``send`` callback (the - broker never holds its lock across a send). Late completions — after - ``cancel`` or ``detach`` already resolved the command — are ignored - and report ``False``. + Safe from inside the controller's own ``send`` callback. Late + completions after ``cancel``/``detach`` are ignored (``False``). """ with self._lock: pending = self._pending.get(command_id) @@ -800,43 +642,28 @@ class BrowserControlBroker: def cancel(self, scope: ControllerScope, *, tool_call_id: Optional[str]) -> bool: """Cancel exactly the pending command matching ``scope`` + tool_call_id. - Emits one ``browser.controller.cancel`` frame naming the cancelled - command's id. Returns ``True`` when a command was cancelled and - ``False`` when nothing matched (so transports can answer idempotently - without inventing state). + Emits one cancel frame; returns ``False`` when nothing matched so + transports can answer idempotently. """ with self._lock: - controller = next( - ( - candidate - for candidate in self._controllers.values() - if _same_scope_identity(candidate.scope, scope) - ), - None, - ) + controller = self._controller_for_identity_locked(scope) if controller is None or not controller.connected: return False with controller.send_lock: with self._lock: - attached = next( + attached = self._controller_for_identity_locked(scope) + if attached is not controller or not controller.connected: + return False + target = next( ( - candidate - for candidate in self._controllers.values() - if _same_scope_identity(candidate.scope, scope) + p + for p in self._pending.values() + if _same_scope_identity(p.scope, scope) + and p.tool_call_id == tool_call_id + and not p.done ), None, ) - if attached is not controller or not controller.connected: - return False - target = None - for pending in self._pending.values(): - if ( - _same_scope_identity(pending.scope, scope) - and pending.tool_call_id == tool_call_id - and not pending.done - ): - target = pending - break if target is None: return False self._resolve_pending(target, cancelled=True) @@ -844,11 +671,10 @@ class BrowserControlBroker: return True # ------------------------------------------------------------------ - # Internals (all callers must hold the lock) + # Internals (callers hold the lock unless noted) # ------------------------------------------------------------------ def _resolve_pending(self, pending: _PendingCommand, *, cancelled: bool) -> None: - """Mark ``pending`` resolved and drop it from the registry.""" pending.cancelled = cancelled pending.done = True del self._pending[pending.command_id] @@ -860,19 +686,14 @@ class BrowserControlBroker: action: str, arguments: dict, ) -> None: - """Fail closed unless ``arguments`` carries an approved artifact id. + """Fail closed unless ``arguments`` carries a store-approved artifact id. - The store is consulted through the duck-typed ``validate`` contract - (raises :class:`ArtifactError` subclasses on any problem), so the - broker never guesses at artifact validity: missing store, missing id, - traversal, expiry, checksum, or scope mismatch all surface as - :class:`ControllerRejected` before any frame is emitted. + Missing store/id, traversal, expiry, checksum, or scope mismatch all + surface as :class:`ControllerRejected` before any frame is emitted. """ store = self._artifact_store_for_scope(scope) if store is None: - raise ControllerRejected( - f"{action} requires an attached artifact store" - ) + raise ControllerRejected(f"{action} requires an attached artifact store") artifact_id = arguments.get("artifact_id") if not isinstance(artifact_id, str) or not artifact_id.strip(): raise ControllerRejected(f"{action} requires a non-empty artifact_id") @@ -895,34 +716,37 @@ class BrowserControlBroker: }, } - def _defer_cancel_locked( - self, - controller: _Controller, - pending: _PendingCommand, - ) -> None: + def _defer_cancel_locked(self, controller: _Controller, pending: _PendingCommand) -> None: controller.deferred_cancels.append(self._cancel_frame(pending)) if len(controller.deferred_cancels) > MAX_DEFERRED_CANCELS: del controller.deferred_cancels[:-MAX_DEFERRED_CANCELS] def _pending_for_scope_locked(self, scope: ControllerScope) -> list[_PendingCommand]: - return [ - pending - for pending in list(self._pending.values()) - if _same_scope_identity(pending.scope, scope) - ] + return [p for p in list(self._pending.values()) if _same_scope_identity(p.scope, scope)] - def _emit_cancel_frames( - self, controller: _Controller, pendings: list[_PendingCommand] - ) -> None: + def _emit_cancel_frames(self, controller: _Controller, pendings: list[_PendingCommand]) -> None: + """Send cancel frames (caller holds ``send_lock``, never the broker lock).""" for pending in pendings: - frame = self._cancel_frame(pending) try: - controller.send(frame) + controller.send(self._cancel_frame(pending)) except Exception: logger.exception( "failed to emit cancel frame for command %r", pending.command_id ) + # ------------------------------------------------------------------ + # Session-lane lookups / bulk teardown + # ------------------------------------------------------------------ + + @staticmethod + def _lane_key(session_id, task_id, principal_id, transport_family) -> Optional[tuple[str, str, str]]: + target = str(session_id or task_id or "").strip() + principal = str(principal_id or "").strip() + family = str(transport_family or "").strip() + if not target or not principal or not family: + return None + return target, principal, family + def scope_for_session( self, *, @@ -933,24 +757,15 @@ class BrowserControlBroker: ) -> Optional[ControllerScope]: """Return one unambiguous attached scope for a server-owned session. - A public session id is only a lookup hint. The caller must also supply - its server-derived principal and transport family; missing identity, - no match, or multiple matches fail closed rather than selecting by - insertion order. + The public session id is only a hint; the caller must also supply its + server-derived principal and transport family. Missing identity, no + match, or multiple matches fail closed rather than picking by order. """ - target = str(session_id or task_id or "").strip() - principal = str(principal_id or "").strip() - family = str(transport_family or "").strip() - if not target or not principal or not family: + key = self._lane_key(session_id, task_id, principal_id, transport_family) + if key is None: return None with self._lock: - matches = [ - scope - for scope in self._controllers - if scope.session_id == target - and scope.principal_id == principal - and scope.transport_family == family - ] + matches = self._lane_scopes_locked(*key) return matches[0] if len(matches) == 1 else None def lane_registered( @@ -961,55 +776,23 @@ class BrowserControlBroker: principal_id: Optional[str] = None, transport_family: Optional[str] = None, ) -> bool: - """Return whether ANY controller (even offline) registered for this lane. + """Whether ANY controller (even offline) registered for this lane. - Distinguishes "a controller bound this session lane and is currently - unavailable" (fail closed — the extension lane stays authoritative) - from "no controller ever registered here" (the caller keeps the - legacy browser backend). Ambiguous lanes report True so the caller - still fails closed rather than silently switching browsers. + Distinguishes "lane bound but currently unavailable" (fail closed; + the extension lane stays authoritative) from "never registered" + (caller keeps the legacy backend). Ambiguous lanes report True. """ - target = str(session_id or task_id or "").strip() - principal = str(principal_id or "").strip() - family = str(transport_family or "").strip() - if not target or not principal or not family: + key = self._lane_key(session_id, task_id, principal_id, transport_family) + if key is None: return False with self._lock: - return any( - scope.session_id == target - and scope.principal_id == principal - and scope.transport_family == family - for scope in self._controllers - ) + return bool(self._lane_scopes_locked(*key)) def disconnect_owner(self, owner: Any) -> int: """Mark every controller owned by one lost transport offline.""" with self._lock: - scopes = [ - scope - for scope, controller in self._controllers.items() - if controller.owner is owner - ] - disconnected = 0 - for scope in scopes: - disconnected += int(self.disconnect(scope, owner=owner)) - return disconnected - - def detach_owner(self, owner: Any, *, notify_controller: bool = True) -> int: - """Hard-detach every controller owned by one transport connection.""" - with self._lock: - scopes = [ - scope - for scope, controller in self._controllers.items() - if controller.owner is owner - ] - for scope in scopes: - self.detach( - scope, - owner=owner, - notify_controller=notify_controller, - ) - return len(scopes) + scopes = [s for s, c in self._controllers.items() if c.owner is owner] + return sum(int(self.disconnect(scope, owner=owner)) for scope in scopes) def reset(self) -> None: """Fail all live work closed and clear tickets (tests/shutdown).""" @@ -1019,8 +802,7 @@ class BrowserControlBroker: self.detach(scope) with self._lock: self._tickets.clear() - # Defensive cleanup for any pending entry whose controller was - # concurrently removed by a transport teardown. + # Pending entries whose controller a concurrent teardown removed. for pending in list(self._pending.values()): self._resolve_pending(pending, cancelled=True) @@ -1042,26 +824,3 @@ _GLOBAL_BROKER = BrowserControlBroker() def get_browser_control_broker() -> BrowserControlBroker: """Process-local broker shared by API and dashboard Gateway transports.""" return _GLOBAL_BROKER - - -def browser_control_enabled(config: Optional[dict] = None) -> bool: - """Return the explicit browser-control feature flag (disabled by default).""" - if config is None: - try: - # Read-only flag probe on every browser tool call / check_fn - # evaluation: skip load_config()'s defensive deepcopy (~135us); - # this function only reads nested dicts and never mutates. - from hermes_cli.config import load_config_readonly - - config = load_config_readonly() - except Exception: - return False - if not isinstance(config, dict): - return False - browser = config.get("browser") - if not isinstance(browser, dict): - return False - extension_control = browser.get("extension_control") - if not isinstance(extension_control, dict): - return False - return extension_control.get("enabled", False) is True diff --git a/gateway/code_skew.py b/gateway/code_skew.py index f7bc4ef3ce..9b77237c3a 100644 --- a/gateway/code_skew.py +++ b/gateway/code_skew.py @@ -1,18 +1,15 @@ """Detect when the gateway is running stale code after a hot ``git pull``. -The gateway is a single long-lived process; its ``sys.modules`` is frozen at -boot. If the checkout is updated underneath it (a manual ``git pull``, or the -window before ``hermes update``'s graceful restart fires), a first-time lazy -import on a new code path can resolve a freshly-pulled consumer module against a -stale cached dependency -> ImportError (see -``tests/test_stale_utils_module_import.py`` for the exact failure). +The gateway's ``sys.modules`` is frozen at boot. If the checkout is updated +underneath it (manual ``git pull``, or the window before ``hermes update``'s +graceful restart), a first-time lazy import can resolve a freshly-pulled module +against a stale cached dependency -> ImportError +(``tests/test_stale_utils_module_import.py``). We snapshot the revision at +startup so risky callers (e.g. ``/model`` switching) can refuse with a clear +"restart the gateway" message instead. -We snapshot the checkout revision at gateway startup and compare on demand, so -risky callers (e.g. ``/model`` switching) can refuse with a clear "restart the -gateway" message instead of crashing on a cryptic import error. - -If the revision can't be read (non-git install, IO error), the boot snapshot -stays ``None`` and skew detection no-ops — it never produces a false positive. +If the revision can't be read (non-git install, IO error) the boot snapshot +stays ``None`` and detection no-ops — never a false positive. """ from __future__ import annotations @@ -24,12 +21,8 @@ _boot_fingerprint: str | None = None def _fingerprint() -> str | None: - """Current checkout fingerprint, reusing the CLI's git-rev reader. - - ``hermes_cli.main`` is always already imported in a gateway process (it's - the entry point), so this import is free and avoids duplicating the - worktree-aware ref resolution. - """ + """Current checkout fingerprint via the CLI's worktree-aware git-rev reader + (``hermes_cli.main`` is always already imported in a gateway process).""" try: from hermes_cli.main import _read_git_revision_fingerprint diff --git a/gateway/cwd_placeholder.py b/gateway/cwd_placeholder.py index f102595e33..b064104c6a 100644 --- a/gateway/cwd_placeholder.py +++ b/gateway/cwd_placeholder.py @@ -12,10 +12,6 @@ from __future__ import annotations CWD_PLACEHOLDERS = frozenset({".", "auto", "cwd"}) -def _truthy_env(value: str | None) -> bool: - return (value or "").strip().lower() in {"true", "1", "yes"} - - def resolve_placeholder_terminal_cwd( *, configured_cwd: str, @@ -37,13 +33,9 @@ def resolve_placeholder_terminal_cwd( return configured_cwd backend = (terminal_backend or "local").strip().lower() + messaging = (messaging_cwd or "").strip() if backend == "local": - messaging = (messaging_cwd or "").strip() return messaging or home_fallback - - if backend == "docker" and docker_mount_cwd_to_workspace: - messaging = (messaging_cwd or "").strip() - if messaging and messaging not in CWD_PLACEHOLDERS: - return messaging - + if backend == "docker" and docker_mount_cwd_to_workspace and messaging and messaging not in CWD_PLACEHOLDERS: + return messaging return None diff --git a/gateway/dead_targets.py b/gateway/dead_targets.py index 15f9b90879..b0ffa3cdd6 100644 --- a/gateway/dead_targets.py +++ b/gateway/dead_targets.py @@ -1,25 +1,18 @@ """Persistent registry of delivery targets that are confirmed unreachable. -When a messaging platform reports that a target chat is permanently gone — a -deleted group (``Forbidden: the group chat was deleted``), a bot kicked/blocked, -or a deactivated user — re-sending to it on every cron tick or every fan-out -delivery wastes a send attempt against the platform's flood-control envelope and -spams the logs. This registry lets the delivery layer short-circuit a target it -has already proven dead, while staying self-healing: any successful send to that -target clears the flag, so a user who re-adds the bot (or restores the chat) -recovers automatically with no manual cleanup. +When a platform reports a target chat is permanently gone (deleted group, +bot kicked/blocked, deactivated user), re-sending on every cron tick wastes a +send against flood control and spams logs. The delivery layer short-circuits +targets proven dead; any later successful send clears the flag (self-healing). -Scope is deliberately narrow. Only *whole-chat* deaths are recorded — the -``forbidden`` and chat-level ``not_found`` (``chat not found``) error kinds. -Thread/topic-level ``not_found`` is NOT recorded here: the adapters already -self-heal that by retrying without ``reply_to`` (see the Telegram adapter's -reply-target-deleted path), and a deleted topic does not mean the parent chat is -dead. +Scope is deliberately narrow: only *whole-chat* deaths (``forbidden`` and +chat-level ``not_found``) are recorded. Thread/topic-level ``not_found`` is +NOT — adapters self-heal that by retrying without ``reply_to``, and a deleted +topic does not mean the parent chat is dead. -The store is a small JSON file under the active profile's HERMES_HOME so each -profile keeps its own dead set. Reads/writes are best-effort: a corrupt or -unwritable file degrades to an in-memory-only registry rather than raising on -the delivery path. +Storage is a JSON file under the active profile's HERMES_HOME (each profile +keeps its own dead set). Reads/writes are best-effort: a corrupt or unwritable +file degrades to in-memory-only rather than raising on the delivery path. """ from __future__ import annotations @@ -35,8 +28,8 @@ from hermes_cli.config import get_hermes_home logger = logging.getLogger(__name__) -# Error kinds (from gateway.platforms.base.classify_send_error) that mean the -# *whole chat* is unreachable, not a transient or thread-level problem. +# Error kinds (gateway.platforms.base.classify_send_error) meaning the whole +# chat is unreachable, not a transient or thread-level problem. _DEAD_ERROR_KINDS = frozenset({"forbidden", "not_found"}) @@ -46,33 +39,24 @@ def _normalize(platform: str, chat_id: str) -> str: class DeadTargetRegistry: - """Thread-safe, persistent set of confirmed-dead delivery targets. + """Thread-safe, persistent set of confirmed-dead targets keyed ``platform:chat_id``. - Keyed on ``platform:chat_id``. Stores the reason and a timestamp for - observability. Self-healing: :meth:`clear` (called on a successful send) - removes the flag. + Each entry stores reason + timestamp for observability; :meth:`clear` + (called on a successful send) removes the flag. """ def __init__(self, path: Optional[Path] = None) -> None: self._lock = threading.RLock() self._dead: Dict[str, Dict[str, object]] = {} - if path is not None: - self._path = path - else: - self._path = get_hermes_home() / "gateway" / "dead_targets.json" + self._path = path if path is not None else get_hermes_home() / "gateway" / "dead_targets.json" self._load() - # -- persistence ------------------------------------------------------- - def _load(self) -> None: try: if self._path.exists(): raw = json.loads(self._path.read_text(encoding="utf-8")) if isinstance(raw, dict): - # Only keep well-shaped entries. - self._dead = { - k: v for k, v in raw.items() if isinstance(v, dict) - } + self._dead = {k: v for k, v in raw.items() if isinstance(v, dict)} except (OSError, ValueError) as exc: logger.debug("dead_targets: could not load %s (%s) — starting empty", self._path, exc) @@ -85,14 +69,12 @@ class DeadTargetRegistry: tmp.write_text(json.dumps(self._dead, indent=2), encoding="utf-8") tmp.replace(self._path) except OSError as exc: - # Best-effort: keep the in-memory state, don't break delivery. + # Best-effort: keep in-memory state, never break delivery. logger.debug("dead_targets: could not persist %s (%s)", self._path, exc) - # -- public API -------------------------------------------------------- - @staticmethod def is_dead_error_kind(error_kind: Optional[str]) -> bool: - """Return True when ``error_kind`` denotes a permanent whole-chat death.""" + """True when ``error_kind`` denotes a permanent whole-chat death.""" return bool(error_kind) and error_kind in _DEAD_ERROR_KINDS def is_dead(self, platform: str, chat_id: Optional[str]) -> bool: @@ -103,7 +85,7 @@ class DeadTargetRegistry: def mark_dead(self, platform: str, chat_id: Optional[str], reason: str = "") -> bool: - """Record a target as confirmed-dead. Returns True if newly added.""" + """Record a target as confirmed-dead. Returns True if newly added.""" if not chat_id: return False key = _normalize(platform, chat_id) @@ -125,7 +107,7 @@ class DeadTargetRegistry: return not existed def clear(self, platform: str, chat_id: Optional[str]) -> bool: - """Remove a target's dead flag (self-healing). Returns True if it was set.""" + """Remove a target's dead flag (self-healing). Returns True if it was set.""" if not chat_id: return False key = _normalize(platform, chat_id) @@ -136,8 +118,3 @@ class DeadTargetRegistry: logger.info("dead_targets: cleared %s (delivery succeeded again)", key) return True return False - - def all_dead(self) -> Dict[str, Dict[str, object]]: - """Snapshot of the current dead set (for diagnostics / `hermes` CLI).""" - with self._lock: - return {k: dict(v) for k, v in self._dead.items()} diff --git a/gateway/delivery.py b/gateway/delivery.py index f63c769aeb..e5ded9c5f8 100644 --- a/gateway/delivery.py +++ b/gateway/delivery.py @@ -1,11 +1,9 @@ """ Delivery routing for cron job outputs and agent responses. -Routes messages to the appropriate destination based on: -- Explicit targets (e.g., "telegram:123456789") -- Platform home channels (e.g., "telegram" → home channel) -- Origin (back to where the job was created) -- Local (always saved to files) +Routes messages by target: explicit ("telegram:123456789"), platform home +channel ("telegram"), origin (back to where the job was created), or local +(saved to files). """ import logging @@ -21,18 +19,15 @@ from hermes_cli.config import get_hermes_home logger = logging.getLogger(__name__) # Cap before gateway-level truncation of cron output for non-chunking platform -# delivery. Telegram's hard API limit is 4096; the headroom covers the "full -# output saved to …" footer appended on truncation. Adapters that split long -# messages natively (BasePlatformAdapter.splits_long_messages) bypass this -# entirely — the adapter chunks in its own send() and the full output is -# preserved. +# delivery. Telegram's hard API limit is 4096; the headroom covers the "full +# output saved to …" footer. Adapters that split long messages natively +# (BasePlatformAdapter.splits_long_messages) bypass this entirely. MAX_PLATFORM_OUTPUT = 4000 # Matches strings that are *only* a "silence" narration with optional markdown -# wrappers. Covers: *(silent)*, _silent_, `silent`, ~silent~, (silent), silent, -# 🔇, a bare ".", "…", and the whitespace/marker-padded variants seen in the -# wild. Anchored to start/end so substantive messages that merely *contain* the -# word "silent" are never matched. +# wrappers: *(silent)*, _silent_, `silent`, (silent), silent, 🔇, a bare ".", +# "…", and marker-padded variants. Anchored so substantive messages that merely +# *contain* the word "silent" never match. _SILENCE_NARRATION = re.compile( r'^[\s*_~`]*\(?\s*(silent|silence|no\s+response|no\s+reply)\s*\.?\)?[\s*_~`]*$' r'|^[\s*_~`]*[\U0001F507\.\u2026]+[\s*_~`]*$', @@ -41,16 +36,11 @@ _SILENCE_NARRATION = re.compile( def _is_silence_narration(content: Optional[str]) -> bool: - """Return True when ``content`` is *only* a silence-narration token. - - Length-guarded (real messages are longer) and anchored to the whole string - so legitimate prose like "The deployment ran silently" or "Silence is - golden — here is the plan..." is never flagged. - """ + """True when ``content`` is *only* a silence-narration token (length-guarded).""" if not content: return False stripped = content.strip() - if not stripped or len(stripped) > 64: # length guard + if not stripped or len(stripped) > 64: return False return bool(_SILENCE_NARRATION.match(stripped)) @@ -81,10 +71,7 @@ class DeliveryTransport: """Send through this transport while preserving the logical platform.""" if self.is_relay: return await self.adapter.send_for_platform( - logical_platform, - chat_id, - content, - metadata=metadata, + logical_platform, chat_id, content, metadata=metadata, ) return await self.adapter.send(chat_id, content, metadata=metadata) @@ -98,21 +85,16 @@ def resolve_delivery_transport( A concrete native adapter always wins. Relay is eligible only when its authenticated transport explicitly advertises that it fronts the logical - platform, which keeps restart-time delivery independent of per-chat caches - without letting Relay hijack unrelated platform targets. + platform, so restart-time delivery is independent of per-chat caches without + letting Relay hijack unrelated platform targets. """ live_adapters = adapters or {} native = live_adapters.get(platform) native_config = config.platforms.get(platform) - # Preserve DeliveryRouter's historical support for explicitly supplied live - # adapters with no config block, but never let an explicitly disabled native - # adapter shadow an enabled Relay transport. + # Explicitly supplied live adapters with no config block are honored, but an + # explicitly disabled native adapter never shadows an enabled Relay transport. if native is not None and (native_config is None or native_config.enabled): - return DeliveryTransport( - adapter=native, - config=native_config, - transport_platform=platform, - ) + return DeliveryTransport(native, native_config, platform) relay = live_adapters.get(Platform.RELAY) relay_config = config.platforms.get(Platform.RELAY) @@ -123,51 +105,47 @@ def resolve_delivery_transport( and callable(fronts_platform) and fronts_platform(platform) ): - return DeliveryTransport( - adapter=relay, - config=relay_config, - transport_platform=Platform.RELAY, - ) + return DeliveryTransport(relay, relay_config, Platform.RELAY) return None +def _as_int(value: Any) -> Optional[int]: + try: + return int(value) + except (TypeError, ValueError): + return None + + def looks_like_telegram_private_chat_id(chat_id: Optional[str]) -> bool: """True when ``chat_id`` is a positive int — Telegram's private-chat shape. - Telegram private chats use positive chat IDs; groups/channels/supergroups - use negative IDs. This is the single source of truth for that heuristic, - reused by the handoff seed path in ``gateway/run.py`` so handoff-created - DM topics key the same way as inbound DM-topic messages. + Groups/channels/supergroups use negative IDs. Single source of truth for the + heuristic, reused by the handoff seed path in ``gateway/run.py`` so + handoff-created DM topics key the same way as inbound DM-topic messages. """ if chat_id is None: return False - try: - return int(chat_id) > 0 - except (TypeError, ValueError): - return False + parsed = _as_int(chat_id) + return parsed is not None and parsed > 0 def _looks_like_int(value: Optional[str]) -> bool: - if value is None: - return False - try: - int(value) - return True - except (TypeError, ValueError): - return False + return value is not None and _as_int(value) is not None + + +def _result_field(result: Any, name: str, default: Any = None) -> Any: + """Read ``name`` from a SendResult object or a plain result dict.""" + if isinstance(result, dict): + return result.get(name, default) + return getattr(result, name, default) def _send_result_failed(result: Any) -> bool: - if isinstance(result, dict): - return result.get("success") is False - return getattr(result, "success", True) is False + return _result_field(result, "success", True) is False def _send_result_error(result: Any) -> Optional[str]: - if isinstance(result, dict): - error = result.get("error") - else: - error = getattr(result, "error", None) + error = _result_field(result, "error") return str(error) if error else None @@ -176,21 +154,12 @@ def _is_thread_not_found_delivery_error(result: Any) -> bool: return bool(error and "thread not found" in error.lower()) -def _send_result_error_kind(result: Any) -> Optional[str]: - """Return the machine-readable error_kind from a SendResult/dict, if any.""" - if isinstance(result, dict): - kind = result.get("error_kind") - else: - kind = getattr(result, "error_kind", None) - return str(kind) if kind else None - - def _classify_dead_from_error_text(error_text: Optional[str]) -> Optional[str]: """Best-effort dead-target classification from a raised error's text. - ``_deliver_to_platform`` raises (it does not return a SendResult) on a hard - failure, so the ``deliver()`` loop only has the exception string. Reuse the - platform-neutral classifier to recover the error_kind from that text. + ``_deliver_to_platform`` raises on hard failure (no SendResult), so the + ``deliver()`` loop only has the exception string; reuse the platform-neutral + classifier to recover the error_kind from it. """ if not error_text: return None @@ -202,9 +171,9 @@ def _classify_dead_from_error_text(error_text: Optional[str]) -> Optional[str]: if not DeadTargetRegistry.is_dead_error_kind(kind): return None # ``not_found`` collapses chat-level and thread/topic/message-level failures. - # Only a whole-chat not_found means the target is dead — a deleted forum topic - # or an edited-away message must not mark the entire chat (and all of its future - # deliveries) dead. See gateway.dead_targets' documented scope. + # Only a whole-chat not_found means the target is dead — a deleted forum + # topic or an edited-away message must not mark the entire chat (and all its + # future deliveries) dead (see gateway.dead_targets). if kind == "not_found" and not is_chat_level_not_found(error_text=error_text): return None return kind @@ -212,35 +181,20 @@ def _classify_dead_from_error_text(error_text: Optional[str]) -> Optional[str]: @dataclass class DeliveryTarget: - """ - A single delivery target. - - Represents where a message should be sent: - - "origin" → back to source - - "local" → save to local files - - "telegram" → Telegram home channel - - "telegram:123456" → specific Telegram chat - """ + """A single delivery target: "origin", "local", "telegram" (home channel), + or "telegram:123456[:thread]" (specific chat).""" platform: Platform chat_id: Optional[str] = None # None means use home channel thread_id: Optional[str] = None is_origin: bool = False is_explicit: bool = False # True if chat_id was explicitly specified - + @classmethod def parse(cls, target: str, origin: Optional[SessionSource] = None) -> "DeliveryTarget": - """ - Parse a delivery target string. - - Formats: - - "origin" → back to source - - "local" → local files only - - "telegram" → Telegram home channel - - "telegram:123456" → specific Telegram chat - """ + """Parse "origin" | "local" | "" | ":[:]".""" target_stripped = target.strip() target_lower = target_stripped.lower() - + if target_lower == "origin": if origin: return cls( @@ -249,35 +203,28 @@ class DeliveryTarget: thread_id=origin.thread_id, is_origin=True, ) - else: - # Fallback to local if no origin - return cls(platform=Platform.LOCAL, is_origin=True) - + return cls(platform=Platform.LOCAL, is_origin=True) + if target_lower == "local": return cls(platform=Platform.LOCAL) - - # Check for platform:chat_id or platform:chat_id:thread_id format - # Use the original case for chat_id/thread_id to preserve case-sensitive IDs + + # Platform names are case-insensitive; chat/thread ids keep original case. + # Unknown platforms are treated as local. if ":" in target_stripped: parts = target_stripped.split(":", 2) - platform_str = parts[0].lower() # Platform names are case-insensitive chat_id = parts[1] if len(parts) > 1 else None thread_id = parts[2] if len(parts) > 2 else None try: - platform = Platform(platform_str) - return cls(platform=platform, chat_id=chat_id, thread_id=thread_id, is_explicit=True) + platform = Platform(parts[0].lower()) except ValueError: - # Unknown platform, treat as local return cls(platform=Platform.LOCAL) - - # Just a platform name (use home channel) + return cls(platform=platform, chat_id=chat_id, thread_id=thread_id, is_explicit=True) + try: - platform = Platform(target_lower) - return cls(platform=platform) + return cls(platform=Platform(target_lower)) except ValueError: - # Unknown platform, treat as local return cls(platform=Platform.LOCAL) - + def to_string(self) -> str: """Convert back to string format.""" if self.is_origin: @@ -291,30 +238,33 @@ class DeliveryTarget: return self.platform.value +async def _ensure_named_dm_topic(adapter: Any, chat_id: str, name: str, *, refresh: bool) -> str: + """Create (or force-recreate) a named Telegram private DM topic; return its thread id.""" + verb = "refresh" if refresh else "create" + ensure_dm_topic = getattr(adapter, "ensure_dm_topic", None) + if ensure_dm_topic is None: + raise RuntimeError(f"Telegram adapter cannot {verb} named private DM topics") + if refresh: + thread_id = await ensure_dm_topic(chat_id, name, force_create=True) + else: + thread_id = await ensure_dm_topic(chat_id, name) + if not thread_id: + raise RuntimeError(f"Failed to {verb} Telegram private DM topic '{name}'") + return str(thread_id) + + class DeliveryRouter: - """ - Routes messages to appropriate destinations. - - Handles the logic of resolving delivery targets and dispatching - messages to the right platform adapters. - """ - + """Resolves delivery targets and dispatches messages to platform adapters.""" + def __init__(self, config: GatewayConfig, adapters: Dict[Platform, Any] = None, dead_targets: Optional[DeadTargetRegistry] = None): - """ - Initialize the delivery router. - - Args: - config: Gateway configuration - adapters: Dict mapping platforms to their adapter instances - dead_targets: Optional shared registry of confirmed-unreachable - targets. When omitted, a profile-local registry is created. - """ + """``dead_targets``: shared registry of confirmed-unreachable targets; + a profile-local registry is created when omitted.""" self.config = config self.adapters = adapters or {} self.output_dir = get_hermes_home() / "cron" / "output" self.dead_targets = dead_targets or DeadTargetRegistry() - + async def deliver( self, content: str, @@ -323,32 +273,16 @@ class DeliveryRouter: job_name: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None ) -> Dict[str, Any]: - """ - Deliver content to all specified targets. - - Args: - content: The message/output to deliver - targets: List of delivery targets - job_id: Optional job ID (for cron jobs) - job_name: Optional job name - metadata: Additional metadata to include - - Returns: - Dict with delivery results per target - """ + """Deliver content to all targets; returns per-target results keyed by target string.""" results = {} - + for target in targets: - # Skip targets we've already proven permanently unreachable - # (deleted group, blocked/kicked bot, deactivated user). Re-sending - # to them on every tick wastes a send against flood control and - # spams logs. Self-healing: a later successful send clears the flag. + # Skip targets proven permanently unreachable (deleted group, blocked + # bot, deactivated user) — re-sending each tick wastes flood-control + # budget. Self-healing: a later successful send clears the flag. # LOCAL/origin-without-chat targets are never dead-tracked. - if ( - target.platform != Platform.LOCAL - and target.chat_id - and self.dead_targets.is_dead(target.platform.value, target.chat_id) - ): + tracked = target.platform != Platform.LOCAL and target.chat_id + if tracked and self.dead_targets.is_dead(target.platform.value, target.chat_id): logger.info( "Skipping delivery to known-dead target %s:%s " "(send to it again to clear)", @@ -365,31 +299,23 @@ class DeliveryRouter: result = self._deliver_local(content, job_id, job_name, metadata) else: result = await self._deliver_to_platform(target, content, metadata) - # Successful platform delivery — clear any stale dead flag. if target.chat_id and not _send_result_failed(result): self.dead_targets.clear(target.platform.value, target.chat_id) - - results[target.to_string()] = { - "success": True, - "result": result - } + results[target.to_string()] = {"success": True, "result": result} except Exception as e: - # A hard failure raises here. If the platform reported a - # whole-chat death, record it so future deliveries short-circuit. - if target.platform != Platform.LOCAL and target.chat_id: + # Hard failures raise. Record a whole-chat death so future + # deliveries short-circuit. + if tracked: dead_kind = _classify_dead_from_error_text(str(e)) if dead_kind: self.dead_targets.mark_dead( target.platform.value, target.chat_id, reason=f"{dead_kind}: {str(e)[:120]}", ) - results[target.to_string()] = { - "success": False, - "error": str(e) - } - + results[target.to_string()] = {"success": False, "error": str(e)} + return results - + def _deliver_local( self, content: str, @@ -399,43 +325,23 @@ class DeliveryRouter: ) -> Dict[str, Any]: """Save content to local files.""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - - if job_id: - output_path = self.output_dir / job_id / f"{timestamp}.md" - else: - output_path = self.output_dir / "misc" / f"{timestamp}.md" - + output_path = self.output_dir / (job_id or "misc") / f"{timestamp}.md" output_path.parent.mkdir(parents=True, exist_ok=True) - - # Build the output document - lines = [] - if job_name: - lines.append(f"# {job_name}") - else: - lines.append("# Delivery Output") - - lines.append("") - lines.append(f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - + + lines = [ + f"# {job_name}" if job_name else "# Delivery Output", + "", + f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + ] if job_id: lines.append(f"**Job ID:** {job_id}") - - if metadata: - for key, value in metadata.items(): - lines.append(f"**{key}:** {value}") - - lines.append("") - lines.append("---") - lines.append("") - lines.append(content) - + for key, value in (metadata or {}).items(): + lines.append(f"**{key}:** {value}") + lines += ["", "---", "", content] + output_path.write_text("\n".join(lines), encoding="utf-8") - - return { - "path": str(output_path), - "timestamp": timestamp - } - + return {"path": str(output_path), "timestamp": timestamp} + def _save_full_output(self, content: str, job_id: str) -> Path: """Save full cron output to disk and return the file path.""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") @@ -446,17 +352,55 @@ class DeliveryRouter: return path def _filter_silence_narration_enabled(self) -> bool: - """Whether the outbound silence-narration filter is active. - - ``HERMES_FILTER_SILENCE_NARRATION`` env var overrides config when set; - otherwise the ``gateway.filter_silence_narration`` config flag wins - (default True). - """ + """``HERMES_FILTER_SILENCE_NARRATION`` env overrides the + ``gateway.filter_silence_narration`` config flag (default True).""" env = os.getenv("HERMES_FILTER_SILENCE_NARRATION") if env is not None: return env.strip().lower() in ("1", "true", "yes", "on") return bool(getattr(self.config, "filter_silence_narration", True)) + def _cap_oversized_output(self, adapter: Any, content: str, job_id: str) -> str: + """Audit-save oversized cron output; truncate it for non-chunking adapters. + + Two independent decisions: (1) above MAX_PLATFORM_OUTPUT the full output + is always written to disk as an audit trail, best-effort — a failed save + (full disk, permissions) never blocks delivery; (2) non-chunking adapters + get the content truncated with a footer pointing to the saved file, while + ``splits_long_messages`` adapters receive the full payload. + """ + if len(content) <= MAX_PLATFORM_OUTPUT: + return content + saved_path: Optional[Path] = None + try: + saved_path = self._save_full_output(content, job_id) + except OSError as exc: + logger.warning( + "Audit save failed for cron output (%d chars, job=%s): %s — " + "delivery proceeds without audit copy", + len(content), job_id, exc, + ) + + if getattr(adapter, "splits_long_messages", False): + if saved_path: + logger.info( + "Cron output preserved for chunking adapter (%d chars) — " + "full output saved to %s", + len(content), saved_path, + ) + return content + + # The footer needs a valid path: if the best-effort save failed, retry + # (a failure now is a real delivery problem and propagates). + if saved_path is None: + saved_path = self._save_full_output(content, job_id) + footer = f"\n\n... [truncated, full output saved to {saved_path}]" + visible = max(0, MAX_PLATFORM_OUTPUT - len(footer)) + logger.info( + "Cron output truncated (%d chars) — full output: %s", + len(content), saved_path, + ) + return content[:visible] + footer + async def _deliver_to_platform( self, target: DeliveryTarget, @@ -471,71 +415,21 @@ class DeliveryRouter: if not target.chat_id: raise ValueError(f"No chat ID for {target.platform.value} delivery") - - # Guard: handle oversized cron output. - # - # Two independent decisions: - # 1. AUDIT SAVE — when content exceeds MAX_PLATFORM_OUTPUT, the full - # output is always written to disk as a recoverable audit trail. - # This fires regardless of adapter capability (best-effort). - # 2. TRUNCATION — for non-chunking adapters, content above the cap is - # truncated with a footer pointing to the saved file. Chunking- - # capable adapters (splits_long_messages=True) receive the full - # payload and split natively in their send(). - job_id = (metadata or {}).get("job_id", "unknown") - saved_path: Optional[Path] = None - if len(content) > MAX_PLATFORM_OUTPUT: - # Step 1 — audit save (best-effort). The save is a side-effect - # audit trail, not essential to delivery. If it fails (full disk, - # permissions), delivery proceeds — the content reaches the adapter - # regardless. - try: - saved_path = self._save_full_output(content, job_id) - except OSError as exc: - logger.warning( - "Audit save failed for cron output (%d chars, job=%s): %s — " - "delivery proceeds without audit copy", - len(content), job_id, exc, - ) + content = self._cap_oversized_output( + adapter, content, (metadata or {}).get("job_id", "unknown") + ) - # Step 2 — truncation (only for non-chunking adapters). - if getattr(adapter, "splits_long_messages", False): - # Adapter chunks natively — deliver full payload. - if saved_path: - logger.info( - "Cron output preserved for chunking adapter (%d chars) — " - "full output saved to %s", - len(content), saved_path, - ) - else: - # Non-chunking adapter — truncate with footer. The footer - # needs a valid path, so if the best-effort save above failed, - # retry it here (a failure now is a real delivery problem). - if saved_path is None: - saved_path = self._save_full_output(content, job_id) - footer = f"\n\n... [truncated, full output saved to {saved_path}]" - visible = max(0, MAX_PLATFORM_OUTPUT - len(footer)) - logger.info( - "Cron output truncated (%d chars) — full output: %s", - len(content), saved_path, - ) - content = content[:visible] + footer - # Substrate-level anti-loop guard: drop hallucinated "silence narration" - # (*(silent)*, 🔇, a bare ".", etc.) before it ever reaches the adapter. - # In bot-to-bot channels these tokens mirror back and forth until a - # model crashes with "no content after all retries". Behavioral prompt - # rules drift across providers; this single chokepoint covers every - # platform adapter regardless of which persona's prompt failed. - # Local/file delivery (_deliver_local) is a separate path and is never - # filtered — saved silence has no loop risk. - # Cron output is an ARTIFACT, not model chatter: a job whose brief is - # legitimately terse ("...", a single 🔇 from a script) has no bot-to-bot - # mirror loop to guard against, and dropping it here while returning - # {"success": True} is exactly how a cron was logged as delivered with - # nothing on the wire (#77763). Cron sends carry job_id in metadata; - # every other caller keeps the filter unchanged. + # (*(silent)*, 🔇, a bare ".") before it reaches any adapter — in + # bot-to-bot channels these mirror back and forth until a model crashes + # with "no content after all retries"; prompt rules drift across + # providers, so this single chokepoint covers every platform. Local/file + # delivery is never filtered (saved silence has no loop risk). + # Cron output is an ARTIFACT, not model chatter: a legitimately terse job + # ("...", a single 🔇) has no mirror loop, and dropping it while returning + # success is how a cron gets logged as delivered with nothing on the + # wire. Cron sends carry job_id in metadata; everything else is filtered. is_cron_artifact = "job_id" in (metadata or {}) if ( self._filter_silence_narration_enabled() @@ -562,96 +456,52 @@ class DeliveryRouter: send_metadata["user_id"] = home.user_id if home.scope_id: send_metadata["scope_id"] = home.scope_id - is_named_telegram_private_topic = False + + # Caller-supplied thread routing always wins over target.thread_id. named_telegram_private_topic_name: Optional[str] = None - if target.thread_id: - has_explicit_direct_topic = ( - "direct_messages_topic_id" in send_metadata - or "telegram_direct_messages_topic_id" in send_metadata - ) - target_thread_id = target.thread_id - is_named_telegram_private_topic = ( + thread_id = target.thread_id + thread_unrouted = thread_id and not any( + key in send_metadata + for key in ("thread_id", "message_thread_id", + "direct_messages_topic_id", "telegram_direct_messages_topic_id") + ) + if thread_unrouted: + is_telegram_private = ( target.platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(target.chat_id) - and not _looks_like_int(target_thread_id) - and "thread_id" not in send_metadata - and "message_thread_id" not in send_metadata - and not has_explicit_direct_topic ) - if is_named_telegram_private_topic: - named_telegram_private_topic_name = target_thread_id - ensure_dm_topic = getattr(adapter, "ensure_dm_topic", None) - if ensure_dm_topic is None: - raise RuntimeError( - "Telegram adapter cannot create named private DM topics" - ) - created_thread_id = await ensure_dm_topic(target.chat_id, target_thread_id) - if not created_thread_id: - raise RuntimeError( - f"Failed to create Telegram private DM topic '{target_thread_id}'" - ) - target_thread_id = str(created_thread_id) - send_metadata["thread_id"] = target_thread_id + if is_telegram_private and not _looks_like_int(thread_id): + # Named topic: create via createForumTopic, use message_thread_id directly. + named_telegram_private_topic_name = thread_id + send_metadata["thread_id"] = await _ensure_named_dm_topic( + adapter, target.chat_id, thread_id, refresh=False + ) send_metadata["telegram_dm_topic_created_for_send"] = True - elif ( - target.platform == Platform.TELEGRAM - and looks_like_telegram_private_chat_id(target.chat_id) - and "thread_id" not in send_metadata - and "message_thread_id" not in send_metadata - and not has_explicit_direct_topic - ): - # Legacy private topic/thread ids that were not created by this - # send path may still need a reply anchor to stay visible in the - # requested lane. Named targets are created above via - # createForumTopic and can use message_thread_id directly. - reply_anchor = send_metadata.get("telegram_reply_to_message_id") - if reply_anchor is None: + elif is_telegram_private: + # Legacy numeric private topic ids not created by this send path + # need a reply anchor to stay visible in the requested lane. + if send_metadata.get("telegram_reply_to_message_id") is None: raise RuntimeError( "Telegram private DM topic delivery requires telegram_reply_to_message_id; " "send to the bare chat or provide a reply anchor" ) - send_metadata["thread_id"] = target_thread_id + send_metadata["thread_id"] = thread_id send_metadata["telegram_dm_topic_reply_fallback"] = True - elif "thread_id" not in send_metadata and "message_thread_id" not in send_metadata and not has_explicit_direct_topic: - send_metadata["thread_id"] = target_thread_id + else: + send_metadata["thread_id"] = thread_id + result = await transport.send( - target.platform, - target.chat_id, - content, - metadata=send_metadata or None, + target.platform, target.chat_id, content, metadata=send_metadata or None, ) if _send_result_failed(result): - if ( - is_named_telegram_private_topic - and named_telegram_private_topic_name - and _is_thread_not_found_delivery_error(result) - ): - ensure_dm_topic = getattr(adapter, "ensure_dm_topic", None) - if ensure_dm_topic is None: - raise RuntimeError( - "Telegram adapter cannot refresh named private DM topics" - ) - refreshed_thread_id = await ensure_dm_topic( - target.chat_id, - named_telegram_private_topic_name, - force_create=True, + if named_telegram_private_topic_name and _is_thread_not_found_delivery_error(result): + send_metadata["thread_id"] = await _ensure_named_dm_topic( + adapter, target.chat_id, named_telegram_private_topic_name, refresh=True ) - if not refreshed_thread_id: - raise RuntimeError( - f"Failed to refresh Telegram private DM topic '{named_telegram_private_topic_name}'" - ) - send_metadata["thread_id"] = str(refreshed_thread_id) send_metadata["telegram_dm_topic_created_for_send"] = True result = await transport.send( - target.platform, - target.chat_id, - content, - metadata=send_metadata or None, + target.platform, target.chat_id, content, metadata=send_metadata or None, ) if _send_result_failed(result): raise RuntimeError(_send_result_error(result) or f"{target.platform.value} delivery failed") return result - - - - diff --git a/gateway/delivery_ledger.py b/gateway/delivery_ledger.py index 16f4633076..7d5fa26d64 100644 --- a/gateway/delivery_ledger.py +++ b/gateway/delivery_ledger.py @@ -1,48 +1,42 @@ """Durable delivery-obligation ledger for gateway final responses. -A final agent response that was generated but not yet confirmed-delivered -to the messaging platform is the one artifact the gateway can lose without -a trace: the turn already burned its tokens, the text exists only in a -Python local, and a crash / planned restart between finalize and platform -ACK drops it silently (#58818, #41696, #63695). +A final agent response generated but not yet confirmed-delivered is the one +artifact the gateway can lose without a trace: the turn already burned its +tokens, the text exists only in a Python local, and a crash / restart between +finalize and platform ACK drops it silently. -This module records a small durable row per outbound final response in the -shared ``state.db`` (same file and conventions as -``tools.async_delegation`` — WAL, owner pid + process-start-time liveness, -bounded retention). The gateway writes three checkpoints around the send: +Each outbound final response gets a small durable row in the shared ``state.db`` +(same conventions as ``tools.async_delegation`` — WAL, owner pid + process-start +liveness, bounded retention). The gateway writes three checkpoints: record_obligation() state='pending' before any send attempt mark_attempting() state='attempting' immediately before the await mark_delivered() / state='delivered' only on SendResult.success mark_failed() state='failed' on a definitive rejection -On startup, ``sweep_recoverable()`` claims rows whose owning process is -dead and hands them to the gateway for redelivery. After a platform adapter -reconnects without a process restart, ``sweep_failed_for_runtime()`` may claim -only the same live process's explicitly allowlisted transient failures. Crash -semantics are explicit about ambiguity (the contract review of the earlier -delivery-outbox attempt, #61790, closed it for silently resending ambiguous -sends): +On startup ``sweep_recoverable()`` claims rows whose owner is dead and hands +them back for redelivery. After an adapter reconnects without a restart, +``sweep_failed_for_runtime()`` may claim only the same live process's +allowlisted transient failures. Crash semantics are explicit about ambiguity +(never silently resend an ambiguous send): -- ``pending`` — the send never started: redeliver plainly, no dup risk. -- ``attempting`` — crashed mid-await: the platform MAY already have the - message. Redelivered WITH a visible recovered-reply marker so the - contract is honest at-least-once, never a silent duplicate. -- ``failed`` — definitively rejected once; the restart is a natural - retry boundary. Also carries the marker. +- ``pending`` — send never started: redeliver plainly, no dup risk. +- ``attempting`` — crashed mid-await: the platform MAY already have it. + Redelivered WITH a visible recovered-reply marker (honest at-least-once). +- ``failed`` — definitively rejected once; restart is a natural retry + boundary. Also carries the marker. - ``delivered`` — nothing to do; retention prunes. -Poison rows cannot spin: attempts are capped, stale rows expire, and both -transition to ``abandoned`` (kept briefly for inspection, then pruned). +Poison rows cannot spin: attempts are capped and stale rows expire, both +transitioning to ``abandoned`` (kept briefly, then pruned). -Everything here is best-effort by design: ledger failures must never block -or delay an actual send. Callers wrap every call in try/except. +Everything here is best-effort: ledger failures must never block or delay an +actual send. Callers wrap every call in try/except. """ from __future__ import annotations import hashlib -import json import logging import os import sqlite3 @@ -57,9 +51,8 @@ logger = logging.getLogger(__name__) _DB_LOCK = threading.Lock() -# Redelivery policy knobs (module constants; deliberately not config — the -# ledger itself is gated by ``gateway.delivery_ledger`` and these bounds -# only matter in the rare recovery path). +# Redelivery policy knobs (deliberately not config — the ledger is gated by +# ``gateway.delivery_ledger`` and these only matter in the rare recovery path). MAX_ATTEMPTS = 3 STALE_AFTER_SECONDS = 24 * 60 * 60 _RETENTION_SECONDS = 7 * 24 * 60 * 60 @@ -72,18 +65,16 @@ RECOVERED_MARKER = ( "so this may be a duplicate:\n\n" ) -# Runtime recovery uses a distinct marker because no gateway restart occurred. -# Keep the ambiguity explicit: a network rejection normally means the platform -# did not accept the message, but an acknowledgement can be lost independently. +# Runtime recovery uses a distinct marker: no restart occurred, but a network +# rejection's acknowledgement can still have been lost independently. RECONNECTED_MARKER = ( "♻️ Recovered reply — the messaging platform reconnected after the original " "delivery failed, so this may be a duplicate:\n\n" ) -# Runtime replay is deliberately fail-closed. Only errors whose send contract -# proves they are transient reconnect failures belong here; permanent rejects -# (blocked bot, bad auth, missing chat) must not be retried merely because an -# adapter reconnected. +# Runtime replay is fail-closed: only errors whose send contract proves they are +# transient reconnect failures. Permanent rejects (blocked bot, bad auth, missing +# chat) must not be retried merely because an adapter reconnected. _RUNTIME_RETRYABLE_ERRORS = frozenset({"send_path_degraded"}) @@ -98,8 +89,7 @@ def _connect() -> sqlite3.Connection: try: _initialize_schema(conn) except Exception: - # A PRAGMA/DDL failure after a successful connect() must not leak the - # just-opened connection back to the caller. + # A PRAGMA/DDL failure after connect() must not leak the connection. conn.close() raise return conn @@ -127,14 +117,10 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: adapter_profile TEXT )""" ) - columns = { - row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)") - } + columns = {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")} if "adapter_profile" not in columns: try: - conn.execute( - "ALTER TABLE delivery_obligations ADD COLUMN adapter_profile TEXT" - ) + conn.execute("ALTER TABLE delivery_obligations ADD COLUMN adapter_profile TEXT") except sqlite3.OperationalError as exc: # Concurrent first-use connections can both observe the old schema. if "duplicate column" not in str(exc).lower(): @@ -145,13 +131,10 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: def _transaction() -> Iterator[sqlite3.Connection]: """Open a connection, commit/rollback on exit, and ALWAYS close it. - ``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the - transaction; they do not close the connection. Using ``with _connect()`` - alone therefore leaks a connection — and its WAL/SHM file descriptors — on - every call, deferring the close to the garbage collector. On a long-running - gateway that exhausts ``RLIMIT_NOFILE`` (the cron-ledger sibling of this - bug was #69567 / PR #69594). ``record_obligation`` runs on every outbound - final response, so this ledger is the highest-frequency leaker. + ``sqlite3.Connection`` as a context manager only commits/rolls back; it does + not close. ``with _connect()`` alone would leak a connection (and its WAL/SHM + fds) per call until GC — ``record_obligation`` runs on every outbound final + response, so this ledger would exhaust ``RLIMIT_NOFILE`` on a long-running gateway. """ conn = _connect() try: @@ -186,20 +169,16 @@ def _owner_alive(pid: Any, started_at: Any) -> bool: except Exception: current_start = None if current_start is None: - # No such process (or unreadable) — treat unreadable-but-extant - # processes as alive only if the pid exists. Route through the - # cross-platform probe: ``os.kill(pid, 0)`` on Windows is NOT a - # no-op (bpo-14484 — CPython maps sig=0 to - # ``GenerateConsoleCtrlEvent(0, pid)``), so a raw probe here could - # Ctrl+C the gateway's own console group whenever psutil failed to - # read the start time of a live pid. ``_pid_exists`` keeps the - # EPERM-means-alive semantics (exists but owned by another user). + # Start time unreadable: alive iff the pid exists. Route through the + # cross-platform probe — ``os.kill(pid, 0)`` on Windows is NOT a no-op + # (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could + # Ctrl+C the gateway's own console group). ``_pid_exists`` keeps the + # EPERM-means-alive semantics (pid exists but is owned by another user). try: from gateway.status import _pid_exists except Exception: if os.name == "nt": - # Never fall back to a raw sig-0 probe on Windows. - return False + return False # never fall back to a raw sig-0 probe on Windows try: os.kill(pid, 0) # windows-footgun: ok — POSIX-only fallback branch except ProcessLookupError: @@ -222,10 +201,9 @@ def _owner_alive(pid: Any, started_at: Any) -> bool: def compute_obligation_id(session_key: str, message_ref: str, content: str) -> str: - """Stable id: same turn + same content re-records idempotently, while - distinct threads/topics on the same chat can never collide (the - session_key carries platform, chat and thread; ``message_ref`` is the - triggering inbound message id, distinguishing turns in one session).""" + """Stable id: same turn + same content re-records idempotently, while distinct + threads/topics on one chat never collide (session_key carries platform, chat and + thread; ``message_ref`` is the triggering inbound message id).""" payload = f"{session_key}|{message_ref}|{content}" return hashlib.sha256(payload.encode("utf-8", "replace")).hexdigest()[:24] @@ -273,11 +251,10 @@ def mark_failed(obligation_id: str, error: str = "") -> None: def release_runtime_claim(obligation_id: str, error: str = "") -> bool: """Return an unsent runtime claim to ``failed`` without spending an attempt. - Runtime recovery claims before clearing ``resume_pending`` so that two - reconnect paths cannot send the same row. If the session flag cannot be - cleared, no platform send was attempted and the claim must not consume the - bounded redelivery budget. Release is fail-closed to the exact current - process instance and the ``attempting`` state. + Runtime recovery claims before clearing ``resume_pending`` so two reconnect + paths cannot send the same row. If the session flag cannot be cleared, no + send was attempted and the claim must not consume the redelivery budget. + Fail-closed to the exact current process instance and ``attempting`` state. """ pid, started = _owner_stamp() if started is None: @@ -306,31 +283,32 @@ def _update_state(obligation_id: str, state: str, error: str = "") -> None: ) +def _exhausted(attempts: int, created_at: float, now: float) -> bool: + return attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS + + def sweep_recoverable( now: Optional[float] = None, *, deliverable_platforms: Optional[set] = None, deliverable_targets: Optional[set] = None, ) -> List[Dict[str, Any]]: - """Claim undelivered rows owned by dead processes; return them for - redelivery. + """Claim undelivered rows owned by dead processes; return them for redelivery. Claiming atomically re-stamps the owner to THIS process and increments - ``attempts``, so a second gateway racing the same sweep cannot - double-claim (the UPDATE is guarded on the previous owner stamp). - Rows over the attempts cap or older than the stale cutoff transition to - 'abandoned' instead of being returned. + ``attempts`` (the UPDATE is guarded on the previous owner stamp, so a second + gateway racing the same sweep cannot double-claim). Rows over the attempts + cap or stale cutoff transition to 'abandoned' instead of being returned. - ``deliverable_platforms`` (platform value strings) restricts claiming to - platforms the caller can actually send on this boot. ``attempts`` is the - redelivery budget, so it must only be spent on a real send: a platform - that failed to connect would otherwise burn one attempt per boot and hit - the cap having never been sent once. Rows for absent platforms are left - untouched for a later boot; the stale cutoff still bounds them. + ``deliverable_platforms`` restricts claiming to platforms the caller can + actually send on this boot: ``attempts`` is the redelivery budget and must + only be spent on a real send, else a platform that failed to connect burns + one attempt per boot and hits the cap having never been sent once. Rows for + absent platforms are left untouched (the stale cutoff still bounds them). ``deliverable_targets`` further scopes multiplexed gateways by exact - ``(platform, adapter_profile)`` identity, preventing one connected bot from - spending another disconnected bot's retry budget. + ``(platform, adapter_profile)`` so one connected bot cannot spend another + disconnected bot's retry budget. """ now = now if now is not None else time.time() pid, started = _owner_stamp() @@ -348,24 +326,16 @@ def sweep_recoverable( adapter_profile) in rows: if _owner_alive(owner_pid, owner_started_at): continue # a live gateway still owns this row - if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: + if _exhausted(attempts, created_at, now): conn.execute( """UPDATE delivery_obligations SET state='abandoned', updated_at=? WHERE obligation_id=?""", (now, oid), ) continue - if ( - deliverable_platforms is not None - and platform not in deliverable_platforms - ): - # No adapter for this platform this boot — the caller cannot - # send, so claiming would spend an attempt on a no-op. - continue - if ( - deliverable_targets is not None - and (platform, adapter_profile) not in deliverable_targets - ): + if deliverable_platforms is not None and platform not in deliverable_platforms: + continue # no adapter this boot — claiming would spend an attempt on a no-op + if deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets: continue cursor = conn.execute( """UPDATE delivery_obligations @@ -382,8 +352,7 @@ def sweep_recoverable( "chat_id": chat_id, "thread_id": thread_id, "content": content, - # pending = send never started, redeliver plainly; - # attempting/failed = ambiguous or rejected, carry marker. + # pending = never started, redeliver plainly; else carry marker. "needs_marker": state != "pending", "profile": adapter_profile, "attempts": attempts + 1, @@ -399,32 +368,27 @@ def sweep_failed_for_runtime( ) -> List[Dict[str, Any]]: """Claim this process's reconnect-retryable failed rows for one adapter. - ``profile`` scopes multiplexed gateways to the bot identity that actually - owned the failed send; ``None`` means the primary/default adapter. The - persisted adapter owner is independent of the routed session namespace. + ``profile`` scopes multiplexed gateways to the bot identity that owned the + failed send; ``None`` means the primary/default adapter. The persisted + adapter owner is independent of the routed session namespace. Unowned rows + and rows owned by another process are left for the startup/dead-owner sweep. - Startup recovery intentionally ignores rows owned by a live gateway. That - protects concurrent processes, but it also means a final response rejected - with ``send_path_degraded`` remains stranded when only the platform adapter - reconnects. This runtime sweep closes that gap without weakening ownership: - - - only rows stamped to this exact process instance are eligible; - - only explicitly allowlisted transient errors are eligible; - - attempts/staleness bounds match startup recovery; - - every update is guarded by the prior owner stamp and ``failed`` state. - - Unowned rows and rows owned by another process are left untouched for the - normal startup/dead-owner sweep. Claimed rows always carry the reconnect + Startup recovery ignores rows owned by a live gateway, so a final response + rejected with ``send_path_degraded`` would stay stranded when only the + adapter reconnects. This sweep closes that gap without weakening ownership: + only rows stamped to this exact process instance, only allowlisted transient + errors, same attempts/staleness bounds, every update guarded by the prior + owner stamp and ``failed`` state. Claimed rows always carry the reconnect marker because the failed send's acknowledgement is not safe to infer. """ now = now if now is not None else time.time() pid, started = _owner_stamp() if started is None: - # PID equality alone cannot distinguish this process from a stale row - # left by an earlier process incarnation after PID reuse. Runtime replay - # is optional recovery, so fail closed when the process fingerprint is - # unavailable; startup recovery remains the durable fallback. + # PID alone cannot distinguish this process from a stale row left after + # PID reuse. Runtime replay is optional, so fail closed; startup + # recovery remains the durable fallback. return [] + expected_profile = "default" if not profile or profile == "default" else str(profile) claimed: List[Dict[str, Any]] = [] with _DB_LOCK, _transaction() as conn: rows = conn.execute( @@ -435,33 +399,18 @@ def sweep_failed_for_runtime( WHERE state='failed' AND platform=?""", (platform,), ).fetchall() - for ( - oid, - session_key, - row_platform, - chat_id, - thread_id, - content, - attempts, - created_at, - owner_pid, - owner_started_at, - last_error, - adapter_profile, - ) in rows: - expected_profile = ( - "default" if not profile or profile == "default" else str(profile) - ) + for (oid, session_key, row_platform, chat_id, thread_id, content, + attempts, created_at, owner_pid, owner_started_at, last_error, + adapter_profile) in rows: if adapter_profile != expected_profile: continue - # Runtime reconnect recovery may act only on its own rows. Exact - # process-start matching prevents PID reuse from stealing work. + # Exact process-start matching prevents PID reuse from stealing work. if owner_pid != pid or owner_started_at != started: continue if str(last_error or "").strip().lower() not in _RUNTIME_RETRYABLE_ERRORS: continue owner_guard = (oid, owner_pid, owner_started_at) - if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: + if _exhausted(attempts, created_at, now): conn.execute( """UPDATE delivery_obligations SET state='abandoned', updated_at=? @@ -504,9 +453,7 @@ def _prune(now: Optional[float] = None) -> None: WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", (cutoff,), ) - total = conn.execute( - "SELECT COUNT(*) FROM delivery_obligations" - ).fetchone()[0] + total = conn.execute("SELECT COUNT(*) FROM delivery_obligations").fetchone()[0] excess = max(0, total - _MAX_ROWS) if excess: conn.execute( @@ -538,25 +485,3 @@ def ledger_enabled(config: Optional[Dict[str, Any]] = None) -> bool: return bool(value) except Exception: return True - - -def debug_rows(limit: int = 20) -> str: - """Human-readable dump for ad-hoc inspection (sqlite3-free path).""" - with _DB_LOCK, _transaction() as conn: - rows = conn.execute( - """SELECT obligation_id, session_key, state, attempts, - created_at, updated_at, last_error - FROM delivery_obligations - ORDER BY updated_at DESC LIMIT ?""", - (limit,), - ).fetchall() - return json.dumps( - [ - { - "id": r[0], "session": r[1], "state": r[2], "attempts": r[3], - "created_at": r[4], "updated_at": r[5], "last_error": r[6], - } - for r in rows - ], - indent=2, - ) diff --git a/gateway/disk_status.py b/gateway/disk_status.py index b762233cf3..22d0102b57 100644 --- a/gateway/disk_status.py +++ b/gateway/disk_status.py @@ -1,30 +1,21 @@ -"""Disk-usage rollup for ``/api/status`` (NS-656). +"""Disk-usage rollup for ``/api/status``. -Companion to :mod:`gateway.memory_status`, closing the same class of gap -for storage: a hosted agent can fill its data volume completely — SQLite -writes failing, session persistence dead, config saves lost — while its -dashboard and the NAS agent card both look perfectly healthy. Fleet -incidents OOF-2 (unrecoverable disk-full) and OOF-107 (fleet-wide disk -exhaustion, remediated by hand) are exactly this failure mode. +Companion to :mod:`gateway.memory_status`: a hosted agent can fill its data +volume (SQLite writes failing, sessions and config saves lost) while its +dashboard and the NAS agent card look healthy. ``gateway/readiness.py`` +already probes disk, but readiness is a component verdict, not user-facing +telemetry — this module produces the public block the dashboard SPA and the +NAS availability sweep consume. -The readiness endpoint already probes disk (``gateway/readiness.py:: -_probe_disk``), but readiness is a component verdict, not user-facing -telemetry — nothing renders it. This module produces the public block -the dashboard SPA and the NAS availability sweep actually consume. +Disk is sampled live via :func:`shutil.disk_usage` (one ``statvfs`` call), so +there is no staleness dimension and no ``sampled_at``. -Unlike the memory block (which distills already-persisted heartbeat -files), disk is sampled live via :func:`shutil.disk_usage` — a single -``statvfs`` call, the same thing the readiness probe does per request. -There is no meaningful "staleness" dimension, so no ``sampled_at``. +Public-safety: ``/api/status`` is unauthenticated (``PUBLIC_API_PATHS``); this +block carries only coarse numbers (MB, one-decimal percent) and an enum — the same +disclosure class as the ``memory`` block. -Public-safety note: ``/api/status`` is an unauthenticated liveness probe -(``PUBLIC_API_PATHS``). This block carries only coarse numbers (MB -granularity, whole-percent usage) and an enum — the same disclosure -class as the ``memory`` block. - -Everything is best-effort and read-only: an unreadable filesystem -degrades to ``pressure="unknown"`` rather than raising into the status -endpoint. +Best-effort and read-only: an unreadable filesystem degrades to +``pressure="unknown"`` rather than raising into the status endpoint. """ from __future__ import annotations @@ -36,12 +27,10 @@ from typing import Any, Dict, Optional logger = logging.getLogger(__name__) -# Disk-pressure thresholds. Percent alone misleads in both directions: -# 90% used on a 100 GB volume leaves a comfortable 10 GB, while 50% used -# on a tiny volume can be one image download from write failures. So the -# percent triggers are gated on absolute headroom also being low, and a -# hard absolute floor applies regardless of size — below it, SQLite -# journaling and config writes are at genuine risk on any volume. +# Percent alone misleads both ways: 90% used on 100 GB leaves 10 GB, while 50% +# on a tiny volume is one download from write failures. So percent triggers are +# gated on absolute headroom also being low, and a hard absolute floor applies +# regardless of size (below it SQLite journaling / config writes are at risk). _CRITICAL_FREE_MB = 256 # < 256 MB free: critical on any volume _CRITICAL_PERCENT = 95.0 # >= 95% used AND < 1 GB free: critical _CRITICAL_HEADROOM_MB = 1024 @@ -69,28 +58,22 @@ def classify_disk_pressure(free_mb: Any, total_mb: Any) -> str: if free is None or total is None or total <= 0: return "unknown" used_percent = (1 - free / total) * 100.0 - if free < _CRITICAL_FREE_MB or ( - used_percent >= _CRITICAL_PERCENT and free < _CRITICAL_HEADROOM_MB + for level, free_floor, percent_floor, headroom in ( + ("critical", _CRITICAL_FREE_MB, _CRITICAL_PERCENT, _CRITICAL_HEADROOM_MB), + ("elevated", _ELEVATED_FREE_MB, _ELEVATED_PERCENT, _ELEVATED_HEADROOM_MB), ): - return "critical" - if free < _ELEVATED_FREE_MB or ( - used_percent >= _ELEVATED_PERCENT and free < _ELEVATED_HEADROOM_MB - ): - return "elevated" + if free < free_floor or (used_percent >= percent_floor and free < headroom): + return level return "ok" def collect_disk_status(home: Optional[Path] = None) -> Dict[str, Any]: """Build the ``disk`` block for ``/api/status``. - ``home`` scopes the sample to a profile's HERMES_HOME (the status - endpoint's ``?profile=`` handling passes it through); on hosted - images every profile shares the ``/opt/data`` volume, so the answer - is the same — but scoping keeps the contract identical to the - ``memory`` block's. - - Always returns a dict — an unreadable/unmounted filesystem yields - ``{"pressure": "unknown", ...}``. Never raises. + ``home`` scopes the sample to a profile's HERMES_HOME (same contract as the + ``memory`` block; on hosted images every profile shares one volume). + Always returns a dict and never raises — an unreadable/unmounted filesystem + yields ``{"pressure": "unknown", ...}``. """ status: Dict[str, Any] = { "pressure": "unknown", diff --git a/gateway/display_config.py b/gateway/display_config.py index fe07268b7f..a02a62c137 100644 --- a/gateway/display_config.py +++ b/gateway/display_config.py @@ -1,12 +1,10 @@ """Per-platform display/verbosity configuration resolver. -Provides ``resolve_display_setting()`` — the single entry-point for reading -display settings with platform-specific overrides and sensible defaults. - -Resolution order (first non-None wins): - 1. ``display.platforms..`` — explicit per-platform user override +``resolve_display_setting()`` is the single entry-point. Resolution order +(first non-None wins): + 1. ``display.platforms..`` — explicit per-platform override 2. ``display.`` — global user setting - 3. ``_PLATFORM_DEFAULTS[][]`` — built-in sensible default + 3. ``_PLATFORM_DEFAULTS[][]`` — built-in platform default 4. ``_GLOBAL_DEFAULTS[]`` — built-in global default Exception: ``display.streaming`` is CLI-only. Gateway streaming follows the @@ -14,70 +12,52 @@ top-level ``streaming`` config unless ``display.platforms..streaming`` sets an explicit per-platform override. Backward compatibility: ``display.tool_progress_overrides`` is still read as a -fallback for ``tool_progress`` when no ``display.platforms`` entry exists. A -config migration (version bump) automatically moves the old format into the new -``display.platforms`` structure. +fallback for ``tool_progress`` when no ``display.platforms`` entry exists (a +config migration moves the old format into ``display.platforms``). """ from __future__ import annotations from typing import Any -# --------------------------------------------------------------------------- -# Overrideable display settings and their global defaults -# --------------------------------------------------------------------------- -# These are the settings that can be configured per-platform. -# Other display settings (compact, personality, skin, etc.) are CLI-only -# and don't participate in per-platform resolution. - +# Settings configurable per-platform, with global defaults. Other display +# settings (compact, personality, skin, ...) are CLI-only. _GLOBAL_DEFAULTS: dict[str, Any] = { "tool_progress": "all", "tool_progress_grouping": "accumulate", # "accumulate" = edit one bubble; "separate" = one msg per tool "show_reasoning": False, - # How a reasoning/thinking summary is rendered when show_reasoning is on. - # "code" -> 💭 **Reasoning:** + fenced code block (legacy default) - # "blockquote"-> each line prefixed with "> " - # "subtext" -> each line prefixed with "-# " (Discord small grey subtext) - # Discord defaults to "subtext"; everywhere else defaults to "code". + # Reasoning summary rendering: "code" (💭 **Reasoning:** + fenced block), + # "blockquote" ("> " lines), "subtext" ("-# " Discord small grey text). "reasoning_style": "code", "tool_preview_length": 0, "streaming": None, # None = follow top-level streaming config - # Gateway-only assistant/status chatter controls. These default on for - # back-compat, but mobile platforms can opt down to final-answer-first. + # Gateway-only assistant/status chatter; default on for back-compat, mobile + # platforms opt down to final-answer-first. "interim_assistant_messages": True, "long_running_notifications": True, "busy_ack_detail": True, - # Whether busy_input_mode=steer sends a visible "Steered into current run" - # acknowledgment after successfully injecting the user's mid-turn message. - # Disable when the platform should steer silently (the text still lands in - # the active run; only the confirmation echo is suppressed). + # busy_input_mode=steer confirmation echo ("Steered into current run"). + # Disabling only suppresses the echo; the text still lands in the run. "busy_steer_ack_enabled": True, - # When true, delete tool-progress / "⏳ Working — N min" / status bubbles - # after the final response lands on platforms that support message - # deletion (e.g. Telegram). Off by default — progress is still shown - # live, just cleaned up after success so the chat doesn't fill up with - # stale breadcrumbs. Failed runs leave bubbles in place as breadcrumbs. + # Delete tool-progress / "⏳ Working" / status bubbles after the final + # response on platforms that support deletion (e.g. Telegram). Off by + # default; progress is still shown live, only cleaned up after success so + # the chat doesn't fill with stale breadcrumbs. Failed runs leave bubbles + # in place as breadcrumbs. "cleanup_progress": False, - # Live working-state status on platforms whose typing indicator renders - # text (Slack's assistant status line). Values: - # "full" / true -> verb + argument preview ("is running pytest…") - # "verb" -> verb only ("is running…") — keeps file paths and - # commands out of shared channels - # "off" / false -> static text (typing_status_text or "is thinking...") - # Independent of tool_progress: works even when progress bubbles are off - # (Slack's default), and costs no extra API calls — the existing typing - # refresh cadence just renders different text. + # Live working-state text on platforms whose typing indicator renders text + # (Slack assistant status): "full"/true = verb + argument preview, + # "verb" = verb only (keeps paths/commands out of shared channels), + # "off"/false = static text. Independent of tool_progress and free: the + # existing typing refresh cadence just renders different text. "live_status": "full", } -# --------------------------------------------------------------------------- -# Sensible per-platform defaults — tiered by platform capability -# --------------------------------------------------------------------------- -# Tier 1 (high): Supports message editing, typically personal/team use -# Tier 2 (medium): Supports editing but often workspace/customer-facing -# Tier 3 (low): No edit support — each progress msg is permanent -# Tier 4 (minimal): Batch/non-interactive delivery - +# Per-platform defaults tiered by capability: +# HIGH — message editing, personal/team use +# MEDIUM — editing, but often workspace/customer-facing +# LOW — no edit support: each progress message is permanent +# MINIMAL — batch / non-interactive delivery _TIER_HIGH = { "tool_progress": "all", "show_reasoning": False, @@ -87,108 +67,63 @@ _TIER_HIGH = { "long_running_notifications": True, "busy_ack_detail": True, } - -_TIER_MEDIUM = { - "tool_progress": "new", - "show_reasoning": False, - "tool_preview_length": 40, - "streaming": None, - "interim_assistant_messages": True, - "long_running_notifications": True, - "busy_ack_detail": True, -} - +_TIER_MEDIUM = {**_TIER_HIGH, "tool_progress": "new"} _TIER_LOW = { + **_TIER_HIGH, "tool_progress": "off", - "show_reasoning": False, - "tool_preview_length": 40, - "streaming": False, - "interim_assistant_messages": False, - "long_running_notifications": False, - "busy_ack_detail": False, -} - -_TIER_MINIMAL = { - "tool_progress": "off", - "show_reasoning": False, - "tool_preview_length": 0, "streaming": False, "interim_assistant_messages": False, "long_running_notifications": False, "busy_ack_detail": False, } +_TIER_MINIMAL = {**_TIER_LOW, "tool_preview_length": 0} _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = { - # Tier 1 — full edit support, personal/team use - # Telegram is usually a mobile inbox: keep tool_progress quiet and skip - # the verbose busy-ack iteration counter, but DO surface real mid-turn - # assistant commentary (interim_assistant_messages) and DO send periodic - # heartbeats (long_running_notifications) so the user has signal between - # turn start and final answer. Otherwise it looks like "typing..." for - # 30 minutes with nothing happening. Opt in to verbose iteration detail - # via display.platforms.telegram.busy_ack_detail / tool_progress. - "telegram": { - **_TIER_HIGH, - "tool_progress": "off", - "busy_ack_detail": False, - }, - # Discord has a native "subtext" primitive (-# small grey text) that reads - # as metadata rather than content, so reasoning summaries default to it - # here instead of the fenced code block used elsewhere. - "discord": {**_TIER_HIGH, "reasoning_style": "subtext"}, + # Telegram is usually a mobile inbox: quiet tool_progress and no busy-ack + # iteration counter, but DO surface interim assistant commentary and + # heartbeats so it doesn't look like "typing..." for 30 minutes. + "telegram": {**_TIER_HIGH, "tool_progress": "off", "busy_ack_detail": False}, + # Discord's "-# " subtext reads as metadata, so reasoning defaults to it. + "discord": {**_TIER_HIGH, "reasoning_style": "subtext"}, - # Tier 2 — edit support, often customer/workspace channels - # Slack: tool_progress off by default — Bolt posts cannot be edited like CLI; - # "new"/"all" spam permanent lines in channels (hermes-agent#14663). - "slack": { + # Slack: Bolt posts cannot be edited like CLI; "new"/"all" spam permanent lines. + "slack": { **_TIER_MEDIUM, "tool_progress": "off", "long_running_notifications": False, "busy_ack_detail": False, }, - "mattermost": _TIER_MEDIUM, - "matrix": _TIER_MEDIUM, - "feishu": _TIER_MEDIUM, - # Buzz (Nostr relay via buzz-cli): messages can be edited in place - # (`buzz messages edit`), so grouped/accumulating progress works, but - # channels are shared community spaces — keep the medium tier. Without - # this entry Buzz inherited the verbose _GLOBAL_DEFAULTS and every - # interim update became a separate permanent channel post (#95841). - "buzz": _TIER_MEDIUM, + "mattermost": _TIER_MEDIUM, + "matrix": _TIER_MEDIUM, + "feishu": _TIER_MEDIUM, + # Buzz (Nostr via buzz-cli) can edit in place but channels are shared + # community spaces; without an entry it inherited the verbose globals. + "buzz": _TIER_MEDIUM, - # Tier 3 — no edit support, progress messages are permanent - "signal": _TIER_LOW, - "whatsapp": _TIER_MEDIUM, # Baileys bridge supports /edit - # WhatsApp Cloud API: Meta added message editing in 2023 but the - # Hermes Cloud adapter doesn't implement edit_message yet, so we - # stay on TIER_LOW (tool_progress off) to avoid spamming each - # status update as a separate message. Promote to TIER_MEDIUM once - # Cloud's edit_message lands. - "whatsapp_cloud": _TIER_LOW, - # Photon (managed iMessage over the gRPC sidecar) and BlueBubbles are both - # permanent-message iMessage inboxes with no message-edit support, so both - # stay TIER_LOW. This keeps tool progress, interim scratch commentary, - # "still working" heartbeats, and busy-ack iteration detail out of the - # user's iMessage thread. Without this entry Photon inherited the noisy - # global ("all") defaults and compacted/narrated on nearly every turn. - "photon": _TIER_LOW, - "bluebubbles": _TIER_LOW, - "weixin": _TIER_LOW, - # WeCom is technically non-editable but exposes a native streaming - # transport (msgtype: "stream" via aibot_respond_msg) that the gateway - # consumer routes mid-stream content through. Enable streaming by default - # so the WeCom client renders the typing animation and cumulative content - # updates instead of a single one-shot markdown drop. - "wecom": {**_TIER_LOW, "streaming": True}, - "wecom_callback": _TIER_LOW, - "dingtalk": _TIER_LOW, + "signal": _TIER_LOW, + "whatsapp": _TIER_MEDIUM, # Baileys bridge supports /edit + # Cloud API supports editing but our adapter lacks edit_message; promote + # to MEDIUM once it lands. + "whatsapp_cloud": _TIER_LOW, + # Photon and BlueBubbles are permanent-message iMessage inboxes (no edit); + # without an entry Photon inherited the noisy "all" globals and narrated + # on nearly every turn. + "photon": _TIER_LOW, + "bluebubbles": _TIER_LOW, + "weixin": _TIER_LOW, + # WeCom is non-editable but has a native streaming transport (msgtype + # "stream") the consumer routes mid-stream content through; streaming on + # gives the client a typing animation + cumulative updates instead of a + # single one-shot markdown drop. + "wecom": {**_TIER_LOW, "streaming": True}, + "wecom_callback": _TIER_LOW, + "dingtalk": _TIER_LOW, - # Tier 4 — batch or non-interactive delivery - "email": _TIER_MINIMAL, - "sms": _TIER_MINIMAL, - "webhook": _TIER_MINIMAL, - "homeassistant": _TIER_MINIMAL, - "api_server": {**_TIER_HIGH, "tool_preview_length": 0}, + "email": _TIER_MINIMAL, + "sms": _TIER_MINIMAL, + "webhook": _TIER_MINIMAL, + "homeassistant": _TIER_MINIMAL, + "api_server": {**_TIER_HIGH, "tool_preview_length": 0}, } # Canonical set of per-platform overrideable keys (for validation). @@ -203,120 +138,121 @@ def resolve_display_setting( ) -> Any: """Resolve a display setting with per-platform override support. - Parameters - ---------- - user_config : dict - The full parsed config.yaml dict. - platform_key : str - Platform config key (e.g. ``"telegram"``, ``"slack"``). Use - ``_platform_config_key(source.platform)`` from gateway/run.py. - setting : str - Display setting name (e.g. ``"tool_progress"``, ``"show_reasoning"``). - fallback : Any - Fallback value when the setting isn't found anywhere. - - Returns - ------- - The resolved value, or *fallback* if nothing is configured. + ``platform_key`` is the platform config key (``"telegram"``, ``"slack"``; + see ``_platform_config_key`` in gateway/run.py). Returns *fallback* when + nothing is configured. """ display_cfg = user_config.get("display") or {} - # 1. Explicit per-platform override (display.platforms..) - platforms = display_cfg.get("platforms") or {} - plat_overrides = platforms.get(platform_key) - if isinstance(plat_overrides, dict): - val = plat_overrides.get(setting) - if val is not None: - return _normalise(setting, val) + # 1. Explicit per-platform override + plat_overrides = (display_cfg.get("platforms") or {}).get(platform_key) + if isinstance(plat_overrides, dict) and plat_overrides.get(setting) is not None: + return _normalise(setting, plat_overrides[setting]) # 1b. Backward compat: display.tool_progress_overrides. if setting == "tool_progress": legacy = display_cfg.get("tool_progress_overrides") - if isinstance(legacy, dict): - val = legacy.get(platform_key) - if val is not None: - return _normalise(setting, val) + if isinstance(legacy, dict) and legacy.get(platform_key) is not None: + return _normalise(setting, legacy[platform_key]) - # 2. Global user setting (display.). Skip display.streaming because - # that key controls only CLI terminal streaming; gateway token streaming is - # governed by the top-level streaming config plus per-platform overrides. - if setting != "streaming": - val = display_cfg.get(setting) - if val is not None: - return _normalise(setting, val) + # 2. Global user setting. display.streaming controls only CLI terminal + # streaming; gateway streaming follows the top-level config + platform overrides. + if setting != "streaming" and display_cfg.get(setting) is not None: + return _normalise(setting, display_cfg[setting]) - # 3. Built-in platform default - plat_defaults = _PLATFORM_DEFAULTS.get(platform_key) - if plat_defaults: - val = plat_defaults.get(setting) - if val is not None: - return val - - # 4. Built-in global default - val = _GLOBAL_DEFAULTS.get(setting) - if val is not None: - return val - - return fallback + # 3. Built-in platform default, 4. built-in global default + val = _PLATFORM_DEFAULTS.get(platform_key, {}).get(setting) + if val is None: + val = _GLOBAL_DEFAULTS.get(setting) + return fallback if val is None else val # --------------------------------------------------------------------------- -# Helpers +# Normalisation of YAML quirks (bare ``off`` → False in YAML 1.1, etc.) # --------------------------------------------------------------------------- +_TRUTHY = {"true", "1", "yes", "on"} +_FALSY = {"false", "0", "no"} + + +def _norm_tool_progress(value: Any) -> str: + if value is False: + return "off" + if value is True: + return "all" + val = str(value).strip().lower() + if val in _FALSY: + return "off" + if val in _TRUTHY: + return "all" + return val if val in {"off", "new", "all", "verbose", "log"} else "all" + + +def _norm_bool(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in _TRUTHY | {"raw", "verbose"} + return bool(value) + + +def _norm_long_running(value: Any) -> Any: + if isinstance(value, str) and value.strip().lower() == "generic": + return "generic" + return _norm_bool(value) + + +def _norm_cleanup_progress(value: Any) -> bool: + if isinstance(value, str): + return value.lower() in _TRUTHY + return bool(value) + + +def _norm_live_status(value: Any) -> str: + """Tri-state: "full" (verb + preview), "verb" (verb only), "off".""" + if value is True: + return "full" + if value is False: + return "off" + val = str(value).strip().lower() + if val in _TRUTHY | {"all"}: + return "full" + if val in _FALSY: + return "off" + return val if val in {"full", "verb", "off"} else "full" + + +def _norm_choice(choices: tuple[str, ...]) -> Any: + def norm(value: Any) -> str: + val = str(value).lower() + return val if val in choices else choices[0] + + return norm + + +def _norm_int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +_NORMALISERS: dict[str, Any] = { + "tool_progress": _norm_tool_progress, + "show_reasoning": _norm_bool, + "streaming": _norm_bool, + "interim_assistant_messages": _norm_bool, + "long_running_notifications": _norm_long_running, + "busy_ack_detail": _norm_bool, + "busy_steer_ack_enabled": _norm_bool, + "thinking_progress": _norm_bool, + "cleanup_progress": _norm_cleanup_progress, + "live_status": _norm_live_status, + "tool_progress_grouping": _norm_choice(("accumulate", "separate")), + "reasoning_style": _norm_choice(("code", "blockquote", "subtext")), + "tool_preview_length": _norm_int, +} + + def _normalise(setting: str, value: Any) -> Any: - """Normalise YAML quirks (bare ``off`` → False in YAML 1.1).""" - if setting == "tool_progress": - if value is False: - return "off" - if value is True: - return "all" - val = str(value).strip().lower() - if val in {"false", "0", "no"}: - return "off" - if val in {"true", "1", "yes", "on"}: - return "all" - return val if val in {"off", "new", "all", "verbose", "log"} else "all" - if setting in { - "show_reasoning", - "streaming", - "interim_assistant_messages", - "long_running_notifications", - "busy_ack_detail", - "busy_steer_ack_enabled", - "thinking_progress", - }: - if isinstance(value, str): - val = value.strip().lower() - if val == "generic" and setting == "long_running_notifications": - return "generic" - return val in {"true", "1", "yes", "on", "raw", "verbose"} - return bool(value) - if setting == "cleanup_progress": - if isinstance(value, str): - return value.lower() in {"true", "1", "yes", "on"} - return bool(value) - if setting == "live_status": - # Tri-state: "full" (verb + preview), "verb" (verb only), "off". - if value is True: - return "full" - if value is False: - return "off" - val = str(value).strip().lower() - if val in {"true", "1", "yes", "on", "all"}: - return "full" - if val in {"false", "0", "no"}: - return "off" - return val if val in {"full", "verb", "off"} else "full" - if setting == "tool_progress_grouping": - val = str(value).lower() - return val if val in ("accumulate", "separate") else "accumulate" - if setting == "reasoning_style": - val = str(value).lower() - return val if val in ("code", "blockquote", "subtext") else "code" - if setting == "tool_preview_length": - try: - return int(value) - except (TypeError, ValueError): - return 0 - return value + """Normalise a user-supplied value for *setting*; unknown settings pass through.""" + norm = _NORMALISERS.get(setting) + return norm(value) if norm else value diff --git a/gateway/drain_control.py b/gateway/drain_control.py index ba7d9f6bf7..32d6c7e1d3 100644 --- a/gateway/drain_control.py +++ b/gateway/drain_control.py @@ -1,65 +1,35 @@ """External drain-control marker contract (dashboard → gateway). -Task 2.2 of the safe-shutdown plan (decisions.md Q-B, option A): the dashboard -has no way to call into a running gateway — there is no HTTP control channel -into the gateway process (guardrails: "there is NO external control channel -into a running gateway"). Restart/drain is driven only by the gateway reacting -to its own inputs: slash commands, process signals, and file markers it writes -itself (``.restart_notify.json``). - -So the begin/cancel-drain dashboard endpoint communicates with the running -gateway the same way: it writes (or removes) a marker file, and a gateway -background watcher reacts to it. This module owns that marker contract so both -sides — the dashboard endpoint (writer) and the gateway watcher (reader) — -share one definition and can never disagree. - -Contract (presence-based, mirroring ``.restart_notify.json``): +There is no control channel into a running gateway, so the dashboard's +begin/cancel-drain endpoint talks to it the same way ``.restart_notify.json`` +does: it writes (or removes) a marker file and a gateway watcher reacts. This +module owns the contract so writer and reader can never disagree. +Contract (presence-based): * begin-drain → write ``{HERMES_HOME}/.drain_request.json`` with ``{"action": "drain", "requested_at": , "principal": , "epoch": , "suppress_notification": }``. * cancel-drain → remove the marker. - * The gateway watcher treats **presence of a marker stamped with the current - instantiation epoch** as "external drain active": flip - ``gateway_state -> "draining"`` and stop accepting new turns. Absence (or a - marker from a *prior* instantiation) means "not draining" (revert to - ``running`` if we had flipped it). + * The watcher treats presence of an ACTIVE marker as "external drain": flip + ``gateway_state -> "draining"`` and stop accepting new turns. Absence or a + stale marker means "not draining" (revert to ``running``). -Why the epoch (NS-570). ``HERMES_HOME`` is a **durable** store — on Hermes -Cloud it is a persistent Fly volume (``/opt/data``). A begin-drain marker -written there *survives a machine restart*. But the disruptive lifecycle -actions a drain protects (auto-update / image migrate / env edit / profile -change) all **restart the machine**, which is exactly the signal that the drain -is over. Without the epoch, a freshly-restarted gateway re-reads the orphaned -marker on boot and parks itself right back in ``draining`` forever (NS-570: an -auto-updated instance refused every turn for ~52 min). Stamping the marker with -an identity of *this* container/VM instantiation, and ignoring a marker whose -epoch doesn't match, makes "a deliberate restart clears the drain" true by -construction — while a marker written during the *current* instantiation (the -live drain) still matches, and an s6 respawn of just the gateway (PID 1 / init -unchanged) still honours an in-flight drain. +Staleness — two independent, individually-lenient signals (either suffices): + * epoch mismatch: HERMES_HOME is a durable volume on Hermes Cloud, so a + marker survives the machine restart that the drain-gated action (update, + migrate, env edit) ends in; honouring it would park the fresh gateway in + ``draining`` forever. A marker is stamped with this instantiation's epoch + and ignored only on a *definite* mismatch. + * expiry: a same-epoch orphan (action completed without a restart, writer + never cancelled) is ignored once ``requested_at`` is older than + :data:`DRAIN_REQUEST_MAX_AGE_SECONDS`. Re-calling :func:`write_drain_request` + refreshes the timestamp — the sanctioned keep-alive for a long drain. -Reading the marker never raises: a malformed/half-written file reads as -"present but contentless", which the watcher still treats as drain-active -(fail-safe toward quiescing — a corrupt begin marker must not be ignored). The -epoch check is deliberately **lenient**: it ignores a marker only on a -*definite* epoch mismatch. A marker with no epoch (legacy/corrupt/contentless), -or an environment where the epoch cannot be computed (non-Linux, no ``/proc``), -both degrade to the original presence-only behaviour — never fail-closed. - -Why the max-age (#85433). The epoch handles the restart case, but it bakes in -the assumption that the action a drain protects always ends in a machine -restart. When a drain-gated action completes *without* recreating the container -and the writer never cancels the drain (writer crash, forgotten cleanup), the -orphaned marker still carries the *current* epoch — so the epoch check honours -it and the gateway bounces every inbound message forever (observed in the -field: a cloud instance refused all Telegram turns for ~3 days). The marker's -``requested_at`` timestamp is therefore also checked: a marker older than -:data:`DRAIN_REQUEST_MAX_AGE_SECONDS` reads as stale. Same leniency contract as -the epoch — only a *definite* expiry (timestamp present, parseable, and too -old) is ignored; a missing/corrupt timestamp still reads as drain-active. A -deliberately long drain has a sanctioned keep-alive: re-calling -:func:`write_drain_request` refreshes ``requested_at``. +Reading never raises: a malformed/half-written file reads as "present but +contentless" (``{}``), which still counts as drain-active — fail-safe toward +quiescing. Both staleness checks only reject on a *definite* verdict; a marker +with no epoch/timestamp, or a host without ``/proc``, degrades to the +presence-only behaviour and never fails closed. """ from __future__ import annotations @@ -77,76 +47,41 @@ _log = logging.getLogger(__name__) _DRAIN_REQUEST_FILENAME = ".drain_request.json" -# Max-age fallback for a same-epoch orphaned marker (#85433). Drain-gated -# lifecycle actions complete in minutes; an hour is comfortably past any -# legitimate drain while still bounding the wedge a leaked marker can cause -# (vs. the unbounded outage observed in the field). Long-running drains -# refresh the marker via write_drain_request() (idempotent re-write bumps -# ``requested_at``) rather than raising this bound. +# Drain-gated lifecycle actions complete in minutes; an hour bounds the wedge a +# leaked marker can cause. Long drains refresh the marker instead of raising this. DRAIN_REQUEST_MAX_AGE_SECONDS = 3600.0 -# Dedup guard for the expired-marker warning: the drain watcher re-reads the -# marker every second, and an expired orphan sits on disk until removed, so -# an unconditional warning would fire ~86k times/day. Keyed by the marker's -# ``requested_at`` string — a keep-alive re-write (new timestamp) that later -# expires again logs again, which is the desired behaviour. +# Dedup for the expired-marker warning: the watcher re-reads every second and an +# expired orphan sits on disk until removed. Keyed by ``requested_at`` so a +# keep-alive re-write that later expires again logs again. _expiry_logged_for: Optional[str] = None @functools.lru_cache(maxsize=1) def current_instantiation_epoch() -> str: - """Identity of THIS container / VM instantiation. + """Identity of THIS container / VM instantiation (":"). - Stable for the life of the PID-1 init process — so an s6 respawn of just - the gateway keeps the same epoch and an in-flight drain is honoured — but - changes when the machine/container is recreated (a fresh PID 1 → a fresh - epoch). Composed from two ``/proc`` facts: + Stable for the life of PID 1 (so an s6 respawn of just the gateway, or a + host ``hermes gateway restart`` under systemd/launchd, keeps honouring an + in-flight drain) but changes whenever the machine is recreated: boot_id + changes on a VM/microVM reboot, PID 1's start time on a plain + ``docker restart`` (same host kernel, brand-new ``/init``). - * the kernel **boot id** (``/proc/sys/kernel/random/boot_id``) — changes - on a VM / microVM reboot (e.g. a Fly Firecracker machine restart); - * **PID 1's start time** (field 22 of ``/proc/1/stat``) — changes on a - plain ``docker restart`` (the host kernel, hence boot_id, is unchanged, - but ``/init`` is a brand-new process). - - Together they discriminate every restart mode that matters: - - | event | boot_id | pid1 start | epoch | marker | - |--------------------------------|---------|------------|--------|--------| - | Fly microVM reboot (auto-upd.) | changes | changes | NEW | reject | - | plain ``docker restart`` | same | changes | NEW | reject | - | s6 respawn of the gateway only | same | same | SAME | honour | - | host ``hermes gateway restart``| same | same(init) | SAME | honour | - - The last row is intentional: a host install has no durable-volume drain - bug, and honouring a drain across a deliberate process restart is the - intended reversible behaviour (D4a) — PID 1 there is the long-lived init - (systemd/launchd), so the epoch is stable. - - Returns ``""`` when neither identity source is readable (non-Linux, no - ``/proc``). An empty epoch disables the staleness check downstream, - degrading to the released presence-only behaviour — never fail-closed. - Memoised: the epoch is constant for the life of the process. + Returns ``""`` when neither source is readable (non-Linux, no ``/proc``), + which disables the epoch check downstream — never fail-closed. """ boot_id = "" try: - boot_id = ( - Path("/proc/sys/kernel/random/boot_id") - .read_text(encoding="utf-8") - .strip() - ) + boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8").strip() except OSError: pass pid1_start = "" try: - # /proc/1/stat: " () ... ...". - # comm can contain spaces and parens, so split on the LAST ')' and - # index into the whitespace-delimited tail. starttime is field 22 - # (1-indexed); after the comm the tail starts at field 3, so it is the - # tail's index 19. + # " () ...": comm may contain spaces/parens, so split + # on the LAST ')'. starttime is field 22 (1-indexed) = tail index 19. stat = Path("/proc/1/stat").read_text(encoding="utf-8") - tail = stat.rsplit(")", 1)[1].split() - pid1_start = tail[19] + pid1_start = stat.rsplit(")", 1)[1].split()[19] except (OSError, IndexError): pass @@ -167,27 +102,16 @@ def write_drain_request( suppress_notification: bool = False, home: Optional[Path] = None, ) -> dict[str, Any]: - """Write the begin-drain marker. Returns the payload written. + """Write the begin-drain marker atomically. Returns the payload written. - Atomic write so the gateway watcher never reads a half-written file. - Idempotent: re-writing while a drain is already in progress refreshes - ``requested_at`` — the sanctioned keep-alive for a drain that legitimately - needs longer than :data:`DRAIN_REQUEST_MAX_AGE_SECONDS`. - - Stamps the marker with :func:`current_instantiation_epoch` so a marker that - later survives a machine restart on the durable HERMES_HOME volume can be - recognised as stale and ignored (NS-570). - - ``suppress_notification`` is a generic "be quiet on the shutdown that ends - this drain" flag. When the drain culminates in a process exit (e.g. NAS - recreates the machine for an auto-update image migration), the gateway's - shutdown path reads it via :func:`drain_notification_suppressed` and skips - the *home-channel* "gateway shutting down" broadcast — the operator-flavoured - ping that would otherwise fire on every routine auto-update, potentially - dozens of times a day. It NEVER suppresses the per-active-session interrupt - ping. The gateway stays agnostic about *why* the drain is quiet; the policy - of which drain causes set the flag lives entirely in the caller (NAS). The - field defaults False so legacy/operator drains behave exactly as before. + Idempotent: re-writing refreshes ``requested_at`` (keep-alive past the + max-age). ``suppress_notification`` asks the shutdown that ends this drain + to skip ONLY the home-channel "gateway shutting down" broadcast (the + per-active-session interrupt ping is never suppressed); the policy of + which drains are quiet lives entirely in the caller. Defaults False so + legacy/operator drains behave exactly as before. The marker is stamped + with :func:`current_instantiation_epoch` so a copy surviving a machine + restart on the durable volume can be recognised as stale. """ payload = { "action": "drain", @@ -201,10 +125,7 @@ def write_drain_request( def clear_drain_request(*, home: Optional[Path] = None) -> bool: - """Remove the drain marker (cancel-drain). Returns True if one existed. - - Best-effort: a missing file is not an error (cancel is idempotent). - """ + """Remove the drain marker (cancel-drain, idempotent). Returns True if one existed.""" path = drain_request_path(home) try: path.unlink() @@ -217,48 +138,19 @@ def clear_drain_request(*, home: Optional[Path] = None) -> bool: def _marker_epoch_is_stale(body: dict[str, Any]) -> bool: - """True iff ``body``'s epoch is a *definite* mismatch with this process. - - Lenient by design — returns False (i.e. "not stale, honour it") whenever it - can't be sure: - * the current epoch can't be computed ("" fallback, no /proc), OR - * the marker carries no epoch (legacy marker, or a corrupt/contentless - ``{}`` body). - Only a marker whose epoch is present AND differs from the current - instantiation epoch is considered stale. This preserves the - fail-safe-toward-quiescing contract for malformed markers. - """ + """True iff ``body``'s epoch is a *definite* mismatch (both epochs known and differ).""" current = current_instantiation_epoch() - if not current: - return False marker_epoch = body.get("epoch") - if not marker_epoch: - return False - return marker_epoch != current + return bool(current and marker_epoch and marker_epoch != current) def _marker_is_expired(body: dict[str, Any]) -> bool: - """True iff ``body``'s ``requested_at`` is *definitely* too old (#85433). + """True iff ``requested_at`` parses AND is older than the max-age. - The max-age fallback for a same-epoch orphan: a drain-gated action that - completes WITHOUT a machine restart leaves a marker the epoch check cannot - reject, and if the writer never cancels the drain the gateway is wedged in - ``draining`` for the life of the container. Bounding the marker's lifetime - by :data:`DRAIN_REQUEST_MAX_AGE_SECONDS` converts that unbounded outage - into a self-healing one. - - Same leniency contract as :func:`_marker_epoch_is_stale` — returns False - ("not expired, honour it") whenever it can't be sure: - * the marker carries no ``requested_at`` (legacy/corrupt/contentless - body), OR - * the timestamp isn't a parseable ISO-8601 string. - Only a timestamp that parses AND lies more than the max-age in the past is - considered expired. A future-dated timestamp (clock skew) is honoured. The - expiry is logged loudly — but once per marker, not per poll: the gateway's - drain watcher re-reads the marker every second, and an expired orphan stays - on disk until an operator or writer removes it, so an unconditional warning - would repeat ~86k times/day. This path only fires when a writer leaked a - marker, and the log is the operator's breadcrumb. + Missing/unparseable timestamps and future-dated ones (clock skew) are + honoured. The expiry is logged once per marker, not per poll: this path + only fires when a writer leaked a marker, and the warning is the + operator's breadcrumb. """ global _expiry_logged_for raw = body.get("requested_at") @@ -289,72 +181,37 @@ def _marker_is_expired(body: dict[str, Any]) -> bool: def _marker_is_stale(body: dict[str, Any]) -> bool: - """True iff the marker is definitely from a drain that is already over. - - Two independent, individually-lenient signals (either suffices): - * epoch mismatch — the marker survived a machine restart (NS-570); - * expiry — a same-epoch orphan outlived any legitimate drain (#85433). - """ + """True iff the marker is definitely from a drain that is already over.""" return _marker_epoch_is_stale(body) or _marker_is_expired(body) -def drain_requested(*, home: Optional[Path] = None) -> bool: - """True iff a begin-drain marker for THIS instantiation is present. - - A marker whose ``epoch`` does not match the current instantiation epoch is - treated as absent: it survived a container/VM restart (HERMES_HOME is a - durable Fly volume on Hermes Cloud) and the lifecycle action that triggered - the drain has already completed — honouring it would wedge the - freshly-restarted gateway in ``draining`` (NS-570). A marker whose - ``requested_at`` is older than :data:`DRAIN_REQUEST_MAX_AGE_SECONDS` is - likewise treated as absent: it is a same-epoch orphan whose drain-gated - action completed without a restart and was never cancelled (#85433). Both - staleness checks are lenient (see :func:`_marker_epoch_is_stale` / - :func:`_marker_is_expired`): a legacy/corrupt marker with no epoch and no - timestamp, or an environment without ``/proc``, still reads as - drain-active. - """ +def _active_drain_body(home: Optional[Path]) -> Optional[dict[str, Any]]: + """Marker body if present AND not stale, else None.""" body = read_drain_request(home=home) - if body is None: - return False - if _marker_is_stale(body): - return False - return True + if body is None or _marker_is_stale(body): + return None + return body + + +def drain_requested(*, home: Optional[Path] = None) -> bool: + """True iff an active (present, same-epoch, unexpired) begin-drain marker exists.""" + return _active_drain_body(home) is not None def drain_notification_suppressed(*, home: Optional[Path] = None) -> bool: - """True iff an ACTIVE drain marker asks to suppress the shutdown broadcast. + """True iff an ACTIVE drain marker explicitly asks to suppress the shutdown broadcast. - "Active" means exactly what :func:`drain_requested` means — a marker present - AND stamped with the current instantiation epoch AND not past its max-age. - A stale (other-epoch) marker that survived a machine restart on the durable - HERMES_HOME volume, or an expired same-epoch orphan (#85433), is - ignored here just as it is for drain state (NS-570): we must never let an - orphaned marker's flag silence a *fresh* gateway's legitimate shutdown - broadcast. - - Only honours the flag when it is explicitly truthy in the marker body. A - legacy marker without the field, a corrupt/contentless ``{}`` body, or an - absent marker all read as "not suppressed" (False) — fail toward the louder, - more-visible behaviour, consistent with :func:`read_drain_request`'s - never-raise contract. The gateway's shutdown path uses this to skip ONLY the - home-channel broadcast; the per-active-session interrupt ping is unaffected. + Uses the same activeness rule as :func:`drain_requested`, so an orphaned + marker can never silence a fresh gateway's legitimate broadcast. A legacy + marker without the field or a contentless ``{}`` body reads as False + (fail toward the louder behaviour). """ - body = read_drain_request(home=home) - if body is None: - return False - if _marker_is_stale(body): - return False - return bool(body.get("suppress_notification")) + body = _active_drain_body(home) + return bool(body and body.get("suppress_notification")) def read_drain_request(*, home: Optional[Path] = None) -> Optional[dict[str, Any]]: - """Return the marker payload, or ``None`` if absent. - - A present-but-unparseable marker returns ``{}`` (truthy-presence preserved - via :func:`drain_requested`; callers that need the body get an empty dict - rather than an exception). Never raises. - """ + """Return the marker payload, ``{}`` if present but unparseable, ``None`` if absent. Never raises.""" path = drain_request_path(home) try: raw = path.read_text(encoding="utf-8") diff --git a/gateway/hosted_room_links.py b/gateway/hosted_room_links.py index 66a5a972a9..83277cc61f 100644 --- a/gateway/hosted_room_links.py +++ b/gateway/hosted_room_links.py @@ -36,6 +36,21 @@ _LEGACY_FIELDS = { "trace_id", "updated_at", } +_OPTIONAL_FIELDS = {"transport_security", "status"} +# SQLite record columns that map 1:1 onto mapping fields, in record order +# (``catalog_json`` is the serialized ``catalog``). +_RECORD_FIELDS = ( + "room_id", + "member_id", + "target_url", + "target_profile", + "grant", + "cancellation_scope_id", + "trace_id", + "transport_security", + "status", + "updated_at", +) _STATUSES = {"ready", "unavailable", "needs_reauthorization"} @@ -55,8 +70,7 @@ class StoredRoomLink: @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> "StoredRoomLink": - allowed = _LEGACY_FIELDS | {"transport_security", "status"} - if set(value) - allowed or not _LEGACY_FIELDS.issubset(value): + if set(value) - _LEGACY_FIELDS - _OPTIONAL_FIELDS or not _LEGACY_FIELDS.issubset(value): raise HostedRoomPeerError("stored room link fields are invalid") room_id = _short_string(value["room_id"], "room_id") member_id = _short_string(value["member_id"], "member_id") @@ -96,54 +110,19 @@ class StoredRoomLink: catalog = json.loads(str(value["catalog_json"])) except Exception as exc: raise HostedRoomPeerError("stored room link catalog is unreadable") from exc - return cls.from_mapping({ - "room_id": value["room_id"], - "member_id": value["member_id"], - "target_url": value["target_url"], - "target_profile": value["target_profile"], - "grant": value["grant"], - "catalog": catalog, - "cancellation_scope_id": value["cancellation_scope_id"], - "trace_id": value["trace_id"], - "transport_security": value["transport_security"], - "status": value["status"], - "updated_at": value["updated_at"], - }) + return cls.from_mapping( + {**{name: value[name] for name in _RECORD_FIELDS}, "catalog": catalog} + ) def catalog_mapping(self) -> dict[str, Any]: - value = { - "installation_id": self.catalog.installation_id, - "protocol_versions": list(self.catalog.protocol_versions), - "link_modes": list(self.catalog.link_modes), - "persistent_process": self.catalog.persistent_process, - "text": self.catalog.text, - "attachments": self.catalog.attachments, - "execution_policy": self.catalog.execution_policy.as_mapping(), - "catalog_digest": self.catalog.catalog_digest, - } - if ( - self.catalog.endpoint_url is not None - or self.catalog.endpoint_reason is not None - ): - value["endpoint"] = self.catalog.endpoint_mapping() - return value + return self.catalog.as_mapping() def as_record(self) -> dict[str, Any]: - return { - "room_id": self.room_id, - "member_id": self.member_id, - "target_url": self.target_url, - "target_profile": self.target_profile, - "grant": self.grant, - "catalog_json": json.dumps( - self.catalog_mapping(), sort_keys=True, separators=(",", ":") - ), - "cancellation_scope_id": self.cancellation_scope_id, - "trace_id": self.trace_id, - "transport_security": self.transport_security, - "status": self.status, - "updated_at": self.updated_at, - } + record = {name: getattr(self, name) for name in _RECORD_FIELDS} + record["catalog_json"] = json.dumps( + self.catalog_mapping(), sort_keys=True, separators=(",", ":") + ) + return record def _short_string(value: Any, field: str) -> str: @@ -153,23 +132,24 @@ def _short_string(value: Any, field: str) -> str: return normalized -def load_room_links(db_path: Path | str) -> tuple[StoredRoomLink, ...]: +def _link_rows(db_path: Path | str) -> list[dict[str, Any]]: rows = hosted_rooms.list_room_link_records(db_path) if len(rows) > MAX_LINKS: raise HostedRoomPeerError("stored room link list is invalid") - return tuple(StoredRoomLink.from_record(row) for row in rows) + return rows + + +def load_room_links(db_path: Path | str) -> tuple[StoredRoomLink, ...]: + return tuple(StoredRoomLink.from_record(row) for row in _link_rows(db_path)) def load_room_links_tolerant( db_path: Path | str, ) -> tuple[tuple[StoredRoomLink, ...], tuple[str, ...]]: """Load healthy routes while quarantining malformed rows by identity.""" - rows = hosted_rooms.list_room_link_records(db_path) - if len(rows) > MAX_LINKS: - raise HostedRoomPeerError("stored room link list is invalid") links = [] errors = [] - for row in rows: + for row in _link_rows(db_path): try: links.append(StoredRoomLink.from_record(row)) except Exception: @@ -225,22 +205,7 @@ def make_stored_link( "target_url": target_url, "target_profile": target_profile, "grant": grant, - "catalog": { - "installation_id": catalog.installation_id, - "protocol_versions": list(catalog.protocol_versions), - "link_modes": list(catalog.link_modes), - "persistent_process": catalog.persistent_process, - "text": catalog.text, - "attachments": catalog.attachments, - "execution_policy": catalog.execution_policy.as_mapping(), - "catalog_digest": catalog.catalog_digest, - **( - {"endpoint": catalog.endpoint_mapping()} - if catalog.endpoint_url is not None - or catalog.endpoint_reason is not None - else {} - ), - }, + "catalog": catalog.as_mapping(), "cancellation_scope_id": cancellation_scope_id, "trace_id": trace_id, "transport_security": transport_security, diff --git a/gateway/hosted_room_peer.py b/gateway/hosted_room_peer.py index 5cbc102e07..a881b9e594 100644 --- a/gateway/hosted_room_peer.py +++ b/gateway/hosted_room_peer.py @@ -1,8 +1,8 @@ """Typed contracts for autonomous cross-gateway hosted-room members. -The Desktop may bootstrap an invitation, but it is never the issuer or the -runtime courier. The target gateway verifies a scoped grant and the full task -coordinates before it admits any model or tool work. +The Desktop may bootstrap an invitation but is never the issuer or runtime +courier: the target gateway verifies a scoped grant and the full task +coordinates before admitting any model or tool work. """ from __future__ import annotations @@ -18,10 +18,10 @@ import re import stat import time import urllib.parse -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import lru_cache from pathlib import Path -from typing import Any, Iterable, Literal, Mapping +from typing import Any, Callable, Iterable, Literal, Mapping from gateway.hosted_room_execution_policy import ( RoomExecutionPolicy, @@ -29,21 +29,15 @@ from gateway.hosted_room_execution_policy import ( ) -# Version 2 adds authority/member lineage to scoped grants. It is intentionally -# not wire-compatible with the unpublished v1 draft; mixed gateways must fall -# back to Desktop-driven rooms instead of accepting a weaker token shape. +# v2 adds authority/member lineage to scoped grants and is deliberately not +# wire-compatible with the unpublished v1 draft; mixed gateways must fall back +# to Desktop-driven rooms instead of accepting a weaker token shape. PROTOCOL_VERSION = 2 MAX_TOKEN_BYTES = 16 * 1024 MAX_PROMPT_BYTES = 256 * 1024 _IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$") _DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") -_LINK_PRIORITY = { - "direct": 0, - "overlay": 1, - "relay": 2, - "pull": 3, - "desktop": 4, -} +_LINK_MODES = frozenset({"direct", "overlay", "relay", "pull", "desktop"}) LinkMode = Literal["direct", "overlay", "relay", "pull", "desktop"] TransportSecurity = Literal["tls", "loopback"] @@ -255,28 +249,16 @@ class GatewayRoomCatalog: raise HostedRoomPeerError("link_modes must be a non-empty list") links: list[LinkMode] = [] for item in links_raw: - if item not in _LINK_PRIORITY: + if item not in _LINK_MODES: raise HostedRoomPeerError("link_modes contains an unsupported mode") if item not in links: links.append(item) for field in ("persistent_process", "text", "attachments"): if not isinstance(value[field], bool): raise HostedRoomPeerError(f"{field} must be a boolean") + policy = RoomExecutionPolicy.from_mapping(value["execution_policy"]) - unsigned = { - "installation_id": installation_id, - "protocol_versions": list(versions), - "link_modes": links, - "persistent_process": value["persistent_process"], - "text": value["text"], - "attachments": value["attachments"], - "execution_policy": RoomExecutionPolicy.from_mapping( - value["execution_policy"] - ).as_mapping(), - } - endpoint_url = None - endpoint_reason = None - transport_security = None + endpoint_url = endpoint_reason = transport_security = None if "endpoint" in value: endpoint = value["endpoint"] if not isinstance(endpoint, Mapping) or not isinstance( @@ -296,11 +278,6 @@ class GatewayRoomCatalog: raise HostedRoomPeerError( "endpoint transport_security does not match its URL" ) - normalized_endpoint = { - "available": True, - "url": endpoint_url, - "transport_security": transport_security, - } else: _exact_fields( endpoint, @@ -310,30 +287,41 @@ class GatewayRoomCatalog: endpoint_reason = _identifier( endpoint["reason"], field="endpoint.reason" ) - normalized_endpoint = { - "available": False, - "reason": endpoint_reason, - } - unsigned["endpoint"] = normalized_endpoint - expected = hashlib.sha256(_canonical_json(unsigned)).hexdigest() - supplied = _digest(value["catalog_digest"], field="catalog_digest") - if not hmac.compare_digest(expected, supplied): - raise HostedRoomPeerError("catalog_digest does not match the catalog") - return cls( + catalog = cls( installation_id=installation_id, protocol_versions=versions, link_modes=tuple(links), persistent_process=value["persistent_process"], text=value["text"], attachments=value["attachments"], - execution_policy=RoomExecutionPolicy.from_mapping( - value["execution_policy"] - ), - catalog_digest=supplied, + execution_policy=policy, + catalog_digest=_digest(value["catalog_digest"], field="catalog_digest"), endpoint_url=endpoint_url, endpoint_reason=endpoint_reason, transport_security=transport_security, ) + unsigned = catalog.as_mapping() + del unsigned["catalog_digest"] + expected = hashlib.sha256(_canonical_json(unsigned)).hexdigest() + if not hmac.compare_digest(expected, catalog.catalog_digest): + raise HostedRoomPeerError("catalog_digest does not match the catalog") + return catalog + + def as_mapping(self) -> dict[str, Any]: + """Canonical catalog mapping; ``endpoint`` appears only when advertised.""" + value = { + "installation_id": self.installation_id, + "protocol_versions": list(self.protocol_versions), + "link_modes": list(self.link_modes), + "persistent_process": self.persistent_process, + "text": self.text, + "attachments": self.attachments, + "execution_policy": self.execution_policy.as_mapping(), + "catalog_digest": self.catalog_digest, + } + if self.endpoint_url is not None or self.endpoint_reason is not None: + value["endpoint"] = self.endpoint_mapping() + return value def endpoint_mapping(self) -> dict[str, Any]: """Return the normalized self-advertised endpoint capability.""" @@ -362,9 +350,8 @@ def catalog_mapping( execution_policy: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Build a canonical catalog mapping with its digest.""" - # A Desktop-managed gateway exits with the app. Treat the caller's flag - # as an upper bound so every local catalog construction site stays honest, - # including older call sites that still pass ``True`` explicitly. + # A Desktop-managed gateway exits with the app: the caller's flag is only an + # upper bound, so call sites that still pass ``True`` stay honest. persistent_process = bool(persistent_process and os.getenv("HERMES_DESKTOP") != "1") checked_policy = RoomExecutionPolicy.from_mapping( execution_policy @@ -377,9 +364,8 @@ def catalog_mapping( ) ) # A RoomLink run is initiated by another installation. Process-wide YOLO - # mode bypasses the scoped approval ContextVar, so it cannot be made safe by - # rewriting the advertised policy. Refuse to advertise or accept remote - # room execution until the target enables manual or smart approvals. + # mode bypasses the scoped approval ContextVar, so rewriting the advertised + # policy cannot make it safe: refuse until manual or smart approvals are on. if checked_policy.approval_mode == "off": raise HostedRoomPeerError( "remote room execution requires manual or smart approvals" @@ -392,7 +378,7 @@ def catalog_mapping( # Direct HTTPS/loopback is the only RoomLink transport implemented by # this backend slice. Do not advertise pull/relay placeholders. "link_modes": [mode for mode in dict.fromkeys(link_modes) if mode == "direct"], - "persistent_process": bool(persistent_process), + "persistent_process": persistent_process, "text": bool(text), "attachments": bool(attachments), "execution_policy": checked_policy.as_mapping(), @@ -423,7 +409,6 @@ def local_catalog_mapping( persistent_process=True, text=text, attachments=attachments, - endpoint=local_room_link_endpoint(), target_profile=target_profile, execution_policy=execution_policy, ) @@ -467,7 +452,7 @@ def _room_link_url_from_config(home: str) -> str | None: def _configured_room_link_url() -> str | None: - """Resolve the explicit endpoint with environment override precedence.""" + """Resolve the explicit endpoint: env override > profile config > root config.""" override = os.getenv("HERMES_ROOM_LINK_URL") if override is not None: return override @@ -479,9 +464,8 @@ def _configured_room_link_url() -> str | None: return configured # RoomLink is a gateway reachability property, not a Bot personality - # setting. Named profiles may override it, but otherwise inherit the - # process gateway's root endpoint so adding a Bot does not require - # repeating network configuration in every profile. + # setting: named profiles may override it but otherwise inherit the process + # gateway's root endpoint, so adding a Bot needs no repeated network config. root = get_default_hermes_root() if root != home: return _room_link_url_from_config(str(root)) @@ -491,9 +475,8 @@ def _configured_room_link_url() -> str | None: def validate_room_link_url(value: Any) -> tuple[str, TransportSecurity]: """Validate a RoomLink endpoint and classify its transport protection. - Scoped grants and room prompts may use plaintext HTTP only when the peer - is reached through the local loopback interface. Every non-loopback - endpoint must use HTTPS. + Plaintext HTTP is allowed only toward the local loopback interface; every + non-loopback endpoint must use HTTPS. """ raw = str(value or "").strip().rstrip("/") try: @@ -523,6 +506,27 @@ def validate_room_link_url(value: Any) -> tuple[str, TransportSecurity]: return raw, "loopback" +# Validator per dispatch field, in validation order. ``prompt`` and +# ``prompt_digest`` are checked together against each other in from_mapping. +_DISPATCH_FIELDS: dict[str, Callable[..., Any]] = { + "protocol_version": _positive_int, + "room_id": _identifier, + "home_install_id": _identifier, + "authority_gateway_id": _identifier, + "authority_epoch": _positive_int, + "member_id": _identifier, + "target_install_id": _identifier, + "target_profile": _identifier, + "task_id": _identifier, + "execution_generation": _positive_int, + "source_event_seq": _positive_int, + "cancellation_scope_id": _identifier, + "capability_digest": _digest, + "execution_policy_digest": _digest, + "trace_id": _identifier, +} + + @dataclass(frozen=True) class HostedMemberDispatch: """Recipient-validated identity for one remote room member attempt.""" @@ -547,48 +551,15 @@ class HostedMemberDispatch: def as_mapping(self) -> dict[str, Any]: """Return the canonical wire mapping used for fingerprinting.""" - return { - "protocol_version": self.protocol_version, - "room_id": self.room_id, - "home_install_id": self.home_install_id, - "authority_gateway_id": self.authority_gateway_id, - "authority_epoch": self.authority_epoch, - "member_id": self.member_id, - "target_install_id": self.target_install_id, - "target_profile": self.target_profile, - "task_id": self.task_id, - "execution_generation": self.execution_generation, - "source_event_seq": self.source_event_seq, - "cancellation_scope_id": self.cancellation_scope_id, - "prompt": self.prompt, - "prompt_digest": self.prompt_digest, - "capability_digest": self.capability_digest, - "execution_policy_digest": self.execution_policy_digest, - "trace_id": self.trace_id, - } + return asdict(self) @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> "HostedMemberDispatch": - required = { - "protocol_version", - "room_id", - "home_install_id", - "authority_gateway_id", - "authority_epoch", - "member_id", - "target_install_id", - "target_profile", - "task_id", - "execution_generation", - "source_event_seq", - "cancellation_scope_id", - "prompt", - "prompt_digest", - "capability_digest", - "execution_policy_digest", - "trace_id", - } - _exact_fields(value, required=required, label="dispatch") + _exact_fields( + value, + required=set(_DISPATCH_FIELDS) | {"prompt", "prompt_digest"}, + label="dispatch", + ) prompt = value["prompt"] if not isinstance(prompt, str) or not prompt.strip(): raise HostedRoomPeerError("prompt must be a non-empty string") @@ -599,87 +570,15 @@ class HostedMemberDispatch: if not hmac.compare_digest(expected_prompt_digest, prompt_digest): raise HostedRoomPeerError("prompt_digest does not match prompt") return cls( - protocol_version=_positive_int( - value["protocol_version"], field="protocol_version" - ), - room_id=_identifier(value["room_id"], field="room_id"), - home_install_id=_identifier( - value["home_install_id"], field="home_install_id" - ), - authority_gateway_id=_identifier( - value["authority_gateway_id"], field="authority_gateway_id" - ), - authority_epoch=_positive_int( - value["authority_epoch"], field="authority_epoch" - ), - member_id=_identifier(value["member_id"], field="member_id"), - target_install_id=_identifier( - value["target_install_id"], field="target_install_id" - ), - target_profile=_identifier(value["target_profile"], field="target_profile"), - task_id=_identifier(value["task_id"], field="task_id"), - execution_generation=_positive_int( - value["execution_generation"], field="execution_generation" - ), - source_event_seq=_positive_int( - value["source_event_seq"], field="source_event_seq" - ), - cancellation_scope_id=_identifier( - value["cancellation_scope_id"], field="cancellation_scope_id" - ), prompt=prompt, prompt_digest=prompt_digest, - capability_digest=_digest( - value["capability_digest"], field="capability_digest" - ), - execution_policy_digest=_digest( - value["execution_policy_digest"], - field="execution_policy_digest", - ), - trace_id=_identifier(value["trace_id"], field="trace_id"), + **{ + name: check(value[name], field=name) + for name, check in _DISPATCH_FIELDS.items() + }, ) -@dataclass(frozen=True) -class RoomLinkProbe: - """One gateway-verified route candidate.""" - - mode: LinkMode - verified: bool - encrypted: bool - latency_ms: float - - -def select_room_link( - probes: Iterable[RoomLinkProbe], - *, - desktop_available: bool, -) -> RoomLinkProbe | None: - """Choose the fastest safe route without weakening encryption.""" - candidates = [ - probe - for probe in probes - if probe.verified - and probe.encrypted - and probe.mode != "desktop" - and math.isfinite(probe.latency_ms) - and probe.latency_ms >= 0 - ] - if candidates: - return min( - candidates, - key=lambda item: (_LINK_PRIORITY[item.mode], item.latency_ms), - ) - if desktop_available: - return RoomLinkProbe( - mode="desktop", - verified=True, - encrypted=True, - latency_ms=0, - ) - return None - - _GRANT_FIELDS = { "version", "grant_id", diff --git a/gateway/hosted_room_policy_checkpoint.py b/gateway/hosted_room_policy_checkpoint.py index e141b0e158..ab1234aa4d 100644 --- a/gateway/hosted_room_policy_checkpoint.py +++ b/gateway/hosted_room_policy_checkpoint.py @@ -11,7 +11,7 @@ import json import sqlite3 from dataclasses import dataclass from pathlib import Path -from typing import Any, Mapping +from typing import Any, Callable, Mapping from gateway import hosted_rooms @@ -26,6 +26,68 @@ _TERMINAL_KINDS = frozenset({ "turn.deferred", }) +_SCHEMA_DDL = ( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_cursors ( + room_id TEXT PRIMARY KEY, + through_seq INTEGER NOT NULL DEFAULT 0, + stopped_through_seq INTEGER NOT NULL DEFAULT 0, + updated_at REAL NOT NULL DEFAULT 0 + )""", + """CREATE TABLE IF NOT EXISTS hosted_room_policy_threads ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + discussion_event_id TEXT NOT NULL, + latest_user_seq INTEGER NOT NULL, + completed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(room_id, thread_id) + )""", + """CREATE INDEX IF NOT EXISTS idx_hosted_room_policy_pending + ON hosted_room_policy_threads( + room_id, completed, latest_user_seq, thread_id + )""", + """CREATE TABLE IF NOT EXISTS hosted_room_policy_events ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + discussion_event_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + PRIMARY KEY(room_id, seq) + )""", + """CREATE INDEX IF NOT EXISTS idx_hosted_room_policy_events_active + ON hosted_room_policy_events( + room_id, discussion_event_id, seq + )""", + """CREATE TABLE IF NOT EXISTS hosted_room_policy_watermarks ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + member_id TEXT NOT NULL, + seen_through_seq INTEGER NOT NULL, + PRIMARY KEY(room_id, thread_id, member_id) + )""", + """CREATE TABLE IF NOT EXISTS hosted_room_policy_publications ( + room_id TEXT NOT NULL, + task_id TEXT NOT NULL, + kind TEXT NOT NULL, + execution_generation INTEGER NOT NULL DEFAULT 0, + seq INTEGER NOT NULL, + PRIMARY KEY(room_id, task_id, kind, execution_generation) + )""", + # Transcript stores only references into the already bounded room log, so + # prompt payloads are never duplicated outside room byte limits. + """CREATE TABLE IF NOT EXISTS hosted_room_policy_transcript ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + seq INTEGER NOT NULL, + kind TEXT NOT NULL, + settled_seq INTEGER, + PRIMARY KEY(room_id, thread_id, seq) + )""", + """CREATE TABLE IF NOT EXISTS hosted_room_policy_transcript_state ( + room_id TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL + )""", +) + @dataclass(frozen=True) class PolicySnapshot: @@ -37,12 +99,36 @@ class PolicySnapshot: watermarks: Mapping[tuple[str, str], int] +def _event_from_room_row(row: sqlite3.Row) -> dict[str, Any]: + return { + "room_id": str(row["room_id"]), + "seq": int(row["seq"]), + "event_id": str(row["event_id"]), + "kind": str(row["kind"]), + "actor": json.loads(row["actor_json"]), + "authority_epoch": row["authority_epoch"], + "payload": json.loads(row["payload_json"]), + "created_at": float(row["created_at"]), + "idempotent": False, + } + + +def _require_room(conn: sqlite3.Connection, room_id: str) -> None: + if conn.execute( + "SELECT 1 FROM hosted_rooms WHERE room_id=?", + (room_id,), + ).fetchone() is None: + raise hosted_rooms.RoomNotFoundError("hosted room not found") + + class HostedRoomPolicyCheckpoint: """Incrementally index room policy without compacting visible history.""" def __init__(self, db_path: Path | str) -> None: self.db_path = Path(db_path) - self._initialize() + with self._connect() as conn: + for ddl in _SCHEMA_DDL: + conn.execute(ddl) def _connect(self) -> sqlite3.Connection: from hermes_state import apply_wal_with_fallback @@ -53,97 +139,8 @@ class HostedRoomPolicyCheckpoint: apply_wal_with_fallback(conn, db_label="state.db (room policy checkpoint)") return conn - def _initialize(self) -> None: - with self._connect() as conn: - conn.execute( - """CREATE TABLE IF NOT EXISTS hosted_room_policy_cursors ( - room_id TEXT PRIMARY KEY, - through_seq INTEGER NOT NULL DEFAULT 0, - stopped_through_seq INTEGER NOT NULL DEFAULT 0, - updated_at REAL NOT NULL DEFAULT 0 - )""" - ) - conn.execute( - """CREATE TABLE IF NOT EXISTS hosted_room_policy_threads ( - room_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - discussion_event_id TEXT NOT NULL, - latest_user_seq INTEGER NOT NULL, - completed INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY(room_id, thread_id) - )""" - ) - conn.execute( - """CREATE INDEX IF NOT EXISTS idx_hosted_room_policy_pending - ON hosted_room_policy_threads( - room_id, completed, latest_user_seq, thread_id - )""" - ) - conn.execute( - """CREATE TABLE IF NOT EXISTS hosted_room_policy_events ( - room_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - discussion_event_id TEXT NOT NULL, - seq INTEGER NOT NULL, - event_json TEXT NOT NULL, - PRIMARY KEY(room_id, seq) - )""" - ) - conn.execute( - """CREATE INDEX IF NOT EXISTS idx_hosted_room_policy_events_active - ON hosted_room_policy_events( - room_id, discussion_event_id, seq - )""" - ) - conn.execute( - """CREATE TABLE IF NOT EXISTS hosted_room_policy_watermarks ( - room_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - member_id TEXT NOT NULL, - seen_through_seq INTEGER NOT NULL, - PRIMARY KEY(room_id, thread_id, member_id) - )""" - ) - conn.execute( - """CREATE TABLE IF NOT EXISTS hosted_room_policy_publications ( - room_id TEXT NOT NULL, - task_id TEXT NOT NULL, - kind TEXT NOT NULL, - execution_generation INTEGER NOT NULL DEFAULT 0, - seq INTEGER NOT NULL, - PRIMARY KEY(room_id, task_id, kind, execution_generation) - )""" - ) - conn.execute( - # Store only references into the already bounded room log. This - # avoids duplicating prompt payloads outside room byte limits. - """CREATE TABLE IF NOT EXISTS hosted_room_policy_transcript ( - room_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - seq INTEGER NOT NULL, - kind TEXT NOT NULL, - settled_seq INTEGER, - PRIMARY KEY(room_id, thread_id, seq) - )""" - ) - conn.execute( - """CREATE TABLE IF NOT EXISTS hosted_room_policy_transcript_state ( - room_id TEXT PRIMARY KEY, - schema_version INTEGER NOT NULL - )""" - ) - @staticmethod - def _event_json(event: Mapping[str, Any]) -> str: - return json.dumps( - dict(event), - ensure_ascii=True, - sort_keys=True, - separators=(",", ":"), - ) - def _store_active_event( - self, conn: sqlite3.Connection, *, event: Mapping[str, Any], @@ -159,12 +156,17 @@ class HostedRoomPolicyCheckpoint: thread_id, discussion_event_id, int(event["seq"]), - self._event_json(event), + json.dumps( + dict(event), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ), ), ) + @staticmethod def _store_transcript_event( - self, conn: sqlite3.Connection, *, event: Mapping[str, Any], @@ -207,20 +209,6 @@ class HostedRoomPolicyCheckpoint: (event["room_id"], thread_id, int(cutoff["seq"])), ) - @staticmethod - def _event_from_room_row(row: sqlite3.Row) -> dict[str, Any]: - return { - "room_id": str(row["room_id"]), - "seq": int(row["seq"]), - "event_id": str(row["event_id"]), - "kind": str(row["kind"]), - "actor": json.loads(row["actor_json"]), - "authority_epoch": row["authority_epoch"], - "payload": json.loads(row["payload_json"]), - "created_at": float(row["created_at"]), - "idempotent": False, - } - def _backfill_transcript( self, conn: sqlite3.Connection, @@ -259,7 +247,7 @@ class HostedRoomPolicyCheckpoint: and row["event_id"] not in settled_seq_by_message ): continue - event = self._event_from_room_row(row) + event = _event_from_room_row(row) thread_id = str(event["payload"].get("thread_id") or "") if thread_id: self._store_transcript_event( @@ -269,13 +257,29 @@ class HostedRoomPolicyCheckpoint: settled_seq=settled_seq_by_message.get(str(row["event_id"])), ) - def _transcript_events( + def _discussion_events( self, conn: sqlite3.Connection, *, room_id: str, thread_id: str, + discussion_event_id: str, + bound_error: str, ) -> list[dict[str, Any]]: + """Merge the thread transcript with the active projection, ordered by seq.""" + + active_rows = conn.execute( + """SELECT event_json FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=? + ORDER BY seq LIMIT ?""", + ( + room_id, + discussion_event_id, + MAX_ACTIVE_POLICY_EVENTS + 1, + ), + ).fetchall() + if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS: + raise RuntimeError(bound_error) rows = conn.execute( """WITH transcript_events(seq) AS ( SELECT seq FROM hosted_room_policy_transcript @@ -294,22 +298,30 @@ class HostedRoomPolicyCheckpoint: ORDER BY events.seq""", (room_id, thread_id, room_id, thread_id, room_id), ).fetchall() - return [self._event_from_room_row(row) for row in rows] + events_by_seq = { + int(event["seq"]): event + for event in ( + *(_event_from_room_row(row) for row in rows), + *(json.loads(row["event_json"]) for row in active_rows), + ) + } + return [events_by_seq[seq] for seq in sorted(events_by_seq)] - def _apply_event(self, conn: sqlite3.Connection, event: Mapping[str, Any]) -> None: + # -- per-kind projection handlers (dispatched by _apply_event) ----------- + + def _apply_user_message( + self, + conn: sqlite3.Connection, + event: Mapping[str, Any], + payload: Mapping[str, Any], + ) -> None: room_id = str(event["room_id"]) - seq = int(event["seq"]) - kind = str(event.get("kind") or "") - payload = event.get("payload") - payload = payload if isinstance(payload, Mapping) else {} - - if kind == "message.user": - thread_id = str(payload.get("thread_id") or "") - event_id = str(event.get("event_id") or "") - if not thread_id or not event_id: - return - conn.execute( - """INSERT INTO hosted_room_policy_threads( + thread_id = str(payload.get("thread_id") or "") + event_id = str(event.get("event_id") or "") + if not thread_id or not event_id: + return + conn.execute( + """INSERT INTO hosted_room_policy_threads( room_id, thread_id, discussion_event_id, latest_user_seq, completed ) VALUES (?, ?, ?, ?, 0) @@ -317,92 +329,92 @@ class HostedRoomPolicyCheckpoint: discussion_event_id=excluded.discussion_event_id, latest_user_seq=excluded.latest_user_seq, completed=0""", - (room_id, thread_id, event_id, seq), - ) - self._store_active_event( - conn, - event=event, - thread_id=thread_id, - discussion_event_id=event_id, - ) - self._store_transcript_event( - conn, - event=event, - thread_id=thread_id, - ) - return + (room_id, thread_id, event_id, int(event["seq"])), + ) + self._store_active_event( + conn, + event=event, + thread_id=thread_id, + discussion_event_id=event_id, + ) + self._store_transcript_event(conn, event=event, thread_id=thread_id) - if kind in {"message.member", *_TERMINAL_KINDS}: - thread_id = str(payload.get("thread_id") or "") - discussion_event_id = str(payload.get("discussion_event_id") or "") - source = conn.execute( - """SELECT 1 FROM hosted_room_policy_events + def _apply_discussion_event( + self, + conn: sqlite3.Connection, + event: Mapping[str, Any], + payload: Mapping[str, Any], + ) -> None: + """Index member messages and terminal turn outcomes of a known discussion.""" + + room_id = str(event["room_id"]) + seq = int(event["seq"]) + kind = str(event.get("kind") or "") + thread_id = str(payload.get("thread_id") or "") + discussion_event_id = str(payload.get("discussion_event_id") or "") + source = conn.execute( + """SELECT 1 FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=? LIMIT 1""", - (room_id, discussion_event_id), - ).fetchone() - if source is None: - return - self._store_active_event( - conn, - event=event, - thread_id=thread_id, - discussion_event_id=discussion_event_id, - ) - if kind in _TERMINAL_KINDS: - task_id = str(payload.get("task_id") or "") - execution_generation = ( - int(payload.get("execution_generation") or 0) - if kind == "turn.deferred" - else 0 - ) - if task_id: - conn.execute( - """INSERT OR IGNORE INTO hosted_room_policy_publications( + (room_id, discussion_event_id), + ).fetchone() + if source is None: + return + self._store_active_event( + conn, + event=event, + thread_id=thread_id, + discussion_event_id=discussion_event_id, + ) + if kind not in _TERMINAL_KINDS: + return + task_id = str(payload.get("task_id") or "") + execution_generation = ( + int(payload.get("execution_generation") or 0) + if kind == "turn.deferred" + else 0 + ) + if task_id: + conn.execute( + """INSERT OR IGNORE INTO hosted_room_policy_publications( room_id, task_id, kind, execution_generation, seq ) VALUES (?, ?, ?, ?, ?)""", - ( - room_id, - task_id, - kind, - execution_generation, - seq, - ), - ) - member_id = str(payload.get("member_id") or "") - seen_through_seq = int(payload.get("seen_through_seq") or 0) - if kind == "turn.settled" and payload.get("message_event_id"): - messages = conn.execute( - """SELECT seq, event_json FROM hosted_room_policy_events + ( + room_id, + task_id, + kind, + execution_generation, + seq, + ), + ) + member_id = str(payload.get("member_id") or "") + seen_through_seq = int(payload.get("seen_through_seq") or 0) + if kind == "turn.settled" and payload.get("message_event_id"): + messages = conn.execute( + """SELECT seq, event_json FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?""", - (room_id, discussion_event_id), - ).fetchall() - decoded_messages = [ - json.loads(message["event_json"]) - for message in messages - ] - committed = next( - ( - message - for message in decoded_messages - if message.get("event_id") - == payload["message_event_id"] - ), - None, + (room_id, discussion_event_id), + ).fetchall() + committed = next( + ( + message + for message in ( + json.loads(row["event_json"]) for row in messages ) - if committed is not None: - seen_through_seq = max( - seen_through_seq, - int(committed["seq"]), - ) - self._store_transcript_event( - conn, - event=committed, - thread_id=thread_id, - settled_seq=int(event["seq"]), - ) - if member_id and seen_through_seq > 0: - conn.execute( - """INSERT INTO hosted_room_policy_watermarks( + if message.get("event_id") == payload["message_event_id"] + ), + None, + ) + if committed is not None: + seen_through_seq = max(seen_through_seq, int(committed["seq"])) + self._store_transcript_event( + conn, + event=committed, + thread_id=thread_id, + settled_seq=int(event["seq"]), + ) + if member_id and seen_through_seq > 0: + conn.execute( + """INSERT INTO hosted_room_policy_watermarks( room_id, thread_id, member_id, seen_through_seq ) VALUES (?, ?, ?, ?) ON CONFLICT(room_id, thread_id, member_id) DO UPDATE SET @@ -410,43 +422,63 @@ class HostedRoomPolicyCheckpoint: hosted_room_policy_watermarks.seen_through_seq, excluded.seen_through_seq )""", - (room_id, thread_id, member_id, seen_through_seq), - ) - return + (room_id, thread_id, member_id, seen_through_seq), + ) - if kind == "room.activity": - thread_id = str(payload.get("thread_id") or "") - discussion_event_id = str(payload.get("discussion_event_id") or "") - conn.execute( - """DELETE FROM hosted_room_policy_events + def _apply_room_activity( + self, + conn: sqlite3.Connection, + event: Mapping[str, Any], + payload: Mapping[str, Any], + ) -> None: + room_id = str(event["room_id"]) + thread_id = str(payload.get("thread_id") or "") + discussion_event_id = str(payload.get("discussion_event_id") or "") + conn.execute( + """DELETE FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?""", - (room_id, discussion_event_id), - ) - conn.execute( - """DELETE FROM hosted_room_policy_threads + (room_id, discussion_event_id), + ) + conn.execute( + """DELETE FROM hosted_room_policy_threads WHERE room_id=? AND thread_id=?""", - (room_id, thread_id), - ) - return + (room_id, thread_id), + ) - if kind == "room.stop_requested": - conn.execute( - """UPDATE hosted_room_policy_cursors + def _apply_stop_requested( + self, + conn: sqlite3.Connection, + event: Mapping[str, Any], + payload: Mapping[str, Any], + ) -> None: + conn.execute( + """UPDATE hosted_room_policy_cursors SET stopped_through_seq=MAX(stopped_through_seq, ?) WHERE room_id=?""", - (seq, room_id), - ) + (int(event["seq"]), str(event["room_id"])), + ) + + _APPLY_BY_KIND: dict[str, Callable[..., None]] = { + "message.user": _apply_user_message, + "message.member": _apply_discussion_event, + **dict.fromkeys(_TERMINAL_KINDS, _apply_discussion_event), + "room.activity": _apply_room_activity, + "room.stop_requested": _apply_stop_requested, + } + + def _apply_event(self, conn: sqlite3.Connection, event: Mapping[str, Any]) -> None: + handler = self._APPLY_BY_KIND.get(str(event.get("kind") or "")) + if handler is None: + return + payload = event.get("payload") + handler(self, conn, event, payload if isinstance(payload, Mapping) else {}) def sync(self, *, room_id: str, latest_seq: int) -> int: """Materialize each unseen event exactly once by durable cursor.""" with self._connect() as conn: conn.execute("BEGIN IMMEDIATE") - if conn.execute( - "SELECT 1 FROM hosted_rooms WHERE room_id=?", - (room_id,), - ).fetchone() is None: - raise hosted_rooms.RoomNotFoundError("hosted room not found") + _require_room(conn, room_id) conn.execute( """INSERT OR IGNORE INTO hosted_room_policy_cursors( room_id, through_seq, stopped_through_seq, updated_at @@ -499,11 +531,7 @@ class HostedRoomPolicyCheckpoint: raise RuntimeError("hosted room policy cursor did not advance") with self._connect() as conn: conn.execute("BEGIN IMMEDIATE") - if conn.execute( - "SELECT 1 FROM hosted_rooms WHERE room_id=?", - (room_id,), - ).fetchone() is None: - raise hosted_rooms.RoomNotFoundError("hosted room not found") + _require_room(conn, room_id) for event in rows: self._apply_event(conn, event) updated = conn.execute( @@ -545,44 +573,26 @@ class HostedRoomPolicyCheckpoint: events=(), watermarks={}, ) - active_rows = conn.execute( - """SELECT event_json FROM hosted_room_policy_events - WHERE room_id=? AND discussion_event_id=? - ORDER BY seq LIMIT ?""", - ( - room_id, - str(thread["discussion_event_id"]), - MAX_ACTIVE_POLICY_EVENTS + 1, - ), - ).fetchall() - if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS: - raise RuntimeError("active room policy projection exceeded its bound") - transcript_events = self._transcript_events( + thread_id = str(thread["thread_id"]) + events = self._discussion_events( conn, room_id=room_id, - thread_id=str(thread["thread_id"]), + thread_id=thread_id, + discussion_event_id=str(thread["discussion_event_id"]), + bound_error="active room policy projection exceeded its bound", ) watermark_rows = conn.execute( """SELECT member_id, seen_through_seq FROM hosted_room_policy_watermarks WHERE room_id=? AND thread_id=?""", - (room_id, str(thread["thread_id"])), + (room_id, thread_id), ).fetchall() - events_by_seq = { - int(event["seq"]): event - for event in ( - *transcript_events, - *(json.loads(row["event_json"]) for row in active_rows), - ) - } return PolicySnapshot( through_seq=through_seq, stopped_through_seq=stopped_through_seq, - events=tuple(events_by_seq[seq] for seq in sorted(events_by_seq)), + events=tuple(events), watermarks={ - (str(thread["thread_id"]), str(row["member_id"])): int( - row["seen_through_seq"] - ) + (thread_id, str(row["member_id"])): int(row["seen_through_seq"]) for row in watermark_rows }, ) @@ -634,31 +644,13 @@ class HostedRoomPolicyCheckpoint: ).fetchone() if source is None: return [] - active_rows = conn.execute( - """SELECT event_json FROM hosted_room_policy_events - WHERE room_id=? AND discussion_event_id=? - ORDER BY seq LIMIT ?""", - ( - room_id, - str(source["discussion_event_id"]), - MAX_ACTIVE_POLICY_EVENTS + 1, - ), - ).fetchall() - transcript_events = self._transcript_events( + return self._discussion_events( conn, room_id=room_id, thread_id=str(source["thread_id"]), + discussion_event_id=str(source["discussion_event_id"]), + bound_error="task policy projection exceeded its bound", ) - if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS: - raise RuntimeError("task policy projection exceeded its bound") - events_by_seq = { - int(event["seq"]): event - for event in ( - *transcript_events, - *(json.loads(row["event_json"]) for row in active_rows), - ) - } - return [events_by_seq[seq] for seq in sorted(events_by_seq)] def compact_completed(self, *, room_id: str) -> None: """Drop any completed projections left by an interrupted sync.""" diff --git a/gateway/hosted_rooms.py b/gateway/hosted_rooms.py index 0272495295..b01aa8dd46 100644 --- a/gateway/hosted_rooms.py +++ b/gateway/hosted_rooms.py @@ -282,11 +282,7 @@ def local_authority_gateway_id() -> str: install_id = get_install_id() if not install_id: raise HostedRoomError("stable gateway install identity is unavailable") - return _validate_identifier( - f"install:{install_id}", - label="authority_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) + return _actor_id(f"install:{install_id}", "authority_gateway_id") def _canonical_json(value: Any, *, label: str, max_bytes: int) -> str: @@ -313,13 +309,46 @@ def _validate_identifier(value: Any, *, label: str, max_chars: int) -> str: return value +def _room_id(value: Any) -> str: + return _validate_identifier(value, label="room_id", max_chars=MAX_ROOM_ID_CHARS) + + +def _event_id(value: Any, *, label: str = "event_id") -> str: + return _validate_identifier(value, label=label, max_chars=MAX_EVENT_ID_CHARS) + + +def _actor_id(value: Any, label: str) -> str: + return _validate_identifier(value, label=label, max_chars=MAX_ACTOR_ID_CHARS) + + +def _require_positive_int(value: Any, label: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise HostedRoomError(f"{label} must be a positive integer") + + +def _now(now: float | None) -> float: + return time.time() if now is None else float(now) + + +def _system_actor_json(actor_id: str) -> str: + return _canonical_json({"kind": "system", "id": actor_id}, label="actor", max_bytes=4 * 1024) + + +def _claim_payload_json(previous_gateway_id: str, new_gateway_id: str, epoch: int) -> str: + return _canonical_json( + { + "previous_gateway_id": previous_gateway_id, + "authority_gateway_id": new_gateway_id, + "authority_epoch": epoch, + }, + label="payload", + max_bytes=MAX_EVENT_JSON_BYTES, + ) + + def user_event_id(client_event_id: Any) -> str: """Map a client retry key into the server-owned user-event namespace.""" - normalized = _validate_identifier( - client_event_id, - label="event_id", - max_chars=MAX_EVENT_ID_CHARS, - ) + normalized = _event_id(client_event_id) digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest() return f"user:{digest}" @@ -415,11 +444,7 @@ def _validate_actor(value: Any, *, kind: str) -> tuple[dict[str, str], str]: if kind not in _EVENT_KINDS_BY_ACTOR[actor_kind]: raise HostedRoomError(f"actor kind '{actor_kind}' cannot append '{kind}'") - actor_id = _validate_identifier( - value.get("id"), - label="actor.id", - max_chars=MAX_ACTOR_ID_CHARS, - ) + actor_id = _actor_id(value.get("id"), "actor.id") actor = {"kind": actor_kind, "id": actor_id} for field, max_chars in ( ("display_name", MAX_ACTOR_LABEL_CHARS), @@ -557,11 +582,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: # Draft builds before the actor contract carried no identity. Preserve # their inert replay rows explicitly as legacy system events rather # than guessing a user or Bot author. - legacy_actor = _canonical_json( - {"kind": "system", "id": "legacy"}, - label="actor", - max_bytes=4 * 1024, - ) + legacy_actor = _system_actor_json("legacy") escaped_actor = legacy_actor.replace("'", "''") conn.execute( "ALTER TABLE hosted_room_events " @@ -731,7 +752,7 @@ def update_room_link_status( WHERE room_id=? AND member_id=?""", ( status, - float(now if now is not None else time.time()), + _now(now), room_id, member_id, ), @@ -781,7 +802,7 @@ def revoke_room_grant_scope( ) -> None: """Revoke every grant issued at or before now for one exact room scope.""" scope_key = _room_grant_scope_key(claims) - timestamp = float(now if now is not None else time.time()) + timestamp = _now(now) expiry = float(expires_at) if expiry <= timestamp: return @@ -827,27 +848,15 @@ def reserve_peer_room( ) -> None: """Fence direct Desktop prompts before the first peer run is admitted.""" - timestamp = float(now if now is not None else time.time()) + timestamp = _now(now) expiry = float(expires_at) if expiry <= timestamp: raise HostedRoomError("peer room reservation must expire in the future") values = ( - _validate_identifier( - claims.get("room_id"), label="room_id", max_chars=MAX_ROOM_ID_CHARS - ), - _validate_identifier( - claims.get("member_id"), label="member_id", max_chars=MAX_ACTOR_ID_CHARS - ), - _validate_identifier( - claims.get("target_profile"), - label="target_profile", - max_chars=MAX_ACTOR_ID_CHARS, - ), - _validate_identifier( - claims.get("authority_gateway_id"), - label="authority_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ), + _room_id(claims.get("room_id")), + _actor_id(claims.get("member_id"), "member_id"), + _actor_id(claims.get("target_profile"), "target_profile"), + _actor_id(claims.get("authority_gateway_id"), "authority_gateway_id"), int(claims.get("authority_epoch") or 0), ) if values[4] < 1: @@ -919,7 +928,7 @@ def peer_room_is_reserved( ) -> bool: """Return whether a live target-side RoomLink reservation fences Desktop.""" - timestamp = float(now if now is not None else time.time()) + timestamp = _now(now) with _transaction(db_path) as conn: row = conn.execute( """SELECT 1 FROM hosted_room_peer_reservations @@ -927,14 +936,8 @@ def peer_room_is_reserved( AND expires_at>? AND revoked_at IS NULL LIMIT 1""", ( - _validate_identifier( - room_id, label="room_id", max_chars=MAX_ROOM_ID_CHARS - ), - _validate_identifier( - target_profile, - label="target_profile", - max_chars=MAX_ACTOR_ID_CHARS, - ), + _room_id(room_id), + _actor_id(target_profile, "target_profile"), timestamp, ), ).fetchone() @@ -949,23 +952,11 @@ def peer_room_grant_is_current( ) -> bool: """Require a grant to match the target's current live reservation.""" - timestamp = float(now if now is not None else time.time()) - room_id = _validate_identifier( - claims.get("room_id"), label="room_id", max_chars=MAX_ROOM_ID_CHARS - ) - member_id = _validate_identifier( - claims.get("member_id"), label="member_id", max_chars=MAX_ACTOR_ID_CHARS - ) - target_profile = _validate_identifier( - claims.get("target_profile"), - label="target_profile", - max_chars=MAX_ACTOR_ID_CHARS, - ) - authority_gateway_id = _validate_identifier( - claims.get("authority_gateway_id"), - label="authority_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) + timestamp = _now(now) + room_id = _room_id(claims.get("room_id")) + member_id = _actor_id(claims.get("member_id"), "member_id") + target_profile = _actor_id(claims.get("target_profile"), "target_profile") + authority_gateway_id = _actor_id(claims.get("authority_gateway_id"), "authority_gateway_id") authority_epoch = int(claims.get("authority_epoch") or 0) if authority_epoch < 1: raise HostedRoomError("authority_epoch must be positive") @@ -995,7 +986,7 @@ def room_grant_is_revoked( now: float | None = None, ) -> bool: """Return whether a grant predates its exact scope's revocation fence.""" - timestamp = float(now if now is not None else time.time()) + timestamp = _now(now) scope_key = _room_grant_scope_key(claims) issued_at = float(claims.get("issued_at") or 0) with _transaction(db_path) as conn: @@ -1018,7 +1009,7 @@ def upsert_remote_run_receipt( now: float | None = None, ) -> None: """Durably bind one logical peer task attempt to its remote run handle.""" - timestamp = float(now if now is not None else time.time()) + timestamp = _now(now) identity = _remote_run_identity(record) with _transaction(db_path, immediate=True) as conn: existing = conn.execute( @@ -1236,9 +1227,7 @@ def _room_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, A return room -def _event_storage_bytes( - *, event_id: str, kind: str, actor_json: str, payload_json: str -) -> int: +def _event_storage_bytes(event_id: str, kind: str, actor_json: str, payload_json: str) -> int: return len((event_id + kind + actor_json + payload_json).encode("utf-8")) @@ -1390,7 +1379,7 @@ def prune_disbanded_rooms( ) -> int: """Purge deleted Group Chat payloads while reserving their identities.""" - timestamp = time.time() if now is None else float(now) + timestamp = _now(now) with _transaction(db_path, immediate=True) as conn: return _prune_disbanded_rooms_locked(conn, now=timestamp) @@ -1421,19 +1410,11 @@ def create_room( now: float | None = None, ) -> dict[str, Any]: """Create a room, or return the identical existing room idempotently.""" - room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) + room_id = _room_id(room_id) name = _validate_room_name(name) normalized_members, members_json = _validate_members(members) - authority_gateway_id = _validate_identifier( - authority_gateway_id, - label="authority_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) - now = time.time() if now is None else float(now) + authority_gateway_id = _actor_id(authority_gateway_id, "authority_gateway_id") + now = _now(now) with _transaction(db_path, immediate=True) as conn: if conn.execute( @@ -1464,32 +1445,10 @@ def create_room( if legacy_adoption: target_epoch = int(existing["authority_epoch"]) + 1 seq = int(existing["next_seq"]) - claim_actor_json = _canonical_json( - {"kind": "system", "id": "authority-control"}, - label="actor", - max_bytes=4 * 1024, - ) - claim_payload_json = _canonical_json( - { - "previous_gateway_id": "legacy", - "authority_gateway_id": authority_gateway_id, - "authority_epoch": target_epoch, - }, - label="payload", - max_bytes=MAX_EVENT_JSON_BYTES, - ) - claim_bytes = _event_storage_bytes( - event_id="system:authority-adopted", - kind="authority.claimed", - actor_json=claim_actor_json, - payload_json=claim_payload_json, - ) - _assert_event_capacity( - conn, - room=existing, - additional_bytes=claim_bytes, - allow_control=True, - ) + claim_actor_json = _system_actor_json("authority-control") + claim_payload_json = _claim_payload_json("legacy", authority_gateway_id, target_epoch) + claim_bytes = _event_storage_bytes("system:authority-adopted", "authority.claimed", claim_actor_json, claim_payload_json) + _assert_event_capacity(conn, room=existing, additional_bytes=claim_bytes, allow_control=True) conn.execute( """INSERT INTO hosted_room_events (room_id, seq, event_id, kind, actor_json, @@ -1627,19 +1586,11 @@ def rename_room( now: float | None = None, ) -> dict[str, Any]: """Rename a live room and append its replay event atomically.""" - room_id = _validate_identifier( - room_id, label="room_id", max_chars=MAX_ROOM_ID_CHARS - ) - event_id = _validate_identifier( - event_id, label="event_id", max_chars=MAX_EVENT_ID_CHARS - ) + room_id = _room_id(room_id) + event_id = _event_id(event_id) name = _validate_room_name(name) - now = time.time() if now is None else float(now) - actor_json = _canonical_json( - {"kind": "system", "id": "room-control"}, - label="actor", - max_bytes=4 * 1024, - ) + now = _now(now) + actor_json = _system_actor_json("room-control") payload_json = _canonical_json( {"name": name}, label="payload", max_bytes=MAX_EVENT_JSON_BYTES ) @@ -1671,12 +1622,7 @@ def rename_room( return result seq = int(room["next_seq"]) epoch = int(room["authority_epoch"]) - event_bytes = _event_storage_bytes( - event_id=event_id, - kind="room.renamed", - actor_json=actor_json, - payload_json=payload_json, - ) + event_bytes = _event_storage_bytes(event_id, "room.renamed", actor_json, payload_json) _assert_event_capacity(conn, room=room, additional_bytes=event_bytes) conn.execute( """UPDATE hosted_rooms @@ -1727,16 +1673,8 @@ def append_event( Repeating the same ``event_id`` and immutable content returns the original event. Reusing the id for different content fails closed. """ - room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) - event_id = _validate_identifier( - event_id, - label="event_id", - max_chars=MAX_EVENT_ID_CHARS, - ) + room_id = _room_id(room_id) + event_id = _event_id(event_id) kind = _validate_event_kind(kind) normalized_actor, actor_json = _validate_actor(actor, kind=kind) authority_scoped = normalized_actor["kind"] in { @@ -1748,22 +1686,13 @@ def append_event( normalized_authority_gateway_id: str | None = None normalized_authority_epoch: int | None = None if authority_scoped: - normalized_authority_gateway_id = _validate_identifier( - authority_gateway_id, - label="authority_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) + normalized_authority_gateway_id = _actor_id(authority_gateway_id, "authority_gateway_id") if ( normalized_actor["kind"] == "gateway" and normalized_actor["id"] != normalized_authority_gateway_id ): raise HostedRoomError("gateway actor.id must match authority_gateway_id") - if ( - isinstance(authority_epoch, bool) - or not isinstance(authority_epoch, int) - or authority_epoch < 1 - ): - raise HostedRoomError("authority_epoch must be a positive integer") + _require_positive_int(authority_epoch, "authority_epoch") normalized_authority_epoch = authority_epoch elif authority_gateway_id is not None or authority_epoch is not None: raise HostedRoomError( @@ -1771,12 +1700,8 @@ def append_event( ) if not isinstance(payload, dict): raise HostedRoomError("payload must be an object") - payload_json = _canonical_json( - payload, - label="payload", - max_bytes=MAX_EVENT_JSON_BYTES, - ) - now = time.time() if now is None else float(now) + payload_json = _canonical_json(payload, label="payload", max_bytes=MAX_EVENT_JSON_BYTES) + now = _now(now) with _transaction(db_path, immediate=True) as conn: existing = conn.execute( @@ -1811,12 +1736,7 @@ def append_event( ): raise AuthorityConflictError("stale hosted room authority") seq = int(room["next_seq"]) - event_bytes = _event_storage_bytes( - event_id=event_id, - kind=kind, - actor_json=actor_json, - payload_json=payload_json, - ) + event_bytes = _event_storage_bytes(event_id, kind, actor_json, payload_json) _assert_event_capacity( conn, room=room, @@ -1874,11 +1794,7 @@ def probe_hosted_room(db_path: Path | str, *, room_id: Any) -> bool: WebSocket reader for SQLite's normal ten-second timeout. """ - checked_room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) + checked_room_id = _room_id(room_id) path = Path(db_path) if not path.is_file(): return False @@ -1916,20 +1832,12 @@ def probe_peer_room_reservation( ) -> bool: """Check a peer reservation without creating or migrating shared state.""" - checked_room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) - checked_profile = _validate_identifier( - target_profile, - label="target_profile", - max_chars=MAX_ACTOR_ID_CHARS, - ) + checked_room_id = _room_id(room_id) + checked_profile = _actor_id(target_profile, "target_profile") path = Path(db_path) if not path.is_file(): return False - checked_now = float(now if now is not None else time.time()) + checked_now = _now(now) try: conn = sqlite3.connect(path, timeout=0.05) try: @@ -1964,11 +1872,7 @@ def room_state( include_disbanded: bool = False, ) -> dict[str, Any]: """Return durable replay and authority state for one room.""" - room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) + room_id = _room_id(room_id) with _transaction(db_path) as conn: row = conn.execute( """SELECT room_id, name, members_json, authority_gateway_id, @@ -2006,11 +1910,7 @@ def request_room_stop( ) -> dict[str, Any]: """Append an idempotent fence that supersedes earlier user turns.""" - cancel_id = _validate_identifier( - cancel_id, - label="cancel_id", - max_chars=MAX_EVENT_ID_CHARS, - ) + cancel_id = _event_id(cancel_id, label="cancel_id") digest = hashlib.sha256(cancel_id.encode()).hexdigest()[:32] return append_event( db_path, @@ -2040,50 +1940,15 @@ def claim_authority( replicated driver must call it only after its lease/quorum policy has established that the previous owner can no longer commit. """ - room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) - expected_gateway_id = _validate_identifier( - expected_gateway_id, - label="expected_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) - new_gateway_id = _validate_identifier( - new_gateway_id, - label="new_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) - event_id = _validate_identifier( - event_id, - label="event_id", - max_chars=MAX_EVENT_ID_CHARS, - ) - if ( - isinstance(expected_epoch, bool) - or not isinstance(expected_epoch, int) - or expected_epoch < 1 - ): - raise HostedRoomError("expected_epoch must be a positive integer") - now = time.time() if now is None else float(now) + room_id = _room_id(room_id) + expected_gateway_id = _actor_id(expected_gateway_id, "expected_gateway_id") + new_gateway_id = _actor_id(new_gateway_id, "new_gateway_id") + event_id = _event_id(event_id) + _require_positive_int(expected_epoch, "expected_epoch") + now = _now(now) target_epoch = expected_epoch + 1 - claim_actor = {"kind": "system", "id": "authority-control"} - claim_actor_json = _canonical_json( - claim_actor, - label="actor", - max_bytes=4 * 1024, - ) - claim_payload = { - "previous_gateway_id": expected_gateway_id, - "authority_gateway_id": new_gateway_id, - "authority_epoch": target_epoch, - } - claim_payload_json = _canonical_json( - claim_payload, - label="payload", - max_bytes=MAX_EVENT_JSON_BYTES, - ) + claim_actor_json = _system_actor_json("authority-control") + claim_payload_json = _claim_payload_json(expected_gateway_id, new_gateway_id, target_epoch) with _transaction(db_path, immediate=True) as conn: row = conn.execute( @@ -2121,18 +1986,8 @@ def claim_authority( raise AuthorityConflictError("hosted room authority changed") else: seq = int(row["next_seq"]) - claim_bytes = _event_storage_bytes( - event_id=event_id, - kind="authority.claimed", - actor_json=claim_actor_json, - payload_json=claim_payload_json, - ) - _assert_event_capacity( - conn, - room=row, - additional_bytes=claim_bytes, - allow_control=True, - ) + claim_bytes = _event_storage_bytes(event_id, "authority.claimed", claim_actor_json, claim_payload_json) + _assert_event_capacity(conn, room=row, additional_bytes=claim_bytes, allow_control=True) conn.execute( """INSERT INTO hosted_room_events (room_id, seq, event_id, kind, actor_json, authority_epoch, @@ -2201,23 +2056,10 @@ def disband_room( now: float | None = None, ) -> dict[str, Any]: """Tombstone a room id permanently and idempotently.""" - room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) - expected_gateway_id = _validate_identifier( - expected_gateway_id, - label="expected_gateway_id", - max_chars=MAX_ACTOR_ID_CHARS, - ) - if ( - isinstance(expected_epoch, bool) - or not isinstance(expected_epoch, int) - or expected_epoch < 1 - ): - raise HostedRoomError("expected_epoch must be a positive integer") - now = time.time() if now is None else float(now) + room_id = _room_id(room_id) + expected_gateway_id = _actor_id(expected_gateway_id, "expected_gateway_id") + _require_positive_int(expected_epoch, "expected_epoch") + now = _now(now) with _transaction(db_path, immediate=True) as conn: room = conn.execute( @@ -2268,28 +2110,14 @@ def disband_room( ): raise AuthorityConflictError("stale hosted room authority") seq = int(room["next_seq"]) - actor_json = _canonical_json( - {"kind": "system", "id": "room-control"}, - label="actor", - max_bytes=4 * 1024, - ) + actor_json = _system_actor_json("room-control") payload_json = _canonical_json( {"room_id": room_id}, label="payload", max_bytes=MAX_EVENT_JSON_BYTES, ) - disband_bytes = _event_storage_bytes( - event_id="system:room-disbanded", - kind="room.disbanded", - actor_json=actor_json, - payload_json=payload_json, - ) - _assert_event_capacity( - conn, - room=room, - additional_bytes=disband_bytes, - allow_control=True, - ) + disband_bytes = _event_storage_bytes("system:room-disbanded", "room.disbanded", actor_json, payload_json) + _assert_event_capacity(conn, room=room, additional_bytes=disband_bytes, allow_control=True) conn.execute( """INSERT INTO hosted_room_events (room_id, seq, event_id, kind, actor_json, authority_epoch, @@ -2357,11 +2185,7 @@ def read_events( include_disbanded: bool = False, ) -> dict[str, Any]: """Read a monotonic room-log delta after ``since_seq``.""" - room_id = _validate_identifier( - room_id, - label="room_id", - max_chars=MAX_ROOM_ID_CHARS, - ) + room_id = _room_id(room_id) if isinstance(since_seq, bool) or not isinstance(since_seq, int) or since_seq < 0: raise HostedRoomError("since_seq must be a non-negative integer") if ( diff --git a/gateway/media_policy.py b/gateway/media_policy.py index 5b4d671aae..2d6019a34d 100644 --- a/gateway/media_policy.py +++ b/gateway/media_policy.py @@ -7,24 +7,15 @@ from environment variables: - ``HERMES_MEDIA_ALLOW_DIRS`` <- gateway.media_delivery_allow_dirs - ``HERMES_MEDIA_TRUST_RECENT_FILES`` <- gateway.trust_recent_files -Historically the config.yaml -> env translation ran ONLY in gateway startup -(gateway/run.py), so any process that delivers media without booting the -gateway — a manual ``hermes cron run`` in the CLI, ``hermes send``, a -standalone cron tick — filtered MEDIA paths under DIFFERENT policy than the -gateway's scheduled deliveries. In strict/allowlisted enterprise deployments -that divergence silently dropped attachments from manual cron runs while -scheduled runs delivered them (text is unaffected — only media goes through -path validation). +The translation used to run only in gateway startup, so standalone delivery +paths (``hermes cron run``, ``hermes send``, a standalone cron tick) filtered +MEDIA paths under a different policy and silently dropped attachments in +strict/allowlisted deployments (text is unaffected -- only media goes through +path validation). ``apply_media_policy_env()`` is the shared, +idempotent helper every delivery entrypoint calls before filtering media paths. -``apply_media_policy_env()`` is that same translation as a shared, idempotent -helper. Gateway startup calls it, and every standalone delivery entrypoint -calls it immediately before filtering media paths. - -Precedence: an explicitly-set environment variable WINS over config.yaml. -This preserves both the operator contract (env overrides are how deployments -pin behavior) and gateway/run.py's historical shape (it only wrote the env -var when the config key was present; we additionally refuse to overwrite a -pre-existing env value so a shell-exported override survives). +Precedence: an explicitly-set environment variable WINS over config.yaml, so a +shell-exported override (and gateway startup's own earlier run) survives. """ from __future__ import annotations @@ -52,37 +43,37 @@ def _load_gateway_cfg(config: Optional[Dict[str, Any]] = None) -> Dict[str, Any] return gateway_cfg if isinstance(gateway_cfg, dict) else {} +def _set_env_default(env: str, value: str) -> None: + """Set ``env`` only when unset/empty and ``value`` is non-empty (env wins).""" + if value and not os.environ.get(env): + os.environ[env] = value + + +def _allow_dirs_str(allow_dirs: Any) -> str: + if isinstance(allow_dirs, str): + return allow_dirs + if isinstance(allow_dirs, (list, tuple)): + return os.pathsep.join(str(p) for p in allow_dirs if p) + return "" + + def apply_media_policy_env(config: Optional[Dict[str, Any]] = None) -> None: """Bridge gateway media-policy settings from config.yaml into the env. - Idempotent and env-wins: a variable already present in the environment is - never overwritten, so gateway startup (which runs this same helper) and - operator shell exports keep precedence. Never raises — a policy-bridge - failure must not break delivery; the validator falls back to its - defaults exactly as before. + Idempotent and env-wins: a variable already present is never overwritten. + Never raises — a policy-bridge failure must not break delivery; the + validator falls back to its defaults exactly as before. """ try: gateway_cfg = _load_gateway_cfg(config) if not gateway_cfg: return - - strict = gateway_cfg.get("strict") - if strict is not None and not os.environ.get(_STRICT_ENV): - os.environ[_STRICT_ENV] = "1" if strict else "0" - + for key, env in (("strict", _STRICT_ENV), ("trust_recent_files", _TRUST_RECENT_ENV)): + flag = gateway_cfg.get(key) + if flag is not None: + _set_env_default(env, "1" if flag else "0") allow_dirs = gateway_cfg.get("media_delivery_allow_dirs") - if allow_dirs and not os.environ.get(_ALLOW_DIRS_ENV): - if isinstance(allow_dirs, str): - allow_dirs_str = allow_dirs - elif isinstance(allow_dirs, (list, tuple)): - allow_dirs_str = os.pathsep.join(str(p) for p in allow_dirs if p) - else: - allow_dirs_str = "" - if allow_dirs_str: - os.environ[_ALLOW_DIRS_ENV] = allow_dirs_str - - trust_recent = gateway_cfg.get("trust_recent_files") - if trust_recent is not None and not os.environ.get(_TRUST_RECENT_ENV): - os.environ[_TRUST_RECENT_ENV] = "1" if trust_recent else "0" + if allow_dirs: + _set_env_default(_ALLOW_DIRS_ENV, _allow_dirs_str(allow_dirs)) except Exception: # noqa: BLE001 - policy bridge must never break delivery logger.debug("apply_media_policy_env failed", exc_info=True) diff --git a/gateway/media_repair.py b/gateway/media_repair.py index 26c107b4e8..adfc3c3bdc 100644 --- a/gateway/media_repair.py +++ b/gateway/media_repair.py @@ -1,21 +1,19 @@ """Repair model-mangled ``computer_use`` screenshot paths in final responses. -``computer_use`` persists a bounded screenshot into the Hermes image cache and -tells the model its absolute path. Some models rewrite a Windows path into a -POSIX-looking one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) when -emitting an explicit ``MEDIA:`` directive, so delivery-path validation rejects -the nonexistent path and the attachment is dropped. +``computer_use`` persists a screenshot into the Hermes image cache and tells the +model its absolute path. Some models rewrite a Windows path into a POSIX-looking +one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) inside an explicit +``MEDIA:`` directive, so delivery-path validation rejects it and drops the +attachment. -The repair is deliberately narrow: it only rewrites paths inside a response -that *already* carries an explicit ``MEDIA:`` directive, and only when the -directive's generated ``computer_use_`` basename exactly matches a -canonical screenshot path returned by ``computer_use`` in the current turn. -It never auto-attaches captures, and normal media path validation still runs -after the repair. +The repair is deliberately narrow: it only rewrites paths inside a response that +*already* carries an explicit ``MEDIA:`` directive, and only when the directive's +generated ``computer_use_`` basename exactly matches a canonical screenshot +path returned by ``computer_use`` in the current turn. It never auto-attaches +captures; normal media path validation still runs afterwards. -This lives in its own module (mirroring ``gateway/media_policy.py``) so every -delivery surface — the messaging gateway's main turn path, gateway background -tasks, and cron job delivery — shares one implementation. +Own module (like ``gateway/media_policy.py``) so the gateway turn path, gateway +background tasks and cron delivery share one implementation. """ from __future__ import annotations @@ -27,9 +25,8 @@ from typing import Any, Dict, Iterator, List logger = logging.getLogger(__name__) -# Absolute-path prefix accepted for canonical capture paths: Windows drive -# letter, POSIX root, or UNC share. Kept as a pattern string so the summary -# regex below and the compiled prefix check stay in sync. +# Absolute-path prefix for canonical capture paths: Windows drive letter, POSIX +# root, or UNC share. Shared string so the summary regex stays in sync. _ABS_PATH_PREFIX_PATTERN = r"(?:[A-Za-z]:[/\\]|/|\\\\)" _ABS_PATH_PREFIX_RE = re.compile(r"^" + _ABS_PATH_PREFIX_PATTERN) @@ -61,30 +58,25 @@ def tool_name_by_call_id(messages: List[Dict[str, Any]]) -> Dict[str, str]: def _computer_use_capture_basename(path: Any) -> str: - """Return a canonical capture basename for either path separator style.""" + """Canonical (lowercased) capture basename for either separator style, or ''.""" value = str(path or "").strip().strip("`\"'") basename = re.split(r"[/\\]", value)[-1] - if _COMPUTER_USE_CAPTURE_BASENAME_RE.fullmatch(basename): - return basename.lower() - return "" + return basename.lower() if _COMPUTER_USE_CAPTURE_BASENAME_RE.fullmatch(basename) else "" def _iter_computer_use_capture_paths(content: Any) -> Iterator[str]: """Yield persisted screenshot paths from computer_use result content. - The tool can return JSON, a multimodal content list, or a text fallback. - The latter two retain the canonical path in the human-readable summary - even though the multimodal envelope's ``meta`` dictionary is not stored in - the tool message. + The tool can return JSON, a multimodal content list, or a text fallback; the + latter two keep the canonical path in the human-readable summary even though + the multimodal envelope's ``meta`` is not stored in the tool message. """ if isinstance(content, str): stripped = content.strip() if stripped.startswith(("{", "[")): - # JSON-looking content: parse first, never regex-scan the raw - # text. JSON escaping doubles backslashes, so a summary-regex hit - # on the raw string would yield ``C:\\\\Users\\\\...`` — a path - # that exists nowhere. Fail closed on unparseable JSON (tool - # output truncation) rather than repair to a corrupted path. + # Parse JSON first, never regex-scan it: JSON escaping doubles + # backslashes, so a raw-text hit would yield a path that exists + # nowhere. Fail closed on unparseable (truncated) JSON. try: payload = json.loads(stripped) except Exception: @@ -111,15 +103,29 @@ def _iter_computer_use_capture_paths(content: Any) -> Iterator[str]: if isinstance(meta, dict) and isinstance(meta.get("screenshot_path"), str): yield meta["screenshot_path"] # Producer shapes (tools/computer_use/tool.py::_capture_response): - # "content"/"text" — multimodal envelope parts; "text_summary"/"summary" — - # the human-readable summary carrying the "(shareable screenshot saved - # to ...)" line. + # content/text = multimodal parts; text_summary/summary = the line carrying + # "(shareable screenshot saved to ...)". for field in ("content", "text", "text_summary", "summary"): nested = content.get(field) if isinstance(nested, (str, dict, list)): yield from _iter_computer_use_capture_paths(nested) +def _current_turn_messages(messages: List[Dict[str, Any]], history_offset: int) -> List[Dict[str, Any]]: + if not history_offset: + return messages + if len(messages) >= history_offset: + return messages[history_offset:] + # Compression can invalidate the slice boundary: recover the turn from its + # last user message, fail closed if none remains. Deliberately narrower than + # gateway/run.py::_collect_auto_append_media_tags' scan-everything fallback — + # that decides whether to ATTACH; this only rewrites paths already emitted. + for index in range(len(messages) - 1, -1, -1): + if messages[index].get("role") == "user": + return messages[index:] + return [] + + def repair_explicit_computer_use_media_paths( response: str, messages: List[Dict[str, Any]], @@ -127,13 +133,10 @@ def repair_explicit_computer_use_media_paths( ) -> str: """Recover model-mangled paths for explicitly requested screenshots. - Repair only an already-explicit ``MEDIA:`` directive whose unique - generated basename case-insensitively matches a canonical screenshot - path from this turn. This does not auto-attach ordinary computer-use - captures, and normal media path validation still runs after the repair. - - Fail-open: the repair is cosmetic, so an unexpected error returns the - response unchanged rather than aborting delivery. + Repairs only an already-explicit ``MEDIA:`` directive whose generated + basename case-insensitively matches a canonical screenshot path from this + turn. Fail-open: the repair is cosmetic, so any unexpected error returns + the response unchanged rather than aborting delivery. """ try: return _repair_explicit_computer_use_media_paths_inner( @@ -152,27 +155,7 @@ def _repair_explicit_computer_use_media_paths_inner( if "MEDIA:" not in response: return response - if history_offset and len(messages) >= history_offset: - turn_messages = messages[history_offset:] - elif history_offset: - # Compression can invalidate the original slice boundary. Recover the - # current turn from its last user message; fail closed if none - # remains. (Deliberately narrower than the scan-everything fallback - # in gateway/run.py::_collect_auto_append_media_tags — that helper - # decides whether to ATTACH, this one only rewrites paths the model - # already explicitly emitted.) - last_user = next( - ( - index - for index in range(len(messages) - 1, -1, -1) - if messages[index].get("role") == "user" - ), - None, - ) - turn_messages = messages[last_user:] if last_user is not None else [] - else: - turn_messages = messages - + turn_messages = _current_turn_messages(messages, history_offset) call_id_names = tool_name_by_call_id(turn_messages) canonical_by_basename: Dict[str, str] = {} @@ -181,10 +164,7 @@ def _repair_explicit_computer_use_media_paths_inner( continue call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "") tool_name = str( - msg.get("name") - or msg.get("tool_name") - or call_id_names.get(call_id) - or "" + msg.get("name") or msg.get("tool_name") or call_id_names.get(call_id) or "" ) if tool_name != "computer_use": continue @@ -196,18 +176,14 @@ def _repair_explicit_computer_use_media_paths_inner( if not canonical_by_basename: return response - # Lazy on purpose: keeps `import gateway.media_repair` cheap for - # standalone cron processes that may never hit a MEDIA: response. - # No import cycle either way (base.py imports neither this module - # nor gateway.run at module level). + # Lazy: keeps `import gateway.media_repair` cheap for standalone cron + # processes that may never hit a MEDIA: response. No import cycle either way. from gateway.platforms.base import BasePlatformAdapter media_files, _ = BasePlatformAdapter.extract_media(response) repaired = response for emitted_path, _is_voice in media_files: - canonical = canonical_by_basename.get( - _computer_use_capture_basename(emitted_path) - ) + canonical = canonical_by_basename.get(_computer_use_capture_basename(emitted_path)) if canonical and emitted_path != canonical: repaired = repaired.replace(emitted_path, canonical) return repaired diff --git a/gateway/memory_status.py b/gateway/memory_status.py index 76af13bd46..9170083346 100644 --- a/gateway/memory_status.py +++ b/gateway/memory_status.py @@ -1,36 +1,18 @@ -"""Memory status rollup for ``/api/status`` (NS-656). +"""Memory status rollup for ``/api/status``. -The gateway already *produces* every memory-pressure signal a user would -want to know about, but all of it dies in log files: +Read side for memory-pressure signals the gateway already persists but only +logs: the 30s ``state/gateway.heartbeat`` (RSS + MemAvailable/MemTotal + swap, +from :func:`gateway.shutdown_watchdog.write_loop_heartbeat`) and the lifecycle +sentinel's ``suspected_oom`` flag (:func:`gateway.lifecycle_ledger.record_startup`). +Distills them into a compact block the dashboard SPA and the NAS availability +sweep consume — no new sampling, no IPC, two small file reads. -* :func:`gateway.shutdown_watchdog.write_loop_heartbeat` embeds a - :func:`gateway.lifecycle_ledger.sample_memory` snapshot (gateway RSS + - system MemAvailable/MemTotal + swap) in ``state/gateway.heartbeat`` - every 30 seconds. -* :func:`gateway.lifecycle_ledger.record_startup` detects an unclean - previous death and flags ``suspected_oom`` — but only into - ``gateway-exit-diag.log`` and a WARNING line. -* ``gateway/agent_cache_pressure.py`` evicts transcripts under pressure, - again log-only. +Public-safety: ``/api/status`` is unauthenticated (``PUBLIC_API_PATHS``), so this +block carries only coarse numbers (MB granularity), enums, and booleans — the +same disclosure class as ``active_agents`` / ``nous_session_valid``. -So a hosted agent can be OOM-killed hourly (the BlueAtlas incident, -NS-608) while its dashboard and the NAS agent card both look perfectly -healthy. This module is the read side that closes the gap: it distills -the *already-persisted* heartbeat + lifecycle sentinel into a compact, -public-safe block that ``/api/status`` can serve to the dashboard SPA -and the NAS availability sweep — no new sampling, no IPC with the -gateway process, just two small file reads. - -Public-safety note: ``/api/status`` is an unauthenticated liveness probe -(``PUBLIC_API_PATHS``), which is exactly why NAS can consume it. This -block therefore carries only coarse numbers (MB granularity), enums, and -booleans — the same disclosure class as the existing ``active_agents`` -count and ``nous_session_valid`` field (which was added for the same -NAS-sweep audience). - -Everything here is best-effort and read-only: a missing/corrupt file -degrades to ``pressure="unknown"`` rather than raising into the status -endpoint. +Best-effort and read-only: a missing/corrupt file degrades to +``pressure="unknown"`` rather than raising into the status endpoint. """ from __future__ import annotations @@ -42,30 +24,32 @@ from typing import Any, Dict, Optional logger = logging.getLogger(__name__) -# Pressure thresholds on system MemAvailable. ``critical`` deliberately -# mirrors the lifecycle ledger's OOM-suspicion heuristics -# (:data:`gateway.lifecycle_ledger._LOW_MEM_AVAILABLE_KIB` / -# ``_LOW_MEM_AVAILABLE_FRACTION``): if a memory level would make a -# subsequent unclean death "suspected OOM", the user should already have -# been warned at that level while the process was still alive. +# Thresholds on system MemAvailable. ``critical`` mirrors the lifecycle ledger's +# OOM-suspicion heuristics (_LOW_MEM_AVAILABLE_KIB / _LOW_MEM_AVAILABLE_FRACTION): +# a level that would make a later unclean death "suspected OOM" should already +# warn while the process is alive. _CRITICAL_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available _CRITICAL_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal _ELEVATED_AVAILABLE_KIB = 128 * 1024 # < 128 MiB available _ELEVATED_AVAILABLE_FRACTION = 0.15 # < 15% of MemTotal -# A heartbeat older than this no longer describes the present. The writer -# cadence is 30s (DEFAULT_HEARTBEAT_INTERVAL_S); 150s of slack tolerates a -# briefly stalled loop without letting a long-dead gateway's last sample -# masquerade as current pressure. +# Writer cadence is 30s (DEFAULT_HEARTBEAT_INTERVAL_S); 150s tolerates a briefly +# stalled loop without letting a long-dead gateway's last sample pose as current. _HEARTBEAT_FRESH_TTL_S = 150.0 _KIB_PER_MB = 1024 -def _mb(kib: Any) -> Optional[int]: - if isinstance(kib, bool) or not isinstance(kib, int) or kib < 0: +def _nonneg_int(value: Any) -> Optional[int]: + """Return *value* if it is a non-negative int (bools rejected), else None.""" + if isinstance(value, bool) or not isinstance(value, int) or value < 0: return None - return kib // _KIB_PER_MB + return value + + +def _mb(kib: Any) -> Optional[int]: + kib = _nonneg_int(kib) + return None if kib is None else kib // _KIB_PER_MB def _parse_iso(value: Any) -> Optional[datetime]: @@ -75,40 +59,26 @@ def _parse_iso(value: Any) -> Optional[datetime]: parsed = datetime.fromisoformat(value) except ValueError: return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) -def classify_pressure( - available_kib: Any, total_kib: Any -) -> str: +def classify_pressure(available_kib: Any, total_kib: Any) -> str: """Map a MemAvailable/MemTotal pair to ``ok``/``elevated``/``critical``. ``unknown`` when the sample is missing or malformed — the caller must not treat "we could not read it" as "memory is fine". """ - if ( - isinstance(available_kib, bool) - or not isinstance(available_kib, int) - or available_kib < 0 - ): + available = _nonneg_int(available_kib) + if available is None: return "unknown" - fraction: Optional[float] = None - if ( - not isinstance(total_kib, bool) - and isinstance(total_kib, int) - and total_kib > 0 + total = _nonneg_int(total_kib) + fraction = available / total if total else None + for level, kib_floor, frac_floor in ( + ("critical", _CRITICAL_AVAILABLE_KIB, _CRITICAL_AVAILABLE_FRACTION), + ("elevated", _ELEVATED_AVAILABLE_KIB, _ELEVATED_AVAILABLE_FRACTION), ): - fraction = available_kib / total_kib - if available_kib < _CRITICAL_AVAILABLE_KIB or ( - fraction is not None and fraction < _CRITICAL_AVAILABLE_FRACTION - ): - return "critical" - if available_kib < _ELEVATED_AVAILABLE_KIB or ( - fraction is not None and fraction < _ELEVATED_AVAILABLE_FRACTION - ): - return "elevated" + if available < kib_floor or (fraction is not None and fraction < frac_floor): + return level return "ok" @@ -124,10 +94,7 @@ def _read_heartbeat(home: Optional[Path]) -> Optional[Dict[str, Any]]: def _read_sentinel(home: Optional[Path]) -> Optional[Dict[str, Any]]: try: - from gateway.lifecycle_ledger import ( - _read_json, - get_lifecycle_sentinel_path, - ) + from gateway.lifecycle_ledger import _read_json, get_lifecycle_sentinel_path return _read_json(get_lifecycle_sentinel_path(home)) except Exception: @@ -141,14 +108,10 @@ def collect_memory_status( ) -> Dict[str, Any]: """Build the ``memory`` block for ``/api/status``. - ``home`` scopes the read to a profile's HERMES_HOME (the status - endpoint's ``?profile=`` handling passes it through); ``None`` means - the active profile. ``now`` is injectable for tests. - - Always returns a dict — a gateway that is down, has never written a - heartbeat, or whose files are corrupt yields - ``{"pressure": "unknown", ...}`` with whatever fields could still be - recovered. Never raises. + ``home`` scopes the read to a profile's HERMES_HOME (``None`` = active + profile); ``now`` is injectable for tests. Always returns a dict and never + raises — a down/never-started gateway or corrupt files yield + ``{"pressure": "unknown", ...}`` plus whatever fields could be recovered. """ moment = now or datetime.now(timezone.utc) status: Dict[str, Any] = { @@ -160,11 +123,9 @@ def collect_memory_status( "sampled_at": None, "last_boot_unclean": False, "last_boot_suspected_oom": False, - # Identity of the CURRENT gateway life (the sentinel's started_at). - # A suspected-OOM restart writes a fresh sentinel, so this changes on - # every boot — the dashboard keys banner dismissal on it so that - # acknowledging one OOM restart does not mute reports of the NEXT - # one (the hourly-restart-loop case is exactly the one that matters). + # Identity of the CURRENT gateway life (sentinel started_at) — changes on + # every boot, so the dashboard can key banner dismissal on it and + # acknowledging one OOM restart does not mute the NEXT one. "boot_id": None, } @@ -179,23 +140,16 @@ def collect_memory_status( status["swap_used_mb"] = _mb(mem.get("swap_used_kib")) if sampled_at is not None: status["sampled_at"] = sampled_at.isoformat() - age_s = (moment - sampled_at).total_seconds() - if 0 <= age_s <= _HEARTBEAT_FRESH_TTL_S: - status["pressure"] = classify_pressure( - mem.get("mem_available_kib"), - mem.get("mem_total_kib"), - ) - # else: stale sample — numbers are still reported (they are - # honest about *when* via sampled_at) but pressure stays - # "unknown" so a dead gateway's final gasp cannot render a - # live "critical" banner forever. + # Stale sample: numbers still reported (sampled_at says when) but + # pressure stays "unknown" so a dead gateway's final gasp cannot + # render a live "critical" banner forever. + if 0 <= (moment - sampled_at).total_seconds() <= _HEARTBEAT_FRESH_TTL_S: + status["pressure"] = classify_pressure(mem.get("mem_available_kib"), mem.get("mem_total_kib")) sentinel = _read_sentinel(home) if sentinel: status["last_boot_unclean"] = bool(sentinel.get("prior_unclean_exit")) - status["last_boot_suspected_oom"] = bool( - sentinel.get("prior_suspected_oom") - ) + status["last_boot_suspected_oom"] = bool(sentinel.get("prior_suspected_oom")) started_at = sentinel.get("started_at") if isinstance(started_at, str) and started_at: status["boot_id"] = started_at diff --git a/gateway/message_timestamps.py b/gateway/message_timestamps.py index 8023099a38..4bb5fe4579 100644 --- a/gateway/message_timestamps.py +++ b/gateway/message_timestamps.py @@ -21,55 +21,71 @@ _HUMAN_TIMESTAMP_RE = re.compile( ) # Older gateway format: [2026-04-13T17:02:06+0200] or [+02:00] -_ISO_TIMESTAMP_RE = re.compile( - r"^\[(?P\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*" -) +_ISO_TIMESTAMP_RE = re.compile(r"^\[(?P\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*") + + +def _localize(dt: datetime, tz) -> float: + """Epoch for ``dt``; naive values take ``tz`` if given, else the local zone.""" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tz) if tz is not None else dt.astimezone() + return float(dt.timestamp()) + + +def _parse_iso(text: str, tz=None) -> Optional[float]: + """Parse an ISO-8601 string (incl. ``+0200`` offsets fromisoformat rejects).""" + try: + dt = datetime.fromisoformat(text) + except (TypeError, ValueError): + try: + dt = datetime.strptime(text, "%Y-%m-%dT%H:%M:%S%z") + except (TypeError, ValueError): + return None + return _localize(dt, tz) + + +def _parse_timestamp_match(match: re.Match, tz=None) -> Optional[float]: + if match.groupdict().get("iso"): + return _parse_iso(match.group("iso"), tz) + try: + dt = datetime.strptime(f"{match.group('date')} {match.group('time')}", "%Y-%m-%d %H:%M:%S") + except ValueError: + return None + return _localize(dt, tz) + + +def _match_timestamp_prefix(text: str) -> Optional[re.Match]: + return _HUMAN_TIMESTAMP_RE.match(text) or _ISO_TIMESTAMP_RE.match(text) def coerce_message_timestamp(ts_value: Any, tz=None) -> Optional[float]: """Coerce a timestamp-like value to Unix epoch seconds. - Accepts Unix epoch numbers, datetime objects, ISO strings, and the gateway's - bracketed human-readable timestamp format. Returns ``None`` when the value - cannot be interpreted. + Accepts epoch numbers, datetime objects, ISO strings, and the gateway's + bracketed human-readable format. Returns ``None`` when uninterpretable. """ if ts_value is None: return None - if isinstance(ts_value, (int, float)): return float(ts_value) - if hasattr(ts_value, "timestamp"): try: return float(ts_value.timestamp()) except Exception: return None - if isinstance(ts_value, str): text = ts_value.strip() if not text: return None - parsed = _parse_timestamp_prefix(text, tz=tz) - if parsed is not None: - return parsed + match = _match_timestamp_prefix(text) + if match is not None: + parsed = _parse_timestamp_match(match, tz=tz) + if parsed is not None: + return parsed try: return float(text) except (TypeError, ValueError): pass - try: - dt = datetime.fromisoformat(text) - except (TypeError, ValueError): - try: - dt = datetime.strptime(text, "%Y-%m-%dT%H:%M:%S%z") - except (TypeError, ValueError): - return None - if dt.tzinfo is None: - if tz is not None: - dt = dt.replace(tzinfo=tz) - else: - dt = dt.astimezone() - return float(dt.timestamp()) - + return _parse_iso(text, tz) return None @@ -78,89 +94,38 @@ def format_message_timestamp(ts_value: Any, tz=None) -> str: epoch = coerce_message_timestamp(ts_value, tz=tz) if epoch is None: return "" - if tz is not None: - dt = datetime.fromtimestamp(epoch, tz=tz) - else: - dt = datetime.fromtimestamp(epoch).astimezone() + dt = datetime.fromtimestamp(epoch, tz=tz) if tz is not None else datetime.fromtimestamp(epoch).astimezone() return "[" + dt.strftime("%a %Y-%m-%d %H:%M:%S %Z") + "]" def strip_leading_message_timestamps(content: str, tz=None) -> Tuple[str, Optional[float]]: """Strip one or more leading gateway timestamp prefixes from ``content``. - Returns ``(clean_content, embedded_epoch)``. If multiple timestamp prefixes - are present, the timestamp closest to the actual message text wins. That - preserves the original platform-send time for legacy contaminated rows like - ``[processing time] [platform time] [sender] message``. + Returns ``(clean_content, embedded_epoch)``. With multiple prefixes the one + closest to the message text wins, preserving the original platform-send + time for legacy contaminated rows like ``[processing time] [platform time] [sender] message``. """ if not isinstance(content, str) or not content: return content, None - text = content embedded_epoch: Optional[float] = None - - while True: - match = _HUMAN_TIMESTAMP_RE.match(text) or _ISO_TIMESTAMP_RE.match(text) - if not match: - break + while (match := _match_timestamp_prefix(text)) is not None: parsed = _parse_timestamp_match(match, tz=tz) if parsed is not None: embedded_epoch = parsed text = text[match.end():] - return text, embedded_epoch def render_user_content_with_timestamp(content: str, ts_value: Any = None, tz=None) -> str: """Render a user message for LLM context with exactly one timestamp prefix. - Existing leading timestamp prefixes are removed first. If such a prefix was - present, its parsed time wins over ``ts_value``; otherwise ``ts_value`` is - formatted and prepended. If no timestamp is available, the cleaned content is - returned unchanged. + An existing leading prefix is stripped and its parsed time wins over + ``ts_value``. If no timestamp is available the cleaned content is returned. """ clean_content, embedded_epoch = strip_leading_message_timestamps(content, tz=tz) effective_ts = embedded_epoch if embedded_epoch is not None else ts_value prefix = format_message_timestamp(effective_ts, tz=tz) if not prefix: return clean_content - if clean_content: - return f"{prefix} {clean_content}" - return prefix - - -def _parse_timestamp_prefix(text: str, tz=None) -> Optional[float]: - match = _HUMAN_TIMESTAMP_RE.match(text) or _ISO_TIMESTAMP_RE.match(text) - if not match: - return None - return _parse_timestamp_match(match, tz=tz) - - -def _parse_timestamp_match(match: re.Match, tz=None) -> Optional[float]: - if "iso" in match.groupdict() and match.group("iso"): - iso_text = match.group("iso") - try: - dt = datetime.fromisoformat(iso_text) - except ValueError: - try: - dt = datetime.strptime(iso_text, "%Y-%m-%dT%H:%M:%S%z") - except ValueError: - return None - if dt.tzinfo is None: - if tz is not None: - dt = dt.replace(tzinfo=tz) - else: - dt = dt.astimezone() - return float(dt.timestamp()) - - date_part = match.group("date") - time_part = match.group("time") - try: - dt = datetime.strptime(f"{date_part} {time_part}", "%Y-%m-%d %H:%M:%S") - except ValueError: - return None - if tz is not None: - dt = dt.replace(tzinfo=tz) - else: - dt = dt.astimezone() - return float(dt.timestamp()) + return f"{prefix} {clean_content}" if clean_content else prefix diff --git a/gateway/platform_registry.py b/gateway/platform_registry.py index ca818b0921..80a513e282 100644 --- a/gateway/platform_registry.py +++ b/gateway/platform_registry.py @@ -1,30 +1,23 @@ """ Platform Adapter Registry -Allows platform adapters (built-in and plugin) to self-register so the gateway -can discover and instantiate them without hardcoded if/elif chains. +Platform adapters (built-in and plugin) self-register here so the gateway can +discover and instantiate them without hardcoded if/elif chains. Plugins +register via ``PluginContext.register_platform()``; ``GatewayRunner +._create_adapter()`` consults the registry first and falls back to the legacy +built-in path. -Built-in adapters continue to use the existing if/elif in _instantiate_adapter() -for now. Plugin adapters register here via PluginContext.register_platform() -and are looked up first -- if nothing is found the gateway falls through to -the legacy code path. GatewayRunner._create_adapter() wraps both paths and -binds every successful adapter to its runner. - -Usage (plugin side): - - from gateway.platform_registry import platform_registry, PlatformEntry +Usage (plugin side):: platform_registry.register(PlatformEntry( - name="irc", - label="IRC", + name="irc", label="IRC", adapter_factory=lambda cfg: IRCAdapter(cfg), check_fn=check_requirements, validate_config=lambda cfg: bool(cfg.extra.get("server")), - required_env=["IRC_SERVER"], - install_hint="pip install irc", + required_env=["IRC_SERVER"], install_hint="pip install irc", )) -Usage (gateway side): +Usage (gateway side):: adapter = platform_registry.create_adapter("irc", platform_config) """ @@ -39,6 +32,9 @@ from hermes_constants import hermes_home_key logger = logging.getLogger(__name__) +_LoadKey = tuple[Optional[str], str] +_Loader = Callable[[], None] + def _plugin_scope_from_callable(callback: Callable) -> Optional[str]: """Infer a plugin profile from code registered outside PluginContext.""" @@ -64,169 +60,113 @@ def _caller_plugin_scope() -> Optional[str]: class PlatformEntry: """Metadata and factory for a single platform adapter.""" - # Identifier used in config.yaml (e.g. "irc", "viber"). + # Identifier used in config.yaml (e.g. "irc") and human-readable label. name: str - - # Human-readable label (e.g. "IRC", "Viber"). label: str - # Factory callable: receives a PlatformConfig, returns an adapter instance. - # Using a factory instead of a bare class lets plugins do custom init - # (e.g. passing extra kwargs, wrapping in try/except). + # Receives a PlatformConfig, returns an adapter instance. A factory (not a + # bare class) lets plugins do custom init / wrapping. adapter_factory: Callable[[Any], Any] - # PASSIVE dependency probe: returns True when the platform's dependencies - # are available RIGHT NOW. Must be side-effect free — it is called from - # status displays (``hermes setup``, ``hermes status``, the dashboard - # readiness probe) and the config enablement pass, none of which may - # trigger a pip install. Put install logic in ``ensure_deps_fn`` instead. + # PASSIVE dependency probe: True when deps are importable RIGHT NOW. Must + # be side-effect free -- it runs from status displays (`hermes setup`, + # `hermes status`, dashboard readiness) and the config enablement pass, + # none of which may pip-install. Install logic belongs in ensure_deps_fn. check_fn: Callable[[], bool] - # Optional: given a PlatformConfig, is it properly configured? - # If None, the registry skips config validation and lets the adapter - # fail at connect() time with a descriptive error. + # Optional config check. None = skip and let connect() fail descriptively. validate_config: Optional[Callable[[Any], bool]] = None - # ACTIVE dependency installer: make the platform's dependencies available, - # installing them (pip / lazy_deps) if needed. Returns True once deps are - # importable, False if they could not be installed. Called by - # ``create_adapter()`` when ``check_fn`` returns False — i.e. exactly at - # the moment the gateway is about to bring the platform up and the user - # has it enabled/configured. None = no auto-install; a False ``check_fn`` - # is then a hard block (correct for platforms with no optional deps). - # - # Why two fields (#79812): when the ACTIVE installer was registered as - # ``check_fn``, every status display pip-installed SDKs as a side effect - # (desktop boot-loop at 94%, see gateway/config.py enablement comments); - # when the PASSIVE probe was registered instead, ``create_adapter()`` - # returned None before ``connect()`` could lazy-install, so the deps - # never installed at all (Teams deadlock). Splitting the two roles makes - # both call sites correct by construction. + # ACTIVE dependency installer: make deps importable (pip / lazy_deps), + # returning True on success. Called by ``create_adapter()`` only when + # ``check_fn`` is False -- the one moment the user has the platform + # enabled+configured and the gateway is about to connect it. None = a + # False ``check_fn`` is a hard block (correct for platforms without + # optional deps). The passive/active split exists because a single field + # either pip-installed from every status display or never installed at all. ensure_deps_fn: Optional[Callable[[], bool]] = None - # Optional: given a PlatformConfig, is the platform connected/enabled? - # Used by ``GatewayConfig.get_connected_platforms()`` and setup UI status. - # If None, falls back to ``validate_config`` or ``check_fn``. + # Is the platform connected/enabled for this PlatformConfig? Used by + # ``GatewayConfig.get_connected_platforms()`` and setup UI status; None + # falls back to ``validate_config`` or ``check_fn``. is_connected: Optional[Callable[[Any], bool]] = None - # Env vars this platform needs (for ``hermes setup`` display). + # Env vars this platform needs (``hermes setup`` display) and the hint + # shown when check_fn returns False. required_env: list = field(default_factory=list) - - # Hint shown when check_fn returns False. install_hint: str = "" - # Optional setup function for interactive configuration. - # Signature: () -> None (prompts user, saves env vars). - # If None, falls back to _setup_standard_platform (needs token_var + vars) - # or a generic "set these env vars" display. + # Interactive setup ``() -> None``. None falls back to + # _setup_standard_platform (needs token_var + vars) or a generic env display. setup_fn: Optional[Callable[[], None]] = None - # "builtin" or "plugin" + # "builtin" or "plugin"; plugin_name is the owning manifest (empty for + # built-ins) so ``hermes gateway setup`` can auto-enable the plugin. source: str = "plugin" - - # Name of the plugin manifest that registered this entry (empty for - # built-ins). Used by ``hermes gateway setup`` to auto-enable the - # owning plugin when the user configures its platform. plugin_name: str = "" - # ── Auth env var names (for _is_user_authorized integration) ── - # E.g. "IRC_ALLOWED_USERS" — checked for comma-separated user IDs. + # Auth env var names for _is_user_authorized: comma-separated allowed user + # IDs, and a truthy "allow everyone" switch. allowed_users_env: str = "" - # E.g. "IRC_ALLOW_ALL_USERS" — if truthy, all users authorized. allow_all_env: str = "" - # ── Message limits ── - # Max message length for smart-chunking. 0 = no limit. + # Max message length for smart-chunking; 0 = no limit. max_message_length: int = 0 - # ── Privacy ── - # If True, session descriptions redact PII (phone numbers, etc.) + # If True, session descriptions redact PII (phone numbers, etc.). pii_safe: bool = False - # ── Display ── - # Emoji for CLI/gateway display (e.g. "💬") + # Emoji for CLI/gateway display. emoji: str = "🔌" - # Whether this platform should appear in _UPDATE_ALLOWED_PLATFORMS - # (allows /update command from this platform). + # Whether /update may be issued from this platform (_UPDATE_ALLOWED_PLATFORMS). allow_update_command: bool = True - # ── LLM guidance ── - # Platform hint injected into the system prompt (e.g. "You are on IRC. - # Do not use markdown."). Empty string = no hint. + # Platform hint injected into the system prompt; empty = none. platform_hint: str = "" - # ── Env-driven auto-configuration ── - # Optional: read env vars, return a dict of ``PlatformConfig.extra`` fields - # to seed when the platform is auto-enabled. Called during - # ``_apply_env_overrides`` BEFORE the adapter is constructed, so - # ``gateway status`` etc. can reflect env-only configuration without - # instantiating the adapter. Return ``None`` (or an empty dict) to skip. - # Signature: () -> Optional[dict[str, Any]] + # Env-driven auto-configuration ``() -> Optional[dict]``: returns + # ``PlatformConfig.extra`` fields to seed when the platform is auto-enabled. + # Runs during ``_apply_env_overrides`` BEFORE the adapter is constructed so + # ``gateway status`` reflects env-only config. None/empty dict = skip. env_enablement_fn: Optional[Callable[[], Optional[dict]]] = None - # ── YAML→env config bridge ── - # Optional: translate this platform's ``config.yaml`` keys into env vars - # and/or seed ``PlatformConfig.extra`` directly. Lets a plugin own its - # YAML config translation instead of forcing core ``gateway/config.py`` - # to know every platform's schema. - # - # Signature: (yaml_cfg: dict, platform_cfg: dict) -> Optional[dict] - # Called from ``load_gateway_config()`` after the generic shared-key loop - # and before ``_apply_env_overrides``. Mutating ``os.environ`` is allowed - # (use ``not os.getenv(...)`` guards to preserve env > YAML precedence); - # any returned dict is merged into ``PlatformConfig.extra``. Exceptions - # are caught and logged at debug level. - # See website/docs/developer-guide/adding-platform-adapters.md for the - # full contract and a worked example. + # YAML->env bridge ``(yaml_cfg, platform_cfg) -> Optional[dict]``: lets a + # plugin own its config.yaml translation instead of core gateway/config.py. + # Called from ``load_gateway_config()`` after the generic shared-key loop and + # before ``_apply_env_overrides``. May mutate ``os.environ`` (guard with + # ``not os.getenv(...)`` to keep env > YAML precedence); the returned dict is + # merged into ``PlatformConfig.extra``. Exceptions are logged at debug. + # Full contract: website/docs/developer-guide/adding-platform-adapters.md. apply_yaml_config_fn: Optional[Callable[[dict, dict], Optional[dict]]] = None - # Optional: home-channel env var name for cron/notification delivery - # (e.g. ``"IRC_HOME_CHANNEL"``). When set, ``cron.scheduler`` treats this - # platform as a valid ``deliver=`` target and reads the env var to - # resolve the default chat/room ID. Empty = no cron home-channel support. + # Home-channel env var (e.g. "IRC_HOME_CHANNEL"). When set, cron.scheduler + # accepts ``deliver=`` and reads it for the default chat/room ID. cron_deliver_env_var: str = "" - # ── Target parsing ── - # Optional: callable that parses a raw target string for this platform into - # a (chat_id, thread_id) tuple, or None if the string is not a recognized - # explicit target. Invoked by ``tools/send_message_tool._parse_target_ref`` - # before channel-directory fallback so plugin platforms can declare their - # own native target syntax (e.g. ``fmsg:@alice@example.com``) without - # hard-casing in Hermes core. - # - # Signature: - # (target_ref: str) -> Optional[tuple[str, Optional[str]]] - # - # If the callable returns None the target proceeds to channel-directory - # resolution. No opaque fallback is applied. + # Target parsing ``(target_ref) -> Optional[(chat_id, thread_id)]``, run by + # ``tools/send_message_tool._parse_target_ref`` before channel-directory + # fallback so plugins can declare native target syntax + # (e.g. ``fmsg:@alice@example.com``). None result = continue to directory + # resolution; no opaque fallback is applied. parse_target_ref_fn: Optional[Callable[[str], Optional[tuple[str, Optional[str]]]]] = None - # Optional validation applied after parsing/normalization or - # channel-directory resolution. Return True to accept, False to reject, or - # a non-empty string to reject with that diagnostic. + # Validation after parsing/normalization or directory resolution: True to + # accept, False to reject, or a non-empty string to reject with diagnostic. validate_target_ref_fn: Optional[Callable[[str], bool | str]] = None - # Optional whole-request handler for custom platform delivery. Receives - # (args, normalized_chat_id, platform_name, pconfig) and may be sync/async. - # Prefer standalone_sender_fn when the standard send contract is enough. + # Whole-request delivery handler ``(args, normalized_chat_id, platform_name, + # pconfig)``, sync or async. Prefer standalone_sender_fn when the standard + # send contract suffices. send_message_handler: Optional[Callable[[dict, str, str, Any], Any]] = None - # ── Standalone (out-of-process) sending ── - # Optional: async coroutine that delivers a message without a live - # gateway adapter. Called by ``tools/send_message_tool._send_via_adapter`` - # when ``cron`` runs in a separate process from the gateway and the - # in-process adapter weakref is therefore ``None``. - # - # Signature: + # Out-of-process sender used by ``_send_via_adapter`` when cron runs apart + # from the gateway and the in-process adapter weakref is None:: # async (pconfig, chat_id, message, *, thread_id=None, # media_files=None, force_document=False) -> dict - # - # Returns ``{"success": True, "message_id": ...}`` on success or - # ``{"error": str}`` on failure. Plugin authors typically open an - # ephemeral connection / acquire a fresh OAuth token, send, and close. - # Without this hook, plugin platforms cannot serve as cron ``deliver=`` - # targets when the gateway is not co-resident with the cron process. + # Returns ``{"success": True, "message_id": ...}`` or ``{"error": str}``. + # Without it, plugin platforms cannot be cron ``deliver=`` targets when the + # gateway is not co-resident. standalone_sender_fn: Optional[Callable[..., Awaitable[dict]]] = None @@ -239,139 +179,107 @@ class PlatformRegistry: def __init__(self) -> None: self._lock = threading.RLock() - # Process-global registrations (for example the built-in relay). + # Process-global registrations (e.g. the built-in relay). self._entries: dict[str, PlatformEntry] = {} # Plugin adapters are isolated per resolved HERMES_HOME and overlay the # process-global entries for lookups in that profile's runtime scope. self._scoped_entries: dict[str, dict[str, PlatformEntry]] = {} - # Deferred platform loaders: name -> zero-arg callable that imports the - # owning plugin module (which calls register() and populates _entries). - # - # Why this exists: platform adapter modules import heavy, platform- - # specific SDKs at module level (lark_oapi, microsoft_teams, discord.py, - # slack_bolt, ...). Eagerly loading all ~20 bundled platform plugins at - # plugin-discovery time added several seconds to *every* `hermes` - # invocation -- including plain `hermes chat`, which never touches any - # gateway platform. Discovery now registers a cheap deferred loader per - # platform; the real module is imported only when a registry lookup - # actually asks for that platform (gateway start, cron delivery, - # `hermes setup`/`gateway status`, send_message). - self._deferred: dict[str, Callable[[], None]] = {} - self._scoped_deferred: dict[str, dict[str, Callable[[], None]]] = {} - self._inflight: dict[tuple[Optional[str], str], threading.Event] = {} - self._inflight_loaders: dict[ - tuple[Optional[str], str], Callable[[], None] - ] = {} - self._inflight_owners: dict[tuple[Optional[str], str], int] = {} - self._cancelled_inflight: set[tuple[Optional[str], str]] = set() + # Deferred loaders: name -> zero-arg callable that imports the owning + # plugin module (which calls register()). Adapter modules import heavy + # SDKs at module level; eagerly loading ~20 bundled platforms added + # seconds to every `hermes` invocation, so the real import happens only + # when a lookup actually asks for that platform. + self._deferred: dict[str, _Loader] = {} + self._scoped_deferred: dict[str, dict[str, _Loader]] = {} + self._inflight: dict[_LoadKey, threading.Event] = {} + self._inflight_loaders: dict[_LoadKey, _Loader] = {} + self._inflight_owners: dict[_LoadKey, int] = {} + self._cancelled_inflight: set[_LoadKey] = set() # A failed loader is no longer discoverable, but its identity remains # until ownership teardown can CAS-restore the displaced predecessor. - self._consumed_loaders: dict[ - tuple[Optional[str], str], Callable[[], None] - ] = {} + self._consumed_loaders: dict[_LoadKey, _Loader] = {} @staticmethod def current_scope_key() -> str: return hermes_home_key() def _scope_maps( - self, - scope: Optional[str], - *, - create: bool = False, - ) -> tuple[dict[str, PlatformEntry], dict[str, Callable[[], None]]]: + self, scope: Optional[str], *, create: bool = False + ) -> tuple[dict[str, PlatformEntry], dict[str, _Loader]]: if scope is None: return self._entries, self._deferred if create: - return ( - self._scoped_entries.setdefault(scope, {}), - self._scoped_deferred.setdefault(scope, {}), - ) - return ( - self._scoped_entries.get(scope, {}), - self._scoped_deferred.get(scope, {}), - ) + return self._scoped_entries.setdefault(scope, {}), self._scoped_deferred.setdefault(scope, {}) + return self._scoped_entries.get(scope, {}), self._scoped_deferred.get(scope, {}) + + def _registration_state( + self, scope: Optional[str], name: str, *, create: bool = False + ) -> tuple[Optional[PlatformEntry], Optional[_Loader]]: + """(entry, loader) for *name*; the loader falls back to in-flight, then consumed.""" + entries, deferred = self._scope_maps(scope, create=create) + entry = entries.get(name) + loader = deferred.get(name) + if entry is None and loader is None: + loader = self._inflight_loaders.get((scope, name)) + if entry is None and loader is None: + loader = self._consumed_loaders.get((scope, name)) + return entry, loader + + def _prune_scope(self, scope: Optional[str]) -> None: + if scope is None: + return + if not self._scoped_entries.get(scope): + self._scoped_entries.pop(scope, None) + if not self._scoped_deferred.get(scope): + self._scoped_deferred.pop(scope, None) # -- deferred loading ---------------------------------------------------- - def register_deferred( - self, - name: str, - loader: Callable[[], None], - *, - scope: Optional[str] = None, - ) -> None: + def register_deferred(self, name: str, loader: _Loader, *, scope: Optional[str] = None) -> None: """Register a lazy loader for a platform that hasn't been imported yet. - *loader* is a zero-arg callable that imports the owning plugin module, - which is expected to call :meth:`register` with the real entry for - *name*. The loader runs at most once, the first time *name* is looked - up (or when the full entry list is materialized). A real entry that is - registered directly (e.g. a built-in) takes precedence -- the deferred - loader is then dropped. + *loader* imports the owning plugin module, which must call + :meth:`register` for *name*. It runs at most once, on first lookup + (or full materialization). A concrete registration takes precedence + and drops the loader. """ with self._lock: entries, deferred = self._scope_maps(scope, create=True) self._consumed_loaders.pop((scope, name), None) - if name in entries: - # Already concretely registered; no need to defer. - return - deferred[name] = loader + if name not in entries: + deferred[name] = loader def snapshot_registration( - self, - name: str, - *, - scope: Optional[str] = None, - ) -> tuple[Optional[PlatformEntry], Optional[Callable[[], None]]]: - """Return the concrete and deferred state for *name* without resolving it. + self, name: str, *, scope: Optional[str] = None + ) -> tuple[Optional[PlatformEntry], Optional[_Loader]]: + """Concrete and deferred state for *name* without resolving it. - This host-facing snapshot lets the plugin ledger restore a deferred - platform loader that a concrete registration displaced, without - importing the displaced adapter as a side effect of taking the - snapshot. + Lets the plugin ledger restore a deferred loader displaced by a concrete + registration without importing the displaced adapter as a side effect. """ with self._lock: - entries, deferred = self._scope_maps(scope) - loader = deferred.get(name) - if entries.get(name) is None and loader is None: - loader = self._inflight_loaders.get((scope, name)) - if entries.get(name) is None and loader is None: - loader = self._consumed_loaders.get((scope, name)) - return entries.get(name), loader + return self._registration_state(scope, name) def restore_registration( self, name: str, - current: tuple[Optional[PlatformEntry], Optional[Callable[[], None]]], - previous: tuple[Optional[PlatformEntry], Optional[Callable[[], None]]], + current: tuple[Optional[PlatformEntry], Optional[_Loader]], + previous: tuple[Optional[PlatformEntry], Optional[_Loader]], *, scope: Optional[str] = None, ) -> bool: - """Restore a platform registration if its full state is still current. + """Restore a registration if its full state is still *current* (CAS). - The identity checks protect a later registration from being removed - while still allowing an unloaded override to reveal the registration - it displaced. Both concrete entries and deferred loaders are part of - the state because bundled platform plugins load lazily. + Identity checks protect a later registration from removal while letting + an unloaded override reveal what it displaced. Deferred loaders are + part of the state because bundled platform plugins load lazily. """ with self._lock: - entries, deferred = self._scope_maps(scope, create=True) - entry = entries.get(name) - loader = deferred.get(name) - load_key = (scope, name) - if entry is None and loader is None: - loader = self._inflight_loaders.get(load_key) - if entry is None and loader is None: - loader = self._consumed_loaders.get(load_key) - current_state = (entry, loader) - is_current = not ( - current_state[0] is not current[0] - or current_state[1] is not current[1] - ) - if not is_current: + entry, loader = self._registration_state(scope, name, create=True) + if entry is not current[0] or loader is not current[1]: return False - + entries, deferred = self._scope_maps(scope) + load_key = (scope, name) previous_entry, previous_loader = previous if previous_entry is None: entries.pop(name, None) @@ -384,18 +292,14 @@ class PlatformRegistry: if load_key in self._inflight: self._cancelled_inflight.add(load_key) self._consumed_loaders.pop(load_key, None) - if scope is not None: - if not entries: - self._scoped_entries.pop(scope, None) - if not deferred: - self._scoped_deferred.pop(scope, None) + self._prune_scope(scope) return True def _resolve(self, name: str, scope: Optional[str] = None) -> None: """Run the deferred loader for *name* if one is pending.""" - loader: Optional[Callable[[], None]] = None + loader: Optional[_Loader] = None event: Optional[threading.Event] = None - load_key: tuple[Optional[str], str] + load_key: _LoadKey is_loader = False with self._lock: active_scope = scope or self.current_scope_key() @@ -420,43 +324,27 @@ class PlatformRegistry: is_loader = True if event is None: return - if ( - not is_loader - and self._inflight_owners.get(load_key) == threading.get_ident() - ): - logger.warning( - "Deferred platform '%s' recursively requested while loading", - name, - ) + if not is_loader and self._inflight_owners.get(load_key) == threading.get_ident(): + logger.warning("Deferred platform '%s' recursively requested while loading", name) return if not is_loader: event.wait() # Teardown may have restored an older deferred generation while - # cancelling the one we waited for. Resolve that predecessor in - # the same lookup instead of returning a one-shot false negative. + # cancelling the one we waited for; resolve that predecessor now + # instead of returning a one-shot false negative. self._resolve(name, active_scope) return try: loader() except Exception as e: - logger.warning( - "Deferred load of platform '%s' failed: %s", - name, - e, - exc_info=True, - ) + logger.warning("Deferred load of platform '%s' failed: %s", name, e, exc_info=True) finally: with self._lock: was_cancelled = load_key in self._cancelled_inflight - load_scope, _load_name = load_key - entries, deferred = self._scope_maps(load_scope) - if ( - not was_cancelled - and name not in entries - and name not in deferred - ): + entries, deferred = self._scope_maps(load_key[0]) + if not was_cancelled and name not in entries and name not in deferred: self._consumed_loaders[load_key] = loader self._inflight.pop(load_key, None) self._inflight_loaders.pop(load_key, None) @@ -466,12 +354,7 @@ class PlatformRegistry: if was_cancelled: self._resolve(name, active_scope) - def is_deferred_load_cancelled( - self, - name: str, - *, - scope: Optional[str] = None, - ) -> bool: + def is_deferred_load_cancelled(self, name: str, *, scope: Optional[str] = None) -> bool: """Return whether ownership teardown cancelled an in-flight loader.""" with self._lock: return (scope, name) in self._cancelled_inflight @@ -479,10 +362,10 @@ class PlatformRegistry: def _resolve_all(self) -> None: """Run every pending deferred loader. - Used by the iterate-all accessors (``all_entries``/``plugin_entries``), - which are only called by paths that genuinely need every adapter: - gateway startup, ``hermes setup``/``gateway status``, channel - directory. CLI chat never iterates the full set. + Only the iterate-all accessors (``all_entries``/``plugin_entries``) call + this, from paths that genuinely need every adapter (gateway startup, + ``hermes setup``/``gateway status``, channel directory). CLI chat never + iterates the full set. """ active_scope = self.current_scope_key() with self._lock: @@ -496,22 +379,11 @@ class PlatformRegistry: global_names.add(name) # Load outside the registry lock; each name has an in-flight event so # concurrent readers wait for the same materialization. - for name in sorted(scoped_names): - self._resolve(name, active_scope) - for name in sorted(global_names): + for name in (*sorted(scoped_names), *sorted(global_names)): self._resolve(name, active_scope) - def register( - self, - entry: PlatformEntry, - *, - scope: Optional[str] = None, - ) -> None: - """Register a platform adapter entry. - - If an entry with the same name exists, it is replaced (last writer - wins -- this lets plugins override built-in adapters if desired). - """ + def register(self, entry: PlatformEntry, *, scope: Optional[str] = None) -> None: + """Register a platform adapter entry (last writer wins on name clash).""" with self._lock: if scope is None and entry.source == "plugin": scope = _caller_plugin_scope() @@ -523,13 +395,10 @@ class PlatformRegistry: entries, deferred = self._scope_maps(scope, create=True) self._consumed_loaders.pop((scope, entry.name), None) deferred.pop(entry.name, None) - if entry.name in entries: - prev = entries[entry.name] + prev = entries.get(entry.name) + if prev is not None: logger.info( - "Platform '%s' re-registered (was %s, now %s)", - entry.name, - prev.source, - entry.source, + "Platform '%s' re-registered (was %s, now %s)", entry.name, prev.source, entry.source ) entries[entry.name] = entry logger.debug("Registered platform adapter: %s (%s)", entry.name, entry.source) @@ -543,25 +412,27 @@ class PlatformRegistry: if inferred_scope is not None or name in entries or name in deferred: deferred.pop(name, None) removed = entries.pop(name, None) is not None - if not entries: - self._scoped_entries.pop(active_scope, None) - if not deferred: - self._scoped_deferred.pop(active_scope, None) + self._prune_scope(active_scope) return removed self._deferred.pop(name, None) return self._entries.pop(name, None) is not None + def _load_pending(self, scope: str, name: str) -> bool: + """True when a lookup of *name* must run/await a deferred loader (lock held).""" + _entries, deferred = self._scope_maps(scope) + return ( + name in deferred + or (name not in self._entries and name in self._deferred) + or (scope, name) in self._inflight + or (None, name) in self._inflight + ) + def get(self, name: str) -> Optional[PlatformEntry]: """Look up a platform entry by name.""" scope = self.current_scope_key() with self._lock: - entries, deferred = self._scope_maps(scope) - needs_resolve = name not in entries and ( - name in deferred - or (name not in self._entries and name in self._deferred) - or (scope, name) in self._inflight - or (None, name) in self._inflight - ) + entries, _deferred = self._scope_maps(scope) + needs_resolve = name not in entries and self._load_pending(scope, name) if needs_resolve: self._resolve(name, scope) with self._lock: @@ -578,53 +449,33 @@ class PlatformRegistry: def plugin_entries(self) -> list[PlatformEntry]: """Return only plugin-registered platform entries.""" - self._resolve_all() return [e for e in self.all_entries() if e.source == "plugin"] def registered_names(self) -> set[str]: - """Return concrete and deferred platform names without loading adapters. + """Concrete and deferred platform names without loading adapters. - Mirrors ``is_registered()``'s scope semantics: names registered under - the current profile scope AND process-global names both count. Plugin - platforms register deferred loaders under a profile scope, so reading - only the global maps would miss every plugin platform. + Same scope semantics as ``is_registered()``: current profile scope AND + process-global names. Plugin platforms register deferred loaders under + a profile scope, so the global maps alone would miss every plugin. """ with self._lock: - scope = self.current_scope_key() - entries, deferred = self._scope_maps(scope) - return ( - entries.keys() - | deferred.keys() - | self._entries.keys() - | self._deferred.keys() - ) + entries, deferred = self._scope_maps(self.current_scope_key()) + return entries.keys() | deferred.keys() | self._entries.keys() | self._deferred.keys() def is_registered(self, name: str) -> bool: - # A deferred (not-yet-imported) platform still counts as registered -- - # the loader will materialize it on first real use. This keeps cheap - # membership checks (toolset resolution, webhook deliver-target checks) - # from triggering a heavy import. + # A deferred (not-yet-imported) platform still counts as registered so + # cheap membership checks (toolset resolution, webhook deliver-target + # checks) never trigger a heavy import. with self._lock: scope = self.current_scope_key() - entries, deferred = self._scope_maps(scope) - return ( - name in entries - or name in deferred - or name in self._entries - or name in self._deferred - or (scope, name) in self._inflight - or (None, name) in self._inflight - ) + entries, _deferred = self._scope_maps(scope) + return name in entries or name in self._entries or self._load_pending(scope, name) def create_adapter(self, name: str, config: Any) -> Optional[Any]: - """Create an adapter instance for the given platform name. + """Create an adapter instance for *name*. - Returns None if: - - No entry registered for *name* - - check_fn() returns False and deps can't be installed - (no ensure_deps_fn, or ensure_deps_fn() returned False) - - validate_config() returns False (misconfigured) - - The factory raises an exception + Returns None when no entry exists, deps are missing and cannot be + installed, ``validate_config`` fails, or the factory raises. """ entry = self.get(name) if entry is None: @@ -634,63 +485,38 @@ class PlatformRegistry: try: deps_ok = bool(entry.check_fn()) except Exception as e: - logger.warning( - "Platform '%s' check_fn raised: %s", entry.label, e - ) + logger.warning("Platform '%s' check_fn raised: %s", entry.label, e) if not deps_ok and entry.ensure_deps_fn is not None: - # Deps missing but the platform can install them on demand. - # This is the ONE place the active installer runs in the adapter - # path: the platform is enabled+configured and the gateway is - # about to connect it, so an install is what the user wants - # (#79812 — Teams' installer previously lived behind this very - # gate inside connect(), which could never be reached). - logger.info( - "Platform '%s' dependencies missing — attempting install...", - entry.label, - ) + # The ONE place the active installer runs in the adapter path: the + # platform is enabled+configured and about to connect, so an install + # is what the user wants. (An installer inside connect() is never + # reached when check_fn is False -- create_adapter() would have + # already returned None.) + logger.info("Platform '%s' dependencies missing — attempting install...", entry.label) try: deps_ok = bool(entry.ensure_deps_fn()) except Exception as e: - logger.warning( - "Platform '%s' dependency install raised: %s", - entry.label, - e, - ) + logger.warning("Platform '%s' dependency install raised: %s", entry.label, e) deps_ok = False if not deps_ok: hint = f" ({entry.install_hint})" if entry.install_hint else "" - logger.warning( - "Platform '%s' requirements not met%s", - entry.label, - hint, - ) + logger.warning("Platform '%s' requirements not met%s", entry.label, hint) return None if entry.validate_config is not None: try: if not entry.validate_config(config): - logger.warning( - "Platform '%s' config validation failed", - entry.label, - ) + logger.warning("Platform '%s' config validation failed", entry.label) return None except Exception as e: - logger.warning( - "Platform '%s' config validation error: %s", - entry.label, - e, - ) + logger.warning("Platform '%s' config validation error: %s", entry.label, e) return None try: - adapter = entry.adapter_factory(config) - return adapter + return entry.adapter_factory(config) except Exception as e: logger.error( - "Failed to create adapter for platform '%s': %s", - entry.label, - e, - exc_info=True, + "Failed to create adapter for platform '%s': %s", entry.label, e, exc_info=True ) return None diff --git a/gateway/profile_routing.py b/gateway/profile_routing.py index c72f8837ea..9b533b5274 100644 --- a/gateway/profile_routing.py +++ b/gateway/profile_routing.py @@ -1,6 +1,6 @@ """Profile-based routing for the gateway with hierarchical matching. -Allows a single Hermes instance to route specific Discord guilds/channels/threads +Lets a single Hermes instance route specific Discord guilds/channels/threads to different profiles — each with their own model, tools, memory, and persona. Matching priority (most specific first): @@ -9,13 +9,11 @@ Matching priority (most specific first): 3. platform + guild_id (guild/server route) — specificity 2 4. No match → default profile -Parent-chain matching: -For Discord threads and forum posts, ``parent_chat_id`` carries the -direct parent (the channel for a thread, the forum channel for a post). -Routes keyed on a channel match both direct messages and messages in -any thread/post whose parent is that channel. +Parent-chain matching: for Discord threads and forum posts, ``parent_chat_id`` +carries the direct parent, so a route keyed on a channel also matches messages +in any thread/post under it. -Configuration (config.yaml): +Configuration (config.yaml):: gateway: profile_routes: @@ -23,19 +21,11 @@ Configuration (config.yaml): platform: discord guild_id: "YOUR_GUILD_ID" profile: server-profile - - - name: special-channel - platform: discord - guild_id: "YOUR_GUILD_ID" - chat_id: "YOUR_CHANNEL_ID" - profile: channel-profile - - name: thread-route platform: discord chat_id: "YOUR_CHANNEL_ID" thread_id: "YOUR_THREAD_ID" profile: thread-profile - - name: owner-whatsapp platform: whatsapp chat_id: "15551234567" # phone, JID, or LID — all equivalent @@ -51,9 +41,8 @@ import logging logger = logging.getLogger(__name__) -# Baileys and Cloud share phone/JID/LID identity rules. Other platforms keep -# exact string compare so Telegram numeric ids and Discord snowflakes stay -# unchanged. +# Baileys and Cloud share phone/JID/LID identity rules. Other platforms keep +# exact string compare so Telegram numeric ids and Discord snowflakes stay unchanged. _WHATSAPP_IDENTITY_PLATFORMS = {"whatsapp", "whatsapp_cloud"} _WHATSAPP_NON_USER_SUFFIXES = ("@g.us", "@broadcast", "@newsletter") @@ -69,11 +58,10 @@ def _is_whatsapp_non_user_chat(chat_id: Optional[str]) -> bool: def _whatsapp_user_chat_ids_match(platform: str, left: Optional[str], right: Optional[str]) -> bool: """True when two WhatsApp *user* chat_ids refer to the same person. - Reuses :func:`gateway.whatsapp_identity.expand_whatsapp_aliases` so a - bare phone number, a ``@s.whatsapp.net`` JID, and a ``@lid`` LID collapse - to one identity — the same helper session keys and adapter allowlists - already use. Group/broadcast JIDs are excluded: those are chats, not - senders. Returns False for non-WhatsApp platforms (exact match only). + Uses ``expand_whatsapp_aliases`` (same helper as session keys and adapter + allowlists) so a bare number, ``@s.whatsapp.net`` JID and ``@lid`` LID + collapse to one identity. Group/broadcast JIDs are chats, not senders, + and non-WhatsApp platforms always return False (exact match only). """ if (platform or "").strip().lower() not in _WHATSAPP_IDENTITY_PLATFORMS: return False @@ -84,9 +72,7 @@ def _whatsapp_user_chat_ids_match(platform: str, left: Optional[str], right: Opt from gateway.whatsapp_identity import expand_whatsapp_aliases left_aliases = expand_whatsapp_aliases(str(left)) - if not left_aliases: - return False - return bool(left_aliases & expand_whatsapp_aliases(str(right))) + return bool(left_aliases and left_aliases & expand_whatsapp_aliases(str(right))) class ProfileRouteRejected(RuntimeError): @@ -108,14 +94,7 @@ class ProfileRoute: @property def specificity(self) -> int: """Higher value = more specific match.""" - s = 0 - if self.guild_id: - s += 2 - if self.chat_id: - s += 4 - if self.thread_id: - s += 8 - return s + return 2 * bool(self.guild_id) + 4 * bool(self.chat_id) + 8 * bool(self.thread_id) def matches( self, @@ -125,50 +104,36 @@ class ProfileRoute: thread_id: Optional[str] = None, parent_chat_id: Optional[str] = None, ) -> bool: - """Return True if this route matches the given source fields. + """Return True if every discriminator the route declares holds (AND). - All configured discriminators are matched conjunctively (AND): every - discriminator that the route declares must hold. ``chat_id`` supports - hierarchical matching for Discord forums/threads: - - Direct channel match: chat_id == route.chat_id - - Thread in channel: parent_chat_id == route.chat_id - A route declaring both ``guild_id`` and ``chat_id`` requires both to - match (a chat match alone does not satisfy a guild constraint). - - WhatsApp / WhatsApp Cloud ``chat_id`` also matches across user-identity - forms (bare number, JID, LID) after the exact-string check. Exact - matches always win first, so existing configs keep working. Groups - (``@g.us``) and broadcasts stay exact-only. + ``chat_id`` matches the channel directly or as the parent of a + thread/forum post; a route with both ``guild_id`` and ``chat_id`` + requires both. WhatsApp ``chat_id`` also matches across user-identity + forms (number/JID/LID) after the exact check; groups/broadcasts stay + exact-only. """ - if not self.enabled: - return False - if self.platform != platform: + if not self.enabled or self.platform != platform: return False if self.thread_id and self.thread_id != thread_id: return False - if self.chat_id and self.chat_id != chat_id and self.chat_id != parent_chat_id: - if not ( - _whatsapp_user_chat_ids_match(platform, self.chat_id, chat_id) - or _whatsapp_user_chat_ids_match(platform, self.chat_id, parent_chat_id) - ): - return False - if self.guild_id and self.guild_id != guild_id: + if ( + self.chat_id + and self.chat_id not in (chat_id, parent_chat_id) + and not _whatsapp_user_chat_ids_match(platform, self.chat_id, chat_id) + and not _whatsapp_user_chat_ids_match(platform, self.chat_id, parent_chat_id) + ): return False - return True + return not (self.guild_id and self.guild_id != guild_id) def _coerce_route_id(value: Any) -> Optional[str]: """Normalize a route discriminator to str for strict equality matching. - PyYAML loads unquoted numeric IDs (Discord snowflakes, Telegram negative - chat ids) as ``int``. Inbound ``SessionSource`` fields are always ``str`` - via ``build_source``, so leaving ints here makes ``matches()`` fail silently. - - Only ``int`` is coerced (the legitimate YAML-numeric case). ``bool`` is an - ``int`` subclass but never a valid id; floats and other types stringify to - something (``"123.0"``) that can never equal an inbound id — recreating the - silent no-match this exists to fix — so they are passed through with a - load-time warning instead of being silently "fixed" (#86470). + PyYAML loads unquoted numeric IDs as ``int`` while inbound ``SessionSource`` + fields are always ``str``, so ints must be coerced or ``matches()`` fails + silently. Only ``int`` (not ``bool``) is coerced; floats and other types + stringify to something (``"123.0"``) that can never match, so they pass + through with a load-time warning instead of being silently "fixed". """ if value is None or isinstance(value, str): return value @@ -183,10 +148,7 @@ def _coerce_route_id(value: Any) -> Optional[str]: def parse_profile_routes(raw: Optional[List[Dict[str, Any]]]) -> List[ProfileRoute]: - """Parse profile_routes from config.yaml into ProfileRoute objects. - - Returns routes sorted by specificity (most specific first). - """ + """Parse profile_routes from config.yaml, sorted most-specific-first.""" if not raw: return [] routes: List[ProfileRoute] = [] @@ -197,18 +159,13 @@ def parse_profile_routes(raw: Optional[List[Dict[str, Any]]]) -> List[ProfileRou platform = entry.get("platform", "") profile = entry.get("profile", "") if not platform or not profile: - logger.warning( - "Skipping profile route %s: missing platform or profile", - name, - ) + logger.warning("Skipping profile route %s: missing platform or profile", name) continue - # Validate profile name to prevent path traversal. Lazy import avoids a + # Validate profile name to prevent path traversal. Lazy import avoids a # circular dependency at module load time. try: - from hermes_cli.profiles import ( - normalize_profile_name, - validate_profile_name, - ) + from hermes_cli.profiles import normalize_profile_name, validate_profile_name + profile = normalize_profile_name(profile) validate_profile_name(profile) except (ValueError, ImportError): @@ -225,7 +182,6 @@ def parse_profile_routes(raw: Optional[List[Dict[str, Any]]]) -> List[ProfileRou enabled=entry.get("enabled", True), ) ) - # Sort: most specific first so the first match wins. routes.sort(key=lambda r: r.specificity, reverse=True) logger.debug("Loaded %d profile routes (most-specific-first)", len(routes)) return routes @@ -239,7 +195,7 @@ def match_profile_route( thread_id: Optional[str] = None, parent_chat_id: Optional[str] = None, ) -> Optional[ProfileRoute]: - """Return the best-matching route, or None for no match.""" + """Return the first (most specific) matching route, or None.""" for route in routes: if route.matches(platform, guild_id=guild_id, chat_id=chat_id, thread_id=thread_id, parent_chat_id=parent_chat_id): return route diff --git a/gateway/relay/__init__.py b/gateway/relay/__init__.py index 959c9b4147..570c72538b 100644 --- a/gateway/relay/__init__.py +++ b/gateway/relay/__init__.py @@ -1,82 +1,91 @@ """Relay/connector support package for the Hermes gateway. -EXPERIMENTAL. This package implements the gateway side of the "Gateway Gateway" -relay design: a generic ``RelayAdapter`` plus the wire-serializable -``CapabilityDescriptor`` the connector hands it at handshake time, and the -production ``WebSocketRelayTransport`` that dials the connector. The public API -(module names, descriptor field set, transport protocol) MAY CHANGE without a -deprecation cycle until at least two real Class-1 platforms (Discord + Telegram) -have shaken out the schema. +EXPERIMENTAL. Gateway side of the "Gateway Gateway" relay design: a generic +``RelayAdapter`` plus the wire-serializable ``CapabilityDescriptor`` the connector +hands it at handshake time, and the production ``WebSocketRelayTransport`` that +dials the connector. The public API (module names, descriptor field set, transport +protocol) MAY CHANGE without a deprecation cycle until at least two real Class-1 +platforms have shaken out the schema. See ``docs/relay-connector-contract.md``. -See ``docs/relay-connector-contract.md`` for the formal cross-repo interface. - -Activation is driven by configuration, not a separate feature flag: the relay -platform is registered when a connector relay URL is configured -(``GATEWAY_RELAY_URL`` env or ``gateway.relay_url`` in config.yaml). Deployments -that don't set it are unaffected — exactly the same shape as ``gateway.proxy_url``. +Activation is config-driven, not a feature flag: the relay platform is registered +when a connector relay URL is configured (``GATEWAY_RELAY_URL`` env or +``gateway.relay_url`` in config.yaml) — the same shape as ``gateway.proxy_url``. """ from __future__ import annotations +import json +import logging import os import re +import socket +import urllib.error +import urllib.parse +import urllib.request from typing import Optional +logger = logging.getLogger("gateway.relay") + # Shape gate for ambient-endpoint token bodies (mode 1b in -# _resolve_relay_identity_token). Accepts a bearer-token-shaped string: -# either a multi-segment dotted token (JWT: header.payload.signature) or a -# single long opaque token (>= 32 chars of the base64url alphabet). Short -# bare words — 'unauthorized', 'error', 'null' — match the alphabet but are -# plain-text error bodies, not credentials, and must fail closed. +# _resolve_relay_identity_token): a JWT (3+ dotted segments) or a long opaque +# base64url token (>= 32 chars). Short bare words — 'unauthorized', 'error', +# 'null' — match the alphabet but are plain-text error bodies and must fail closed. _AMBIENT_TOKEN_SHAPE = re.compile( r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+){2,}" # JWT-like: 3+ dotted segments r"|[A-Za-z0-9_-]{32,}" # long opaque bearer token ) +_HTTP_TIMEOUT_S = 15.0 -def relay_url() -> Optional[str]: - """The connector relay endpoint URL, or None when relay is not configured. - Checks ``GATEWAY_RELAY_URL`` (convenient for Docker) first, then - ``gateway.relay_url`` in config.yaml. A non-empty value activates the relay - platform; absence means a normal direct/single-tenant gateway. - """ - url = os.environ.get("GATEWAY_RELAY_URL", "").strip() - if url: - return url.rstrip("/") +# ─────────────────────────── config access ─────────────────────────── + + +def _load_cfg() -> dict: + """The full gateway config, or ``{}`` — config absence/parse must never crash boot.""" try: from gateway.run import _load_gateway_config # late import to avoid cycle cfg = _load_gateway_config() - url = (cfg.get("gateway") or {}).get("relay_url") - url = (url or "").strip() - if url: - return url.rstrip("/") - except Exception: # noqa: BLE001 - config absence/parse must never crash registration - pass - return None + except Exception: # noqa: BLE001 + return {} + return cfg if isinstance(cfg, dict) else {} + + +def _gateway_cfg() -> dict: + """The ``gateway:`` block of config.yaml, or ``{}``.""" + block = _load_cfg().get("gateway") + return block if isinstance(block, dict) else {} + + +def _env_or_cfg(env_var: str, cfg_key: str) -> str: + """Env var first (Docker/NAS stamp), then ``gateway.`` in config.yaml. + + Returns the stripped value, ``""`` when neither is set. + """ + value = os.environ.get(env_var, "").strip() + return value or str(_gateway_cfg().get(cfg_key, "") or "").strip() + + +def relay_url() -> Optional[str]: + """The connector relay endpoint URL, or None when relay is not configured. + + ``GATEWAY_RELAY_URL`` env, then ``gateway.relay_url``. A non-empty value + activates the relay platform; absence means a normal direct gateway. + """ + return _env_or_cfg("GATEWAY_RELAY_URL", "relay_url").rstrip("/") or None def relay_platform_identities() -> list[tuple[str, str]]: - """The (platform, bot_id) pairs this gateway fronts over the relay (Phase 1.5). + """The ordered (platform, bot_id) pairs this gateway fronts over the relay. - Shape A (multi-platform-per-agent, D-Q1.5c — CUT OVER, no scalar fallback): - one gateway fronts a SET of platforms on one WS connection. The set is the - env-stamped deploy config: - - - ``GATEWAY_RELAY_PLATFORMS`` — comma-sep list (e.g. ``discord,telegram``). - - ``GATEWAY_RELAY_BOT_IDS`` — JSON keyed map - ``{"discord": {"botId": "..."}, "telegram": {"botId": "...", "username": "..."}}``. - - Returns the ordered list of ``(platform, bot_id)`` pairs (the FIRST is the - default the handshake/descriptor falls back to). The connector accepts N - hellos accumulating into its advertised set; outbound frames discriminate - per-frame on the platform (gateway-gateway D-Q1.5b.1). A platform present in - the list but absent from the ids map resolves with an empty bot_id (the - connector rejects an unprovisioned platform with a structured failure). - - Defaults to ``[("relay", "")]`` when nothing is configured (the generic - single-plane fallback for a connector that didn't stamp a platform set). + One gateway fronts a SET of platforms on one WS connection, from the env-stamped + deploy config: ``GATEWAY_RELAY_PLATFORMS`` (comma-sep, e.g. ``discord,telegram``) + and ``GATEWAY_RELAY_BOT_IDS`` (JSON map ``{"discord": {"botId": "..."}, ...}``). + The FIRST pair is the default the handshake/descriptor falls back to. A platform + listed but absent from the ids map resolves with an empty bot_id (the connector + rejects an unprovisioned platform with a structured failure). Defaults to + ``[("relay", "")]`` — the generic single-plane fallback — when nothing is set. """ platforms_raw = os.environ.get("GATEWAY_RELAY_PLATFORMS", "").strip() platforms = [p.strip() for p in platforms_raw.split(",") if p.strip()] @@ -92,305 +101,186 @@ def relay_platform_identities() -> list[tuple[str, str]]: def relay_fronted_platforms() -> set[str]: - """The logical platform names the relay connector fronts for this gateway. + """Logical platform names the relay fronts, minus the generic ``relay`` fallback. - Thin, env-derived wrapper over :func:`relay_platform_identities` (the - ``GATEWAY_RELAY_PLATFORMS`` deploy stamp) minus the generic ``relay`` - fallback. This is the SAME source ``ws_transport`` seeds the live - adapter's identity set from (``RelayAdapter.fronts_platform``), so - config-time validation (e.g. cron delivery preflight) and fire-time - routing can never disagree — and it needs no live adapter handle, so a - standalone scheduler process can consult it too. Empty set when the - relay fronts nothing. + Same env source ``ws_transport`` seeds the live adapter's identity set from, so + config-time validation (cron delivery preflight) and fire-time routing can never + disagree — and it needs no live adapter, so a standalone scheduler can use it. """ return {p for p, _ in relay_platform_identities() if p != "relay"} def _relay_bot_ids_map() -> dict: - """Parse ``GATEWAY_RELAY_BOT_IDS`` (JSON keyed map). Never raises — a malformed - map yields ``{}`` so a bad config degrades to empty bot ids (the connector - rejects an unprovisioned platform) rather than crashing boot.""" - import json - import logging - + """Parse ``GATEWAY_RELAY_BOT_IDS``; a malformed map yields ``{}`` rather than crashing boot.""" raw = os.environ.get("GATEWAY_RELAY_BOT_IDS", "").strip() if not raw: return {} try: parsed = json.loads(raw) return parsed if isinstance(parsed, dict) else {} - except Exception: # noqa: BLE001 - a bad map must not crash boot - logging.getLogger("gateway.relay").warning( - "GATEWAY_RELAY_BOT_IDS is not valid JSON; treating as empty" - ) + except Exception: # noqa: BLE001 + logger.warning("GATEWAY_RELAY_BOT_IDS is not valid JSON; treating as empty") return {} -def relay_bot_username(platform: str) -> Optional[str]: - """The bot's deep-link username/handle for a platform (e.g. Telegram's - ``@handle`` for ``t.me/``), read from the per-platform entry in - ``GATEWAY_RELAY_BOT_IDS``. None when absent (most platforms don't need one). - """ - entry = _relay_bot_ids_map().get(platform) - if isinstance(entry, dict): - username = entry.get("username") - if username: - return str(username).lstrip("@") - return None - - def relay_platform_identity() -> tuple[str, str]: - """The PRIMARY (platform, bot_id) — the first identity in the configured set. - - Kept for call sites that need a single representative identity (the default - descriptor platform, the policy projection's primary). The full set is - ``relay_platform_identities()``. Defaults to ``("relay", "")``. - """ + """The PRIMARY (platform, bot_id) — first of ``relay_platform_identities()``.""" return relay_platform_identities()[0] def relay_connection_auth() -> tuple[Optional[str], Optional[str]]: """The (gateway_id, upgrade_secret) this gateway authenticates the WS upgrade with. - Both come from enrollment (``hermes gateway enroll`` writes them to - ``~/.hermes/.env``): ``GATEWAY_RELAY_ID`` identifies the enrolled instance, - ``GATEWAY_RELAY_SECRET`` is the per-gateway signing secret. Either absent -> - ``(None, None)`` and the transport dials unauthenticated (dev/test, or a - connector that doesn't enforce auth). Checks env first (Docker), then - ``gateway.relay_id`` / ``gateway.relay_secret`` in config.yaml. + Both come from enrollment (``GATEWAY_RELAY_ID`` / ``GATEWAY_RELAY_SECRET`` env, + then ``gateway.relay_id`` / ``gateway.relay_secret``). Either absent -> + ``(None, None)`` and the transport dials unauthenticated. """ - gateway_id = os.environ.get("GATEWAY_RELAY_ID", "").strip() - secret = os.environ.get("GATEWAY_RELAY_SECRET", "").strip() - if not (gateway_id and secret): - try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - cfg = (_load_gateway_config().get("gateway") or {}) - gateway_id = gateway_id or str(cfg.get("relay_id", "") or "").strip() - secret = secret or str(cfg.get("relay_secret", "") or "").strip() - except Exception: # noqa: BLE001 - config absence/parse must never crash registration - pass + gateway_id = _env_or_cfg("GATEWAY_RELAY_ID", "relay_id") + secret = _env_or_cfg("GATEWAY_RELAY_SECRET", "relay_secret") return (gateway_id or None, secret or None) def relay_endpoint() -> Optional[str]: """The gateway's own PUBLIC inbound URL, asserted to the connector at provision. - The connector delivers signed inbound POSTs to this URL and stores it on the - tenant's route rows. It is gateway-asserted (the connector scopes it to the - verified tenant, so a dishonest gateway can only misdirect its OWN inbound). - The *source* of the value differs by deployment but the code path is uniform: - a self-hosted operator sets ``GATEWAY_RELAY_ENDPOINT`` (mirrors how they set - ``HERMES_DASHBOARD_PUBLIC_URL``); a hosted/NAS container has the same var - stamped in (NAS knows the public URL only in that case). Absent -> the + The connector delivers signed inbound POSTs here and stores it on the tenant's + route rows; gateway-asserted but scoped to the verified tenant, so a dishonest + gateway can only misdirect its OWN inbound. ``GATEWAY_RELAY_ENDPOINT`` env + (self-hosted or NAS-stamped), then ``gateway.relay_endpoint``. Absent -> the gateway provisions outbound-only (no inbound routes written). - - Env first (Docker), then ``gateway.relay_endpoint`` in config.yaml. """ - url = os.environ.get("GATEWAY_RELAY_ENDPOINT", "").strip() - if not url: - try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - cfg = (_load_gateway_config().get("gateway") or {}) - url = str(cfg.get("relay_endpoint", "") or "").strip() - except Exception: # noqa: BLE001 - config absence/parse must never crash boot - url = "" - return url.rstrip("/") or None + return _env_or_cfg("GATEWAY_RELAY_ENDPOINT", "relay_endpoint").rstrip("/") or None def relay_route_keys() -> list[str]: """Discriminators (scope_ids / chat_ids / paths) this gateway's tenant owns. - Gateway-provided config, paired with ``relay_endpoint()``: the connector - writes one route row per (routeKey -> tenant, endpoint), so route keys only - take effect alongside an endpoint. Empty -> outbound-only provisioning (the - connector accepts an empty set and writes no route rows). - - ``GATEWAY_RELAY_ROUTE_KEYS`` is comma-separated; config.yaml - ``gateway.relay_route_keys`` may be a list or a comma string. + Paired with ``relay_endpoint()``: the connector writes one route row per + (routeKey -> tenant, endpoint), so keys only take effect alongside an endpoint. + ``GATEWAY_RELAY_ROUTE_KEYS`` is comma-separated; ``gateway.relay_route_keys`` + may be a list or a comma string. Empty -> outbound-only provisioning. """ raw = os.environ.get("GATEWAY_RELAY_ROUTE_KEYS", "").strip() if not raw: - try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - cfg = (_load_gateway_config().get("gateway") or {}) - val = cfg.get("relay_route_keys", "") - if isinstance(val, (list, tuple)): - return [str(k).strip() for k in val if str(k).strip()] - raw = str(val or "").strip() - except Exception: # noqa: BLE001 - raw = "" + val = _gateway_cfg().get("relay_route_keys", "") + if isinstance(val, (list, tuple)): + return [str(k).strip() for k in val if str(k).strip()] + raw = str(val or "").strip() return [k.strip() for k in raw.split(",") if k.strip()] def relay_instance_id() -> Optional[str]: - """Stable per-instance id this gateway forwards at provision (Phase 6 Unit α). + """Stable per-instance id forwarded at provision (``GATEWAY_RELAY_INSTANCE_ID`` / ``gateway.relay_instance_id``). - Binds the connector's ``gatewayId -> instanceId`` so the connector can route - inbound per-instance (not tenant-broadcast) once Phase 6 delivery lands. The - value is the NAS ``AgentInstance.id`` for a managed agent (NAS stamps - ``GATEWAY_RELAY_INSTANCE_ID`` into the container env, beside - ``GATEWAY_RELAY_URL``); a self-hosted operator may set it explicitly. It is - gateway-asserted but safely scoped: the org/tenant stays token-verified, so a - dishonest gateway can only bind ITS OWN tenant's instance — the same posture - as ``relay_endpoint()``. Absent -> the connector stores null and per-instance - routing simply has no binding for this connection yet (back-compat). - - Env first (Docker/NAS), then ``gateway.relay_instance_id`` in config.yaml. + Binds the connector's ``gatewayId -> instanceId`` so inbound can route + per-instance rather than tenant-broadcast. NAS stamps its ``AgentInstance.id`` + for a managed agent; a self-hosted operator may set it. Gateway-asserted but + tenant-scoped like ``relay_endpoint()``. Absent -> the connector stores null + (back-compat: no per-instance binding yet). """ - value = os.environ.get("GATEWAY_RELAY_INSTANCE_ID", "").strip() - if not value: - try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - cfg = (_load_gateway_config().get("gateway") or {}) - value = str(cfg.get("relay_instance_id", "") or "").strip() - except Exception: # noqa: BLE001 - config absence/parse must never crash boot - value = "" - return value or None + return _env_or_cfg("GATEWAY_RELAY_INSTANCE_ID", "relay_instance_id") or None def relay_wake_url() -> Optional[str]: - """The gateway's WAKE URL, forwarded at provision (Phase 5 §5.2 wake PRIMITIVE). + """The gateway's WAKE URL forwarded at provision (``GATEWAY_RELAY_WAKE_URL`` / ``gateway.relay_wake_url``). - A poke target the connector issues a payload-free GET to when a buffered-only - (going-idle) destination for this instance receives its first buffered event, - so a suspended gateway wakes, reconnects its relay WS, and drains its - delivery-leg backlog. The value's *source* differs by deployment but the code - path is uniform: a managed/NAS container has ``GATEWAY_RELAY_WAKE_URL`` stamped - in (NAS knows the Fly autostart / dashboard hostname); a self-hosted operator - sets it explicitly (or passes ``--wake-url`` to ``hermes gateway enroll``). - - Gateway-asserted but safely scoped: the org/tenant stays token-verified, so a - dishonest gateway can only register a wake target for ITS OWN instance — the - same posture as ``relay_instance_id()`` / the retired ``relay_endpoint()``. - Absent -> the connector stores null and simply can't wake this instance - (buffering still works; the gateway drains whenever it next reconnects). - - Env first (Docker/NAS), then ``gateway.relay_wake_url`` in config.yaml. + A poke target the connector GETs (payload-free) when a going-idle destination for + this instance receives its first buffered event, so a suspended gateway wakes, + reconnects its relay WS and drains its backlog. NAS stamps it for a managed + container; a self-hosted operator sets it (or ``hermes gateway enroll --wake-url``). + Tenant-scoped like ``relay_instance_id()``. Absent -> the connector can't wake + this instance (buffering still works; the gateway drains on next reconnect). """ - value = os.environ.get("GATEWAY_RELAY_WAKE_URL", "").strip() - if not value: - try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - cfg = (_load_gateway_config().get("gateway") or {}) - value = str(cfg.get("relay_wake_url", "") or "").strip() - except Exception: # noqa: BLE001 - config absence/parse must never crash boot - value = "" - return value.rstrip("/") or None + return _env_or_cfg("GATEWAY_RELAY_WAKE_URL", "relay_wake_url").rstrip("/") or None def relay_display_name() -> Optional[str]: - """The human-facing agent display name, forwarded at provision (Phase 1 parity). + """The human-facing agent display name forwarded at provision. - The PRIMARY source for the connector's multi-agent reply-attribution prefix - (gateway-gateway #171): in a multi-agent scope the shared bot prepends - ``**:** `` to this instance's replies. Gateway-asserted but - safely scoped exactly like ``relay_instance_id()`` / ``relay_wake_url()`` — - the tenant stays token-verified, so a dishonest gateway can only label its - OWN instance. Absent -> the connector stores null and attribution falls - back to the instance's linked-owner identity, else skips the prefix. - - Env first (Docker/NAS stamps ``GATEWAY_RELAY_DISPLAY_NAME``), then the - skin's branded agent name (``get_branding("agent_name")`` — the same value - the CLI banner shows), so a self-hosted rename via skin config propagates - on the next boot's re-provision (the connector rotates on change, same as - a wake-url move). + Primary source of the connector's multi-agent reply-attribution prefix + (``**:** ``). ``GATEWAY_RELAY_DISPLAY_NAME`` env, then the skin's + branded agent name (the CLI banner value), so a skin rename propagates on the + next boot's re-provision. Absent -> the connector falls back to the instance's + linked-owner identity, else skips the prefix. """ value = os.environ.get("GATEWAY_RELAY_DISPLAY_NAME", "").strip() if not value: try: from hermes_cli.skin_engine import get_active_skin # late import: boot-safe - value = str( - get_active_skin().get_branding("agent_name", "") or "" - ).strip() + value = str(get_active_skin().get_branding("agent_name", "") or "").strip() except Exception: # noqa: BLE001 - branding absence must never crash boot value = "" - # The stock brand name is IDENTICAL on every default install, so in a - # multi-agent scope it would prefix every reply "**Hermes Agent:**" — - # shadowing the connector's linked-owner fallback, which actually - # disambiguates. Only a deliberately customized name is forwarded. + # The stock brand is identical on every default install: forwarding it would + # prefix every reply "**Hermes Agent:**" and shadow the connector's + # linked-owner fallback, which actually disambiguates. Only a customized + # name is forwarded. if value == "Hermes Agent": value = "" - # Mirror the connector's ingest sanitization (trim + 64-char cap) so what - # we send is what gets stored. + # Mirror the connector's ingest sanitization (trim + 64-char cap). return value[:64] or None +# ─────────────────────────── connector HTTP ─────────────────────────── + + +def _http_base(relay_dial_url: str) -> str: + """``ws(s)://…/relay`` dial URL -> ``http(s)://…`` connector base (shared with media).""" + from gateway.relay.media import media_base_url + + return media_base_url(relay_dial_url) + + def _provision_url(relay_dial_url: str) -> str: - """Map the ``ws(s)://…/relay`` dial URL to the ``http(s)://…/relay/provision`` POST URL.""" - raw = relay_dial_url.rstrip("/") - if raw.startswith("ws://"): - raw = "http://" + raw[len("ws://"):] - elif raw.startswith("wss://"): - raw = "https://" + raw[len("wss://"):] - if raw.endswith("/relay"): - raw = raw[: -len("/relay")] - return f"{raw}/relay/provision" + """Map the dial URL to the ``/relay/provision`` POST URL.""" + return f"{_http_base(relay_dial_url)}/relay/provision" def _policy_url(relay_dial_url: str) -> str: - """Map the ``ws(s)://…/relay`` dial URL to the ``http(s)://…/relay/policy`` POST URL. + """Map the dial URL to the ``/relay/policy`` POST URL (relevance-policy channel).""" + return f"{_http_base(relay_dial_url)}/relay/policy" - Same host derivation as ``_provision_url``; the connector mounts the - relevance-policy update channel at ``/relay/policy`` (Phase 6 Unit ζ). - """ - raw = relay_dial_url.rstrip("/") - if raw.startswith("ws://"): - raw = "http://" + raw[len("ws://"):] - elif raw.startswith("wss://"): - raw = "https://" + raw[len("wss://"):] - if raw.endswith("/relay"): - raw = raw[: -len("/relay")] - return f"{raw}/relay/policy" + +def _json_post_request(url: str, token: str, body: dict) -> urllib.request.Request: + """A bearer-authenticated JSON POST request to the connector.""" + return urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) def relay_relevance_policy(platform: Optional[str] = None) -> Optional[dict]: """Project a fronted platform's RELEVANCE config into the connector's generic vocabulary. - The connector's relevance gate (Phase 6 Unit ζ) reasons over a - platform-agnostic policy — ``requireAddress`` / ``freeResponseScopes`` / - ``allowOtherBots`` — NOT over Discord/Telegram words. This is the gateway - side of that contract: it reads the agent's existing relevance knobs and - emits the generic shape the connector stores per-instance (Phase 1.5: the - connector keys the policy by ``(tenant, platform, instanceId)``, so each - fronted platform gets its own row — pass its name here). + The connector's relevance gate reasons over a platform-agnostic policy, keyed by + ``(tenant, platform, instanceId)``. Mapping from the agent's existing knobs: + - ``requireAddress`` <- the platform's ``require_mention`` + - ``freeResponseScopes`` <- the platform's ``free_response_channels`` + - ``allowOtherBots`` <- ``{PLATFORM}_ALLOW_BOTS`` in {"mentions","all"} - Mapping (the connector vocabulary ← the gateway's existing config): - - ``requireAddress`` ← the platform's ``require_mention`` (the agent - only engages a non-owner message that @mentions it / replies to it). - - ``freeResponseScopes`` ← the platform's ``free_response_channels`` (the - channel/scope ids where ``require_mention`` is waived — same scope - vocabulary the connector's δ scope grants + ε floor use). - - ``allowOtherBots`` ← ``{PLATFORM}_ALLOW_BOTS`` in {"mentions","all"} - (whether bot-authored messages are admitted; default off). - - Read from the relay platform's config block (the platform the connector - fronts, e.g. ``discord:``), falling back to the bridged top-level keys, then - the ``{PLATFORM}_*`` env. ``platform`` defaults to the PRIMARY fronted - platform (back-compat). Returns the generic dict, or None when relay isn't - configured or the platform exposes no relevance knobs (⇒ the connector's - default — mention-gated — applies unchallenged; an EXPLICIT - ``require_mention: false`` IS a knob and is declared so the connector - doesn't mention-gate an agent configured to free-respond). + Read from the platform's config block (``discord:``), falling back to the bridged + top-level keys, then env. ``platform`` defaults to the PRIMARY fronted platform. + Returns None when relay isn't configured or nothing is CONFIGURED to declare, so + the connector's default (mention-gated) applies. The condition is "require_mention + is unset", NOT "falsy": an EXPLICIT ``require_mention: false`` is a non-default + choice that MUST be declared or the connector would mention-gate an agent + configured to free-respond. """ if platform is None: platform, _bot_id = relay_platform_identity() if not platform or platform == "relay": - # No concrete fronted platform resolved ⇒ nothing platform-specific to project. return None - # Resolve the platform's config block + the bridged top-level keys. require_mention = None free_response: list[str] = [] try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - cfg = _load_gateway_config() or {} + cfg = _load_cfg() plat_cfg = cfg.get(platform) if not isinstance(plat_cfg, dict): _gw_platforms = (cfg.get("gateway") or {}).get("platforms") or {} @@ -416,26 +306,18 @@ def relay_relevance_policy(platform: Optional[str] = None) -> Optional[dict]: except Exception: # noqa: BLE001 - config absence/parse must never crash boot pass - # allow_other_bots ← {PLATFORM}_ALLOW_BOTS in {"mentions","all"} (same gate as - # the gateway's own authz_mixin DISCORD_ALLOW_BOTS bypass). + # Same gate as the gateway's own authz_mixin DISCORD_ALLOW_BOTS bypass. allow_bots_env = os.environ.get(f"{platform.upper()}_ALLOW_BOTS", "").lower().strip() allow_other_bots = allow_bots_env in {"mentions", "all"} - # Nothing CONFIGURED to declare ⇒ let the connector keep its default policy - # (mention-gated with agent-thread continuation — matches absence-of-row - # semantics on the connector side). NOTE the condition is "require_mention - # is unset", NOT "require_mention is falsy": the connector's default is now - # requireAddress=true, so an EXPLICIT `require_mention: false` is a - # non-default choice that MUST be declared or the connector would - # mention-gate an agent configured to free-respond. + # Nothing CONFIGURED to declare => keep the connector's default. The test is + # "require_mention is unset", NOT "falsy": an EXPLICIT `require_mention: false` + # is a non-default choice that MUST be declared (see docstring). if require_mention is None and not free_response and not allow_other_bots: return None - - require_address = bool(require_mention) if require_mention is not None else False - return { "platform": platform, - "requireAddress": require_address, + "requireAddress": bool(require_mention), "freeResponseScopes": free_response, "allowOtherBots": allow_other_bots, } @@ -453,20 +335,15 @@ def _post_provision( instance_id: Optional[str] = None, wake_url: Optional[str] = None, display_name: Optional[str] = None, - timeout: float = 15.0, + timeout: float = _HTTP_TIMEOUT_S, ) -> dict: """POST to the connector's ``/relay/provision`` and return the JSON body. - The connector validates ``access_token`` against NAS, derives the - authoritative tenant, mints the per-gateway secret + per-tenant delivery key, - upserts the tenant's route rows, and returns - ``{secret, deliveryKey, tenant, gatewayId, routeKeys}``. Raises RuntimeError - with a user-facing message on any non-2xx / transport failure. + The connector validates ``access_token`` against NAS, derives the authoritative + tenant, mints the per-gateway secret + per-tenant delivery key, upserts route rows + and returns ``{secret, deliveryKey, tenant, gatewayId, routeKeys}``. Raises + RuntimeError with a user-facing message on any non-2xx / transport failure. """ - import json - import urllib.error - import urllib.request - body: dict = { "gatewayId": gateway_id, "platform": platform, @@ -474,30 +351,12 @@ def _post_provision( "gatewayEndpoint": gateway_endpoint or "", "routeKeys": route_keys, } - # Only send instanceId when we actually have one — omitting it lets the - # connector store null (back-compat) rather than binding an empty string. - if instance_id: - body["instanceId"] = instance_id - # Same for the wake URL (Phase 5 §5.2): omit when absent so the connector - # stores null and simply can't wake this instance (buffering still works). - if wake_url: - body["wakeUrl"] = wake_url - # Same for the display name (Phase 1 parity, gg#171): omit when absent so - # the connector stores null and attribution falls back to the linked-owner - # identity. - if display_name: - body["displayName"] = display_name - data = json.dumps(body).encode("utf-8") - req = urllib.request.Request( - provision_url, - data=data, - method="POST", - headers={ - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) + # Optional fields are OMITTED when absent so the connector stores null + # (back-compat) rather than binding an empty string. + for key, value in (("instanceId", instance_id), ("wakeUrl", wake_url), ("displayName", display_name)): + if value: + body[key] = value + req = _json_post_request(provision_url, access_token, body) try: with urllib.request.urlopen(req, timeout=timeout) as resp: payload = json.loads(resp.read().decode()) @@ -505,7 +364,7 @@ def _post_provision( detail = "" try: detail = (json.loads(exc.read().decode()) or {}).get("error", "") - except Exception: + except Exception: # noqa: BLE001 pass raise RuntimeError( f"connector returned HTTP {exc.code}" + (f": {detail}" if detail else "") @@ -521,65 +380,44 @@ def _post_provision( def _resolve_relay_identity_token() -> str: """Resolve the caller-identity bearer token the connector introspects to a tenant. - Canonical resolver shared by the runtime self-provision path and the - ``hermes gateway enroll`` CLI. Three modes, in precedence order: + Canonical resolver shared by runtime self-provision and ``hermes gateway enroll``. + Modes, in precedence order: + 1. Generic OIDC client-credentials (self-hosted IdP, no Nous Portal): when + ``gateway.idp.token_url`` (``GATEWAY_RELAY_IDP_TOKEN_URL``) is set together + with client id + secret, POST the OAuth2 ``client_credentials`` grant. + 1b. Ambient token endpoint: ``token_url`` with NEITHER client_id nor + client_secret is a metadata-server-style endpoint (e.g. Domino's + ``$DOMINO_API_PROXY/access-token``): a plain GET whose body IS the token, + raw JWT or a JSON envelope with ``access_token``. Possession of the + (typically loopback) endpoint is the credential. + 2. Nous Portal (default): ``resolve_nous_access_token()``. - 1. **Generic OIDC client-credentials** (air-gapped / self-hosted-IdP, NO - Nous Portal): when ``gateway.idp.token_url`` (or - ``GATEWAY_RELAY_IDP_TOKEN_URL``) is configured together with a client - id/secret, obtain a workload access token via the OAuth2 - ``client_credentials`` grant against the operator's own IdP (Entra; - Keycloak; Authentik in the sandbox). The connector's Seam-A OIDC - verifier reads a claim (default ``tid``) off it as the tenant. - 1b. **Ambient token endpoint**: when ``token_url`` is configured with - NEITHER client_id nor client_secret, the URL is treated as a metadata-server-style - ambient credential endpoint (e.g. Domino's - ``$DOMINO_API_PROXY/access-token``): a plain GET whose response body - IS the token — either a raw JWT string or a JSON envelope with an - ``access_token`` field. No client registration involved; possession - of the (typically loopback) endpoint is the credential. - 2. **Nous Portal** (default): ``resolve_nous_access_token()`` — existing - managed/hosted behaviour. - - Raises on failure; callers decide whether that's fatal (enroll CLI) or a - graceful boot no-op (self-provision). + Raises on failure; callers decide whether that's fatal (enroll CLI) or a graceful + boot no-op (self-provision). """ token_url = os.environ.get("GATEWAY_RELAY_IDP_TOKEN_URL", "").strip() client_id = os.environ.get("GATEWAY_RELAY_IDP_CLIENT_ID", "").strip() client_secret = os.environ.get("GATEWAY_RELAY_IDP_CLIENT_SECRET", "").strip() scope = os.environ.get("GATEWAY_RELAY_IDP_SCOPE", "").strip() if not token_url: - try: - from gateway.run import _load_gateway_config # late import to avoid cycle - - idp = ((_load_gateway_config().get("gateway") or {}).get("idp") or {}) - token_url = str(idp.get("token_url", "") or "").strip() - client_id = client_id or str(idp.get("client_id", "") or "").strip() - client_secret = client_secret or str(idp.get("client_secret", "") or "").strip() - scope = scope or str(idp.get("scope", "") or "").strip() - except Exception: # noqa: BLE001 - config absence must not crash - token_url = token_url or "" + idp = _gateway_cfg().get("idp") + idp = idp if isinstance(idp, dict) else {} # malformed gateway.idp degrades to "no token_url" + token_url = str(idp.get("token_url", "") or "").strip() + client_id = client_id or str(idp.get("client_id", "") or "").strip() + client_secret = client_secret or str(idp.get("client_secret", "") or "").strip() + scope = scope or str(idp.get("scope", "") or "").strip() if not token_url: - # Mode 2 — Nous Portal (default, unchanged behaviour). from hermes_cli.auth import resolve_nous_access_token return resolve_nous_access_token() - import json - import urllib.error - import urllib.parse - import urllib.request - if not client_id and not client_secret: - # Mode 1b — ambient token endpoint (no client credentials configured). - # Plain GET; the body is the token, raw or JSON-enveloped. + # Mode 1b — plain GET; the body is the token, raw or JSON-enveloped. req = urllib.request.Request( - token_url, - method="GET", - headers={"Accept": "application/json, text/plain"}, + token_url, method="GET", headers={"Accept": "application/json, text/plain"} ) - with urllib.request.urlopen(req, timeout=15.0) as resp: + with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_S) as resp: body = resp.read().decode().strip() token = "" if body.startswith("{"): @@ -587,10 +425,8 @@ def _resolve_relay_identity_token() -> str: envelope_token = (json.loads(body) or {}).get("access_token") except ValueError: envelope_token = None - # Same contract as the client_credentials path below: the value - # must be a non-empty STRING. No shape gate here — a JSON envelope - # is a deliberate token response (and opaque tokens may use the - # standard-base64 alphabet the raw-body gate would reject). + # No shape gate on an envelope: it is a deliberate token response, and + # opaque tokens may use the standard-base64 alphabet the raw gate rejects. if isinstance(envelope_token, str): token = envelope_token.strip() elif _AMBIENT_TOKEN_SHAPE.fullmatch(body): @@ -605,8 +441,8 @@ def _resolve_relay_identity_token() -> str: return token if not client_id or not client_secret: - # Exactly one credential configured: this is a mistyped client_credentials - # setup, not an ambient endpoint. Keep the loud error (never GET the IdP). + # Exactly one credential: a mistyped client_credentials setup, not an + # ambient endpoint. Fail loud; never GET the IdP. missing = "client_secret" if client_id else "client_id" raise RuntimeError( f"gateway.idp.token_url is configured with a partial client credential " @@ -615,24 +451,17 @@ def _resolve_relay_identity_token() -> str: f"as an ambient token endpoint (plain GET returning the token)." ) - # Mode 1 — generic OAuth2 client_credentials grant. - form = { - "grant_type": "client_credentials", - "client_id": client_id, - "client_secret": client_secret, - } + # Mode 1 — OAuth2 client_credentials grant. + form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret} if scope: form["scope"] = scope req = urllib.request.Request( token_url, data=urllib.parse.urlencode(form).encode("utf-8"), method="POST", - headers={ - "Content-Type": "application/x-www-form-urlencoded", - "Accept": "application/json", - }, + headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"}, ) - with urllib.request.urlopen(req, timeout=15.0) as resp: + with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_S) as resp: payload = json.loads(resp.read().decode()) access_token = (payload or {}).get("access_token") if not isinstance(access_token, str) or not access_token.strip(): @@ -643,50 +472,26 @@ def _resolve_relay_identity_token() -> str: def self_provision_relay() -> bool: """Boot-time relay self-provision: mint relay creds in-process, no human, no disk. - Fires when relay is configured (``relay_url()`` set) and NO per-gateway secret - is already present, AND the agent can resolve its own Nous access token. In - that case the runtime resolves the agent's own Nous access token (the same - ``resolve_nous_access_token()`` the enroll CLI / dashboard register use), - POSTs ``/relay/provision`` asserting its own endpoint + route keys, and sets - ``GATEWAY_RELAY_ID`` / ``GATEWAY_RELAY_SECRET`` / ``GATEWAY_RELAY_DELIVERY_KEY`` - into ``os.environ`` so the subsequent ``register_relay_adapter()`` picks them - up. The creds live ONLY in process memory — never written to ``~/.hermes/.env``. + Fires when ``relay_url()`` is set and NO per-gateway secret is pinned: resolves + the agent's identity token, POSTs ``/relay/provision`` for EACH fronted platform + and sets ``GATEWAY_RELAY_ID`` / ``GATEWAY_RELAY_SECRET`` / + ``GATEWAY_RELAY_DELIVERY_KEY`` in ``os.environ`` so ``register_relay_adapter()`` + picks them up. Creds live ONLY in process memory (never ``~/.hermes/.env``), so a + hosted container re-provisions every boot; the connector's rotation window covers + a still-connected prior instance. - The trigger is deliberately NOT ``is_managed()``: that means - "package-manager/NixOS-managed" and is False on a NAS-hosted Fly agent (which - sets neither ``HERMES_MANAGED`` nor a ``.managed`` marker), so gating on it - blocked the exact hosted case this is for. The real signal is "you pointed me - at a connector and didn't pin a secret" — which is both NAS-independent and - self-guarding: + The trigger is deliberately NOT ``is_managed()`` (False on a NAS-hosted Fly + agent); "pointed at a connector without a pinned secret" is the real signal and + self-guards: an enrolled self-hosted gateway has a PINNED secret -> skipped; a + box with no resolvable identity -> graceful no-op. - - A NAS-hosted agent: has ``GATEWAY_RELAY_URL``, no pinned secret, and a - bootstrapped NAS token -> self-provisions. - - A self-hosted operator who ran ``hermes gateway enroll``: has a PINNED - ``GATEWAY_RELAY_SECRET`` -> skipped (the secret-present guard below). - - A self-hosted box with a relay URL but no NAS identity: - ``resolve_nous_access_token()`` fails -> graceful no-op. - - Stateless: process-env creds don't survive a restart, so a hosted container - re-provisions every boot; the connector's rotation window covers a still- - connected prior instance. An explicitly-pinned ``GATEWAY_RELAY_SECRET`` (env - or config) is RESPECTED — self-provision skips so an operator pin isn't - stomped. - - Returns True if it provisioned, False otherwise. NEVER raises: a provision - failure logs and returns False so the gateway still boots (and - ``register_relay_adapter`` will simply dial unauthenticated / be rejected, - rather than the whole gateway crashing). + Returns True iff it provisioned. NEVER raises: a failure logs and returns False so + the gateway still boots (the adapter then dials unauthenticated / is rejected). """ - import logging - - logger = logging.getLogger("gateway.relay") - dial_url = relay_url() if not dial_url: return False - # Respect an already-present (pinned/stamped) secret — don't stomp it. This - # is also what makes a self-hosted, enrolled gateway skip self-provision. existing_id, existing_secret = relay_connection_auth() if existing_id and existing_secret: logger.info("relay self-provision skipped: GATEWAY_RELAY_SECRET already set") @@ -695,15 +500,11 @@ def self_provision_relay() -> bool: try: access_token = _resolve_relay_identity_token() except Exception as exc: # noqa: BLE001 - boot must survive a token failure - # No resolvable identity (e.g. a self-hosted box that hasn't enrolled and - # configured no IdP) -> nothing to provision with; skip quietly and boot. logger.warning("relay self-provision skipped: could not resolve identity token (%s)", exc) return False identities = relay_platform_identities() # gatewayId default mirrors the enroll CLI's hostname-based slug. - import socket - try: host = socket.gethostname().strip() except Exception: # noqa: BLE001 @@ -715,15 +516,11 @@ def self_provision_relay() -> bool: wake_url = relay_wake_url() display_name = relay_display_name() - # Phase 1.5 (D-Q1.5c): provision EACH fronted platform under the SAME - # gatewayId + the SAME (platform-less) per-gateway secret. The connector's - # secret record is (gatewayId -> tenant) only; platform/botId live on the - # per-platform route rows (relayProvision.ts:124/148), so N provision POSTs - # with one gatewayId add N platforms' routes under one secret. The loop is - # PARTIAL-FAILURE-TOLERANT: a platform that fails to provision is logged and - # skipped (it just isn't fronted) — the others still come up. The FIRST - # successful provision sets the in-process creds; later platforms re-provision - # against the same gatewayId (idempotent on the secret, additive on routes). + # Provision EACH fronted platform under the SAME gatewayId + the SAME per-gateway + # secret: the connector's secret record is (gatewayId -> tenant) only, platform/ + # botId live on per-platform route rows, so N POSTs with one gatewayId are + # idempotent on the secret and additive on routes. PARTIAL-FAILURE-TOLERANT: a + # platform that fails is logged and skipped (just not fronted); the others come up. provisioned: list[str] = [] result: dict = {} for platform, bot_id in identities: @@ -748,12 +545,9 @@ def self_provision_relay() -> bool: ) continue provisioned.append(platform) - # Set creds in-process on the FIRST success so register_relay_adapter() - # reads them from os.environ (the per-gateway secret authenticates the - # outbound WS upgrade). Subsequent platforms share the same gatewayId + - # secret (the connector returns the same record for the same gatewayId). - # Never logged. - if "GATEWAY_RELAY_SECRET" not in os.environ or not os.environ.get("GATEWAY_RELAY_SECRET"): + # Set creds in-process on the FIRST success (the per-gateway secret + # authenticates the outbound WS upgrade). Never logged. + if not os.environ.get("GATEWAY_RELAY_SECRET"): os.environ["GATEWAY_RELAY_ID"] = str(result.get("gatewayId") or gateway_id) os.environ["GATEWAY_RELAY_SECRET"] = str(result.get("secret") or "") os.environ["GATEWAY_RELAY_DELIVERY_KEY"] = str(result.get("deliveryKey") or "") @@ -765,11 +559,10 @@ def self_provision_relay() -> bool: ) return False - tenant = str(result.get("tenant") or "") logger.info( "relay self-provisioned (gateway_id=%s tenant=%s platforms=%s routes=%d inbound=%s instance=%s wake=%s)", os.environ.get("GATEWAY_RELAY_ID", gateway_id), - tenant or "?", + str(result.get("tenant") or "") or "?", ",".join(provisioned), len(route_keys), "yes" if endpoint else "outbound-only", @@ -779,30 +572,14 @@ def self_provision_relay() -> bool: return True -def _post_policy(*, policy_url: str, token: str, policy: dict, timeout: float = 15.0) -> int: - """POST the relevance policy to the connector's ``/relay/policy``; return the HTTP status. +def _post_policy(*, policy_url: str, token: str, policy: dict, timeout: float = _HTTP_TIMEOUT_S) -> int: + """POST the relevance policy to ``/relay/policy``; return the HTTP status. - Authenticated with the gateway's own per-gateway upgrade token (the SAME - bearer shape as the WS upgrade — ``make_upgrade_token``), so the connector - resolves ``{tenant, instanceId}`` from its stored secret record, never the - body. Raises RuntimeError on transport failure (the caller treats any - failure as non-fatal — relevance is an optimization, not a boot dependency). + Authenticated with the gateway's own upgrade token (``make_upgrade_token``), so + the connector resolves ``{tenant, instanceId}`` from its stored secret record, + never the body. Raises RuntimeError on transport failure. """ - import json - import urllib.error - import urllib.request - - data = json.dumps(policy).encode("utf-8") - req = urllib.request.Request( - policy_url, - data=data, - method="POST", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) + req = _json_post_request(policy_url, token, policy) try: with urllib.request.urlopen(req, timeout=timeout) as resp: return int(resp.status) @@ -813,45 +590,28 @@ def _post_policy(*, policy_url: str, token: str, policy: dict, timeout: float = def send_relay_policy() -> bool: - """Declare this gateway's relevance policy to the connector (Phase 6 Unit ζ). + """Declare this gateway's relevance policy to the connector, per fronted platform. - Runs at boot AFTER the per-gateway secret is resolved (self-provisioned or - pinned), projecting the agent's relevance config into the generic vocabulary - (``relay_relevance_policy``) and POSTing it to ``/relay/policy`` with the - gateway's own upgrade token. The connector stores it per-instance and the - relevance gate enforces it on delivery — so the SAME mention-gating / - free-response / allow-bots behavior the agent applies directly also governs - relay delivery, and excluded traffic never wakes a scaled-to-zero agent. + Runs at boot AFTER the per-gateway secret is resolved. The connector stores the + policy per-instance and enforces it on delivery, so the SAME mention-gating / + free-response / allow-bots behavior the agent applies directly also governs relay + delivery, and excluded traffic never wakes a scaled-to-zero agent. Re-declared + every boot (idempotent full replace). A platform with nothing non-default to + declare is skipped; a failed POST for one platform doesn't block the others. - Self-healing: the agent is the source of truth and re-declares every boot - (mirrors the ``routeKeys`` upsert at provision). Idempotent — a full replace. - - NEVER raises and NEVER blocks boot: relevance is an optimization layered on - the δ/ε authorization gate (which already protects isolation), so a failed - declaration just means the connector keeps the prior/quiet policy. Returns - True iff the connector accepted the policy (HTTP 200). + NEVER raises and NEVER blocks boot: relevance is an optimization layered on the + authorization gate, so a failed declaration just leaves the connector's prior + policy. Returns True iff the connector accepted at least one policy (HTTP 200). """ - import logging - - logger = logging.getLogger("gateway.relay") - dial_url = relay_url() if not dial_url: return False gateway_id, secret = relay_connection_auth() if not gateway_id or not secret: - # No resolved per-gateway secret (unenrolled / provision failed) ⇒ we - # can't authenticate the policy POST; skip quietly (the WS upgrade would - # be unauthenticated too, so there's no instance to attach a policy to). + # Can't authenticate the POST (and there's no instance to attach a policy to). return False - # Phase 1.5: declare a policy PER fronted platform — the connector keys the - # relevance policy by (tenant, platform, instanceId), so each platform this - # gateway fronts gets its own row. Per-platform, partial-tolerant: a platform - # with nothing non-default to declare is skipped; a failed POST for one - # platform doesn't block the others. A single-platform gateway declares one - # policy exactly as before. try: from gateway.relay.auth import make_upgrade_token @@ -864,8 +624,6 @@ def send_relay_policy() -> bool: for platform, _bot_id in relay_platform_identities(): policy = relay_relevance_policy(platform) if policy is None: - # Nothing non-default to declare for this platform ⇒ the connector's - # quiet default already matches; don't write a redundant row. continue try: status = _post_policy(policy_url=_policy_url(dial_url), token=token, policy=policy) @@ -896,13 +654,9 @@ def register_relay_adapter(force: bool = False, url: Optional[str] = None) -> bo """Register the generic ``relay`` platform via the platform registry. Registers when a relay URL is configured (or ``force=True`` for tests, which - builds a transport-less adapter — the unit-test posture). Returns True if - registration happened. Additive: uses the same registry path as plugin - adapters, so no core dispatch changes are needed. - - When a URL is present the factory builds a live ``WebSocketRelayTransport``; - the ``RelayAdapter`` negotiates the real ``CapabilityDescriptor`` at - ``connect()`` time via ``transport.handshake()``. + builds a transport-less adapter). Returns True if registration happened. With a + URL the factory builds a live ``WebSocketRelayTransport``; the adapter negotiates + the real ``CapabilityDescriptor`` at ``connect()`` via ``transport.handshake()``. """ resolved_url = url if url is not None else relay_url() if not (force or resolved_url): @@ -915,9 +669,8 @@ def register_relay_adapter(force: bool = False, url: Optional[str] = None) -> bo platform, bot_id = relay_platform_identity() def _factory(config): - # Placeholder descriptor; replaced by the negotiated one at connect time - # when a transport is present. With no URL (force/test) the adapter is - # transport-less and keeps the placeholder. + # Placeholder descriptor; replaced by the negotiated one at connect time. + # With no URL (force/test) the adapter is transport-less and keeps it. placeholder = CapabilityDescriptor( contract_version=CONTRACT_VERSION, platform=platform, @@ -938,18 +691,14 @@ def register_relay_adapter(force: bool = False, url: Optional[str] = None) -> bo resolved_url, platform, bot_id, - # Phase 1.5: the full SET of (platform, bot_id) this gateway fronts. - # The transport sends one hello per identity (the connector - # accumulates them) and resolves the per-frame egress botId from - # this set. A single-platform deploy passes a 1-element list, so - # behaviour is byte-identical to before. + # The full SET of identities: one hello per identity (the connector + # accumulates them) and the per-frame egress botId resolves from it. identities=relay_platform_identities(), gateway_id=gateway_id, upgrade_secret=upgrade_secret, - # Phase 5 §5.3: re-dial + re-handshake after an unexpected socket - # close so a gateway that went idle/suspended re-establishes its - # relay socket — which triggers the connector's buffered-flip drain - # (the delivery-leg onResume) on the new handshake. + # Re-dial + re-handshake after an unexpected close so a gateway that + # went idle re-establishes its socket, which triggers the connector's + # buffered-flip drain on the new handshake. reconnect=True, ) return RelayAdapter(config, placeholder, transport=transport) diff --git a/gateway/relay/transport.py b/gateway/relay/transport.py index 82470780ed..fa0a74597a 100644 --- a/gateway/relay/transport.py +++ b/gateway/relay/transport.py @@ -2,19 +2,11 @@ The ``RelayAdapter`` (gateway side) delegates all wire I/O to a ``RelayTransport``. The gateway dials OUT to the connector, so a production transport is a WebSocket -client; in tests it is an in-memory stub (``tests/gateway/relay/stub_connector.py``). - -This module defines the protocol surface only — no concrete transport. The -contract has four concerns: - - 1. Lifecycle: ``connect`` / ``disconnect``. - 2. Handshake: ``handshake`` returns the ``CapabilityDescriptor`` the connector - advertises for the platform this adapter fronts. - 3. Inbound: ``set_inbound_handler`` registers a callback the transport invokes - with each normalized ``MessageEvent`` the connector delivers. - 4. Outbound: ``send_outbound`` carries send/edit/typing actions back to the - connector; ``get_chat_info`` proxies a chat-info lookup; ``send_interrupt`` - routes a mid-turn /stop down the socket that owns the session_key. +client (``ws_transport.py``); in tests it is an in-memory stub +(``tests/gateway/relay/stub_connector.py``). This module defines the protocol +surface only: lifecycle (connect/disconnect), handshake (the advertised +``CapabilityDescriptor``), inbound (``set_inbound_handler``) and outbound +(``send_outbound`` / ``get_chat_info`` / ``send_interrupt``). EXPERIMENTAL: may change without a deprecation cycle until >=2 Class-1 platforms validate it. See docs/relay-connector-contract.md. @@ -30,11 +22,10 @@ from gateway.relay.descriptor import CapabilityDescriptor # Callback the transport invokes for each inbound normalized event. InboundHandler = Callable[[MessageEvent], Awaitable[None]] -# Callback the transport invokes for each forwarded passthrough request (§5.1). -# The first arg is a PassthroughForward (gateway/relay/ws_transport.py) — typed -# as Any here to keep this protocol module free of a concrete-transport import -# (ws_transport imports FROM this module). The second is an optional bufferId -# (Phase 5 §5.3 buffered flip) the handler acks after durable handoff. +# Callback for each forwarded passthrough request (§5.1): ``(forward, buffer_id)``. +# ``forward`` is a ws_transport.PassthroughForward, typed Any here because +# ws_transport imports FROM this module; ``buffer_id`` (§5.3 buffered flip) is +# acked by the handler after durable handoff. PassthroughHandler = Callable[[Any, Optional[str]], Awaitable[None]] @@ -59,14 +50,11 @@ class RelayTransport(Protocol): ... def set_passthrough_handler(self, handler: "PassthroughHandler") -> None: - """Register the callback invoked with each forwarded passthrough request. + """Register the callback for each forwarded passthrough request (§5.1). - Phase 5 §5.1: the passthrough plane (Discord interactions, Twilio, …) - answers the provider's edge ACK at the connector, then forwards the real - request to the gateway over this same outbound socket (a hosted gateway - has no public inbound port). The transport invokes ``handler(forward, - buffer_id)`` for each ``passthrough_forward`` frame. Optional on a - transport (an in-memory stub may not implement it). + The connector answers the provider's edge ACK itself, then forwards the + real request over this same outbound socket (a hosted gateway has no + public inbound port). Optional on a transport (a stub may not implement it). """ ... @@ -75,14 +63,10 @@ class RelayTransport(Protocol): ) -> Dict[str, Any]: """Carry an outbound action (send/edit/typing) to the connector. - Returns a result dict; for ``op == "send"`` it carries - ``success`` and optionally ``message_id`` / ``error``. - - ``platform`` (Phase 1.5) tags WHICH fronted platform this reply targets, - carried on the OutboundFrame envelope so a gateway fronting N platforms - egresses each reply through the right sender (the transport resolves the - matching advertised botId). Omitted ⇒ the connector falls back to the - session's default platform (single-platform deploys unchanged). + Returns a result dict; for ``op == "send"`` it carries ``success`` and + optionally ``message_id`` / ``error``. ``platform`` tags WHICH fronted + platform this reply targets (the transport resolves the matching botId); + omitted ⇒ the connector uses the session's default platform. """ ... @@ -93,24 +77,20 @@ class RelayTransport(Protocol): async def send_interrupt(self, session_key: str, reason: Optional[str] = None) -> None: """Route a mid-turn /stop to the connector for ``session_key``. - The connector forwards it down the socket owned by the gateway - instance running that session (the /stop routing invariant). On the - gateway side this is the OUTBOUND direction; the actual task - cancellation happens when the connector echoes an interrupt inbound - (handled in Task 1.4). + The connector forwards it down the socket owned by the gateway instance + running that session. This is the OUTBOUND direction; the actual task + cancellation happens when the connector echoes an interrupt inbound. """ ... async def go_idle(self, timeout_s: float = 10.0) -> bool: - """Ask the connector to flip this instance to buffered-only (Phase 5 §5.3). + """Ask the connector to flip this instance to buffered-only (§5.3). - Sends ``going_idle`` and awaits the connector's ``going_idle_ack`` — the - connector-authoritative confirmation that live delivery stopped and inbound - now buffers durably for replay on reconnect (Q-5.3c). Returns True on ack, - False on timeout / not-connected (the caller proceeds to close regardless; - without §5.3 wiring there is simply no buffering). Optional on a transport - (an in-memory stub may not implement it). Emitted as part of the gateway's - EXISTING drain transition — not a new idle path. + Sends ``going_idle`` and awaits the connector-authoritative + ``going_idle_ack`` (live delivery stopped; inbound now buffers for replay + on reconnect). Returns True on ack, False on timeout / not-connected — + the caller closes regardless. Optional on a transport. Emitted as part of + the gateway's EXISTING drain transition -- not a new idle path. """ ... @@ -119,25 +99,16 @@ class RelayTransport(Protocol): ) -> Dict[str, Any]: """Act on a shared-identity capability bound to a session (A2 outbound). - Some platforms hand the connector a credential that acts on the SHARED - bot identity (e.g. a Discord interaction follow-up token, valid ~15min). - Under A2 that credential NEVER reaches the gateway — the connector - stripped it at the edge and bound it in its capability vault keyed by - the session. To use it, the gateway issues a SEMANTIC action against the - session it is already in; it never names or holds a token. + Some platforms hand the connector a credential acting on the SHARED bot + identity (e.g. a Discord interaction follow-up token). Under A2 it NEVER + reaches the gateway: the connector binds it in its capability vault keyed + by session, and the gateway issues a SEMANTIC action against that session. - The action dict carries: - ``op`` == ``"follow_up"`` - ``session_key`` the session whose bound capability to wield - ``kind`` the capability kind (e.g. ``"discord.interaction_token"``) - ``content`` the message content to send via that capability - ``metadata?`` optional extras - - The connector resolves the real capability (``resolveOutboundCapability`` - on its side), enforces the tenant match (tenant B can never wield tenant - A's capability), and egresses. Returns ``{success, message_id?, error?}``; - ``success`` is False when the capability is absent/expired or the tenant - doesn't match — the gateway then has nothing to retry with (by design: a - leaked gateway holds zero capability material). + Action dict: ``op == "follow_up"``, ``session_key``, ``kind`` (e.g. + ``"discord.interaction_token"``), ``content``, optional ``metadata``. + The connector resolves the capability, enforces the tenant match, and + egresses. Returns ``{success, message_id?, error?}``; ``success`` is False + when the capability is absent/expired or the tenant mismatches — nothing to + retry with (by design: a leaked gateway holds zero capability material). """ ... diff --git a/gateway/relay/ws_transport.py b/gateway/relay/ws_transport.py index bba21cd171..416f5cbdc4 100644 --- a/gateway/relay/ws_transport.py +++ b/gateway/relay/ws_transport.py @@ -1,27 +1,15 @@ """Production WebSocket RelayTransport — the gateway's live link to the connector. -The gateway dials OUT to the connector's relay endpoint over a WebSocket and -speaks the newline-delimited JSON frame protocol defined in the connector repo -(``gateway-gateway`` ``src/relay/protocol.ts``) and mirrored in -``docs/relay-connector-contract.md``: +The gateway dials OUT to the connector's relay endpoint and speaks the +newline-delimited JSON frame protocol mirrored in ``docs/relay-connector-contract.md``: - gateway -> connector : hello, outbound, interrupt - connector -> gateway : descriptor, inbound, outbound_result, interrupt_inbound + gateway -> connector : hello, outbound, interrupt, going_idle, inbound_ack + connector -> gateway : descriptor, inbound, outbound_result, interrupt_inbound, + going_idle_ack, passthrough_forward -Frames: - hello {type, platform, botId} - descriptor {type, descriptor} (handshake reply) - inbound {type, event, bufferId?} (a normalized MessageEvent) - outbound {type, requestId, action} (send/edit/typing/follow_up) - outbound_result {type, requestId, result} - interrupt {type, session_key, reason?} (gateway egresses /stop) - interrupt_inbound{type, session_key, chat_id} (connector -> owning gateway) - -This is the concrete transport behind the ``RelayTransport`` Protocol; the -``RelayAdapter`` delegates all wire I/O to it. Outbound calls block on a -per-request future keyed by ``requestId`` until the matching ``outbound_result`` -arrives. A background reader task pumps inbound frames to the registered handler -and resolves pending outbound futures. +Outbound calls block on a per-request future keyed by ``requestId`` until the +matching ``outbound_result`` arrives; a background reader task pumps inbound +frames to the registered handler and resolves pending futures. EXPERIMENTAL: the frame schema may change without a deprecation cycle until at least two Class-1 platforms validate it. @@ -35,7 +23,7 @@ import logging import os import uuid from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Awaitable, Callable, Dict, List, Optional from gateway.platforms.base import MessageEvent, MessageType from gateway.session import SessionSource @@ -51,33 +39,31 @@ except ImportError: # pragma: no cover - exercised only when the extra is absen WEBSOCKETS_AVAILABLE = websockets is not None -# How long to wait for the handshake descriptor and for each outbound result. _HANDSHAKE_TIMEOUT_S = 30.0 _OUTBOUND_TIMEOUT_S = 30.0 -# Bound supervisor/reader/ws.close awaits so a wedged peer cannot stall -# adapter.disconnect. Three sequential awaits at 1.0s stay under the runner's -# default 5s adapter disconnect budget (plus the 2s go_idle ACK budget). +# Bound on each of the three sequential teardown awaits (supervisor, reader, +# ws.close) so a wedged peer cannot stall adapter.disconnect past the runner's +# default 5s budget. _TEARDOWN_AWAIT_TIMEOUT_S = 1.0 -# Bounded drain for in-flight outbound frames at disconnect: long enough for a +# Max drain for in-flight outbound frames at disconnect: long enough for a # platform edit round-trip through the connector, short enough that shutdown -# stays snappy when the connector is gone. The EFFECTIVE grace is clamped at -# disconnect time so drain + the three sequential teardown awaits stay inside -# the runner's adapter-disconnect budget (gateway/run.py wraps disconnect() in -# asyncio.wait_for; blowing that budget cancels teardown mid-drain, skips the -# fail-pending loop, and leaves callers blocked on _OUTBOUND_TIMEOUT_S). +# stays snappy when the connector is gone. Clamped at disconnect +# time so drain + teardown stay inside the runner's adapter-disconnect budget: +# blowing it cancels teardown mid-drain and leaves callers blocked on +# _OUTBOUND_TIMEOUT_S. _DISCONNECT_DRAIN_GRACE_S = 5.0 +# Private-use close code the connector sends when it rejects/revokes a gateway's +# WS upgrade auth. Received AFTER a successful handshake it means the per-gateway +# secret was revoked (opt-out / deprovision) — terminal, no reconnect. +_RELAY_UNAUTHORIZED_CLOSE_CODE = 4401 def _disconnect_drain_grace_s(budget_s: Optional[float] = None) -> float: - """Effective drain grace: clamped to the caller's disconnect budget. + """Effective drain grace: clamped to the caller's REMAINING disconnect budget. - ``budget_s`` is the REMAINING budget threaded down by the caller - (RelayAdapter.disconnect measures what go_idle and monitor teardown - already consumed). When None, mirrors - gateway/run.py:_adapter_disconnect_timeout_secs (env override with - the same variable, same default) rather than importing it — the - transport must stay importable without the gateway runner. Reserves - the three sequential teardown awaits plus a small margin. + When ``budget_s`` is None, mirrors the runner's env-driven default rather than + importing gateway/run.py (the transport must import without the runner). + Reserves the three sequential teardown awaits plus a small margin. """ budget = _env_disconnect_budget_s() if budget_s is None else max(0.0, budget_s) reserved = 3 * _TEARDOWN_AWAIT_TIMEOUT_S + 0.5 @@ -85,11 +71,10 @@ def _disconnect_drain_grace_s(budget_s: Optional[float] = None) -> float: def _env_disconnect_budget_s() -> float: - """The runner's adapter-disconnect budget, read the same way - gateway/run.py:_adapter_disconnect_timeout_secs reads it (same env - variable, same default). Callers above the transport use this to - apportion the budget across go_idle / monitor teardown / drain.""" - budget = 5.0 # _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT in gateway/run.py + """The runner's adapter-disconnect budget (same env var + default as + gateway/run.py:_adapter_disconnect_timeout_secs). Callers above the transport + use it to apportion the budget across go_idle / monitor teardown / drain.""" + budget = 5.0 raw = os.getenv("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT", "").strip() if raw: try: @@ -98,31 +83,13 @@ def _env_disconnect_budget_s() -> float: pass return budget -# Phase 7 Unit 7d-B: the application close code the connector sends when it -# rejects/revokes a gateway's WS upgrade auth (mirrors the connector's -# `4401` "unauthorized" close — a private-use code, not a standard WS code). -# A 4401 received AFTER a successful handshake means the per-gateway secret was -# revoked (opt-out / deprovision), which the transport treats as terminal. -_RELAY_UNAUTHORIZED_CLOSE_CODE = 4401 - def _ws_dial_url(url: str) -> str: - """Normalize a connector URL to the ``ws(s)://…/relay`` dial target. + """Normalize the configured connector BASE URL to the ``ws(s)://…/relay`` dial target. - The relay URL is configured once (``GATEWAY_RELAY_URL`` / ``gateway.relay_url``) - as the connector's BASE URL (e.g. ``https://connector.example``) and shared by - both the provision POST (which needs ``http(s)://…/relay/provision`` — see - ``_provision_url``) and the WS dial (which needs ``ws(s)://…/relay``, the path - the connector mounts its ``WebSocketServer`` on). Two normalizations, both - load-bearing: - - - scheme: ``https -> wss``, ``http -> ws`` (``websockets.connect`` raises - "scheme isn't ws or wss" on an http(s) URL). - - path: ensure it ends in ``/relay`` (the connector returns HTTP 400 on an - upgrade to any other path, since the WS server is mounted at ``/relay``). - - Idempotent: an already-``ws(s)://…/relay`` URL is returned unchanged, so a URL - configured WITH the scheme and/or ``/relay`` still works. + ``https -> wss`` / ``http -> ws`` (websockets.connect rejects http schemes) and + the path must end in ``/relay`` (the connector 400s any other upgrade path). + Idempotent on an already-normalized URL. """ raw = (url or "").strip() if raw.startswith("https://"): @@ -136,22 +103,12 @@ def _ws_dial_url(url: str) -> str: def _render_relay_context(context: Any) -> Optional[str]: - """Render the connector's read-only surrounding-context array into the string - ``MessageEvent.channel_context`` field. + """Flatten the connector's read-only ``context`` list (oldest→newest) into the + ``MessageEvent.channel_context`` string history-backfill already uses. - The connector attaches ``context`` as a list of normalized message objects - (oldest→newest, same channel) for an addressed turn on a context-capable - platform (design relay-channel-context). We flatten each to a - ``: `` line so it rides the SAME read-only injection path that - history-backfill already uses (run.py prepends ``channel_context`` ahead of - the trigger message). This is REFERENCE context only — it never triggers the - agent; the trigger decision was already made connector-side on the addressed - event alone. - - Returns None when there is no usable context (absent/empty list, or a - connector that doesn't send the field), so ``channel_context`` stays unset - and behaviour is byte-identical to today. Never raises — a malformed context - payload must not break inbound delivery of the (already-admitted) turn. + Reference context only — never triggers the agent. Returns None when there is + no usable context so ``channel_context`` stays unset. Never raises: a malformed + payload must not break delivery of the already-admitted turn. """ if not context or not isinstance(context, list): return None @@ -169,8 +126,7 @@ def _render_relay_context(context: Any) -> Optional[str]: lines.append(f"{author}: {text}" if author else str(text)) if not lines: return None - body = "\n".join(lines) - return f"[Recent channel messages]\n{body}" + return "[Recent channel messages]\n" + "\n".join(lines) def _normalize_slack_parent_command( @@ -178,8 +134,7 @@ def _normalize_slack_parent_command( message_type: MessageType, ) -> tuple[str, MessageType]: """Mirror native Slack ``/hermes`` routing for authenticated relay text.""" - stripped = text.strip() - parent_parts = stripped.split(maxsplit=1) + parent_parts = text.strip().split(maxsplit=1) if not parent_parts or parent_parts[0] != "/hermes": return text, message_type @@ -188,57 +143,37 @@ def _normalize_slack_parent_command( payload = parent_parts[1].strip() if len(parent_parts) > 1 else "" subcommand_map = slack_subcommand_map() subcommand_map["compact"] = "/compress" - payload_parts = payload.split() if payload else [] - first_word = payload_parts[0] if payload_parts else "" + first_word = payload.split()[0] if payload else "" if first_word in subcommand_map: rest = payload[len(first_word) :].strip() - normalized = ( - f"{subcommand_map[first_word]} {rest}".strip() - if rest - else subcommand_map[first_word] - ) - elif payload: - normalized = payload + normalized = f"{subcommand_map[first_word]} {rest}".strip() else: - normalized = "/help" + normalized = payload or "/help" - normalized_type = ( - MessageType.COMMAND if normalized.startswith("/") else MessageType.TEXT - ) + normalized_type = MessageType.COMMAND if normalized.startswith("/") else MessageType.TEXT return normalized, normalized_type def _media_types_from_wire(raw: Dict[str, Any]) -> list[str]: - """Per-attachment MIME types, aligned to ``media_urls`` BY URL. + """Per-attachment MIME types, parallel to ``media_urls``. - INVARIANT: the returned list is ALWAYS the same length as ``media_urls`` - (padded with ``""``), or empty when there are no urls. Consumers index the - two lists by the same ``i`` (``_event_media_type_at``) AND concatenate them - pairwise (``merge_pending_message_event`` extends both), so a short - ``media_types`` is not a harmless absence — on a merge it shifts every - subsequent entry onto the wrong url. - - Resolution is BY URL LOOKUP, never by position: ``media_urls`` (legacy, - flat) and ``media`` (rich, Phase 2) are independent wire fields that may - disagree in order, and a length check alone accepts a reordered pair — - attaching one attachment's MIME to another's url, which silently - mis-routes it. A url with no matching ``media[]`` entry keeps its slot as - ``""`` and falls back to message-level classification. + INVARIANT: always the same length as ``media_urls`` (padded with ``""``), or + empty when there are no urls — consumers index and concatenate the two lists + pairwise, so a short list would shift later entries onto the wrong url. + Resolved BY URL LOOKUP into ``media[]``, never by position: the two wire fields + are independent and may disagree in order. A url with no match keeps ``""`` and + falls back to message-level classification. """ urls = raw.get("media_urls") if not isinstance(urls, list) or not urls: return [] media = raw.get("media") - mime_by_url: dict[str, str] = {} - if isinstance(media, list): - for m in media: - if isinstance(m, dict): - url = m.get("url") - if isinstance(url, str) and url: - mime_by_url[url] = m.get("mime") or "" - # One slot per url, ALWAYS — even with no media[] at all (all ""), so the - # parallel-array invariant holds for every consumer. + mime_by_url: dict[str, str] = { + m["url"]: m.get("mime") or "" + for m in (media if isinstance(media, list) else []) + if isinstance(m, dict) and isinstance(m.get("url"), str) and m["url"] + } types = [mime_by_url.get(u, "") if isinstance(u, str) else "" for u in urls] missing = sum(1 for t in types if not t) if missing and mime_by_url: @@ -251,17 +186,15 @@ def _media_types_from_wire(raw: Dict[str, Any]) -> list[str]: def _event_from_wire(raw: Dict[str, Any]) -> MessageEvent: - """Rebuild a MessageEvent from the connector's normalized inbound payload. + """Rebuild a MessageEvent from the connector's normalized inbound payload (§3). - The connector emits SessionSource as the snake_case wire form (§3); map it - back onto the gateway dataclasses. Unknown message types fall back to TEXT. + Unknown platforms fall back to RELAY, unknown message types to TEXT. """ src = raw.get("source", {}) or {} from gateway.config import Platform - platform = src.get("platform", "relay") try: - platform_enum = Platform(platform) + platform_enum = Platform(src.get("platform", "relay")) except ValueError: platform_enum = Platform.RELAY @@ -271,12 +204,10 @@ def _event_from_wire(raw: Dict[str, Any]) -> MessageEvent: chat_type=src.get("chat_type", "dm"), chat_name=src.get("chat_name"), user_id=src.get("user_id"), - # Native adapters surface the human-facing DISPLAY name as user_name - # (e.g. Discord `message.author.display_name`); the connector sends the - # raw platform username as user_name plus optional user_display_name / - # user_handle enrichments (contract §3). Prefer the display name for - # parity with native lanes — session keys derive from user_id, never - # user_name, so this is presentation-only and key-stable. + # The connector sends the raw username as user_name plus optional + # user_display_name / user_handle enrichments; native adapters surface the + # DISPLAY name. Prefer the display name for parity with them. Session keys + # derive from user_id, never user_name, so this is presentation-only. user_name=( src.get("user_display_name") or src.get("user_name") @@ -289,34 +220,19 @@ def _event_from_wire(raw: Dict[str, Any]) -> MessageEvent: scope_id=src.get("scope_id"), parent_chat_id=src.get("parent_chat_id"), message_id=src.get("message_id"), - # The HERMES profile this event is routed to (multiplex mode). The - # connector stamps it on the wire source when NAS resolves the target - # profile for a Team-Gateway message; absent for a single-profile - # gateway, where it stays None and session keys keep the legacy - # ``agent:main`` namespace (SessionStore._resolve_profile_for_key). - # Consumed by build_session_key's profile namespacing + the per-turn - # config/credential scope — the same field the /p// HTTP - # prefix and per-credential polling adapters already set. + # Multiplex mode: the connector stamps the target Hermes profile; None on + # a single-profile gateway keeps the legacy ``agent:main`` namespace. profile=src.get("profile"), - # Auto-thread markers (Phase 4): stamped by the CONNECTOR when this - # event's thread was auto-created by its auto-thread egress policy. - # Lights the SAME semantic-rename lane native Discord uses - # (_is_discord_auto_thread_lane's relay-aware sibling reads these). + # Connector-stamped auto-thread markers; light the same semantic-rename + # lane native Discord uses. auto_thread_created=bool(src.get("auto_thread_created", False)), auto_thread_initial_name=src.get("auto_thread_initial_name"), - # Discord auto-thread session continuity: the connector stamps the - # thread id this channel message's reply WILL be auto-threaded into - # (== the message id) so the gateway keys the initiating channel message - # and its later in-thread follow-ups to ONE session. See - # build_session_key / SessionSource.prospective_thread_id. + # Thread id this channel message's reply WILL be auto-threaded into, so the + # initiating message and its in-thread follow-ups share ONE session. prospective_thread_id=src.get("prospective_thread_id"), - # Authentic upstream-trust signal: this event arrived over the - # per-instance-authenticated relay WS, so the connector already resolved - # it to this instance's owner-bound author. ``platform`` is the - # UNDERLYING platform (e.g. discord), not ``relay`` — authz keys the - # upstream-trust decision off THIS flag, not off ``platform`` (which - # would miss because the relay adapter is registered under - # ``Platform.RELAY``). Stamped here, never read off the wire. + # Stamped here, never read off the wire: this event arrived over the + # authenticated relay WS. Authz keys upstream trust off THIS flag, not + # ``platform`` (which is the UNDERLYING platform, not ``relay``). delivered_via_upstream_relay=True, ) try: @@ -326,74 +242,41 @@ def _event_from_wire(raw: Dict[str, Any]) -> MessageEvent: text = raw.get("text", "") if platform_enum == Platform.SLACK: - # Team Gateway carries Slack slash text over the authenticated message - # relay, bypassing Hermes' native Slack command callback. Normalize at - # the wire boundary so adapter-level active-session gates see the real + # Slack slash text arrives over the relay bypassing the native command + # callback; normalize at the wire boundary so adapter gates see the real # gateway command rather than the legacy `hermes` parent name. text, msg_type = _normalize_slack_parent_command(text, msg_type) + reply_to = raw.get("reply_to") or {} + prompt_response = raw.get("prompt_response") return MessageEvent( text=text, message_type=msg_type, source=source, message_id=raw.get("message_id"), reply_to_message_id=raw.get("reply_to_message_id"), - # Richer quoted-reply context (Phase 4): what the user replied TO, - # when the connector had it in hand (Discord referenced_message, - # Telegram reply_to_message, WhatsApp context + text cache). Maps to - # the SAME MessageEvent fields native adapters populate, so run.py's - # reply-context injection works identically over the relay. - reply_to_text=(raw.get("reply_to") or {}).get("text"), - reply_to_author_name=(raw.get("reply_to") or {}).get("author"), - reply_to_is_own_message=bool((raw.get("reply_to") or {}).get("is_own", False)), + reply_to_text=reply_to.get("text"), + reply_to_author_name=reply_to.get("author"), + reply_to_is_own_message=bool(reply_to.get("is_own", False)), media_urls=raw.get("media_urls") or [], - # Per-attachment MIME types, parallel to media_urls, from the - # connector's rich media[] array. run.py's per-attachment classifiers - # (_event_media_type_at) consult media_types[i] FIRST and only fall - # back to the message-level type when it's empty — so this mapping is - # what lets a relayed image/document/audio attachment route exactly - # like its native-adapter equivalent (and what makes a kind:"voice" - # attachment STT-eligible). Entries without a mime keep positional - # alignment with an empty string. - # - # media_urls and media[] are independent wire fields. Today's - # connector builds both from the same list, but a malformed or - # future producer could disagree — and a LENGTH MISMATCH would - # silently misassociate a MIME with the wrong URL (worse than no - # MIME at all: it mis-routes an attachment). Fail safe: map only - # when the two agree, otherwise leave media_types empty and let the - # message-level type drive classification (pre-fix behaviour). + # Parallel to media_urls; run.py's per-attachment classifiers consult + # media_types[i] FIRST (this is what routes a relayed image/document/voice + # attachment like its native equivalent, e.g. makes kind:"voice" STT-eligible). media_types=_media_types_from_wire(raw), - # Surrounding channel/group CONTEXT the connector attached for this - # addressed turn (design relay-channel-context): a read-only, oldest→ - # newest list of nearby non-addressed messages (Model A pull / Model B - # buffer). Rendered into the existing ``channel_context`` field — the - # same read-only injection path history-backfill already uses - # (run.py prepends it ahead of the trigger message). Absent / empty on a - # connector that doesn't send it, a dm, or a no-context platform, so - # this is purely additive and byte-identical to today when unset. channel_context=_render_relay_context(raw.get("context")), - # Structured interactive-prompt reply (Phase 3): carried verbatim off - # the wire when present ({prompt_id, option_id, label?, - # prompt_message_id?}). The RelayAdapter's inbound bridge consumes it - # to resolve pending approvals/confirms/clarifies; a gateway that - # predates the resolvers just sees the command-shaped text. - prompt_response=( - dict(raw["prompt_response"]) - if isinstance(raw.get("prompt_response"), dict) - else None - ), + # Structured interactive-prompt reply, verbatim off the wire; the adapter + # consumes it to resolve pending approvals/confirms/clarifies. + prompt_response=dict(prompt_response) if isinstance(prompt_response, dict) else None, ) @dataclass class PassthroughForward: - """A connector-forwarded passthrough-plane request (Phase 5 §5.1). + """A connector-forwarded passthrough-plane request (§5.1). The connector answered the provider's latency-critical ACK at its edge, then - forwarded the real (already-sanitized) request to this gateway over the WS. - ``body`` is the exact decoded bytes the connector forwarded (the wire carries - it base64-encoded for byte parity). ``headers`` preserve arrival order. + forwarded the sanitized request here. ``body`` is the exact decoded bytes + (base64 on the wire); ``headers`` preserve arrival order. """ platform: str @@ -402,38 +285,28 @@ class PassthroughForward: path: str headers: list[tuple[str, str]] body: bytes - # The HERMES profile this interaction is routed to (multiplex mode). - # Mirrors the ``profile`` field _event_from_wire already carries on the - # ``inbound`` frame's SessionSource (#60586) — the connector stamps it - # when NAS resolves the target profile for a Team-Gateway interaction; - # absent for a single-profile gateway, where it stays None and session - # keys keep the legacy ``agent:main`` namespace. Without this, a Discord - # slash-command/button/modal relayed through the passthrough plane always - # fell back to agent:main even when the equivalent plain message would - # have been routed to the correct profile. + # Multiplex-mode target profile, mirroring the ``profile`` on the inbound + # frame's SessionSource; None keeps the legacy ``agent:main`` namespace. + # Without it a relayed Discord slash-command/button/modal always fell back to + # agent:main even when the equivalent plain message routed to the right profile. profile: Optional[str] = None def _passthrough_from_wire(raw: Dict[str, Any]) -> PassthroughForward: - """Rebuild a PassthroughForward from the connector's wire frame. - - Mirrors the connector's ``PassthroughForward`` (relay/protocol.ts): the body - is base64-decoded back to the exact bytes the connector forwarded, so the - gateway re-processes byte-identical content (the connector is the trust - boundary; it already verified at the edge). - """ + """Rebuild a PassthroughForward from the wire frame (body base64-decoded to + the exact bytes the connector forwarded). No verification here: the connector + is the trust boundary and already verified at the edge.""" import base64 - body_b64 = raw.get("bodyB64", "") or "" try: - body = base64.b64decode(body_b64) + body = base64.b64decode(raw.get("bodyB64", "") or "") except Exception: # noqa: BLE001 - a malformed body must not crash the reader body = b"" - headers_raw = raw.get("headers", []) or [] - headers: list[tuple[str, str]] = [] - for pair in headers_raw: - if isinstance(pair, (list, tuple)) and len(pair) == 2: - headers.append((str(pair[0]), str(pair[1]))) + headers = [ + (str(pair[0]), str(pair[1])) + for pair in (raw.get("headers", []) or []) + if isinstance(pair, (list, tuple)) and len(pair) == 2 + ] return PassthroughForward( platform=str(raw.get("platform", "")), bot_id=str(raw.get("botId", "")), @@ -445,6 +318,14 @@ def _passthrough_from_wire(raw: Dict[str, Any]) -> PassthroughForward: ) +async def _await_bounded(aw: Awaitable[Any]) -> None: + """Best-effort teardown await: bounded, swallows timeout/cancel/errors.""" + try: + await asyncio.wait_for(aw, timeout=_TEARDOWN_AWAIT_TIMEOUT_S) + except (asyncio.TimeoutError, asyncio.CancelledError, Exception): # noqa: BLE001 + pass + + class WebSocketRelayTransport: """RelayTransport over a WebSocket connection the gateway dials to the connector.""" @@ -471,78 +352,49 @@ class WebSocketRelayTransport: self._url = _ws_dial_url(url) self._platform = platform self._bot_id = bot_id - # Phase 1.5 (Shape A): the full SET of (platform, bot_id) this gateway - # fronts on this one WS. The handshake sends one `hello` per identity so - # the connector accumulates them into its advertised set (gateway-gateway - # D-Q1.5b.1); the first identity (platform/bot_id above) is the default an - # untagged outbound falls back to. Defaults to the single (platform, bot_id) - # so existing single-platform callers are unchanged. + # Every (platform, bot_id) this gateway fronts on this one WS: one `hello` + # per identity; the first is the default an untagged outbound falls back to. self._identities = list(identities) if identities else [(platform, bot_id)] self._connect_timeout_s = connect_timeout_s self._outbound_timeout_s = outbound_timeout_s - # Connection auth (Phase 2): when a per-gateway secret is configured the - # gateway presents an HMAC bearer on the WS upgrade so the connector can - # authenticate it (reject 4401 otherwise). gateway_id identifies the - # enrolled instance — the connector peeks it to index its secret verify - # list, then verifies the signature. Absent -> unauthenticated upgrade - # (dev/test, or a connector that doesn't enforce auth). + # Upgrade auth: with a per-gateway secret the gateway presents an HMAC + # bearer keyed by gateway_id; absent -> unauthenticated upgrade. self._gateway_id = gateway_id self._upgrade_secret = upgrade_secret - # Phase 5 §5.3: a NET-NEW reconnect supervisor. The base transport's - # _read_loop just ends on socket close ("reconnection is caller policy"); - # with reconnect=True the transport re-dials + re-handshakes after an - # UNEXPECTED close (not a deliberate disconnect()), so a gateway that went - # idle/suspended re-establishes its socket — which makes the connector - # drain that instance's buffered-only delivery-leg backlog (onResume) on - # the new handshake. Off by default so existing tests + the stub are - # unaffected; register_relay_adapter turns it on in production. + # Reconnect supervisor: re-dial + re-handshake after an UNEXPECTED close + # (not disconnect()), which makes the connector drain this instance's + # buffered backlog on the new handshake. Off by default (stub/tests). self._reconnect = reconnect self._reconnect_backoff_s = reconnect_backoff_s self._reconnect_max_backoff_s = reconnect_max_backoff_s self._supervisor: Optional[asyncio.Task[None]] = None - # scale-to-zero §Phase 0 (D12/F14): a DORMANT close is distinct from both - # disconnect() (terminal: cancels the supervisor) and an unexpected close - # (re-dials immediately). go_dormant() sets this True, then closes the - # socket WITHOUT setting _closing — so _read_loop's fall-through still - # kicks the reconnect supervisor (the wake path stays armed), but the - # supervisor waits on the longer dormant cadence instead of the fast - # reconnect backoff, so it does not fight the platform's suspend window. - # On resume (process unfrozen) the pending wait completes, the re-dial - # succeeds, and the connector drains this instance's buffered backlog on - # the new handshake. Cleared on a successful re-dial (_dial_and_start). + # Dormant close (go_dormant) is distinct from disconnect() (terminal) and + # an unexpected close (fast re-dial): the socket closes WITHOUT _closing, + # so the reader still arms the supervisor, but it polls on the dormant + # cadence so it does not fight the platform's suspend window. Cleared on + # a successful re-dial. self._dormant = False - # The re-dial poll cadence while dormant. A suspended machine's event - # loop is frozen, so this timer only advances once the machine is awake; - # it just needs to be short enough that a freshly-woken machine re-dials - # promptly (the connector's wake poke is what triggers the platform - # autostart in the first place — §3.4(5)). + # A suspended machine's event loop is frozen, so this timer only advances + # once awake; it just needs to re-dial promptly after wake. self._dormant_redial_s = 1.0 self._ws: Any = None self._reader: Optional[asyncio.Task[None]] = None self._inbound: Optional[InboundHandler] = None + self._interrupt_inbound_handler: Any = None + self._passthrough_handler: Any = None + # `_descriptor` is the FIRST (primary-identity) descriptor; the map holds + # one per hello'd identity, keyed by platform (descriptor_for_platform). self._descriptor: Optional[CapabilityDescriptor] = None - # Phase 1.5 multi-platform: descriptors keyed by the underlying platform - # (one per hello'd identity). `_descriptor` above stays the FIRST - # (primary-identity) descriptor for back-compat; this map is the - # per-platform capability surface read via `descriptor_for_platform`. self._descriptors_by_platform: Dict[str, CapabilityDescriptor] = {} self._descriptor_ready: asyncio.Future[CapabilityDescriptor] | None = None - # requestId -> future awaiting the matching outbound_result. self._pending: Dict[str, asyncio.Future[Dict[str, Any]]] = {} - # Phase 5 §5.3: future awaiting the connector's going_idle_ack. self._going_idle_ack: asyncio.Future[None] | None = None self._closing = False - # Phase 7 Unit 7d-B: a 4401 (unauthorized) close AFTER we have already - # handshaked successfully at least once means the connector REVOKED this - # gateway's per-gateway secret — i.e. the operator opted this instance - # OUT of the relay (Unit 7b deprovision). That is TERMINAL: the secret is - # gone, so re-dialing just spins against a dead credential forever - # (the "retrying 4401" the dashboard showed). We stop reconnecting and - # surface it as a clean, non-retryable "disabled" state. A 4401 BEFORE - # any successful handshake stays retryable — that's a cold-start / - # not-yet-provisioned race, not a revocation. + # A 4401 close AFTER at least one successful handshake means the connector + # REVOKED this gateway's secret (opt-out): terminal, stop reconnecting. + # A 4401 BEFORE any handshake is a cold-start race and stays retryable. self._handshake_succeeded = False self._auth_revoked = False @@ -552,72 +404,44 @@ class WebSocketRelayTransport: return True async def _dial_and_start(self) -> None: - """Open the socket, start the reader, send hello. Used by connect() and + """Open the socket, start the reader, send hello(s). Used by connect() and by the reconnect supervisor on a re-dial.""" - loop = asyncio.get_running_loop() - self._descriptor_ready = loop.create_future() - # A fresh handshake is coming; clear any stale descriptor so handshake() - # awaits the new one (matters on a re-dial). The per-platform map resets - # with it — a reconnected connector re-sends one descriptor per hello. + self._descriptor_ready = asyncio.get_running_loop().create_future() + # Fresh handshake generation: a reconnected connector re-sends one + # descriptor per hello, so stale descriptors must not survive. self._descriptor = None self._descriptors_by_platform = {} - # scale-to-zero (D12): a successful (re-)dial ends any dormant state — we - # are live again, so a subsequent UNEXPECTED close should reconnect on the - # normal fast backoff, not the dormant cadence. + # A successful (re-)dial ends any dormant state. self._dormant = False + # WAN-friendly keepalive: the library default (20s pong deadline) produces + # spurious `1011 keepalive ping timeout` closes under transient latency / + # event-loop stalls; 60s tolerates them while detecting a dead link ~90s. + kwargs: Dict[str, Any] = {"ping_interval": 30, "ping_timeout": 60} headers = self._upgrade_headers() - # WAN-friendly keepalive: customer gateways cross WAN paths to the - # connector; the websockets library default (ping_interval=20, - # ping_timeout=20) gives the peer only a 20s pong deadline, which - # produces spurious `1011 keepalive ping timeout` closes under - # transient latency / event-loop stalls (Coatue incident 2026-08-18). - # ping_timeout=60 tolerates such stalls while still detecting a dead - # link within ~90s worst case (30s interval + 60s pong deadline). if headers: - self._ws = await websockets.connect( # type: ignore[union-attr] - self._url, - additional_headers=headers, - ping_interval=30, - ping_timeout=60, - ) - else: - self._ws = await websockets.connect( # type: ignore[union-attr] - self._url, - ping_interval=30, - ping_timeout=60, - ) + kwargs["additional_headers"] = headers + self._ws = await websockets.connect(self._url, **kwargs) # type: ignore[union-attr] self._reader = asyncio.create_task(self._read_loop(), name="relay-ws-reader") - # Send one hello PER fronted identity (Phase 1.5 Shape A). The connector - # accumulates them into its advertised set (the first sets the session - # default; each adds to the egress-allowed set). A single-platform gateway - # sends exactly one hello — byte-identical to before. The descriptor for - # the FIRST identity resolves handshake(); later descriptors are absorbed. + # One hello PER fronted identity; the connector accumulates them (first + # sets the session default). The FIRST descriptor resolves handshake(). for platform, bot_id in self._identities: hello: Dict[str, Any] = {"type": "hello", "platform": platform, "botId": bot_id} - # Phase 4: declare the gateway's slash-command set on the Discord - # hello. The connector (which holds the bot token) reconciles - # Discord's global registration against it — idempotent, detached, - # best-effort on its side; a connector predating the field ignores - # it (additive). Only Discord has an app-command registry. + # Declare the slash-command set on the Discord hello so the connector + # (which holds the bot token) reconciles Discord's registration. + # Enrichment only — never blocks the handshake; a connector predating the + # field ignores it. Only Discord has an app-command registry. if platform == "discord": try: from gateway.relay.command_manifest import build_relay_command_manifest hello["command_manifest"] = build_relay_command_manifest() - except Exception: # noqa: BLE001 - manifest is enrichment, never blocks the handshake + except Exception: # noqa: BLE001 logger.debug("relay command manifest build failed", exc_info=True) await self._send(hello) def _upgrade_headers(self) -> Dict[str, str]: - """Auth headers for the WS upgrade, or {} when no secret is configured. - - Presents ``Authorization: Bearer *** where the token is a signed - bearer built with the per-gateway secret (``gateway/relay/auth.py`` - ``make_upgrade_token``), keyed by ``gateway_id`` so the connector can - index its verify list. The connector rejects the upgrade (close 4401) - when this is missing/invalid/revoked; an unauthenticated connector - ignores it. - """ + """``Authorization: Bearer `` for the WS upgrade, or {} when + no secret is configured (the connector closes 4401 on a bad/missing one).""" if not (self._upgrade_secret and self._gateway_id): return {} from gateway.relay.auth import make_upgrade_token @@ -628,66 +452,50 @@ class WebSocketRelayTransport: async def disconnect(self, *, budget_s: Optional[float] = None) -> None: """Tear down the socket, draining in-flight outbound frames first. - ``budget_s`` is the REMAINING wall-clock budget the caller can spend - here (RelayAdapter.disconnect threads it down after go_idle / monitor - teardown). When None, the env-mirrored runner default applies. + ``budget_s`` is the REMAINING wall-clock budget the caller can spend here; + None applies the env-mirrored runner default. """ self._closing = True try: - # Drain grace: a trailing outbound frame (typically the turn's - # finalize edit) may still be awaiting its outbound_result. Failing - # it immediately loses a message the connector was about to ack — - # staging incident 2026-08-09 froze a Slack reply at its preview - # snapshot exactly this way. Give in-flight requests a short bounded - # window to resolve before tearing the socket down. + # A trailing outbound frame (typically the turn's finalize edit) may + # still await its outbound_result; failing it immediately loses a + # message the connector was about to ack. asyncio.wait (not + # wait_for+gather): on timeout it must NOT cancel the futures — the + # finally below owns their terminal state. pending = [f for f in self._pending.values() if not f.done()] if pending: - _grace = _disconnect_drain_grace_s(budget_s) - if _grace > 0: + grace = _disconnect_drain_grace_s(budget_s) + if grace > 0: try: - # asyncio.wait (not wait_for+gather): on timeout it must NOT - # cancel the futures — the fail-any-remaining loop below owns - # their terminal state. - await asyncio.wait(pending, timeout=_grace) + await asyncio.wait(pending, timeout=grace) except Exception: # noqa: BLE001 - grace is best-effort pass - if self._supervisor is not None: - self._supervisor.cancel() - try: - await asyncio.wait_for( - self._supervisor, timeout=_TEARDOWN_AWAIT_TIMEOUT_S - ) - except (asyncio.TimeoutError, asyncio.CancelledError, Exception): # noqa: BLE001 - best-effort teardown - pass - self._supervisor = None - if self._reader is not None: - self._reader.cancel() - try: - await asyncio.wait_for(self._reader, timeout=_TEARDOWN_AWAIT_TIMEOUT_S) - except (asyncio.TimeoutError, asyncio.CancelledError, Exception): # noqa: BLE001 - best-effort teardown - pass - self._reader = None + for attr in ("_supervisor", "_reader"): + task = getattr(self, attr) + if task is not None: + task.cancel() + await _await_bounded(task) + setattr(self, attr, None) if self._ws is not None: - try: - await asyncio.wait_for(self._ws.close(), timeout=_TEARDOWN_AWAIT_TIMEOUT_S) - except (asyncio.TimeoutError, asyncio.CancelledError, Exception): # noqa: BLE001 - pass - finally: - self._ws = None + await _await_bounded(self._ws.close()) + self._ws = None finally: - # Fail any in-flight outbound waiters so callers don't hang. - # Runs in a finally so a cancellation landing anywhere in the - # drain/teardown above (the runner's wait_for budget, an outer - # cleanup deadline) can NEVER leave a registered future - # unresolved — a stranded waiter would otherwise block until - # _OUTBOUND_TIMEOUT_S (30s). Idempotent: done futures are - # skipped, so a second disconnect() pass is safe. - for fut in self._pending.values(): - if not fut.done(): - fut.set_exception(RuntimeError("relay transport closed")) - self._pending.clear() + # Fail in-flight waiters so callers don't hang. In a finally so a + # cancellation landing anywhere above (runner budget, outer cleanup + # deadline) can NEVER leave a registered future unresolved for the + # full _OUTBOUND_TIMEOUT_S. Idempotent: done futures are skipped. + closed = RuntimeError("relay transport closed") + self._fail_pending(lambda fut: fut.set_exception(closed)) if self._going_idle_ack is not None and not self._going_idle_ack.done(): - self._going_idle_ack.set_exception(RuntimeError("relay transport closed")) + self._going_idle_ack.set_exception(closed) + + def _fail_pending(self, settle: Callable[[asyncio.Future[Dict[str, Any]]], None]) -> None: + """Settle every unresolved pending future via ``settle`` and clear the map. + list() snapshot: settling wakes waiters whose finally-pop mutates the dict.""" + for fut in list(self._pending.values()): + if not fut.done(): + settle(fut) + self._pending.clear() async def handshake(self) -> CapabilityDescriptor: if self._descriptor is not None: @@ -697,28 +505,27 @@ class WebSocketRelayTransport: return await asyncio.wait_for(self._descriptor_ready, timeout=self._connect_timeout_s) def descriptor_for_platform(self, platform: str) -> Optional[CapabilityDescriptor]: - """The negotiated descriptor for one fronted platform, or None. - - Phase 1.5 multi-platform: the connector replies one descriptor per - hello'd identity; they accumulate here keyed by the descriptor's own - ``platform`` field. Callers (RelayAdapter) use this to resolve PER-CHAT - capabilities — e.g. Discord's 2000-char max_message_length vs - Telegram's 4096 — instead of applying the primary identity's scalar - descriptor to every platform this gateway fronts. - """ + """The negotiated descriptor for one fronted platform (per-chat caps, e.g. + Discord's 2000 vs Telegram's 4096 max_message_length), or None.""" return self._descriptors_by_platform.get(platform) @property def auth_revoked(self) -> bool: - """True once the connector closed the socket with 4401 AFTER a prior - successful handshake — i.e. the per-gateway secret was revoked (the - operator opted this instance out of the relay). Terminal: the transport - stops reconnecting, and the adapter surfaces a clean "disabled" state.""" + """True once the connector closed 4401 AFTER a successful handshake — the + per-gateway secret was revoked. Terminal: no reconnect.""" return self._auth_revoked def set_inbound_handler(self, handler: InboundHandler) -> None: self._inbound = handler + def set_interrupt_inbound_handler(self, handler: Any) -> None: + """Register the callback for connector->gateway interrupt_inbound frames.""" + self._interrupt_inbound_handler = handler + + def set_passthrough_handler(self, handler: Any) -> None: + """Register ``handler(forward, buffer_id)`` for passthrough_forward frames (§5.1).""" + self._passthrough_handler = handler + # ── outbound ───────────────────────────────────────────────────────── async def send_outbound( self, action: Dict[str, Any], *, platform: Optional[str] = None @@ -728,27 +535,22 @@ class WebSocketRelayTransport: async def send_follow_up( self, action: Dict[str, Any], *, platform: Optional[str] = None ) -> Dict[str, Any]: - # follow_up rides the same outbound frame; the connector dispatches by - # action.op. Kept as a distinct method to satisfy the transport Protocol - # and to make the A2 call site explicit. + # Same outbound frame; the connector dispatches by action.op. Kept as a + # distinct method to satisfy the transport Protocol. return await self._request_response(action, platform=platform) def _bot_id_for(self, platform: Optional[str]) -> Optional[str]: - """The bot_id this transport advertised at hello for ``platform`` (Phase 1.5). + """The bot_id hello'd for ``platform``, or None when we don't front it. - The connector validates a per-frame egress target against the SET of - ``platform:botId`` pairs it accumulated from the N hellos, so a per-frame - ``platform`` must ride with its MATCHING ``botId`` (the session default - botId belongs to the first identity and would mis-key for a second - platform). Resolved from the identity set this transport was built with. - None when the platform isn't one we front (the connector then rejects it - with a structured failure — never a wrong-credential send).""" + A per-frame ``platform`` must ride with its MATCHING botId: the connector + validates against its accumulated ``platform:botId`` set, and the session + default botId belongs to the first identity only. For a platform we don't + front the connector rejects the frame with a structured failure — never a + wrong-credential send. + """ if not platform: return None - for p, b in self._identities: - if p == platform: - return b - return None + return next((b for p, b in self._identities if p == platform), None) async def get_chat_info(self, chat_id: str) -> Dict[str, Any]: result = await self._request_response( @@ -761,85 +563,52 @@ class WebSocketRelayTransport: async def send_interrupt(self, session_key: str, reason: Optional[str] = None) -> None: await self._send({"type": "interrupt", "session_key": session_key, "reason": reason}) - # ── going-idle / buffered-flip (Phase 5 §5.3) ──────────────────────── + # ── going-idle / buffered-flip (§5.3) ──────────────────────────────── async def go_idle(self, timeout_s: float = 10.0) -> bool: - """Ask the connector to flip this instance's destination to buffered-only. + """Ask the connector to flip this instance to buffered-only. - Sends ``going_idle`` and awaits the connector's ``going_idle_ack`` — the - connector-AUTHORITATIVE confirmation that live delivery has stopped and - subsequent inbound buffers durably (Q-5.3c). Returns True on ack, False on - timeout / not-connected (the caller proceeds to close anyway — at worst a - live event races a closing socket exactly as before §5.3, no regression). - - The gateway stays serving (the read loop keeps handling inbound) until the - ack, so an event landing in the flip window is delivered live, not lost. + Awaits the connector-AUTHORITATIVE ``going_idle_ack``. Returns False on + timeout / not-connected (the caller closes anyway). The read loop keeps + serving until the ack, so an event landing in the flip window is delivered + live, not lost. """ if self._ws is None: return False - loop = asyncio.get_running_loop() - self._going_idle_ack = loop.create_future() + self._going_idle_ack = asyncio.get_running_loop().create_future() try: await self._send({"type": "going_idle"}) await asyncio.wait_for(self._going_idle_ack, timeout=timeout_s) return True - except (asyncio.TimeoutError, Exception): # noqa: BLE001 - ack is best-effort + except Exception: # noqa: BLE001 - ack is best-effort return False finally: self._going_idle_ack = None async def go_dormant(self, timeout_s: float = 10.0) -> bool: - """Quiesce this transport for a scale-to-zero suspend (D12 / Phase 0). + """Quiesce for a scale-to-zero suspend: go_idle, then close the socket + WITHOUT setting ``_closing``. - Distinct from BOTH ``disconnect()`` and an unexpected close (F14): - - ``disconnect()`` sets ``_closing=True`` and CANCELS the reconnect - supervisor — terminal, "shutting down for good." A machine suspended - after that never re-dials on wake, so its buffered backlog strands. - - An unexpected close re-dials IMMEDIATELY (fast backoff) — the socket - never stays down, so the platform proxy never sees the connection go - away and never suspends the machine. - - ``go_dormant()`` is the third mode the suspend behaviour needs: - 1. ``go_idle()`` → the connector flips this instance to buffered-only - and acks (so inbound that arrives while we sleep buffers durably and - replays on the next handshake). - 2. Close the socket so the platform proxy sees load drop to zero (the - precondition for Fly ``autostop:"suspend"``) — but WITHOUT setting - ``_closing``. The reader's normal end-of-socket fall-through still - arms the reconnect supervisor, so the wake path stays live; the - ``_dormant`` flag just makes that supervisor poll on the dormant - cadence rather than fight the suspend window. - - On resume (process unfrozen) the supervisor's pending wait completes, the - re-dial succeeds, and the connector drains the buffered backlog on the new - handshake. Returns the ``go_idle`` ack result (True on ack); the dormancy - close happens regardless (a missed ack at worst races one live event onto - a closing socket, exactly as §5.3 already tolerates). - - No-op-safe: a transport that never connected (``_ws is None``) just - returns False without closing. + disconnect() cancels the supervisor (never re-dials on wake, stranding the + backlog); an unexpected close re-dials immediately (the platform proxy + never sees load drop, never suspends). Here the reader's fall-through still + arms the supervisor, which polls on the dormant cadence; on resume the + re-dial makes the connector drain the buffered backlog. Returns the go_idle + ack result; the close happens regardless. No-op (False) when never connected. """ if self._ws is None: return False acked = await self.go_idle(timeout_s=timeout_s) - # Mark dormant BEFORE closing so the supervisor (armed by the reader's - # fall-through) takes the dormant cadence, and a racing live event can't - # flip us back to a fast reconnect. + # Mark dormant BEFORE closing so the supervisor takes the dormant cadence. self._dormant = True try: - await asyncio.wait_for( - self._ws.close(), timeout=_TEARDOWN_AWAIT_TIMEOUT_S - ) - except (asyncio.TimeoutError, Exception): # noqa: BLE001 - best-effort; the reader still ends + arms reconnect + await asyncio.wait_for(self._ws.close(), timeout=_TEARDOWN_AWAIT_TIMEOUT_S) + except Exception: # noqa: BLE001 - best-effort; the reader still ends + arms reconnect logger.debug("relay go_dormant: ws.close() raised or timed out", exc_info=True) return acked async def _send_inbound_ack(self, buffer_id: str) -> None: - """Acknowledge durable receipt of a buffered inbound delivery (§5.3). - - Sent after the adapter has durably taken a buffered inbound event the - connector replayed on reconnect; the connector acks the buffer entry only - after this, giving drain-without-dup on the delivery leg. - """ + """Ack durable receipt of a replayed buffered inbound; the connector only + advances its buffer cursor after this (drain-without-dup).""" try: await self._send({"type": "inbound_ack", "bufferId": buffer_id}) except Exception: # noqa: BLE001 - a failed ack just redelivers the entry next time @@ -852,25 +621,19 @@ class WebSocketRelayTransport: *, platform: Optional[str] = None, ) -> Dict[str, Any]: + # Fail fast during teardown: the disconnect() fail-pending loop may already + # have run, so a future registered now would never be settled. if self._closing: - # Teardown in progress: the disconnect() fail-pending loop may - # already have run, so a future registered now would never be - # resolved or failed — the caller would block the full - # _OUTBOUND_TIMEOUT_S for a socket that is going away. Fail fast. return {"success": False, "error": "relay transport closed"} if self._ws is None: return {"success": False, "error": "relay transport not connected"} request_id = uuid.uuid4().hex - loop = asyncio.get_running_loop() - fut: asyncio.Future[Dict[str, Any]] = loop.create_future() + fut: asyncio.Future[Dict[str, Any]] = asyncio.get_running_loop().create_future() self._pending[request_id] = fut frame: Dict[str, Any] = {"type": frame_type, "requestId": request_id, "action": action} - # Phase 1.5: tag the per-frame egress platform on the OutboundFrame - # envelope (gateway-gateway D-Q1.5b.1), with its MATCHING advertised botId - # so the connector's `${platform}:${botId}` advertised-set check passes. - # Only set when a concrete platform was resolved for this chat so a - # single-platform gateway emits the exact frame shape as before (the - # connector falls back to the session's default platform when absent). + # Tag the egress platform with its MATCHING advertised botId only when a + # concrete platform was resolved, so a single-platform gateway emits the + # exact frame shape as before (connector falls back to session default). if platform: frame["platform"] = platform bot_id = self._bot_id_for(platform) @@ -882,37 +645,19 @@ class WebSocketRelayTransport: frame_sent = True return await asyncio.wait_for(fut, timeout=self._outbound_timeout_s) except asyncio.TimeoutError: - # AMBIGUOUS by contract (PR 85796 review): the frame reached the - # wire — only the acknowledgement is missing. The connector may - # well have applied it (draft frame appended, stream sealed). - # Consumers that need to distinguish "connector rejected this" - # from "outcome unknown" key on this flag; the fail-fast paths - # above (closing / not connected) never sent anything and are - # definite non-delivery, so they stay unmarked. - return { - "success": False, - "error": "relay outbound timed out", - "ambiguous": True, - } + # AMBIGUOUS by contract: the frame reached the wire and only the ack is + # missing — the connector may have applied it. The fail-fast paths + # above never sent anything (definite non-delivery) and stay unmarked. + return {"success": False, "error": "relay outbound timed out", "ambiguous": True} except Exception as exc: # noqa: BLE001 - a dead socket is a failed send, not a raise - # No `is None` check can close the window where the socket dies - # BETWEEN the liveness guard above and the actual write — the - # reader's finally hasn't cleared _ws yet, so _send raises - # ConnectionClosed straight into callers whose contract is a - # result dict (RelayAdapter.send consumes it with no try). - # Report it like every other failed send. CancelledError is a - # BaseException, so cancellation still propagates. - # - # Ambiguity contract (PR 85796): a raise from the WRITE means the - # frame never reached the wire — definite non-delivery, no flag. - # A failure surfaced by the FUTURE (e.g. disconnect() failing - # pending mid-flight) means the frame WAS sent and only the - # outcome is unknown — mark it ambiguous like the timeout above. + # The socket can die between the liveness guard and the write, so + # _send may raise into callers whose contract is a result dict. + # A raise from the WRITE = frame never sent (no flag); a failure + # surfaced by the FUTURE (disconnect failing pending mid-flight) = + # frame sent, outcome unknown -> ambiguous. CancelledError still + # propagates (BaseException). logger.debug("relay %s send failed", frame_type, exc_info=True) - result: Dict[str, Any] = { - "success": False, - "error": f"relay send failed: {exc}", - } + result: Dict[str, Any] = {"success": False, "error": f"relay send failed: {exc}"} if frame_sent: result["ambiguous"] = True return result @@ -926,18 +671,14 @@ class WebSocketRelayTransport: await self._ws.send(json.dumps(frame) + "\n") async def _read_loop(self) -> None: - # Bind the socket this reader serves: the finally below must only - # clear _ws if it still points at THIS socket (a supervisor re-dial - # may have already installed a fresh one by the time we unwind). + # Bind the socket this reader serves: the finally must only clear _ws if + # it still points at THIS socket (a re-dial may have installed a fresh one). ws = self._ws buf = "" try: if ws is None: - # Scheduled without a socket (a lifecycle bug, not a normal - # path). The old `assert` here escaped BEFORE the finally - # existed to fail pending futures — the one exit that could - # still strand waiters for the full outbound timeout. Fall - # through to the finally instead; it settles them all. + # Lifecycle bug, not a normal path. Fall through to the finally so + # pending waiters are still settled (an assert here stranded them). logger.error("relay ws read loop started with no socket") return try: @@ -948,14 +689,7 @@ class WebSocketRelayTransport: for line in lines: if line.strip(): await self._handle_frame(line) - except asyncio.CancelledError: - raise except Exception as exc: # noqa: BLE001 - log + let the task end; reconnection handled below - # Phase 7 Unit 7d-B: detect a 4401 (unauthorized) close. After a prior - # successful handshake this is a REVOCATION (opt-out / deprovision) — - # the per-gateway secret is gone, so reconnecting is futile. Latch a - # terminal "auth revoked" state and DON'T re-dial. Before any - # successful handshake a 4401 stays retryable (cold-start race). if self._close_code_of(exc) == _RELAY_UNAUTHORIZED_CLOSE_CODE and self._handshake_succeeded: self._auth_revoked = True if not self._closing: @@ -965,14 +699,9 @@ class WebSocketRelayTransport: ) elif not self._closing: logger.warning("relay ws read loop ended: %s", exc) - # Phase 5 §5.3: the socket closed. If reconnect is enabled and this was - # NOT a deliberate disconnect(), kick the reconnect supervisor so the - # gateway re-dials + re-handshakes (which triggers the connector's - # buffered-flip drain on the new handshake). Self-scheduling: the reader - # ends here, the supervisor re-dials and starts a fresh reader. - # Phase 7 Unit 7d-B: a revoked credential (terminal 4401) is the one case - # we deliberately do NOT reconnect — the secret is dead until the - # instance is recreated, so spinning would just reproduce the failure. + # Socket closed. Unless this was a deliberate disconnect() or a terminal + # revocation (re-dialing a dead credential just spins), arm the + # supervisor: it re-dials and starts a fresh reader. if ( self._reconnect and not self._closing @@ -983,158 +712,117 @@ class WebSocketRelayTransport: self._reconnect_loop(), name="relay-ws-reconnect" ) finally: - # The socket this reader served is dead. Drop the handle (identity- - # guarded: a re-dial that already installed a FRESH socket must not - # be clobbered) so every `self._ws is None` liveness check — send, - # _request_response, go_idle, go_dormant — reports "not connected" - # for the whole outage. Without this, _ws kept pointing at the dead - # socket on every reader exit that arms NO supervisor (terminal - # 4401 revocation, reconnect=False transports), and a send there - # registered a future nothing could resolve: a full - # _outbound_timeout_s (~30s) wedge — including the revocation - # path's own fatal-error notification. disconnect() owns the - # handle during deliberate teardown, so leave it alone then. + # Drop the dead handle (identity-guarded) so every `_ws is None` + # liveness check reports "not connected" for the whole outage — on + # exits that arm NO supervisor (terminal 4401, reconnect=False) a send + # would otherwise register a future nothing can resolve. disconnect() + # owns the handle during deliberate teardown. if self._ws is ws and not self._closing: self._ws = None - # The reader is the ONLY thing that can resolve a pending - # outbound_result future — once it exits (socket dropped, error, - # or cancellation cleanup) every in-flight _request_response waiter - # is unresolvable and would otherwise block the full - # _outbound_timeout_s (~30s) on a dead socket (Coatue incident - # 2026-08-18: stuck sends after a 1011 keepalive close). Fail them - # NOW with the dict shape callers expect (never an exception on - # the outbound path). list() snapshot: set_result wakes waiters - # whose finally-pop would otherwise mutate the dict mid-iteration. - for _rid, fut in list(self._pending.items()): - if not fut.done(): - fut.set_result( - {"success": False, "error": "relay transport connection lost"} - ) - self._pending.clear() + # The reader is the ONLY thing that resolves pending futures; once it + # exits every waiter would block the full outbound timeout. Fail them + # NOW with the dict shape callers expect (never an exception). + self._fail_pending( + lambda fut: fut.set_result( + {"success": False, "error": "relay transport connection lost"} + ) + ) @staticmethod def _close_code_of(exc: BaseException) -> Optional[int]: - """Best-effort extraction of a WebSocket close code from a raised - exception. websockets' ConnectionClosed* expose the peer's Close frame - via `.rcvd`/`.sent` (preferred; `.code` is deprecated in websockets 13+). - Returns None when unknown.""" + """WebSocket close code from a raised exception, or None. websockets' + ConnectionClosed* expose the Close frame via `.rcvd`/`.sent` (`.code` is + deprecated in websockets 13+).""" for attr in ("rcvd", "sent"): - frame = getattr(exc, attr, None) - fcode = getattr(frame, "code", None) + fcode = getattr(getattr(exc, attr, None), "code", None) if isinstance(fcode, int): return fcode code = getattr(exc, "code", None) return code if isinstance(code, int) else None async def _reconnect_loop(self) -> None: - """Re-dial the connector with capped exponential backoff until reconnected - or disconnect() is called. NET-NEW for §5.3: a re-established socket makes - the connector replay this instance's buffered-only backlog on the new - handshake (the delivery-leg onResume). Never raises out (a re-dial failure - just retries); ends when a dial succeeds (its reader takes over) or closing. + """Re-dial with capped exponential backoff until a dial succeeds (its + reader takes over) or disconnect(). Never raises out. - scale-to-zero (D12): when the close was a deliberate go_dormant() rather - than an unexpected drop, start from the dormant poll cadence. On a - suspended machine the event loop is frozen, so this sleep only advances - once the machine is awake — it just needs to be short enough that a - freshly-woken machine re-dials promptly. A successful _dial_and_start() - clears _dormant, so any LATER unexpected drop reconnects on the normal - fast backoff.""" + After go_dormant() start from the dormant cadence; a successful dial + clears _dormant so any LATER unexpected drop uses the fast backoff. + """ backoff = self._dormant_redial_s if self._dormant else self._reconnect_backoff_s while not self._closing: - try: - await asyncio.sleep(backoff) - except asyncio.CancelledError: - raise + await asyncio.sleep(backoff) if self._closing: return try: await self._dial_and_start() logger.info("relay ws reconnected") - return # the fresh reader is running; supervisor's job is done - except asyncio.CancelledError: - raise + return except Exception as exc: # noqa: BLE001 - keep retrying on dial failure logger.warning("relay ws reconnect failed: %s", exc) backoff = min(backoff * 2, self._reconnect_max_backoff_s) + # ── inbound frame dispatch ─────────────────────────────────────────── async def _handle_frame(self, line: str) -> None: try: frame = json.loads(line) except json.JSONDecodeError: logger.warning("relay: skipping malformed frame") return - ftype = frame.get("type") - if ftype == "descriptor": - descriptor = CapabilityDescriptor.from_json(json.dumps(frame.get("descriptor", {}))) - # Phase 1.5 multi-platform: one descriptor frame arrives per hello'd - # identity. Accumulate them keyed by the descriptor's own platform so - # the adapter can resolve PER-CHAT capabilities (e.g. Discord's 2000 - # vs Telegram's 4096 max_message_length) instead of collapsing N - # platforms onto whichever descriptor arrived last. - if descriptor.platform: - self._descriptors_by_platform[descriptor.platform] = descriptor - # The FIRST descriptor of this connection generation is the session - # default (the primary identity's) — later arrivals must NOT - # overwrite it, or the scalar capability surface silently becomes - # last-writer-wins across platforms. - if self._descriptor is None: - self._descriptor = descriptor - # Phase 7 Unit 7d-B: a received descriptor means the WS upgrade auth - # passed and the connector accepted us — record that we've handshaked - # at least once, so a LATER 4401 close is read as a revocation - # (opt-out), not a cold-start race. - self._handshake_succeeded = True - if self._descriptor_ready is not None and not self._descriptor_ready.done(): - self._descriptor_ready.set_result(descriptor) - elif ftype == "inbound": - if self._inbound is not None: - event = _event_from_wire(frame.get("event", {})) - await self._inbound(event) - # Phase 5 §5.3: a buffered delivery (replayed on reconnect) carries - # a bufferId; ack it after the handler has durably taken it so the - # connector advances its delivery-leg buffer cursor (no dup). A live - # delivery has no bufferId — nothing to ack. - buffer_id = frame.get("bufferId") - if buffer_id: - await self._send_inbound_ack(str(buffer_id)) - elif ftype == "going_idle_ack": - # Phase 5 §5.3: the connector confirmed our destination is now - # buffered-only; resolve the waiter go_idle() is blocked on. - if self._going_idle_ack is not None and not self._going_idle_ack.done(): - self._going_idle_ack.set_result(None) - elif ftype == "outbound_result": - fut = self._pending.get(frame.get("requestId", "")) - if fut is not None and not fut.done(): - fut.set_result(frame.get("result", {})) - elif ftype == "interrupt_inbound": - # Bridged into the adapter's interrupt path by the runner wiring. - handler = getattr(self, "_interrupt_inbound_handler", None) - if handler is not None: - await handler(frame.get("session_key", ""), frame.get("chat_id", "")) - elif ftype == "passthrough_forward": - # Phase 5 §5.1: a forwarded passthrough-plane request (Discord - # interaction, Twilio, …) the connector already edge-ACKed. It rides - # the SAME outbound WS as inbound messages so a hosted gateway needs - # no public inbound port. Dispatch to the adapter's handler; the - # bufferId (when present, §5.3 buffered flip) is passed for ack. - handler = getattr(self, "_passthrough_handler", None) - if handler is not None: - fwd = _passthrough_from_wire(frame.get("forward", {})) - await handler(fwd, frame.get("bufferId")) - else: - # hello/outbound/interrupt are gateway->connector; ignore if echoed. - pass + # hello/outbound/interrupt are gateway->connector; ignored if echoed. + handler = self._FRAME_HANDLERS.get(frame.get("type")) + if handler is not None: + await handler(self, frame) - def set_interrupt_inbound_handler(self, handler: Any) -> None: - """Register the callback for connector->gateway interrupt_inbound frames.""" - self._interrupt_inbound_handler = handler + async def _on_descriptor(self, frame: Dict[str, Any]) -> None: + descriptor = CapabilityDescriptor.from_json(json.dumps(frame.get("descriptor", {}))) + # One descriptor per hello'd identity, keyed by platform for per-chat caps. + if descriptor.platform: + self._descriptors_by_platform[descriptor.platform] = descriptor + # The FIRST descriptor of this generation is the session default; later + # arrivals must NOT overwrite it (else last-writer-wins across platforms). + if self._descriptor is None: + self._descriptor = descriptor + # Upgrade auth passed at least once: a LATER 4401 is a revocation. + self._handshake_succeeded = True + if self._descriptor_ready is not None and not self._descriptor_ready.done(): + self._descriptor_ready.set_result(descriptor) - def set_passthrough_handler(self, handler: Any) -> None: - """Register the callback for connector->gateway passthrough_forward frames. + async def _on_inbound(self, frame: Dict[str, Any]) -> None: + if self._inbound is None: + return + await self._inbound(_event_from_wire(frame.get("event", {}))) + # A replayed buffered delivery carries a bufferId; ack AFTER the handler + # has taken it so the connector advances its cursor (no dup). + buffer_id = frame.get("bufferId") + if buffer_id: + await self._send_inbound_ack(str(buffer_id)) - Mirrors set_interrupt_inbound_handler: the runner/adapter wires this so a - forwarded passthrough request (Phase 5 §5.1) reaches the adapter over the - same outbound WS the gateway already holds. ``handler(forward, buffer_id)``. - """ - self._passthrough_handler = handler + async def _on_going_idle_ack(self, frame: Dict[str, Any]) -> None: + if self._going_idle_ack is not None and not self._going_idle_ack.done(): + self._going_idle_ack.set_result(None) + + async def _on_outbound_result(self, frame: Dict[str, Any]) -> None: + fut = self._pending.get(frame.get("requestId", "")) + if fut is not None and not fut.done(): + fut.set_result(frame.get("result", {})) + + async def _on_interrupt_inbound(self, frame: Dict[str, Any]) -> None: + if self._interrupt_inbound_handler is not None: + await self._interrupt_inbound_handler( + frame.get("session_key", ""), frame.get("chat_id", "") + ) + + async def _on_passthrough_forward(self, frame: Dict[str, Any]) -> None: + # Edge-ACKed passthrough request riding the same WS (no public inbound + # port needed); bufferId (§5.3) is passed through for ack. + if self._passthrough_handler is not None: + fwd = _passthrough_from_wire(frame.get("forward", {})) + await self._passthrough_handler(fwd, frame.get("bufferId")) + + _FRAME_HANDLERS = { + "descriptor": _on_descriptor, + "inbound": _on_inbound, + "going_idle_ack": _on_going_idle_ack, + "outbound_result": _on_outbound_result, + "interrupt_inbound": _on_interrupt_inbound, + "passthrough_forward": _on_passthrough_forward, + } diff --git a/gateway/response_filters.py b/gateway/response_filters.py index b11d585ea2..0bfe5ad1c5 100644 --- a/gateway/response_filters.py +++ b/gateway/response_filters.py @@ -1,8 +1,7 @@ """Gateway response filtering helpers. -These helpers operate at the gateway boundary: they decide whether a completed -agent turn should be delivered to the chat, not what should be persisted in the -conversation history. +These decide whether a completed agent turn should be delivered to the chat, +not what should be persisted in conversation history. """ from __future__ import annotations @@ -10,11 +9,8 @@ from __future__ import annotations import unicodedata from typing import Any -# Canonical model-emitted control token for intentional silence. -SILENT_REPLY_TOKEN = "NO_REPLY" - -# Exact whole-response markers that mean "the agent intentionally chose not to -# reply". Keep this list small and explicit; arbitrary empty output remains an +# Exact whole-response markers meaning "the agent intentionally chose not to +# reply". Keep small and explicit; arbitrary empty output remains an # error/empty-response path, not silence. LIVE_GATEWAY_SILENT_MARKERS = frozenset({ "[SILENT]", @@ -23,23 +19,25 @@ LIVE_GATEWAY_SILENT_MARKERS = frozenset({ "NO REPLY", }) +# Longer than any marker could plausibly be, even with stray punctuation. +_MARKER_LENGTH_CAP = 64 + def _canonical_silence_candidate(text: str) -> str: return " ".join(text.strip().upper().split()) -def _strip_edge_silence_punctuation(text: str) -> str: - """Strip stray edge punctuation without erasing marker structure. +def _is_edge_punctuation(ch: str) -> bool: + # Square brackets stay structural so malformed ``[SILENT`` cannot become ``SILENT``. + return ch not in "[]" and unicodedata.category(ch).startswith("P") - Models sometimes emit ``.NO_REPLY`` or ``*NO_REPLY*`` instead of the exact - marker. Keep square brackets structural so malformed ``[SILENT`` does not - become ``SILENT``. - """ - start = 0 - end = len(text) - while start < end and text[start] not in "[]" and unicodedata.category(text[start]).startswith("P"): + +def _strip_edge_silence_punctuation(text: str) -> str: + """Strip stray edge punctuation (``.NO_REPLY``, ``*NO_REPLY*``) without erasing marker structure.""" + start, end = 0, len(text) + while start < end and _is_edge_punctuation(text[start]): start += 1 - while end > start and text[end - 1] not in "[]" and unicodedata.category(text[end - 1]).startswith("P"): + while end > start and _is_edge_punctuation(text[end - 1]): end -= 1 return text[start:end].strip() @@ -49,42 +47,39 @@ def _canonical_silence_candidates(text: str) -> tuple[str, ...]: stripped = _strip_edge_silence_punctuation(text.strip()) if stripped == text.strip(): return (exact,) - fallback = _canonical_silence_candidate(stripped) - return (exact, fallback) + return (exact, _canonical_silence_candidate(stripped)) + + +def _short_stripped(text: Any) -> str: + """Stripped text if it is a non-empty string within the marker cap, else ''.""" + if not isinstance(text, str): + return "" + stripped = text.strip() + return stripped if 0 < len(stripped) <= _MARKER_LENGTH_CAP else "" def is_intentional_silence_response(response: Any) -> bool: - """Return True only when ``response`` is exactly a silence marker. + """True only when ``response`` is exactly a silence marker. - Substantive prose that merely mentions ``NO_REPLY`` or ``[SILENT]`` must be - delivered normally. A blank response is also not silence; blank output is - handled by the empty-response failure path. + Prose that merely mentions ``NO_REPLY`` must be delivered normally. A blank + response is not silence either — that is the empty-response failure path. """ - if not isinstance(response, str): - return False - stripped = response.strip() - if not stripped: - return False - if len(stripped) > 64: - return False - return any(candidate in LIVE_GATEWAY_SILENT_MARKERS for candidate in _canonical_silence_candidates(stripped)) + stripped = _short_stripped(response) + return bool(stripped) and any( + c in LIVE_GATEWAY_SILENT_MARKERS for c in _canonical_silence_candidates(stripped) + ) def is_autonomous_silence_response(response: Any) -> bool: """Loose silence matcher for autonomous lanes (cron, webhook). - Autonomous lanes instruct the agent to emit ``[SILENT]`` when a tick - produced nothing worth a human's attention, and models reliably bracket - the marker with a short note explaining why they stayed quiet. Unlike - :func:`is_intentional_silence_response` (the interactive-chat rule, which - demands the response be EXACTLY a marker), this suppresses when a marker - is the whole response, sits on its own first or last line, or the - bracketed sentinel opens the response (the documented - ``[SILENT] No changes detected`` pattern). A token buried mid-sentence - in a genuine report is still delivered. - - Shares :data:`LIVE_GATEWAY_SILENT_MARKERS` so the interactive and - autonomous marker sets can never drift apart. + Autonomous lanes ask for ``[SILENT]`` when a tick produced nothing worth + attention, and models reliably bracket the marker with a short note. Unlike + :func:`is_intentional_silence_response` (interactive rule: EXACTLY a marker), + this suppresses when a marker is the whole response, sits on its own first or + last line, or the bracketed sentinel opens the response (``[SILENT] No + changes detected``). A token buried mid-sentence is still delivered. + Shares :data:`LIVE_GATEWAY_SILENT_MARKERS` so the two sets cannot drift. """ if not isinstance(response, str): return False @@ -95,53 +90,35 @@ def is_autonomous_silence_response(response: Any) -> bool: def _is_token(line: str) -> bool: return _canonical_silence_candidate(line) in LIVE_GATEWAY_SILENT_MARKERS - # Whole response is exactly a token. if _is_token(stripped): return True - # Marker on its own first or last line (leading/trailing note on a - # separate line — e.g. "2 deals filtered\n\n[SILENT]"). lines = [ln for ln in stripped.splitlines() if ln.strip()] if lines and (_is_token(lines[0]) or _is_token(lines[-1])): return True - # Bracketed sentinel used as a same-line prefix — the documented pattern - # "[SILENT] No changes detected". Restricted to the bracketed form so a - # bare word like "Silent retry succeeded" is NOT swallowed. - if stripped.upper().startswith("[SILENT]"): - return True - return False + # Bracketed form only, so a bare "Silent retry succeeded" is NOT swallowed. + return stripped.upper().startswith("[SILENT]") def is_intentional_silence_agent_result(agent_result: dict | None, response: Any) -> bool: """Silence markers suppress delivery only for successful agent turns.""" - if not isinstance(agent_result, dict): - return False - if agent_result.get("failed"): + if not isinstance(agent_result, dict) or agent_result.get("failed"): return False return is_intentional_silence_response(response) def is_partial_silence_marker(text: Any) -> bool: - """Return True while ``text`` could still resolve to a silence marker. + """True while streamed ``text`` could still resolve to a silence marker. - The streaming path accumulates the reply delta-by-delta and must decide, - before the whole response is known, whether to show what it has so far. - A buffer whose canonical form is a non-empty *prefix* of a silence marker - (e.g. ``"NO"`` on the way to ``"NO_REPLY"``, or an exact marker that has - not yet been terminated by stream-end) is held back so a raw marker is - never edited onto the screen and then belatedly retracted. - - Anything that has already diverged from every marker (ordinary prose) — - and anything longer than the marker cap — returns False so normal - streaming resumes immediately. This is the streaming counterpart to - :func:`is_intentional_silence_response`, sharing the same marker set and - canonicalization so the two never drift. + The streaming path must decide, before the whole response is known, whether + to show its buffer. A buffer whose canonical form is a non-empty *prefix* of + a marker (``"NO"`` on the way to ``"NO_REPLY"``, or an exact marker not yet + terminated by stream-end) is held back so a raw marker is never shown and + then retracted. Anything that has diverged from every marker, or exceeds the + marker cap, returns False so normal streaming resumes. Shares the marker set + and canonicalization with :func:`is_intentional_silence_response`. """ - if not isinstance(text, str): - return False - stripped = text.strip() - if not stripped or len(stripped) > 64: - return False - for candidate in _canonical_silence_candidates(stripped): - if candidate and any(marker.startswith(candidate) for marker in LIVE_GATEWAY_SILENT_MARKERS): - return True - return False + stripped = _short_stripped(text) + return bool(stripped) and any( + c and any(marker.startswith(c) for marker in LIVE_GATEWAY_SILENT_MARKERS) + for c in _canonical_silence_candidates(stripped) + ) diff --git a/gateway/restart.py b/gateway/restart.py index 58e6d15ceb..6923596655 100644 --- a/gateway/restart.py +++ b/gateway/restart.py @@ -6,38 +6,32 @@ from collections.abc import Mapping from hermes_cli.config import DEFAULT_CONFIG -# EX_TEMPFAIL from sysexits.h — used to ask the service manager to restart -# the gateway after a graceful drain/reload path completes. +# EX_TEMPFAIL (sysexits.h): ask the service manager to restart the gateway +# after a graceful drain/reload path completes. GATEWAY_SERVICE_RESTART_EXIT_CODE = 75 -# EX_CONFIG from sysexits.h — fatal configuration error (e.g. token -# collision, no messaging platforms). The s6 finish script translates -# this into exit 125 (permanent failure) so the supervisor stops -# restarting the gateway. See #51228. +# EX_CONFIG (sysexits.h): fatal configuration error (token collision, no +# platforms). The s6 finish script translates this into exit 125 (permanent +# failure) so the supervisor stops restarting the gateway. GATEWAY_FATAL_CONFIG_EXIT_CODE = 78 def is_global_startup_conflict(error_code: str | None) -> bool: - """Return True when an adapter's fatal error is a single-writer ownership conflict. + """True when an adapter's fatal error is a single-writer ownership conflict. - ``BasePlatformAdapter._acquire_platform_lock`` emits ``{scope}_lock`` - with ``retryable=True`` on purpose: a *mid-run* reconnect must be able to - recover once the live holder exits or a stale record is cleared (#54167). - At startup, though, a live foreign holder is a configuration conflict — - two gateways cannot poll one bot token — so the startup router must not - treat that flag as "transient blip, retry-queue forever". This matches by - error CODE only (the ``{scope}_lock`` / ``lock_conflict`` families every - adapter emits for scoped-lock and identity conflicts), never by message - text. + Adapters emit ``{scope}_lock`` with ``retryable=True`` so a *mid-run* + reconnect can recover once the holder exits or a stale record is cleared. + At startup a live foreign + holder is a configuration conflict (two gateways cannot poll one token), so + the startup router must not treat it as a transient blip. Matches by error + CODE only (``lock_conflict`` / ``*_lock``), never by message text. """ code = (error_code or "").strip().lower() - if not code: - return False - return code == "lock_conflict" or code.endswith("_lock") + return bool(code) and (code == "lock_conflict" or code.endswith("_lock")) # Set by ``hermes gateway run --external-supervisor``. Unlike systemd's # INVOCATION_ID and launchd's XPC_SERVICE_NAME, this survives wrappers that -# intentionally replace the child environment (for example ``sudo env -i``). +# replace the child environment (e.g. ``sudo env -i``). EXTERNAL_GATEWAY_SUPERVISOR_ENV = "HERMES_GATEWAY_EXTERNAL_SUPERVISOR" DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float( @@ -48,36 +42,33 @@ DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT = float( ) DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT = 5.0 -# In-band restart (``/restart``, SIGUSR1, self-restart from a child CLI) -# waits for active turns to finish *before* ``stop()`` begins. Distinct -# from ``restart_drain_timeout``, which is the force-interrupt budget -# once ``stop()`` is running (and must stay short under systemd -# TimeoutStopSec). See #77184. +# In-band restart (``/restart``, SIGUSR1, self-restart) waits for active turns +# to finish *before* ``stop()`` begins. Distinct from ``restart_drain_timeout``, +# the force-interrupt budget once ``stop()`` is running (must stay short under +# systemd TimeoutStopSec). DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT = float( DEFAULT_CONFIG["agent"]["restart_after_turn_timeout"] ) -# Cron-only floor under the ``stop()`` drain. ``restart_drain_timeout`` -# defaults to 0 because interrupting a *chat* turn is cheap and recoverable: -# the user is told the gateway is restarting and the session is pre-marked -# resume_pending. An interrupted *cron* run has neither property — nobody is -# waiting on it, it lands in jobs.json as a permanent failure, and a recurring -# job just waits for its next schedule — so a zero-second drain silently -# destroys work. See #82161. +# Cron-only floor under the ``stop()`` drain. ``restart_drain_timeout`` defaults +# to 0 because interrupting a *chat* turn is cheap and recoverable (user is +# told, session pre-marked resume_pending). An interrupted *cron* run has +# neither property — nobody is waiting on it, it lands in jobs.json as a +# permanent failure and a recurring job just waits for its next schedule — so a +# zero-second drain silently destroys work. DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT = float( DEFAULT_CONFIG["agent"]["cron_drain_timeout"] ) -# Seconds of the shutdown watchdog leash held back for the work that still has -# to happen after the drain returns: interrupt agents, kill tool subprocesses, -# mark in-flight jobs interrupted, disconnect adapters. Waiting for cron past -# that point trades a job that is killed *and recorded* for one that is -# SIGKILLed mid-write and stays wedged at ``last_status=running`` forever. +# Seconds of the shutdown watchdog leash held back for post-drain work +# (interrupt agents, kill tool subprocesses, mark jobs interrupted, disconnect +# adapters). Waiting for cron past that trades a job killed *and recorded* for +# one SIGKILLed mid-write and wedged at ``last_status=running`` forever. CRON_DRAIN_CLEANUP_RESERVE_S = 10.0 # systemd TimeoutStopSec headroom after the stop-path drain budget, and the -# floor used when that budget is still the default immediate (0s) chat drain. -# Keep these in lockstep with generate_systemd_unit() / #94759. +# floor when that budget is still the default immediate (0s) chat drain. +# Keep in lockstep with generate_systemd_unit(). SYSTEMD_STOP_HEADROOM_S = 30.0 SYSTEMD_TIMEOUT_STOP_SEC_FLOOR = 60.0 @@ -87,9 +78,7 @@ def is_gateway_supervisor_process( ) -> bool: """Return whether this gateway process is owned by a supervisor.""" env = os.environ if environ is None else environ - if env.get("INVOCATION_ID"): - return True - if env.get("HERMES_S6_SUPERVISED_CHILD"): + if env.get("INVOCATION_ID") or env.get("HERMES_S6_SUPERVISED_CHILD"): return True xpc_service = env.get("XPC_SERVICE_NAME", "") if xpc_service and xpc_service != "0": @@ -103,17 +92,33 @@ def is_gateway_supervisor_process( def is_container_restart_context() -> bool: - """Return whether the gateway is running inside a container for restart - routing purposes (Docker/Podman ⇒ the detached setsid path dies with the - cgroup; exit-75 service restart is the only viable path). - - Extracted from the inline probe in the /restart handler so tests can mock - container detection hermetically — a real ``/.dockerenv`` on a - containerized CI runner otherwise flips the routing under the test. - """ + """Whether the gateway runs in a container (Docker/Podman): the detached + setsid restart path dies with the cgroup, so exit-75 service restart is the + only viable path. Separate function so tests can mock container detection + (a real ``/.dockerenv`` on CI otherwise flips the routing).""" return os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv") +def _seconds(value: object, fallback: float = 0.0) -> float: + """Non-negative float, or ``fallback`` on non-numeric input.""" + try: + return max(float(value), 0.0) # type: ignore[arg-type] + except (TypeError, ValueError): + return fallback + + +def _parse_timeout_keeping_zero(raw: object, default: float) -> float: + """Parse a timeout where ``0`` is a deliberate disable (must NOT fall + through to ``default``), unlike None / blank / non-numeric input.""" + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + try: + value = float(raw) + except (TypeError, ValueError): + return default + return max(0.0, value) + + def parse_restart_drain_timeout(raw: object) -> float: """Parse a configured drain timeout, falling back to the shared default.""" try: @@ -124,38 +129,13 @@ def parse_restart_drain_timeout(raw: object) -> float: def parse_restart_after_turn_timeout(raw: object) -> float: - """Parse the after-turn wait cap for in-band restart, falling back to default. - - ``0`` is a deliberate disable (legacy immediate drain) and must not fall - through to the default — unlike empty/missing input. - """ - if raw is None: - return DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT - if isinstance(raw, str) and not raw.strip(): - return DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT - try: - value = float(raw) - except (TypeError, ValueError): - return DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT - return max(0.0, value) + """Parse the after-turn wait cap for in-band restart (``0`` = legacy immediate drain).""" + return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT) def parse_cron_drain_timeout(raw: object) -> float: - """Parse the cron-only drain floor, falling back to the shared default. - - ``0`` is a deliberate opt-out — cron work is then interrupted on the same - budget as chat work, the pre-#82161 behaviour — and must not fall through - to the default, unlike empty/missing input. - """ - if raw is None: - return DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT - if isinstance(raw, str) and not raw.strip(): - return DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT - try: - value = float(raw) - except (TypeError, ValueError): - return DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT - return max(0.0, value) + """Parse the cron-only drain floor (``0`` = opt out; cron interrupted on the chat budget).""" + return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT) def resolve_cron_drain_budget( @@ -168,26 +148,14 @@ def resolve_cron_drain_budget( ) -> float: """Seconds the shutdown drain may spend waiting on in-flight cron work. - The configured floor is clamped to what this process can actually honour. - The shutdown watchdog hard-exits at ``watchdog_delay`` and the service - manager's ``TimeoutStopSec`` is sized from the full stop budget (drain - vs cron floor + cleanup reserve, plus headroom — see - ``resolve_systemd_timeout_stop_sec``), so waiting past that leash - (minus ``cleanup_reserve_s`` for the teardown that follows the drain) - would swap a cleanly-interrupted job for a SIGKILL that leaves it - wedged mid-run — strictly worse than the bug being fixed. - - Never returns less than ``drain_timeout``: the cron floor only ever - extends the wait, so an operator who deliberately configured a long - ``restart_drain_timeout`` keeps it. + The configured floor is clamped to what this process can honour: the + shutdown watchdog hard-exits at ``watchdog_delay`` (and TimeoutStopSec is + sized from the same budget), so waiting past that leash minus + ``cleanup_reserve_s`` swaps a cleanly-interrupted job for a SIGKILL that + leaves it wedged. Never returns less than ``drain_timeout``: the cron floor + only ever extends the wait, so an operator who deliberately configured a + long ``restart_drain_timeout`` keeps it. """ - - def _seconds(value: object, fallback: float = 0.0) -> float: - try: - return max(float(value), 0.0) # type: ignore[arg-type] - except (TypeError, ValueError): - return fallback - drain = _seconds(drain_timeout) floor = _seconds(cron_drain_timeout) if floor <= 0.0: @@ -210,30 +178,17 @@ def resolve_systemd_timeout_stop_sec( ) -> int: """Seconds systemd ``TimeoutStopSec`` must cover the full stop budget. - ``restart_drain_timeout`` is only the chat-turn interrupt budget (default - 0). The stop path may wait longer for in-flight cron work — - ``cron_drain_timeout`` plus ``cleanup_reserve_s`` — before it even starts - interrupting. Sizing the unit from drain alone lets systemd SIGKILL an - in-budget drain (#94759). - - A zero ``cron_drain_timeout`` is a deliberate opt-out and does not extend - the budget. Non-numeric inputs degrade to 0 rather than raising. + ``restart_drain_timeout`` is only the chat-turn interrupt budget (default 0); + the stop path may first wait ``cron_drain_timeout`` + ``cleanup_reserve_s`` + for cron work, so sizing from drain alone lets systemd SIGKILL an in-budget + drain. A zero cron timeout is an opt-out and does not extend the budget. + Non-numeric inputs degrade to 0. """ - - def _seconds(value: object) -> float: - try: - return max(float(value), 0.0) # type: ignore[arg-type] - except (TypeError, ValueError): - return 0.0 - drain = _seconds(drain_timeout) cron = _seconds(cron_drain_timeout) - reserve = _seconds(cleanup_reserve_s) - headroom = _seconds(headroom_s) - floor = _seconds(floor_s) - cron_budget = (cron + reserve) if cron > 0.0 else 0.0 + cron_budget = (cron + _seconds(cleanup_reserve_s)) if cron > 0.0 else 0.0 stop_budget = max(drain, cron_budget) - return int(max(floor, stop_budget + headroom)) + return int(max(_seconds(floor_s), stop_budget + _seconds(headroom_s))) def resolve_restart_exit_wait_budget( @@ -244,24 +199,11 @@ def resolve_restart_exit_wait_budget( ) -> float: """Seconds a CLI should wait for the gateway PID to exit after SIGUSR1. - In-band restart may defer ``stop()`` until active turns finish - (``after_turn_timeout``) and then spend up to ``drain_timeout`` inside - ``stop()``. Callers that fall back to a hard kill on wait expiry must - cover both phases or they reintroduce #77184. + In-band restart may defer ``stop()`` until turns finish (``after_turn_timeout``) + and then spend ``drain_timeout`` inside ``stop()``; callers that hard-kill on + expiry must cover both phases. """ - try: - drain = max(float(drain_timeout), 0.0) - except (TypeError, ValueError): - drain = 0.0 - try: - after_turn = max(float(after_turn_timeout), 0.0) - except (TypeError, ValueError): - after_turn = 0.0 - try: - margin = max(float(headroom), 0.0) - except (TypeError, ValueError): - margin = 0.0 - return drain + after_turn + margin + return _seconds(drain_timeout) + _seconds(after_turn_timeout) + _seconds(headroom) def parse_signal_interrupt_grace_timeout(raw: object) -> float: diff --git a/gateway/restart_loop_guard.py b/gateway/restart_loop_guard.py index 581a1986cf..78ed396800 100644 --- a/gateway/restart_loop_guard.py +++ b/gateway/restart_loop_guard.py @@ -1,33 +1,23 @@ -"""Auto-resume restart-loop breaker (#30719, defense-3). +"""Auto-resume restart-loop breaker (defense-3). Defenses 1 and 2 (the ``_HERMES_GATEWAY`` guard on ``hermes gateway -stop|restart`` + ``terminal_tool``, and the cron-creation lifecycle -filter) stop the agent from scheduling its own restart via the cron and -CLI paths. They do NOT cover every SIGTERM source: an agent running a -raw ``terminal("launchctl kickstart -k gui//ai.hermes.gateway")``, -an external monitor with a bad trigger, or any other repeated crash can -still drive the supervisor (launchd ``KeepAlive`` / systemd ``Restart=``) -into a tight respawn loop. On each boot the gateway auto-resumes the -restart-interrupted session, whose next turn re-runs the offending -logic — SIGTERM every ~10 seconds until manually broken. +stop|restart`` + ``terminal_tool``, and the cron-creation lifecycle filter) +stop the agent scheduling its own restart via cron/CLI. They do NOT cover +every SIGTERM source (raw ``launchctl kickstart``, a bad external monitor, +any repeated crash): the supervisor respawns, the gateway auto-resumes the +restart-interrupted session, whose next turn re-runs the offending logic. -This module is the last-resort circuit breaker: it records a timestamp -each time the gateway boots with restart-interrupted sessions pending, -keeps the current chain of such boots persisted across processes (each -boot is a fresh process, so in-memory state is useless), and reports the -loop as "tripped" once too many of them chain together. Boots chain -while consecutive gaps stay within ``max_gap_seconds``, so the breaker -sees slow crash cycles (a wedged event loop killed by the liveness -watchdog every ~150s, #81642) exactly as well as the fast ~10s respawn -loop it was originally written for. -When tripped, the caller SKIPS auto-resume for that boot — the gateway -still starts and serves real inbound messages, it just stops replaying -the session that keeps killing it, which breaks the cycle and puts a -human back in the loop. +This module is the last-resort circuit breaker. Each boot with +restart-interrupted sessions pending is timestamped and persisted (each boot +is a fresh process, so in-memory state is useless). Boots CHAIN while +consecutive gaps stay within ``max_gap_seconds``, so slow crash cycles (a +wedged loop killed by the liveness watchdog every ~150s) trip exactly like the +fast ~10s respawn loop. When tripped, the caller SKIPS auto-resume for that +boot — the gateway still serves real inbound messages, it just stops replaying +the session that keeps killing it. -State lives in ``/gateway/restart_loop.json`` so it is -profile-scoped and survives process death. It is intentionally tiny and -best-effort: any read/write failure fails OPEN (no false trip) because a +State lives in ``/gateway/restart_loop.json`` (profile-scoped). +Best-effort: any read/write failure fails OPEN (no false trip) because a broken breaker must never wedge a healthy gateway. """ @@ -42,26 +32,20 @@ from hermes_constants import get_hermes_home logger = logging.getLogger("gateway.run") -# Defaults chosen so a legitimate operator restart (or two) never trips the -# breaker, but the documented ~10s respawn loop does within a few cycles. +# A legitimate operator restart (or two) never trips; a ~10s respawn loop does +# within a few cycles. DEFAULT_MAX_RESTARTS = 3 DEFAULT_WINDOW_SECONDS = 60 -# Longest gap between two consecutive restart-interrupted boots that still -# counts them as the SAME loop (#81642). A fixed ``window_seconds`` prune can -# only see crash cycles faster than the window: a loop whose period exceeds it -# drops its own history on every boot, so the counter never leaves 1 and the -# breaker never trips no matter how long the loop runs. The reported cycle was -# ~150s (wedged event loop -> ~90s liveness watchdog hard-exit -> respawn -> -# auto-resume replays the same session), i.e. structurally invisible to the 60s -# window. Chaining on the inter-boot GAP instead makes the breaker period- -# agnostic: any repeating cycle trips once ``max_restarts`` links accumulate, -# and a single boot followed by real quiet resets the chain. +# Longest gap between consecutive restart-interrupted boots that still counts +# them as the SAME loop. A fixed-window prune only sees cycles faster than the +# window (a slower loop drops its own history every boot and never trips); +# chaining on the inter-boot gap makes the breaker period-agnostic, and a +# single boot followed by real quiet resets the chain. DEFAULT_MAX_GAP_SECONDS = 300 -# Cap the persisted chain so a long-running loop cannot grow the state file -# without bound. Only the newest ``max_restarts`` entries can change a -# verdict; the rest are kept for forensics. +# Cap the persisted chain; only the newest ``max_restarts`` entries can change +# a verdict, the rest are forensics. _MAX_STORED_BOOTS = 50 @@ -71,10 +55,8 @@ def _state_path(): def _load_boots() -> List[float]: try: - raw = _state_path().read_text(encoding="utf-8") - data = json.loads(raw) - boots = data.get("boots", []) - return [float(t) for t in boots if isinstance(t, (int, float))] + data = json.loads(_state_path().read_text(encoding="utf-8")) + return [float(t) for t in data.get("boots", []) if isinstance(t, (int, float))] except (OSError, ValueError, TypeError): return [] @@ -89,28 +71,24 @@ def _save_boots(boots: List[float]) -> None: def _chain_gap(window_seconds: int, max_gap_seconds: int) -> float: - """Effective inter-boot gap that still links two boots into one loop. - - Floored by ``window_seconds`` so an operator who widens the window never - ends up with a breaker that is *less* sensitive than they asked for. - """ + """Inter-boot gap that still links two boots. Floored by ``window_seconds`` so + widening the window never makes the breaker *less* sensitive.""" return float(max(1, window_seconds, max_gap_seconds)) def _chain_ending_at(boots: List[float], ts: float, gap: float) -> List[float]: - """Return the unbroken chain of boots leading up to ``ts``. + """Unbroken chain of boots leading up to ``ts`` (oldest first). - Walks backwards from ``ts`` and keeps boots while each successive gap stays - within ``gap``. The first gap that exceeds it ends the chain: everything - older belongs to a previous, already-resolved episode. A chain broken at - the head (nothing recent enough) yields an empty list, which is how a - healthy gateway forgets an old loop. + Walks backwards keeping boots while each successive gap stays within + ``gap``; the first wider gap ends the chain (older boots belong to an + already-resolved episode). Nothing recent enough -> empty list, which is how + a healthy gateway forgets an old loop. """ chain: List[float] = [] prev = ts for t in sorted(boots, reverse=True): if t > ts: - # Clock moved backwards (NTP step, restored state file). Treat the + # Clock moved backwards (NTP step, restored state file): treat the # future entry as adjacent rather than dropping the whole chain. chain.append(t) continue @@ -128,49 +106,17 @@ def record_restart_interrupted_boot( now: Optional[float] = None, max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS, ) -> List[float]: - """Record that the gateway just booted with restart-interrupted sessions. + """Record a restart-interrupted boot; return the pruned chain + now (most recent last). - Drops boots that belong to an earlier, already-broken chain (any gap wider - than ``max_gap_seconds``) and appends the current time. Returns the - pruned+appended list (most recent last). Best-effort — a persistence - failure returns the in-memory list without raising. + Best-effort — a persistence failure returns the in-memory list without raising. """ ts = time.time() if now is None else now - gap = _chain_gap(window_seconds, max_gap_seconds) - boots = _chain_ending_at(_load_boots(), ts, gap) + boots = _chain_ending_at(_load_boots(), ts, _chain_gap(window_seconds, max_gap_seconds)) boots.append(ts) _save_boots(boots[-_MAX_STORED_BOOTS:]) return boots -def is_restart_loop_tripped( - max_restarts: int = DEFAULT_MAX_RESTARTS, - window_seconds: int = DEFAULT_WINDOW_SECONDS, - *, - now: Optional[float] = None, - max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS, -) -> bool: - """Return True if the gateway has restarted ``>= max_restarts`` times with - restart-interrupted sessions in one unbroken chain ending at ``now``. - - Reads the persisted boot log written by - ``record_restart_interrupted_boot`` and counts the boots that still chain - together (consecutive gaps within ``max_gap_seconds``), so the verdict does - not depend on how fast the crash cycle happens to be. - Fails OPEN (returns False) on any error — a broken breaker must never - wedge a healthy gateway. - """ - if max_restarts <= 0: - return False - ts = time.time() if now is None else now - gap = _chain_gap(window_seconds, max_gap_seconds) - try: - recent = _chain_ending_at(_load_boots(), ts, gap) - except Exception: # pragma: no cover — _load_boots already guards - return False - return len(recent) >= max_restarts - - def clear() -> None: """Remove the persisted boot log (used on clean shutdown / by tests).""" try: @@ -186,13 +132,10 @@ def check_and_record( now: Optional[float] = None, max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS, ) -> bool: - """Record this restart-interrupted boot and report whether the loop is now - tripped. + """Record this boot and return True when auto-resume should be SKIPPED. - This is the single entry point the gateway calls: it appends the current - boot, then checks whether the (now-updated) chain has reached the - threshold. Returns True when auto-resume should be SKIPPED to break the - loop. + The single entry point the gateway calls: appends the current boot, then + checks whether the updated chain has reached ``max_restarts``. """ boots = record_restart_interrupted_boot( window_seconds, now=now, max_gap_seconds=max_gap_seconds diff --git a/gateway/rich_sent_store.py b/gateway/rich_sent_store.py index 264d030941..ebd6a1a71a 100644 --- a/gateway/rich_sent_store.py +++ b/gateway/rich_sent_store.py @@ -1,13 +1,10 @@ """Local index of text we've sent via ``sendRichMessage`` (Bot API 10.1). Telegram does NOT echo a rich message's content back in ``reply_to_message`` -when a user replies to it (verified: ``.text``/``.caption`` empty, -``.api_kwargs`` None). So replies to the launchd briefings / any rich send -arrive with no quotable text and the agent is blind to what was referenced. - -Fix: remember ``message_id -> text`` at send time, look it up by -``reply_to_id`` on inbound. This module is the single source of truth for that -index. +(``.text``/``.caption`` empty, ``.api_kwargs`` None), so replies to rich sends +arrive with no quotable text. We remember ``message_id -> text`` at send time +and look it up by ``reply_to_id`` on inbound. This module is the single source +of truth for that index. Best-effort and dependency-free: every operation swallows errors and degrades to a no-op / ``None`` so it can never break a send or an inbound message. @@ -25,17 +22,25 @@ _MAX_TEXT_CHARS = 2000 def _store_path() -> str: - # Resolve via get_hermes_home() so the active profile override is honored. + # get_hermes_home() honors the active profile override. from hermes_constants import get_hermes_home - home = get_hermes_home() - return os.path.join(str(home), "state", "rich_sent_index.json") + return os.path.join(str(get_hermes_home()), "state", "rich_sent_index.json") def _key(chat_id, message_id) -> str: return f"{chat_id}:{message_id}" +def _load(path: str) -> dict: + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + except (FileNotFoundError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + def record(chat_id, message_id, text: Optional[str]) -> None: """Persist ``text`` for ``(chat_id, message_id)``. No-op on any failure.""" if not text or message_id is None or chat_id is None: @@ -43,22 +48,11 @@ def record(chat_id, message_id, text: Optional[str]) -> None: path = _store_path() try: os.makedirs(os.path.dirname(path), exist_ok=True) - try: - with open(path, "r", encoding="utf-8") as fh: - data = json.load(fh) - if not isinstance(data, dict): - data = {} - except (FileNotFoundError, ValueError): - data = {} - data[_key(chat_id, message_id)] = { - "t": text[:_MAX_TEXT_CHARS], - "ts": int(time.time()), - } - # Trim oldest by timestamp when over cap. - if len(data) > _MAX_ENTRIES: - for k, _ in sorted( - data.items(), key=lambda kv: kv[1].get("ts", 0) - )[: len(data) - _MAX_ENTRIES]: + data = _load(path) + data[_key(chat_id, message_id)] = {"t": text[:_MAX_TEXT_CHARS], "ts": int(time.time())} + if len(data) > _MAX_ENTRIES: # trim oldest by timestamp + by_age = sorted(data.items(), key=lambda kv: kv[1].get("ts", 0)) + for k, _ in by_age[: len(data) - _MAX_ENTRIES]: data.pop(k, None) tmp = f"{path}.tmp.{os.getpid()}" with open(tmp, "w", encoding="utf-8") as fh: @@ -72,12 +66,5 @@ def lookup(chat_id, message_id) -> Optional[str]: """Return stored text for ``(chat_id, message_id)`` or ``None``.""" if message_id is None or chat_id is None: return None - try: - with open(_store_path(), "r", encoding="utf-8") as fh: - data = json.load(fh) - entry = data.get(_key(chat_id, message_id)) - if isinstance(entry, dict): - return entry.get("t") or None - except (FileNotFoundError, ValueError, AttributeError): - return None - return None + entry = _load(_store_path()).get(_key(chat_id, message_id)) + return (entry.get("t") or None) if isinstance(entry, dict) else None diff --git a/gateway/run.py b/gateway/run.py index 5d44f273ea..1656a40cfd 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -71,16 +71,9 @@ from hermes_cli.config import _is_ssh_remote_tilde_cwd, cfg_get from hermes_cli.fallback_config import get_fallback_chain # --- Agent cache tuning --------------------------------------------------- -# Bounds the per-session AIAgent cache to prevent unbounded growth in -# long-lived gateways (each AIAgent holds LLM clients, tool schemas, -# memory providers, etc.). LRU order + idle TTL eviction are enforced -# from _enforce_agent_cache_cap() and _session_expiry_watcher() below. -# -# These are the defaults; `agent.agent_cache.max_size` / -# `agent.agent_cache.idle_ttl_secs` in config.yaml override them per -# deployment. Neither bound knows how many BYTES a cached agent holds, so -# _sweep_agent_cache_under_pressure() adds the missing memory-pressure valve -# (see gateway/agent_cache_pressure.py). +# Bounds the per-session AIAgent cache to prevent unbounded growth in long-lived gateways (each +# AIAgent holds LLM clients, tool schemas, memory providers, etc.). LRU order + idle TTL eviction +# are enforced from _enforce_agent_cache_cap() and _session_expiry_watcher() below. _AGENT_CACHE_MAX_SIZE = 128 _AGENT_CACHE_IDLE_TTL_SECS = 3600.0 # evict agents idle for >1h _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT = 30.0 @@ -88,22 +81,15 @@ _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT = 30.0 # returns. Leave enough outer budget for initialize/deleteWebhook/start_polling # wall deadlines plus readiness; other platforms retain the 30s isolation bound. _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT = 180.0 -# Cold-start cap for Telegram (#85993): the initial connect awaited before the -# gateway reaches `running` must not spend the full 180s budget — an -# unreachable Telegram would hold EVERY platform's serving state hostage for -# the whole window. The initial attempt gets one bounded try; on timeout the -# platform is queued for the reconnect watcher, which retries with the full -# 180s budget (is_reconnect=True preserves the offline update queue, #46621). +# Cold-start cap for Telegram: the initial connect awaited before the gateway reaches `running` must +# not spend the full 180s budget — an unreachable Telegram would hold EVERY platform's serving state +# hostage for the whole window. _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT = 45.0 _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT = 5.0 -# End reasons that mean the USER deliberately closed this thread of work -# (/new -> session_reset / new_session, an explicit exit, or a /switch). -# Shared by _classify_completion_target (pre-flight verdict) and -# _resolve_async_delegation_session (in-pipeline routing) so the two can -# never disagree: every reason the classifier calls "deliver" must be one -# the resolver actually delivers, otherwise the durable row is acked at -# adapter acceptance and then silently dropped inside the pipeline — -# a falsely-acknowledged permanent loss. +# End reasons that mean the USER deliberately closed this thread of work (/new -> session_reset / +# new_session, an explicit exit, or a /switch). Shared by _classify_completion_target and +# _resolve_async_delegation_session so they can never disagree: a reason the classifier calls +# "deliver" that the resolver drops would be acked at adapter acceptance and then silently lost. _USER_BOUNDARY_END_REASONS = ( "session_reset", "user_exit", @@ -156,11 +142,9 @@ _TELEGRAM_NOISY_STATUS_RE = re.compile( _HYGIENE_COOLDOWN_LADDER_MULTIPLIERS = (1, 3, 9) -# Absolute ceiling on an escalated hygiene cooldown, mirroring -# _RECONNECT_BACKOFF_CAP above: with an operator-raised base the multiplier -# ladder alone would reach 9h (base 3600 -> 32400s), which is indistinguishable -# from "compaction silently switched off". 1h is well past the point where a -# retry is cheap and still recovers within a session. +# Absolute ceiling on an escalated hygiene cooldown, mirroring _RECONNECT_BACKOFF_CAP above: with an +# operator-raised base the multiplier ladder alone would reach 9h (base 3600 -> 32400s), which is +# indistinguishable from "compaction silently switched off". _HYGIENE_COOLDOWN_MAX_SECONDS = 3600.0 # Flat retry-after recorded when a hygiene compression is ABANDONED because @@ -180,20 +164,13 @@ def _hygiene_cooldown_for_failure( """Bump the hygiene failure streak and return the escalated cooldown. This is a MULTIPLIER ladder (x1, x3, x9) over the operator's configured - ``hygiene_failure_cooldown_seconds``, clamped to - ``_HYGIENE_COOLDOWN_MAX_SECONDS``, so a tuned base is preserved as rung 1. + ``hygiene_failure_cooldown_seconds``, clamped to ``_HYGIENE_COOLDOWN_MAX_SECONDS``, so a + tuned base is preserved as rung 1. - It exists because the in-agent equivalent is unreachable from here: - ``ContextCompressor.record_timeout_failure`` escalates on an absolute - 60 -> 300 -> 900s ladder driven by the in-memory - ``_consecutive_timeout_failures`` counter, which ``bind_session_state`` - zeroes. Session hygiene constructs a FRESH ``AIAgent`` per run and re-binds - state every time, so from the gateway that streak is structurally always 0 - and only the flat ``hygiene_failure_cooldown_seconds`` could ever be - recorded — a session whose summary model always times out retried on that - same fixed interval forever (#79624). The streak is mirrored to SQLite by - rotation-stable ``session_key`` so it outlives both the per-run agent and - gateway restarts; ``PersistentState`` keeps the hot in-process view. + Needed because the in-agent ladder (``ContextCompressor.record_timeout_failure``) keys off an + in-memory streak that ``bind_session_state`` zeroes; hygiene builds a FRESH ``AIAgent`` per + run, so from here that streak is always 0. The streak is instead mirrored to SQLite by + rotation-stable ``session_key`` so it outlives the per-run agent and gateway restarts. """ streak = 1 state = None @@ -257,28 +234,15 @@ def hygiene_compaction_recovered( ) -> bool: """True when a hygiene run actually recovered the session. - Extracted from ``_handle_message_with_agent`` so the decision is unit - testable: it previously lived inline in a ~2000-line async method, and the - only way to pin it was a source-reading test — which AGENTS.md bans - outright, naming this file. + Extracted from ``_handle_message_with_agent`` so the decision is unit testable: it previously + lived inline in a ~2000-line async method, and the only way to pin it was a source-reading + test — which AGENTS.md bans outright, naming this file. - "Recovered" requires all three: - - * the compressor did not abort (no summary produced at all); - * the transcript was actually rewritten — either rotated into a new session - or compacted in place. The degenerate "did not rotate or compact in - place" path (#21301) reuses the pre-compression counts, so relying on the - numbers alone would read a no-op as success; - * the request materially shrank, per the canonical - :func:`compression_made_progress` (#39548) — a row-count drop counts even - when the summary keeps the token estimate flat, and a sub-5% token wobble - does not count at all. - - The token arguments are deliberately compared through that shared predicate - rather than with a bare ``<``: ``approx_tokens`` can be provider-reported - while ``new_tokens`` is always a rough estimate (documented to run 30-50% - high on code-heavy sessions), so a bare comparison both misses real wins and - counts noise as one. + "Recovered" requires all three: the compressor did not abort; the transcript was actually + rewritten (rotated or compacted in place — the no-op path reuses pre-compression counts, so + numbers alone would read it as success); and the request materially shrank per + :func:`compression_made_progress` (a bare ``<`` would miss row-count wins and count the + 30-50% estimate noise as one). """ if aborted: return False @@ -326,32 +290,15 @@ async def run_codex_hygiene_compaction( timeout_seconds: float, failure_cooldown_seconds: float = 300.0, ) -> str: - """Session hygiene for ``codex_app_server`` sessions (#73503). + """Session hygiene for ``codex_app_server`` sessions. - On this runtime the model's real working context is the app-server's - server-side thread, not Hermes' transcript: ``CodexAppServerSession`` is - constructed with no history and each turn submits only the new user - message (agent/codex_runtime.py), so the persisted transcript is a mirror - that is never replayed into a thread. Two consequences drive this path: - - * Rewriting the local mirror (the detached hygiene agent's normal - compression) shrinks nothing the model actually carries — it was a - permanent no-op ("compressed 150 -> 150 msgs"). - * Evicting the cached live agent afterwards destroys the only real - context: the next turn spawns an EMPTY thread and the model starts - blank while Hermes still mirrors a full history (abrupt amnesia — the - user-facing damage documented on #73503). - - So hygiene must compact the LIVE cached agent's thread via the - app-server's own ``thread/compact/start`` (through - ``_compress_context_via_codex_app_server``) and KEEP that agent cached. - Never build a detached compressor and never evict here. - - Mode contract (``compression.codex_app_server_auto``): only ``hermes`` - lets Hermes' threshold initiate app-server compaction; ``native`` leaves - the schedule to codex itself and ``off`` disables Hermes-initiated - automatic compaction entirely — both return without touching the thread - or the transcript, and neither may fall back to the local compressor. + On this runtime the model's real context is the app-server's server-side thread; the local + transcript is a never-replayed mirror. So rewriting the mirror shrinks nothing (permanent + no-op) and evicting the cached live agent destroys the only real context (next turn starts an + EMPTY thread — abrupt amnesia). Hygiene must therefore compact the LIVE agent's thread via + ``thread/compact/start`` and KEEP that agent cached: never build a detached compressor, never + evict here. Modes ``native``/``off`` return without touching thread or transcript and must + never fall back to the local compressor. Returns an outcome tag for logging/tests: ``compacted``, ``skipped:`` or ``failed:``. @@ -360,11 +307,9 @@ async def run_codex_hygiene_compaction( if mode not in {"native", "hermes", "off"}: mode = "native" if mode != "hermes": - # native: the app-server compacts on its own schedule; off: the - # operator disabled Hermes-initiated automatic compaction. A local - # transcript fallback is wrong in EVERY mode here (it cannot shrink - # the thread), so both modes are a clean skip — crucially without - # the detached-compressor path's cache eviction. + # native: the app-server compacts on its own schedule; off: the operator disabled Hermes- + # initiated automatic compaction. A local transcript fallback is wrong in EVERY mode (it + # cannot shrink the thread), so both are a clean skip — without the detached path's eviction. return f"skipped:mode={mode}" agent = None @@ -482,15 +427,10 @@ def _record_hygiene_cooldown( """Persist a session-hygiene compression-failure cooldown to the state DB. Uses the same ``compression_failure_cooldown_until`` column and - ``record_compression_failure_cooldown`` method that the in-conversation - compression path (``agent/context_compressor.py``) already uses, so the - cooldown survives gateway restarts (#74136). - - ``error`` is forwarded because the recorder writes - ``compression_failure_error`` UNCONDITIONALLY — omitting it clobbers to NULL - any reason the in-conversation path recorded, and readers surface that - reason to the user (falling back to "unknown error"). That matters more now - that an escalated cooldown can last up to an hour. + ``record_compression_failure_cooldown`` method that the in-conversation compression path + (``agent/context_compressor.py``) already uses, so the cooldown survives gateway restarts. + ``error`` is forwarded because the recorder writes ``compression_failure_error`` + UNCONDITIONALLY — omitting it clobbers any reason the in-conversation path recorded to NULL. """ import time as _time session_db = getattr(gateway, "_session_db", None) @@ -509,25 +449,15 @@ def _record_hygiene_cooldown( def _status_template_to_regex(template: str) -> str: """Compile a compression status template constant into a regex source. - Literal text is escaped verbatim (so wording drift in - agent/conversation_compression.py cannot silently diverge from this - matcher — the constants ARE the wording) and each ``{field}`` format - placeholder is replaced with a numeric-ish pattern covering every value - the emit sites format in (ints, ``{:,}`` thousands separators). + Literal text is escaped verbatim (the constants ARE the wording, so drift cannot silently + diverge from this matcher); each ``{field}`` placeholder becomes a numeric-ish pattern. """ parts = re.split(r"\{[^{}]*\}", template) return r"[\d,]+".join(re.escape(part) for part in parts) -# ROUTINE compression progress statuses, derived from the SAME template -# constants the emit sites format (agent/conversation_compression.py, #69550) -# — never re-inlined wording. Used ONLY by the opt-in -# ``compression.progress_notices`` gate below (#52995) to decide which of the -# noisy statuses matched by _TELEGRAM_NOISY_STATUS_RE are compression -# progress (deliverable when the user opted in) versus unrelated aux/retry -# chatter (always suppressed on chat surfaces). Failure notices and manual -# /compress feedback never match _TELEGRAM_NOISY_STATUS_RE in the first -# place, so they are unaffected by this gate. +# ROUTINE compression progress statuses, derived from the SAME template constants the emit sites +# format (agent/conversation_compression.py, #69550) — never re-inlined wording. _COMPRESSION_PROGRESS_STATUS_RE = re.compile( "|".join( _status_template_to_regex(_template) @@ -550,10 +480,9 @@ _COMPRESSION_PROGRESS_STATUS_RE = re.compile( def _gateway_compression_progress_notices_enabled() -> bool: """True when the user opted into routine compression progress notices. - Reads ``compression.progress_notices`` from the gateway's raw YAML config - (#52995). Default False — routine compression stays silent-by-design on - chat platforms unless explicitly enabled. Read live (mtime-cached) so a - config edit on a running gateway takes effect on the next status. + Reads ``compression.progress_notices`` from the gateway's raw YAML config. Default False — + routine compression stays silent-by-design on chat platforms unless explicitly enabled. Read + live (mtime-cached) so a config edit on a running gateway takes effect on the next status. Fail-closed: any config read error keeps the silent default. """ try: @@ -570,12 +499,9 @@ def _gateway_compression_progress_notices_enabled() -> bool: pass return False -# Surfaces that consume gateway text programmatically (CLI/TUI "local" -# diagnostics, API JSON, webhook payloads) and therefore must keep RAW -# status/error text. EVERY other platform is a human-facing chat surface -# where operational lifecycle/provider-error noise (and any secrets in it) -# must be suppressed or sanitized. Widens #28533's Telegram-only filter to -# all chat gateways (#39293). Fail-closed: unknown/empty platform -> chat. +# Surfaces that consume gateway text programmatically (CLI/TUI "local" diagnostics, API JSON, +# webhook payloads) and therefore must keep RAW status/error text. Widens #28533's Telegram-only +# filter to all chat gateways. Fail-closed: unknown/empty platform -> chat. _GATEWAY_RAW_TEXT_PLATFORMS = frozenset( {"local", "api_server", "webhook", "msgraph_webhook"} ) @@ -658,12 +584,8 @@ _GATEWAY_SECRET_PATTERNS = ( def _ensure_windows_gateway_venv_imports() -> None: """Make detached Windows gateway runs see the Hermes venv packages. - Some Windows restart paths run the gateway under uv's base ``pythonw.exe`` - to avoid the venv launcher respawning a visible console interpreter. That - mode can import the source tree via cwd/PYTHONPATH but still miss optional - packages installed only in ``venv/Lib/site-packages`` (notably the MCP SDK). - Patch the live process before MCP discovery so tool injection does not - depend on every launcher preserving PYTHONPATH perfectly. + Patch the live process before MCP discovery so tool injection does not depend on every + launcher preserving PYTHONPATH perfectly. """ if sys.platform != "win32": return @@ -732,16 +654,11 @@ def _interim_metadata( ) -> Dict[str, Any]: """Mark a mid-turn status/advisory send as NOT the turn-final. - Stream-is-the-message adapters (relay Slack native streaming) intercept - the first unmarked send to an armed (chat, turn) key and seal the live - stream with its content. Every gateway-side send that can fire while a - turn is streaming — heartbeats, inactivity warnings, approval fallbacks, - background-review notices — MUST carry this marker or it will seal the - user's answer stream with status text (PR 85796 review, B5: probed live, - the 3-minute heartbeat sealed the stream and the real final arrived as - a duplicate while later frames were silently swallowed by the seal - tombstone). The marker is gateway-internal; adapters strip it before - the wire. + Stream-is-the-message adapters (relay Slack native streaming) intercept the first unmarked + send to an armed (chat, turn) key and seal the live stream with its content. Every gateway- + side send that can fire mid-turn (heartbeats, inactivity warnings, approval fallbacks, + background-review notices) MUST carry this marker or it seals the user's answer stream with + status text. The marker is gateway-internal; adapters strip it before the wire. """ merged = dict(metadata or {}) merged["_interim_send"] = True @@ -754,16 +671,12 @@ def _seed_hygiene_system_prompt( ) -> bool: """Keep gateway hygiene from rebuilding a live session's system prompt. - The hygiene helper runs outside the live session's fully initialized - prompt environment (hygiene-only platform marker, no platform context - files; the memory provider is loaded only when - ``compression.checkpoint_required`` demands it). - Compression is allowed to persist a system prompt, so letting that helper - rebuild one would strip external provider blocks from the live session. - Seed the exact persisted prompt instead. When no usable prompt can be - restored, seed an empty cache entry. Compression either preserves that - unusable value or rebuilds with the hygiene-only platform marker; the real - turn will rebuild either form with its fully initialized providers. + The hygiene helper runs outside the live session's fully initialized prompt environment + (hygiene-only platform marker, no platform context files, memory provider only when required). + Compression is allowed to persist a system prompt, so letting that helper rebuild one would + strip external provider blocks from the live session. Seed the exact persisted prompt + instead. When no usable prompt can be restored, seed an empty cache entry; the real turn + rebuilds either form with its fully initialized providers. """ stored_prompt = "" if isinstance(session_row, dict): @@ -778,15 +691,9 @@ def _seed_hygiene_system_prompt( def _is_transient_network_error(exc: BaseException) -> bool: """Return True for transient network errors safe to log + swallow. - The crash class targeted by #31066 / #31110: an unhandled Telegram - ``TimedOut`` (or peer ``NetworkError`` / ``httpx`` connection error) - propagating to the event loop and killing the entire gateway - process. These are by definition transient — the next poll cycle or - user action recovers — so they must never crash the process. - - Walk the exception cause chain so wrapped errors (e.g. PTB's - ``NetworkError`` wrapping ``httpx.ConnectError``) are still - classified. The chain is bounded to avoid pathological cycles. + These are by definition transient — the next poll cycle or user action recovers — so they + must never crash the process. Walk the exception cause chain so wrapped errors (e.g. PTB's + ``NetworkError`` wrapping ``httpx.ConnectError``) are still classified. """ seen: set[int] = set() cur: Optional[BaseException] = exc @@ -824,13 +731,9 @@ def _gateway_loop_exception_handler( ) -> None: """Loop-level safety net for transient network errors. - Installed once during :func:`start_gateway`. Catches the - ``telegram.error.TimedOut`` crash class (issues #31066 / #31110) - and any peer transient network error before it can kill the - gateway process. Logs at WARNING with full traceback so the - originating call site stays diagnosable; non-transient errors - are forwarded to the default loop handler so real bugs still - surface. + Installed once during :func:`start_gateway`. Logs at WARNING with full traceback so the + originating call site stays diagnosable; non-transient errors are forwarded to the default + loop handler so real bugs still surface. """ exc = context.get("exception") if exc is not None and _is_transient_network_error(exc): @@ -856,17 +759,12 @@ def _gateway_loop_exception_handler( def _redact_gateway_user_facing_secrets(text: str) -> str: """Secret redaction before text can leave the gateway. - Delegates to the authoritative ``agent.redact.redact_sensitive_text`` — the - same Tirith-grade redactor already applied to logs, tool output, and - approval-command prompts — so the outbound chat path masks the full - credential set the startup banner promises ("chat responses are scrubbed - before delivery"), not a divergent subset. ``force=True`` honors redaction - even when ``security.redact_secrets`` is off, matching the - ``_redact_approval_command`` reasoning (#23810). - - The narrow ``_GATEWAY_SECRET_PATTERNS`` set runs as a belt-and-suspenders - second pass so nothing the gateway historically caught can regress, and so - redaction still degrades gracefully if the import ever fails. + Delegates to ``agent.redact.redact_sensitive_text`` (the same redactor used for logs, tool + output and approval prompts) so the chat path masks the full credential set, not a divergent + subset; ``force=True`` redacts even when ``security.redact_secrets`` is off. + The narrow ``_GATEWAY_SECRET_PATTERNS`` set runs as a belt-and-suspenders second pass so + nothing the gateway historically caught can regress, and so redaction still degrades + gracefully if the import ever fails. """ redacted = str(text or "") try: @@ -885,13 +783,10 @@ def _redact_gateway_user_facing_secrets(text: str) -> str: def _redact_approval_command(cmd: "str | None") -> str: """Redact credentials from a command before it goes into an approval prompt. - Tirith's *findings* are already redacted, but the gateway approval prompt - is built from the raw command string, so a credential-shaped value Tirith - flagged would otherwise be echoed verbatim to the chat platform (#48456). - Uses ``redact_sensitive_text(force=True)`` — the same Tirith-grade redactor - — so the prompt honors redaction even when ``security.redact_secrets`` is - off. Module-level so the wiring is unit-testable (the call site is a deeply - nested gateway closure that cannot be driven directly). + Tirith's *findings* are already redacted, but the gateway approval prompt is built from the + raw command string, so a credential-shaped value Tirith flagged would otherwise be echoed + verbatim to the chat platform. ``force=True`` so the prompt is redacted even when + ``security.redact_secrets`` is off. Module-level so the wiring is unit-testable. """ from agent.redact import redact_sensitive_text @@ -978,15 +873,10 @@ _GATEWAY_PROVIDER_ERROR_SHAPE_RE = re.compile( def _looks_like_gateway_provider_error(text: str) -> bool: """True when text is infrastructure/provider failure, not normal content. - Two heuristics combined so the rewrite only fires on actual provider - error envelopes, not on assistant prose that happens to mention an - HTTP status code: - - 1. The text is short — real provider errors are 1–3 lines of envelope - text; assistant answers are usually longer. - 2. AND the error marker appears at the start of the message (optionally - behind a punctuation/symbol prefix), not buried mid-paragraph in an - explanation like "HTTP 404 means 'not found' — ...". + Two heuristics combined so the rewrite only fires on actual provider error envelopes, not on + assistant prose that happens to mention an HTTP status code: the text is short (real + envelopes are 1-3 lines) AND the error marker appears at the start of the message, not + buried mid-paragraph in an explanation. """ if not text: return False @@ -1001,27 +891,20 @@ def _looks_like_gateway_provider_error(text: str) -> bool: def _sanitize_gateway_final_response(platform: Any, text: str) -> str: """Sanitize final gateway replies before sending them to chat surfaces. - Every human-facing chat surface (Telegram, WhatsApp, Discord, Slack, - Signal, Matrix, plugin platforms, etc.) should receive concise, safe - provider failure categories with secrets redacted instead of raw HTTP - bodies, request IDs, leaked credentials, or policy text. Only programmatic - surfaces in ``_GATEWAY_RAW_TEXT_PLATFORMS`` (CLI/TUI ``local`` diagnostics, - API JSON, webhook payloads) keep the raw text unchanged. + Every human-facing chat surface (Telegram, WhatsApp, Discord, Slack, Signal, Matrix, plugin + platforms, etc.) should receive concise, safe provider failure categories with secrets + redacted instead of raw HTTP bodies, request IDs, leaked credentials, or policy text. """ if not text: return text if _gateway_surface_passes_raw_text(platform): return text - # Lone UTF-16 surrogates (U+D800–U+DFFF) in model output crash chat - # surfaces downstream: Telegram's ``utf16_len`` length check and Signal - # formatting both ``.encode()`` the reply and raise UnicodeEncodeError - # before any send (#55143, #55309). The stored-history copy is already - # sanitized by ``build_assistant_message`` and ``finalize_turn`` scrubs - # the returned ``final_response``, but this boundary is the last line of - # defense for every legacy/plugin delivery path that hands us raw text. - # Raw-text/programmatic surfaces above keep passthrough — their JSON - # consumers escape surrogates safely. + # Lone UTF-16 surrogates (U+D800–U+DFFF) in model output crash chat surfaces downstream: + # Telegram's ``utf16_len`` length check and Signal formatting both ``.encode()`` the reply and + # raise UnicodeEncodeError before any send. Stored history is already sanitized upstream; this + # boundary is the last line of defense for legacy/plugin delivery paths that hand us raw text. + # Raw-text/programmatic surfaces above keep passthrough — their JSON consumers escape safely. from agent.message_sanitization import _sanitize_surrogates text = _sanitize_surrogates(str(text)) @@ -1051,12 +934,10 @@ def _prepare_gateway_status_message(platform: Any, event_type: str, message: str text = _redact_gateway_user_facing_secrets(text) if _TELEGRAM_NOISY_STATUS_RE.search(text): - # Opt-in #52995: `compression.progress_notices: true` lets ROUTINE - # compression progress statuses through to chat platforms. The - # membership check is derived from the #69550 template constants, so - # non-compression noise (aux failures, provider retry chatter, ...) - # stays suppressed even when the gate is open. Default False keeps - # the silent-by-design behavior byte-identical. + # Opt-in #52995: `compression.progress_notices: true` lets ROUTINE compression progress + # statuses through to chat platforms. Membership is derived from the template constants, so + # non-compression noise (aux failures, retry chatter) stays suppressed even when the gate is + # open. Default False keeps the silent-by-design behavior byte-identical. if not ( _gateway_compression_progress_notices_enabled() and _COMPRESSION_PROGRESS_STATUS_RE.search(text) @@ -1070,14 +951,11 @@ def _prepare_gateway_status_message(platform: Any, event_type: str, message: str def render_notice_line(notice) -> str: """Render an AgentNotice to a single plaintext line for messaging platforms. - Messaging has no persistent status bar (unlike the TUI), so a notice is a - one-shot standalone push. The notice policy already bakes the level glyph - (⚠ / • / ✕ / ✓) into the text, and the TUI + CLI REPL render that text - verbatim — so we emit it as-is here too. Prepending a per-level glyph would - DOUBLE it ("⚠ ⚠ Credits 90% used", "⛔ ✕ Credit access paused"). Plaintext - only — no markdown — so it renders uniformly across Telegram/Discord/Slack/ - SMS without per-platform escaping. Fail-soft: a malformed/empty notice - degrades to "" rather than raising on the agent's callback path. + Messaging has no persistent status bar (unlike the TUI), so a notice is a one-shot standalone + push. The notice policy already bakes the level glyph (⚠ / • / ✕ / ✓) into the text, and the + TUI + CLI REPL render that text verbatim — so we emit it as-is here too; prepending a per-level + glyph would DOUBLE it. Plaintext only (no markdown) so it renders uniformly across platforms. + Fail-soft: a malformed/empty notice degrades to "" rather than raising in the callback path. """ return str(getattr(notice, "text", "") or "").strip() @@ -1098,20 +976,11 @@ async def _send_or_update_status_coro(adapter, chat_id, status_key, content, met def _approval_send_outcome(future, timeout: float) -> str: """Classify an approval prompt send as ``sent`` / ``failed`` / ``ambiguous``. - ``ambiguous`` == the scheduling future timed out. The card may well have - posted: the connector may only ack after the deadline (slow platform API - call, transient backpressure, event-loop stall), and treating that timeout - as a failure has been observed in live relay testing to re-send the card - repeatedly, leaving the user's tap resolving a prompt whose turn had moved - on. Callers must treat - ``ambiguous`` as possibly-delivered: keep the prompt registration alive - and do NOT re-send or fall back — the boundary rule is that only a - DEFINITIVE failure (error result / non-timeout exception / no future) - re-asks. - - Definitive failures log their detail here (scheduling exception text or - the SendResult error) so callers sharing this classifier keep the - diagnostic breadcrumb the old inline code had. + ``ambiguous`` == the scheduling future timed out; the card may well have posted (late + connector ack), and treating that as failure re-sent the card repeatedly in live testing. + Callers must treat ``ambiguous`` as possibly-delivered: keep the prompt registration alive + and do NOT re-send or fall back — the boundary rule is that only a DEFINITIVE failure (error + result / non-timeout exception / no future) re-asks. Definitive failures log their detail here. """ if future is None: logger.warning("Prompt send failed: no scheduling future (loop unavailable)") @@ -1134,14 +1003,11 @@ def _approval_send_outcome(future, timeout: float) -> str: def _clarify_send_disposition(fut, *, session_key: str, clarify_mod) -> "str | None": """Decide whether a clarify prompt send aborts the wait, per the boundary rule. - Same physics as the exec-approval card: the scheduling future can hit its - deadline while the clarify card HAS already posted (late connector ack). - Treating that timeout as a definitive failure cleared the session out from - under a rendered card — the user answers a question whose registration is - gone. Only a DEFINITIVE failure (error result / non-timeout exception / - no future) tears down the registration and aborts; ``ambiguous`` keeps the - registration armed and proceeds to the normal bounded wait, which already - handles the truly-lost-card case via its response timeout. + Same physics as the exec-approval card: the scheduling future can hit its deadline while the + clarify card HAS already posted (late connector ack); treating that as definitive cleared the + registration under a rendered card. Only a DEFINITIVE failure tears down the registration and + aborts; ``ambiguous`` keeps it armed and proceeds to the bounded wait (whose response timeout + already covers the truly-lost-card case). Returns the abort sentinel string on definitive failure, else ``None`` (proceed to ``wait_for_response``). @@ -1162,14 +1028,7 @@ def _clarify_send_disposition(fut, *, session_key: str, clarify_mod) -> "str | N def _clarify_send_then_wait(fut, *, clarify_id: str, session_key: str, clarify_mod) -> str: - """Resolve a clarify prompt: send disposition, then the bounded wait. - - The full caller contract in one testable seam: a definitive send failure - returns the undeliverable sentinel (registration torn down); ``sent`` and - ``ambiguous`` both proceed to ``wait_for_response`` with the configured - timeout — for ambiguous, the registration stays armed so a late reply to - the (probably rendered) card still resolves. - """ + """Resolve a clarify prompt: send disposition, then the bounded wait.""" abort = _clarify_send_disposition( fut, session_key=session_key, clarify_mod=clarify_mod ) @@ -1192,11 +1051,10 @@ def _resolve_progress_thread_id( ) -> Optional[str]: """Return thread/root ID that progress/status bubbles should target. - ``reply_in_thread=False`` (Slack ``platforms.slack.extra.reply_in_thread``) - disables the synthetic-thread fallback: progress messages must not create - a thread the final flat reply would then inherit. A source.thread_id equal - to the event's own message id is the adapter's synthetic session-keying - thread, not a real thread — treat it as "no thread" too (#18859). + ``reply_in_thread=False`` (Slack ``platforms.slack.extra.reply_in_thread``) disables the + synthetic-thread fallback: progress messages must not create a thread the final flat reply + would then inherit. A source.thread_id equal to the event's own message id is the adapter's + synthetic session-keying thread, not a real one — treat it as "no thread" too. """ platform_value = getattr(platform, "value", platform) platform_key = str(platform_value or "").lower() @@ -1238,10 +1096,9 @@ def _resolve_gateway_display_bool( ) -> bool: """Resolve a boolean display setting with optional platform-only opt-in. - Some display features expose assistant scratch text rather than deliberate - user-facing output. For high-noise threaded chat surfaces such as - Mattermost, a global opt-in is too broad: they must be enabled with an - explicit display.platforms.. override. + Some display features expose assistant scratch text rather than deliberate user-facing + output. For high-noise threaded chat surfaces such as Mattermost, a global opt-in is too + broad: they must be enabled with an explicit display.platforms.. override. """ current_platform = _gateway_platform_value(platform or platform_key) platform_only = { @@ -1306,36 +1163,24 @@ def _telegramize_command_mentions(text: str, platform: Any) -> str: # ``config.yaml`` ``agent.gateway_auto_continue_freshness``. _AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT = 60 * 60 -# Default bound for how long ``_finish_startup_restore`` waits on boot -# auto-resume turns before releasing the inbound gate (see -# ``_startup_restore_drain_timeout_secs``). 30s is comfortably longer than a -# normal resume turn's first response yet short enough that one pathologically -# long resumed turn can't hold every channel's inbound queued for minutes. -# Override via ``config.yaml`` ``agent.gateway_startup_restore_drain_timeout``. +# Default bound for how long ``_finish_startup_restore`` waits on boot auto-resume turns before +# releasing the inbound gate (see ``_startup_restore_drain_timeout_secs``). Override via +# ``config.yaml`` ``agent.gateway_startup_restore_drain_timeout``. _STARTUP_RESTORE_DRAIN_TIMEOUT_SECS_DEFAULT = 30.0 -# Default bound for the boot-time turn-machinery warm-up (#99373). On a -# fresh boot with no resume_pending sessions ``_finish_startup_restore`` -# used to open the inbound gate almost immediately, while the agent-side -# turn machinery (the run_agent/model_tools import graph, the tool-registry -# check_fn probes, the prompt builder) was still completely cold. A message -# arriving in that window was served with a skeleton system prompt: no -# context tier, no tool schemas (~1.7K tokens instead of ~14.6K). The -# warm-up runs BEFORE the gate opens so the first inbound turn starts with -# initialized machinery; the bound keeps a wedged init from making the -# gateway permanently unavailable. Override via ``config.yaml`` -# ``agent.gateway_startup_warmup_timeout`` (non-positive disables warm-up). +# Default bound for the boot-time turn-machinery warm-up. Without it a message arriving right after +# boot was served with a skeleton system prompt (cold run_agent/model_tools import graph, no tool +# schemas). The warm-up runs BEFORE the gate opens so the first inbound turn starts with initialized +# machinery; the bound keeps a wedged init from making the gateway permanently unavailable. +# Override via ``agent.gateway_startup_warmup_timeout`` (non-positive disables warm-up). _STARTUP_WARMUP_TIMEOUT_SECS_DEFAULT = 20.0 def _coerce_gateway_timestamp(value: Any) -> Optional[float]: """Best-effort conversion of stored gateway timestamps to epoch seconds. - Missing/unparseable timestamps return None so legacy transcripts keep the - historical auto-continue behaviour instead of being silently dropped. - Accepts: datetime, epoch seconds (int/float), epoch milliseconds (when - the magnitude exceeds year-2286), ISO-8601 strings (with or without a - trailing ``Z``), and numeric strings. + Missing/unparseable timestamps return None so legacy transcripts keep the historical auto- + continue behaviour instead of being silently dropped. """ if value is None: return None @@ -1365,44 +1210,25 @@ def _coerce_gateway_timestamp(value: Any) -> Optional[float]: def _auto_continue_freshness_window() -> float: """Return the configured auto-continue freshness window in seconds. - Thin wrapper that delegates to the canonical implementation in - ``gateway.session`` (the single source of truth shared with the - routing-time zombie gate in ``get_or_create_session``). Reads - ``HERMES_AUTO_CONTINUE_FRESHNESS`` (bridged from ``config.yaml`` - ``agent.gateway_auto_continue_freshness`` at gateway startup, same - pattern as ``HERMES_AGENT_TIMEOUT``). Falls back to the module default - when unset or malformed. Non-positive values disable the freshness gate - (restores the pre-fix "always fresh" behaviour for users who want to opt - out). Kept here so existing call sites and test patches importing it - from ``gateway.run`` continue to work. + Thin wrapper over the canonical implementation in ``gateway.session`` (shared with the + routing-time zombie gate in ``get_or_create_session``). Falls back to the module default when + unset or malformed; non-positive values disable the freshness gate. Kept here so existing call + sites and test patches importing it from ``gateway.run`` continue to work. """ from gateway.session import auto_continue_freshness_window return auto_continue_freshness_window() def _startup_restore_drain_timeout_secs() -> float: - """Max seconds ``_finish_startup_restore`` waits on boot auto-resume turns - before releasing the inbound gate and draining the queue. + """Max seconds ``_finish_startup_restore`` waits on boot auto-resume turns before releasing the + inbound gate and draining the queue. - While startup restore is in progress the gateway QUEUES every inbound - message (``_queue_startup_restore_event``) instead of processing it, so no - channel gets a reply until the gate opens. The gate is opened by - ``_finish_startup_restore``, which waits for the synthetic boot - auto-resume turns to finish. A single long resumed turn therefore held - the gate shut for every channel — inbound piled up unanswered for as long - as that one turn ran. - - This bounds that wait. Duplicate-agent safety does NOT depend on the - wait: ``_schedule_resume_pending_sessions`` claims each session's - ``_running_agents`` slot SYNCHRONOUSLY (before the gate ever runs), so a - message drained while a resume turn is still running queues behind that - slot rather than spawning a second agent. So on timeout we release the - gate and let the slow turn finish in the background. - - Reads ``HERMES_STARTUP_RESTORE_DRAIN_TIMEOUT`` (bridged from - ``config.yaml`` ``agent.gateway_startup_restore_drain_timeout`` at gateway - startup, same pattern as the other ``agent.*`` knobs). Non-positive - disables the bound (restores the historical "wait forever" behaviour). + While startup restore is in progress the gateway QUEUES every inbound message + (``_queue_startup_restore_event``) instead of processing it, so no channel gets a reply until + the gate opens. This bounds that wait. Duplicate-agent safety does NOT depend on the wait: + ``_schedule_resume_pending_sessions`` claims each session's ``_running_agents`` slot + SYNCHRONOUSLY, so a drained message queues behind a still-running resume turn rather than + spawning a second agent. Non-positive disables the bound. """ raw = os.environ.get("HERMES_STARTUP_RESTORE_DRAIN_TIMEOUT") if raw is None or raw == "": @@ -1414,20 +1240,14 @@ def _startup_restore_drain_timeout_secs() -> float: def _startup_warmup_timeout_secs() -> float: - """Max seconds the boot warm-up may hold the inbound gate shut (#99373). + """Max seconds the boot warm-up may hold the inbound gate shut. - ``GatewayRunner._warm_turn_prerequisites`` initializes the agent-side - turn machinery BEFORE ``_finish_startup_restore`` opens the inbound - gate, so a message arriving seconds after boot can no longer be served - with a skeleton system prompt (no context tier, no tool schemas). The - warm-up is bounded so a wedged import or probe can never make the - gateway permanently unavailable — on timeout the gate opens anyway and - the warm-up finishes in the background. - - Reads ``HERMES_STARTUP_WARMUP_TIMEOUT`` (bridged from ``config.yaml`` - ``agent.gateway_startup_warmup_timeout`` at gateway startup, same - pattern as the other ``agent.*`` knobs). Non-positive disables the - warm-up entirely (restores the historical lazy-init behaviour). + ``GatewayRunner._warm_turn_prerequisites`` initializes the agent-side turn machinery BEFORE + ``_finish_startup_restore`` opens the inbound gate, so a message arriving seconds after boot + can no longer be served with a skeleton system prompt (no context tier, no tool schemas). + Bounded so a wedged import/probe can never make the gateway permanently unavailable — on + timeout the gate opens anyway and the warm-up finishes in the background. Non-positive + disables the warm-up. """ raw = os.environ.get("HERMES_STARTUP_WARMUP_TIMEOUT") if raw is None or raw == "": @@ -1441,15 +1261,11 @@ def _startup_warmup_timeout_secs() -> float: def _warm_turn_machinery_sync() -> int: """Synchronously initialize the turn prerequisites a first turn needs. - Runs on an executor thread from ``_warm_turn_prerequisites``. Covers - exactly the lazy init observed inside skeleton turns (#99373): - - * the ``run_agent`` heavy import graph (the gateway imports it lazily - inside per-request handlers, so nothing else pulls it in at boot); - * ``model_tools.get_tool_definitions`` — materializes tool schemas and - primes the tool-registry ``check_fn`` TTL cache so availability - probes don't run (and fail cold) inside the user's first turn; - * the context-file tier (AGENTS.md / SOUL.md discovery + read). + Runs on an executor thread from ``_warm_turn_prerequisites``. Covers exactly the lazy init + observed inside skeleton turns: the ``run_agent`` import graph (otherwise only pulled in + lazily per request), ``model_tools.get_tool_definitions`` (materializes schemas and primes + the tool-registry ``check_fn`` TTL cache so probes don't run cold inside the first turn), and + the context-file tier. Returns the number of tool schemas materialized (logged for diagnosability). @@ -1514,12 +1330,9 @@ def _is_fresh_gateway_interruption( ) -> bool: """Return True when an interruption marker is fresh enough to auto-continue. - Unknown timestamps are treated as fresh for backward compatibility with - legacy transcripts (pre-dating timestamp persistence) and with in-memory - test scaffolding that constructs history entries without timestamps. - - A non-positive ``window_secs`` disables the gate (always fresh), which - restores the pre-fix behaviour for users who opt out via config. + Unknown timestamps are treated as fresh for backward compatibility with legacy transcripts + (pre-dating timestamp persistence) and with in-memory test scaffolding that constructs + history entries without timestamps. """ window = ( float(window_secs) @@ -1543,19 +1356,12 @@ def build_resume_recovery_note( ) -> str: """Build the resume-pending recovery system note for an interrupted turn. - ``reason`` is the session's ``resume_reason`` (``restart_timeout``, - ``shutdown_timeout``, or anything else → generic interruption phrasing). - ``message`` is the user's NEW message text; empty means this is the - startup auto-resume turn synthesized by - ``_schedule_resume_pending_sessions`` with no human message attached. - - ``interactive`` selects the empty-message guidance: on interactive - platforms a human is present, so "report the restore and ask what next" - is right. On non-interactive event platforms (webhook, API server — - adapters with ``interactive_resume = False``) nobody can answer; the - resumed turn must instead complete the interrupted work, or the task is - silently abandoned behind a "restored" acknowledgement that goes - nowhere (#57056). + Empty ``message`` means the startup auto-resume turn with no human message attached. + ``interactive`` selects the empty-message guidance: on interactive platforms "report the + restore and ask what next" is right. On non-interactive event platforms (webhook, API server — + adapters with ``interactive_resume = False``) nobody can answer; the resumed turn must instead + complete the interrupted work, or the task is silently abandoned behind a "restored" + acknowledgement that goes nowhere. """ reason_phrase = ( "a gateway restart" @@ -1612,14 +1418,11 @@ def _prepare_resume_pending_message( ) -> tuple[str, str]: """Return the recovery message and the user text to persist. - Resume turns replace the startup event's text with a recovery note before - entering the agent. When the original message is empty (the synthesized - auto-resume turn), persist the note too — persisting the empty string - left a blank user row in state.db that the pre-call sanitizer re-healed - on every later call forever (#86580). When the user sent REAL text while - the resume was pending, keep persisting their clean words: the transcript - stays scaffold-free (the model still receives the wrapped note), and a - non-empty row never trips the sanitizer. + When the original message is empty (synthesized auto-resume turn), persist the note too — + persisting "" left a blank user row that the pre-call sanitizer re-healed on every later call. + When the user sent REAL text while the resume was pending, keep persisting their clean words: + the transcript stays scaffold-free (the model still receives the wrapped note), and a non- + empty row never trips the sanitizer. """ recovery_message = build_resume_recovery_note( reason, message or "", interactive=interactive, @@ -1674,39 +1477,23 @@ def _build_replay_entry( msg: Dict[str, Any], preserve_timestamp: bool = False, ) -> Dict[str, Any]: - """Build a replay entry for a non-tool-calling message, preserving the - assistant fields the agent's API builders rely on for multi-turn fidelity. + """Build a replay entry for a non-tool-calling message, preserving the assistant fields the + agent's API builders rely on for multi-turn fidelity. - Lifted out of the inline ``run_sync`` closure so the field whitelist can - be unit-tested in isolation. Mirrors the ``_ASSISTANT_REPLAY_FIELDS`` - contract above. + Lifted out of the inline ``run_sync`` closure so the field whitelist can be unit-tested in + isolation. Mirrors the ``_ASSISTANT_REPLAY_FIELDS`` contract above. - ``preserve_timestamp``: when True, copy the source row's ``timestamp`` - onto the replay entry. Currently only user messages need this — the - stale-dangerous-confirmation stripper in ``agent/replay_cleanup.py`` - reads the timestamp to decide whether a confirmation is too old to - replay safely. Assistant/tool messages are not timestamp-stripped in - the same way, so we keep the existing default of dropping it. - - Empty values: most fields are dropped when falsy (matching the original - PR #2974 behaviour) since an empty list/string for those carries no - information. The exception is ``reasoning_content``: DeepSeek/Kimi - thinking-mode replay treats an empty string as a meaningful sentinel - that ``_copy_reasoning_content_for_api`` upgrades to a single space. - Dropping it here would make the gateway send no ``reasoning_content`` - at all on the next turn, which can cause HTTP 400 from strict thinking - providers. + ``preserve_timestamp``: only user messages need it (the stale-dangerous-confirmation stripper + reads it); assistant/tool messages are not timestamp-stripped, so we keep dropping it. + Falsy fields are dropped EXCEPT ``reasoning_content``: DeepSeek/Kimi thinking-mode replay + treats "" as a sentinel (upgraded to a space downstream); dropping it can cause HTTP 400. """ entry: Dict[str, Any] = {"role": role, "content": content} - # api_content sidecar (persist-what-you-send, prompt-cache stability): - # forward the exact bytes previously sent to the API for this message so - # the agent's api_messages build can substitute them and keep the request - # prefix byte-stable across turns. Forward ONLY when this replay pipeline - # did not rewrite the content (timestamp injection, auto-continue strip, - # mirror prefix): a rewritten clean content means the pipeline decided - # different bytes must replay — resending the stored sidecar would - # reintroduce exactly what was stripped. Dropping it costs one cache - # boundary; resending stripped noise is a behavior regression. + # api_content sidecar (persist-what-you-send, prompt-cache stability): forward the exact bytes + # previously sent to the API for this message so the agent's api_messages build can substitute + # them and keep the request prefix byte-stable across turns. Forward ONLY when this pipeline + # did not rewrite the content — resending the sidecar would reintroduce exactly what was + # stripped. Dropping it costs one cache boundary; resending stripped noise is a regression. _sidecar = msg.get("api_content") if ( role in ("user", "assistant") @@ -1742,12 +1529,9 @@ _CURRENT_ADDRESSED_MESSAGE_HEADER = "[Current addressed message - answer only th def _uses_telegram_observed_group_context(channel_prompt: Optional[str]) -> bool: """Return True for Telegram group turns that may include observed chatter. - Telegram's observe-unmentioned mode persists skipped group chatter so a - later @mention can see it. Those rows must not replay as ordinary user - turns: a weak wake word like ``@bot cambio`` should not make the model treat - old unmentioned chatter as pending work. The Telegram adapter marks these - turns with a channel prompt; this helper keeps the run-path check explicit - and unit-testable. + Telegram's observe-unmentioned mode persists skipped group chatter so a later @mention can + see it. Those rows must not replay as ordinary user turns: a weak wake word like ``@bot + cambio`` should not make the model treat old unmentioned chatter as pending work. """ return bool(channel_prompt and _TELEGRAM_OBSERVED_CONTEXT_PROMPT_MARKER in channel_prompt) @@ -1768,11 +1552,9 @@ def _csv_or_list_to_set(raw: Any) -> set[str]: def _slack_ignored_channels_from_gateway_config(config: Any) -> set[str]: """Return Slack channels that the generic gateway must never dispatch. - The Slack adapter has the first-line drop, but this runner-level guard is - intentionally duplicated as a fail-safe. If a future Slack code path, test - hook, malformed event, or stale adapter instance bypasses the Slack plugin - adapter, ignored channels still cannot reach auth, pairing, sessions, or - the agent/home-channel prompt pipeline. + The Slack adapter has the first-line drop, but this runner-level guard is intentionally + duplicated as a fail-safe: if any code path, test hook, or stale adapter bypasses the plugin + adapter, ignored channels still cannot reach auth, pairing, sessions, or the prompt pipeline. """ platform_cfg = getattr(config, "platforms", {}).get(Platform.SLACK) raw = None @@ -1803,10 +1585,9 @@ def _is_slack_ignored_channel(config: Any, chat_id: Any) -> bool: def _message_timestamps_enabled(user_config: Optional[dict]) -> bool: """True when gateway.message_timestamps.enabled is opted in. - Default OFF: injecting a ``[Tue 2026-04-28 13:40:53 CEST]`` prefix onto - every user message changes what the model sees for all gateway users, so - it must be explicitly enabled in config.yaml under - ``gateway.message_timestamps.enabled``. + Default OFF: injecting a ``[Tue 2026-04-28 13:40:53 CEST]`` prefix onto every user message + changes what the model sees for all gateway users, so it must be explicitly enabled in + config.yaml under ``gateway.message_timestamps.enabled``. """ if not isinstance(user_config, dict): return False @@ -1828,15 +1609,9 @@ def _build_gateway_agent_history( ) -> tuple[List[Dict[str, Any]], Optional[str]]: """Convert stored gateway transcript rows into agent replay messages. - Observed Telegram group rows are returned as API-only context for the - current addressed message instead of being replayed as normal prior user - turns. Keeping that context out of ``conversation_history`` avoids - consecutive-user repair merging it with the live user turn and then hiding - the current message behind ``history_offset`` during persistence. - - When ``inject_timestamps`` is True (gateway.message_timestamps.enabled), - each replayed user message is rendered with a single human-readable - timestamp prefix from its stored metadata. + Keeping that context out of ``conversation_history`` avoids consecutive-user repair merging + it with the live user turn and then hiding the current message behind ``history_offset`` + during persistence. """ from hermes_time import get_timezone as _get_msg_tz @@ -1880,11 +1655,9 @@ def _build_gateway_agent_history( clean_msg = {k: v for k, v in msg.items() if k not in {"timestamp", "observed"}} agent_history.append(clean_msg) elif content: - # Strip gateway-injected auto-continue notes that were persisted - # as part of user messages during interrupted turns. Keep the - # user's real text after the note, but never replay the recovery - # instruction itself — that is what caused infinite re-execution - # loops for interrupted long-running tools. + # Strip gateway-injected auto-continue notes that were persisted as part of user + # messages during interrupted turns. Keep the user's real text but never replay the + # recovery instruction itself — that caused infinite re-execution loops. if role == "user": content = _strip_auto_continue_noise(content) if not content: @@ -1893,11 +1666,8 @@ def _build_gateway_agent_history( if msg.get("mirror"): mirror_src = msg.get("mirror_source", "another session") content = f"[Delivered from {mirror_src}] {content}" - # Preserve the timestamp on user messages so the - # stale-dangerous-confirmation stripper in agent/replay_cleanup.py - # can read it. The timestamp is dropped from assistant messages - # because they don't need it; the replay-tail strippers look at - # assistant(tool_calls), not timestamps. + # Preserve the timestamp on user messages so the stale-dangerous-confirmation stripper + # in agent/replay_cleanup.py can read it. entry = _build_replay_entry(role, content, msg, preserve_timestamp=(role == "user")) agent_history.append(entry) @@ -1905,18 +1675,15 @@ def _build_gateway_agent_history( # tools that were killed mid-flight. agent_history = strip_interrupted_tool_tails(agent_history) - # Strip a dangling assistant(tool_calls) tail with no tool answers — - # the signature of a SIGKILL mid-tool-call (e.g. the tool itself ran - # `docker restart`/`kill` and took the gateway down before the result - # was persisted). Without this the model re-issues the unanswered call - # on resume and loops the restart forever (#49201). + # Strip a dangling assistant(tool_calls) tail with no tool answers — the signature of a SIGKILL + # mid-tool-call (e.g. the tool itself ran `docker restart`/`kill` and took the gateway down + # before the result was persisted). Without this the model re-issues the unanswered call on + # resume and loops the restart forever. agent_history = strip_dangling_tool_call_tail(agent_history) - # Strip stale dangerous-confirmation text in user messages (#59607). - # A high-risk confirmation phrase (e.g. "confirm forced restart") that - # is older than the expiry window must not be replayed to the model, - # otherwise an unrelated follow-up message can be interpreted as a - # fresh confirmation and trigger the destructive action a second time. + # Strip stale dangerous-confirmation text in user messages. A high-risk confirmation phrase + # (e.g. "confirm forced restart") older than the expiry window must not replay, or an unrelated + # follow-up could be read as a fresh confirmation and re-trigger the destructive action. agent_history = strip_stale_dangerous_confirmations( agent_history, now=time.time() ) @@ -1929,14 +1696,13 @@ def _select_cached_agent_history( persisted_history: List[Dict[str, Any]], live_history: Any, ) -> List[Dict[str, Any]]: - """Prefer a cached live transcript only when it is longer and contains at - least one real, non-ephemeral unpersisted row. + """Prefer a cached live transcript only when it is longer and contains at least one real, non- + ephemeral unpersisted row. - Guards the FTS write-corruption case (#50502): when message writes fail - silently through corrupt FTS triggers, the next turn reloads a stale/empty - ``conversation_history`` from disk even though the same cached ``AIAgent`` - still holds unpersisted real rows in ``_session_messages``. Replacing those - rows with the shorter persisted copy causes immediate same-session amnesia. + Guards the FTS write-corruption case: when message writes fail silently through corrupt FTS + triggers, the next turn reloads a stale/empty ``conversation_history`` from disk even though + the same cached ``AIAgent`` still holds unpersisted real rows in ``_session_messages``. + Replacing those with the shorter persisted copy causes immediate same-session amnesia. Length alone does not trigger retention. Returns ``persisted_history`` unchanged unless the live copy is a longer @@ -1988,10 +1754,9 @@ def _wrap_current_message_with_observed_context(message: Any, observed_context: def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: """Return the ``timestamp`` of the last usable transcript row, if any. - Skips metadata-only rows (``session_meta``, system injections) that are - dropped before being handed to the agent. Returns ``None`` when no - usable row carries a timestamp — callers should treat that as "fresh" - for backward compatibility. + Skips metadata-only rows (``session_meta``, system injections) that are dropped before being + handed to the agent. Returns ``None`` when no usable row carries a timestamp — callers should + treat that as "fresh" for backward compatibility. """ if not history: return None @@ -2010,10 +1775,9 @@ def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: return None -# Tool results can contain literal MEDIA: examples in docs, logs, or other -# ordinary outputs. Only tools that intentionally create deliverable media -# artifacts should be eligible for automatic append when the model omits them -# from the final gateway reply. +# Tool results can contain literal MEDIA: examples in docs, logs, or other ordinary outputs. Only +# tools that intentionally create deliverable media artifacts should be eligible for automatic +# append when the model omits them from the final gateway reply. _AUTO_APPEND_MEDIA_TOOL_NAMES = { "text_to_speech", "text_to_speech_tool", @@ -2022,11 +1786,8 @@ _AUTO_APPEND_MEDIA_TOOL_NAMES = { # ---- helpers: detect interrupted tool tails & auto-continue noise ---------- -# Replay-tail sanitization lives in agent/replay_cleanup.py so every resume -# surface (this messaging gateway AND the TUI/WebUI gateway) shares one -# implementation. Import the canonical names directly — the historical -# private ``_``-prefixed aliases were retired once the last external -# consumers (tests) moved to agent.replay_cleanup. +# Replay-tail sanitization lives in agent/replay_cleanup.py so every resume surface (this messaging +# gateway AND the TUI/WebUI gateway) shares one implementation. from agent.replay_cleanup import ( # noqa: E402 strip_interrupted_tool_tails, strip_dangling_tool_call_tail, @@ -2052,10 +1813,9 @@ def _is_auto_continue_noise(content: Any) -> bool: def _strip_auto_continue_noise(content: Any) -> Any: """Remove persisted gateway auto-continue note prefix from user text. - Older gateway builds prepended the recovery note directly to the user - message, so the transcript row can contain both the synthetic note and - the user's real question. Strip one or more leading synthetic notes while - preserving any real text that follows. + Older gateway builds prepended the recovery note directly to the user message, so the + transcript row can contain both the synthetic note and the user's real question. Strip one or + more leading synthetic notes while preserving any real text that follows. """ if not _is_auto_continue_noise(content): return content @@ -2067,18 +1827,14 @@ def _strip_auto_continue_noise(content: Any) -> Any: text = text[end + 1 :].lstrip() return text -# Tools in this set return their deliverable artifact as a JSON payload with a -# local-file path field rather than a literal ``MEDIA:`` tag (e.g. image_generate -# returns ``{"success": true, "image": "/abs/path.png"}``). The auto-append path -# extracts the path from these fields so delivery is deterministic and does not -# depend on the model restating the path in its final reply. +# Tools in this set return their deliverable artifact as a JSON payload with a local-file path field +# rather than a literal ``MEDIA:`` tag (e.g. image_generate returns ``{"success": true, "image": +# "/abs/path.png"}``). _JSON_MEDIA_TOOL_PATH_FIELDS = ("host_image", "image", "agent_visible_image") -# Extension-anchored MEDIA: matcher for tool results. Mirrors the dispatch-site -# pattern so a bare ``MEDIA:`` token in prose (no deliverable extension) is never -# auto-appended. Kept local to the auto-append path; the producer-tool allowlist -# below is the primary guard, this is the secondary precision guard. +# Extension-anchored MEDIA: matcher for tool results. Mirrors the dispatch-site pattern so a bare +# ``MEDIA:`` token in prose (no deliverable extension) is never auto-appended. _TOOL_MEDIA_RE = re.compile( r'MEDIA:((?:[A-Za-z]:[/\\]|/|~\/)\S+\.(?:png|jpe?g|gif|webp|' r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|' @@ -2088,10 +1844,9 @@ _TOOL_MEDIA_RE = re.compile( ) -# Shared with cron delivery and gateway background tasks — the repair must -# run on every surface that feeds a final response into media extraction. -# Canonical names live in gateway.media_repair (same retirement of private -# aliases as the agent.replay_cleanup import above). +# Shared with cron delivery and gateway background tasks — the repair must run on every surface that +# feeds a final response into media extraction. Canonical names live in gateway.media_repair (same +# retirement of private aliases as the agent.replay_cleanup import above). from gateway.media_repair import ( # noqa: E402 repair_explicit_computer_use_media_paths, tool_name_by_call_id as _tool_name_by_call_id, @@ -2105,21 +1860,12 @@ def _collect_auto_append_media_tags( ) -> tuple[List[str], bool]: """Collect real media tags from current-turn producer-tool results only. - Two layered guards keep stale/example MEDIA: strings out of the reply: - - 1. Producer-tool allowlist: only tools that intentionally emit deliverable - artifacts (TTS) are eligible. Documentation, logs, and search results can - contain example strings such as MEDIA:/absolute/path/to/file, which must - never be delivered as attachments. (Fixes the original report behind #16721.) - 2. Current-turn isolation: only messages produced this turn are scanned, so a - tool result from an earlier turn (still present in the full message list) - cannot leak onto a later text-only reply (#34608). - - Mid-run context compression can rewrite/shrink the message list below the - original history length. When that happens the slice boundary is no longer - trustworthy, so fall back to scanning every message and rely on - ``history_media_paths`` for dedup, preserving the compression-safe behaviour - of #160. The producer-tool allowlist still applies on the fallback path. + Two layered guards keep stale/example MEDIA: strings out of the reply: a producer-tool + allowlist (docs/logs/search results contain example MEDIA: strings that must never become + attachments) and current-turn isolation (an earlier turn's tool result must not leak onto a + later text-only reply). If mid-run compression shrank the list below the original history + length the slice boundary is untrustworthy: scan every message and rely on + ``history_media_paths`` for dedup; the allowlist still applies. """ history_media_paths = history_media_paths or set() # Only trust the slice boundary when the message list still contains the @@ -2173,14 +1919,8 @@ def _collect_auto_append_media_tags( def _collect_history_media_paths(agent_history: List[Dict[str, Any]]) -> set: """Collect every media path already delivered in prior assistant/tool output. - Used to dedup auto-appended and model-emitted MEDIA tags so the same file - is not re-sent on later turns. Covers three delivery shapes: - * ``MEDIA:`` text tags in tool results, - * ``MEDIA:`` text tags in assistant messages (model-generated tags), - * ``image_generate`` JSON-payload paths (``host_image`` / ``image`` / - ``agent_visible_image``), which carry no MEDIA: tag. - - Missing the JSON-payload shape caused #46627; missing the assistant-message + Used to dedup auto-appended and model-emitted MEDIA tags so the same file is not re-sent on + later turns. Missing the JSON-payload shape caused #46627; missing the assistant-message shape caused repeated delivery when the model echoed a previous MEDIA tag. """ paths: set = set() @@ -2191,10 +1931,9 @@ def _collect_history_media_paths(agent_history: List[Dict[str, Any]]) -> set: path = match.group(1).strip().rstrip('",}') if path: paths.add(path) - # The regex alone misses quoted and spaced paths that the delivery - # pipeline's extract_media grammar accepts — collect through the same - # extractor so the dedup set sees every path that could actually have - # been delivered. + # The regex alone misses quoted and spaced paths that the delivery pipeline's extract_media + # grammar accepts — collect through the same extractor so the dedup set sees every path that + # could actually have been delivered. media_files, _ = BasePlatformAdapter.extract_media(content) paths.update(path for path, _is_voice in media_files) @@ -2232,10 +1971,8 @@ def _collect_history_media_paths(agent_history: List[Dict[str, Any]]) -> set: def _ensure_ssl_certs() -> None: """Set SSL_CERT_FILE if the system doesn't expose CA certs to Python. - Windows startup paths (Desktop, Scheduled Tasks, installer children) can - occasionally inherit a stale SSL_CERT_FILE. Returning just because the - variable is present makes every later httpx/OpenAI client construction fail - with FileNotFoundError from ssl.load_verify_locations(). Treat a missing + Returning just because the variable is present makes every later httpx/OpenAI client + construction fail with FileNotFoundError from ssl.load_verify_locations(). Treat a missing path as unset and fall back to certifi instead. """ configured_cert = os.environ.get("SSL_CERT_FILE") @@ -2343,16 +2080,12 @@ load_hermes_dotenv(hermes_home=_hermes_home, project_env=Path(__file__).resolve( def _reload_runtime_env_preserving_config_authority() -> None: """Reload .env for fresh credentials without letting stale .env override config. - Gateway processes are long-lived, so per-turn code reloads ~/.hermes/.env to - pick up rotated API keys. config.yaml remains authoritative for agent budget - settings such as agent.max_turns; otherwise a stale HERMES_MAX_ITERATIONS in - .env can replace the startup bridge on later turns. - - In multiplex mode this is a NO-OP for the credential reload: secrets come - from the per-turn ``set_secret_scope`` (installed by ``_profile_runtime_scope``) - which loads the routed profile's ``.env`` into an isolated mapping. Mutating - the process-global ``os.environ`` here would defeat that isolation and leak - the default profile's keys to every profile's turns and subprocesses. + Gateways are long-lived, so per-turn code reloads ~/.hermes/.env to pick up rotated keys; + config.yaml stays authoritative for agent budget settings (else a stale HERMES_MAX_ITERATIONS + in .env would replace the startup bridge). In multiplex mode the credential reload is a NO-OP: + secrets come from the per-turn ``set_secret_scope`` isolated mapping, and mutating the + process-global ``os.environ`` here would defeat that isolation and leak the + default profile's keys to every profile's turns and subprocesses. """ from agent.secret_scope import is_multiplex_active if is_multiplex_active(): @@ -2382,10 +2115,9 @@ def _bridge_max_turns_from_config(home: "Path") -> None: cfg = _expand_env_vars(cfg) if not isinstance(cfg, dict): cfg = {} - # Managed scope: keep administrator-pinned values authoritative on every - # turn too. This per-turn reload re-bridges config→env, so without the - # overlay a managed agent.max_turns / timezone / redact_secrets would be - # replaced by the user's value after the first turn. Fail-open. + # Managed scope: keep administrator-pinned values authoritative on every turn too. This per- + # turn reload re-bridges config→env, so without the overlay a managed agent.max_turns / + # timezone / redact_secrets would be replaced by the user's value after the first turn. try: from hermes_cli import managed_scope cfg = managed_scope.apply_managed_overlay(cfg) @@ -2397,12 +2129,10 @@ def _bridge_max_turns_from_config(home: "Path") -> None: agent_cfg = cfg.get("agent", {}) if isinstance(agent_cfg, dict) and "max_turns" in agent_cfg: raw = agent_cfg["max_turns"] - # Preserve the raw value's spelling (e.g. "none", "unlimited", "120") - # so resolve_turn_limit() in _current_max_iterations can interpret it. - # Skip bridging when the YAML value is Python None (from `null` or bare - # `key:`) — this preserves "absent = default" semantics downstream. - # Without this guard, str(None) → "None" → resolve_turn_limit maps it - # to the unlimited sentinel instead of the default (90/500). + # Preserve the raw value's spelling (e.g. "none", "unlimited", "120") so + # resolve_turn_limit() in _current_max_iterations can interpret it. Skip when the YAML value + # is Python None (`null` / bare `key:`): str(None) -> "None" would map to the unlimited + # sentinel instead of preserving "absent = default". if raw is not None: os.environ["HERMES_MAX_ITERATIONS"] = str(raw) elif "HERMES_MAX_ITERATIONS" in os.environ: @@ -2421,10 +2151,9 @@ def _bridge_max_turns_from_config(home: "Path") -> None: def _current_max_iterations() -> int: """Return the current per-turn iteration budget after runtime env refresh. - Goes through :func:`hermes_cli.config.resolve_turn_limit` so that - ``agent.max_turns: none`` / ``unlimited`` (bridged into - ``HERMES_MAX_ITERATIONS`` as a string) resolves to the unlimited sentinel - instead of crashing ``int()``. + Goes through :func:`hermes_cli.config.resolve_turn_limit` so that ``agent.max_turns: none`` / + ``unlimited`` (bridged into ``HERMES_MAX_ITERATIONS`` as a string) resolves to the unlimited + sentinel instead of crashing ``int()``. """ _reload_runtime_env_preserving_config_authority() from hermes_cli.config import resolve_turn_limit as _resolve_turn_limit @@ -2437,13 +2166,11 @@ from contextlib import ( ) -# Platforms that bind a host TCP port (HTTP/webhook listeners). In a profile -# multiplexer the default profile owns the single shared listener and serves -# every profile through the /p// URL prefix, so a SECONDARY profile -# enabling one of these is always a misconfiguration. We skip that secondary -# profile (SecondaryPortBindingConfigError) so a single bad profile cannot -# take down the whole multiplexer. The set lives in gateway.config so the -# dashboard's pre-write validation enforces the same policy. +# Platforms that bind a host TCP port (HTTP/webhook listeners). In a profile multiplexer the default +# profile owns the single shared listener and serves every profile through the /p// URL +# prefix, so a SECONDARY profile enabling one of these is always a misconfiguration. That profile is +# skipped (SecondaryPortBindingConfigError) so one bad profile cannot take down the multiplexer; the +# set lives in gateway.config so the dashboard's pre-write validation enforces the same policy. from gateway.config import ( PORT_BINDING_PLATFORM_VALUES as _PORT_BINDING_PLATFORM_VALUES, platform_binds_port as _platform_binds_port, @@ -2464,14 +2191,13 @@ class SecondaryPortBindingConfigError(MultiplexConfigError): class HygieneTurnHoldExceeded(Exception): - """The hygiene-compression turn-hold budget elapsed while the summary - model was still streaming progress. + """The hygiene-compression turn-hold budget elapsed while the summary model was still streaming + progress. - This is an availability boundary, not a failure: the compressor is - healthy, but the current user turn cannot wait any longer. It must NOT - be routed through the idle-timeout failure path (which stamps - AGENT_COMPRESSION_TIMEOUT, sends a "no output" message, and advances - the failure cooldown ladder). + This is an availability boundary, not a failure: the compressor is healthy, but the current + user turn cannot wait any longer. It must NOT be routed through the idle-timeout failure path + (which stamps AGENT_COMPRESSION_TIMEOUT, sends a "no output" message, and advances the + failure cooldown ladder). """ @@ -2490,13 +2216,10 @@ def _multiplex_profile_homes(config: object) -> list[tuple[str, "Path"]]: def _enable_multiplex_log_routing(config: object) -> bool: """Route agent.log/errors.log/gateway.log records to their owning profile. - ``setup_logging(mode="gateway")`` binds the queued file handlers to the - launch home, so under ``multiplex_profiles`` every secondary profile's - records (emitted inside ``_profile_runtime_scope``) land in the default - profile's log files (#82936). Swap the static handlers for the - profile routers from #99440 — the same primitive the Desktop cron ticker - uses — once the served-profile set is known. Inert for single-profile - gateways (``enable_profile_log_routing`` is a no-op below two homes). + ``setup_logging(mode="gateway")`` binds the queued file handlers to the launch home, so under + ``multiplex_profiles`` every secondary profile's records (emitted inside + ``_profile_runtime_scope``) land in the default profile's log files. Swap in the profile + routers once the served-profile set is known; inert for single-profile gateways. """ if not getattr(config, "multiplex_profiles", False): return False @@ -2514,25 +2237,13 @@ def _enable_multiplex_log_routing(config: object) -> bool: def _handoff_watch_scopes(runner: object) -> list: """``(profile_name, home)`` pairs whose ``state.db`` the watcher must poll. - ``/handoff`` writes ``handoff_state='pending'`` into the store of the - profile the CLI ran under (``hermes -p medicina``), but the watcher - resolves ``_session_db`` from whatever HERMES_HOME is active on its task. - Unscoped, that is always the ROOT store, so a pending handoff queued by - any secondary profile is never seen and the CLI times out with the - gateway plainly alive. - - ``(None, None)`` means "poll unscoped" (the root/default store — the - legacy single-profile path, always first so its behaviour is unchanged). - A multiplexed gateway additionally yields each SECONDARY profile's - ``(name, home)`` to be polled inside ``_profile_runtime_scope``. The - default profile is deliberately not repeated: its home resolves to the - very same ``state.db`` as the unscoped poll, and polling it twice per - tick is pure waste. - - Module-level and defensive on purpose: the watcher's tests bind - ``_handoff_watcher`` onto a ``SimpleNamespace`` with no ``config``, and a - raising scope resolver would be swallowed by the loop's exception handler - and silently disable the watcher. Any failure degrades to the root poll. + ``/handoff`` writes into the store of the profile the CLI ran under, but the watcher resolves + ``_session_db`` from the active HERMES_HOME. Unscoped, that is always the ROOT store, so a + pending handoff queued by any secondary profile is never seen and the CLI times out with the + gateway plainly alive. ``(None, None)`` = unscoped root poll, always first (legacy behaviour + unchanged); SECONDARY profiles are added, the default profile is not repeated (same state.db). + Defensive on purpose: a raising resolver would be swallowed by the loop handler and silently + disable the watcher. Any failure degrades to the root poll. """ scopes: list = [(None, None)] try: @@ -2550,15 +2261,11 @@ def _handoff_watch_scopes(runner: object) -> list: async def _reclaim_stale(runner: object) -> None: """Fail handoffs left in ``running`` by a gateway that died mid-dispatch. - Runs once per store at watcher startup. ``running`` is only ever set by - the watcher for the duration of one in-process dispatch, so a row still - in that state belongs to a previous process. It can never reach a - terminal state on its own, and ``request_handoff`` refuses a NEW request - while the row sits there — the session would be permanently unable to - hand off again, with nothing surfaced to the user. - - Defensive throughout: the watcher's unit tests bind it onto stand-ins - with no such method, and a raising reclaim would abort watcher startup. + Runs once per store at watcher startup. ``running`` is only ever set by the watcher for the + duration of one in-process dispatch, so a row still in that state belongs to a previous + process. It can never reach a terminal state on its own, and ``request_handoff`` refuses a + NEW request while it sits there — the session could never hand off again. Defensive: a + raising reclaim would abort watcher startup. """ session_db = getattr(runner, "_session_db", None) if session_db is None: @@ -2617,20 +2324,13 @@ def _profile_runtime_scope( ): """Scope config/skills/memory AND credentials to a profile for one turn. - Combines the two seams the multiplexer needs: - 1. ``set_hermes_home_override`` — redirects ``get_hermes_home()`` (config, - skills, memory, SOUL, sessions) to the profile's home. Contextvar, so - it propagates into the agent worker thread via ``copy_context()``. - 2. ``set_secret_scope`` — installs the profile's ``.env`` secrets as the - authoritative credential source, so ``get_secret`` reads this profile's - keys and never the process-global ``os.environ`` (which in a - multiplexer may hold another profile's values). - - Only used on the multiplexed inbound path. Single-profile gateways never - enter this scope, so their behavior is unchanged. Loading the profile's - ``.env`` here does NOT mutate ``os.environ`` — ``build_profile_secret_scope`` - returns an isolated dict — which is what keeps subprocesses (MCP, kanban) - from inheriting cross-profile secrets. + Combines the two seams the multiplexer needs: (1) ``set_hermes_home_override`` redirects + ``get_hermes_home()`` to the profile's home — a contextvar, so it propagates into the agent + worker thread via ``copy_context()``; (2) ``set_secret_scope`` installs the profile's ``.env`` + as the authoritative credential source so ``get_secret`` never reads process-global + ``os.environ``. Only used on the multiplexed inbound path; single-profile gateways never enter + this scope. Loading ``.env`` here does NOT mutate ``os.environ``, which is what keeps + subprocesses (MCP, kanban) from inheriting cross-profile secrets. """ from hermes_constants import set_hermes_home_override, reset_hermes_home_override from agent.secret_scope import ( @@ -2649,11 +2349,10 @@ def _profile_runtime_scope( secrets = build_profile_secret_scope(Path(profile_home)) secret_token = set_secret_scope(secrets) - # Per-turn terminal scope (third seam of the profile boundary): installs - # the routed profile's COMPLETE terminal policy — never ambient env — via - # tools.terminal_scope. Without it terminal_tool reads the process-global - # TERMINAL_* vars a previous profile's turn may have pinned - # (first-writer-wins backend leak; #68559). + # Per-turn terminal scope (third seam of the profile boundary): installs the routed profile's + # COMPLETE terminal policy — never ambient env — via tools.terminal_scope. Without it + # terminal_tool reads process-global TERMINAL_* vars a previous profile's turn may have pinned + # (first-writer-wins backend leak). from tools.terminal_scope import install_and_reset_profile_terminal_scope with install_and_reset_profile_terminal_scope(Path(profile_home)): @@ -2675,19 +2374,11 @@ async def _async_profile_runtime_scope(profile_home: "Path"): def load_gateway_config_for_runner() -> "GatewayConfig": """Load gateway config for the process-level GatewayRunner. - When ``gateway.multiplex_profiles`` is off, this is identical to - ``load_gateway_config()`` (legacy single-profile path). - - When multiplexing is on, reload under the default/active profile's - ``_profile_runtime_scope`` so platform tokens in that profile's ``.env`` - resolve through the secret scope — the same path secondary profiles use - in ``_start_one_profile_adapters``. Without this, primary startup calls - ``load_gateway_config()`` unscoped: ``_getenv`` falls through to - ``os.environ``, which often has no ``TELEGRAM_BOT_TOKEN`` once the token - lives only under ``profiles//.env`` (#64674). - - Single-profile gateways never set ``multiplex_profiles``, so they keep the - unscoped load and are unaffected. + When multiplexing is on, reload under the default/active profile's ``_profile_runtime_scope`` + so platform tokens in that profile's ``.env`` resolve through the secret scope — the same + path secondary profiles use in ``_start_one_profile_adapters``. Unscoped, ``_getenv`` falls + through to ``os.environ``, which often lacks the token once it lives only under + ``profiles//.env``. Off -> identical to ``load_gateway_config()``. """ cfg = load_gateway_config() if not getattr(cfg, "multiplex_profiles", False): @@ -2710,13 +2401,9 @@ def load_gateway_config_for_runner() -> "GatewayConfig": async def _discover_gateway_mcp_tools(config: object) -> None: """Run startup MCP discovery for every profile this gateway serves. - ``discover_mcp_tools`` reads ``mcp_servers`` from ``get_hermes_home()``'s - config, so an unscoped call only ever connects the launch profile's - servers (#95518). Under multiplex, run it once per served profile inside - that profile's ``_profile_runtime_scope`` and carry the scope into the - executor thread with ``copy_context()`` (the same shape as - ``_run_in_executor_with_context``). Single-profile gateways keep the one - unscoped call. + ``discover_mcp_tools`` reads ``mcp_servers`` from ``get_hermes_home()``'s config, so an + unscoped call only ever connects the launch profile's servers. Single-profile gateways keep + the one unscoped call. """ from tools.mcp_tool import discover_mcp_tools @@ -2751,19 +2438,12 @@ def _platform_has_bot_credential(platform: "Platform", platform_config: "Platfor api_key = getattr(platform_config, "api_key", None) or "" if isinstance(api_key, str) and api_key.strip(): return True - # Matrix also authenticates by password login (MATRIX_USER_ID + - # MATRIX_PASSWORD, no MATRIX_ACCESS_TOKEN). Those credentials land in - # ``extra`` rather than ``.token``, so a token-only check reads a - # perfectly reconnectable password-auth config as credential-less and - # evicts it from the retry queue on the first transient failure — after - # which it stays down until the gateway is restarted by hand. Mirror the - # adapter's own gate: homeserver + user_id + password. - # - # Read ONLY from extra, never os.getenv: build_config() already copies all - # three env vars onto extra, and importing this module loads ~/.hermes/.env, - # so an env fallback would report "has credential" for every Matrix config - # on the box — including the empty-primary multiplex case (#64674) this - # check exists to evict. + # Matrix also authenticates by password login (MATRIX_USER_ID + MATRIX_PASSWORD, no + # MATRIX_ACCESS_TOKEN). Those land in ``extra``, so a token-only check would evict a perfectly + # reconnectable config from the retry queue on the first transient failure. Mirror the + # adapter's own gate: homeserver + user_id + password. Read ONLY from extra, never os.getenv: + # build_config() already copies the env vars onto extra, and an env fallback would report "has + # credential" for every Matrix config on the box — including the empty-primary multiplex case. if platform is Platform.MATRIX: extra = getattr(platform_config, "extra", None) or {} if all( @@ -2800,12 +2480,9 @@ if _config_path.exists(): _cfg = _expand_env_vars(_cfg) if not isinstance(_cfg, dict): _cfg = {} - # Managed scope: overlay administrator-pinned values BEFORE bridging to - # env vars, so a managed timezone / redact_secrets / max_turns / terminal - # setting wins over the user's value at the env layer too. This bridge - # reads config.yaml directly (not via load_config), so without the - # overlay every HERMES_*/TERMINAL_* env var below would carry the user's - # value even when an administrator pinned it. Fail-open via the helper. + # Managed scope: overlay administrator-pinned values BEFORE bridging to env vars, so a + # managed timezone / redact_secrets / max_turns / terminal setting wins over the user's + # value at the env layer too. Fail-open via the helper. try: from hermes_cli import managed_scope _cfg = managed_scope.apply_managed_overlay(_cfg) @@ -2859,18 +2536,15 @@ if _config_path.exists(): for _cfg_key, _env_var in _terminal_env_map.items(): if _cfg_key in _terminal_cfg: _val = _terminal_cfg[_cfg_key] - # Skip cwd placeholder values (".", "auto", "cwd") — the - # gateway resolves these to Path.home() later (line ~255). - # Writing the raw placeholder here would just be noise. - # Only bridge explicit absolute paths from config.yaml. + # Skip cwd placeholder values (".", "auto", "cwd") — the gateway resolves these + # to Path.home() later (line ~255). Writing the raw placeholder here would just + # be noise. Only bridge explicit absolute paths from config.yaml. if _cfg_key == "cwd" and str(_val) in {".", "auto", "cwd"}: continue - # Expand shell tilde in local/container cwd so subprocess.Popen - # never receives a literal "~/" which the kernel rejects. - # SSH cwd is interpreted by the remote shell, so preserve - # "~" / "~/..." for the SSH backend instead of expanding it - # to the Hermes host/container HOME (often /opt/data). Shared - # predicate with terminal_tool so the two sites can't drift. + # Expand shell tilde in local/container cwd so subprocess.Popen never receives a + # literal "~/" which the kernel rejects. SSH cwd is interpreted by the remote + # shell, so preserve "~" for the SSH backend instead of expanding to the Hermes + # host HOME. Shared predicate with terminal_tool so the two sites can't drift. if _cfg_key == "cwd" and isinstance(_val, str): if not _is_ssh_remote_tilde_cwd(_terminal_backend, _val.strip()): _val = os.path.expanduser(_val) @@ -2878,15 +2552,9 @@ if _config_path.exists(): os.environ[_env_var] = json.dumps(_val) else: os.environ[_env_var] = str(_val) - # Compression config is read directly from config.yaml by run_agent.py - # and auxiliary_client.py — no env var bridging needed. - # Auxiliary model/direct-endpoint overrides (vision, - # approval, plus any plugin-registered auxiliary tasks). - # Each task has provider/model/base_url/api_key; bridge non-default - # values to env vars named AUXILIARY__*. The legacy - # hard-coded list (vision/approval) is replaced by a - # dynamic loop so plugin-registered tasks benefit from the same - # config→env bridging without core knowing about each one. + # Compression config is read directly from config.yaml by run_agent.py and + # auxiliary_client.py — no env var bridging needed. Auxiliary model/direct-endpoint + # overrides (vision, approval, plus any plugin-registered auxiliary tasks). _auxiliary_cfg = _cfg.get("auxiliary", {}) if _auxiliary_cfg and isinstance(_auxiliary_cfg, dict): # Built-in tasks that previously had explicit env-var bridging. @@ -3009,10 +2677,9 @@ if _config_path.exists(): _redact = _security_cfg.get("redact_secrets") if _redact is not None: os.environ["HERMES_REDACT_SECRETS"] = str(_redact).lower() - # Gateway settings (media delivery allowlist + recency trust + strict mode) - # Delegated to the shared bridge so standalone delivery entrypoints - # (manual `hermes cron run`, ticks without the gateway) apply the SAME - # policy translation — process parity for attachment filtering. + # Gateway settings (media delivery allowlist + recency trust + strict mode) Delegated to the + # shared bridge so standalone delivery entrypoints (manual `hermes cron run`, ticks without + # the gateway) apply the SAME policy translation — process parity for attachment filtering. _gateway_cfg = _cfg.get("gateway", {}) if isinstance(_gateway_cfg, dict): from gateway.media_policy import apply_media_policy_env @@ -3021,11 +2688,9 @@ if _config_path.exists(): _trust_recent_seconds = _gateway_cfg.get("trust_recent_files_seconds") if _trust_recent_seconds is not None: os.environ["HERMES_MEDIA_TRUST_RECENT_SECONDS"] = str(_trust_recent_seconds) - # Bridge gateway.platform_connect_timeout → the internal env var the - # connect path + Discord adapter ready-wait both read (#19776). - # Unlike the agent.*/display.* bridges above (config-authoritative), - # this env var is the manual-override escape hatch, so it WINS if - # already set explicitly; otherwise config.yaml supplies the value. + # Bridge gateway.platform_connect_timeout → the internal env var the connect path + + # Discord adapter ready-wait both read. Unlike the config-authoritative bridges above, + # this env var is the manual-override escape hatch: it WINS if already set explicitly. if ( "platform_connect_timeout" in _gateway_cfg and not os.environ.get("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "").strip() @@ -3034,13 +2699,8 @@ if _config_path.exists(): _gateway_cfg["platform_connect_timeout"] ) except Exception as _bridge_err: - # Previously this was silent (`except Exception: pass`), which - # hid partial bridge failures and let .env defaults shadow - # config.yaml values — users observed max_turns=500 in config - # but a 60-iteration cap in practice. Surface the failure to - # stderr so operators see it even though `logger` is not yet - # initialized at module-import time (logger is defined further - # down this module). + # Surface the failure to stderr so operators see it even though `logger` is not yet + # initialized at module-import time (logger is defined further down this module). print( f" Warning: config.yaml → env bridge failed: " f"{type(_bridge_err).__name__}: {_bridge_err}", @@ -3078,16 +2738,13 @@ except Exception as _bootstrap_exc: # Gateway runs in quiet mode - suppress debug output and use cwd directly (no temp dirs) os.environ["HERMES_QUIET"] = "1" -# HERMES_EXEC_ASK is set in start_gateway(), not at import time. Importing this -# module from CLI tools (e.g. send_message → _gateway_runner_ref) must not flip -# interactive CLI sessions into ask-mode, or Dangerous Command prompts become -# silent pending_approval with no Approve/Deny UI. +# HERMES_EXEC_ASK is set in start_gateway(), not at import time. Importing this module from CLI +# tools (e.g. send_message → _gateway_runner_ref) must not flip interactive CLI sessions into ask- +# mode, or Dangerous Command prompts become silent pending_approval with no Approve/Deny UI. -# Set terminal working directory for messaging platforms. -# config.yaml terminal.cwd is the canonical source (bridged to TERMINAL_CWD -# by the config bridge above). Placeholder values are resolved per-backend — -# see gateway/cwd_placeholder.py for the three-case contract (local vs docker -# mount-off vs docker mount-on). MESSAGING_CWD is a backward-compat fallback. +# Set terminal working directory for messaging platforms. config.yaml terminal.cwd is the canonical +# source (bridged to TERMINAL_CWD by the config bridge above). MESSAGING_CWD is a backward-compat +# fallback. from gateway.cwd_placeholder import CWD_PLACEHOLDERS, resolve_placeholder_terminal_cwd _configured_cwd = os.environ.get("TERMINAL_CWD", "") @@ -3249,10 +2906,9 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: return None -# Sentinel placed into _running_agents immediately when a session starts -# processing, *before* any await. Prevents a second message for the same -# session from bypassing the "already running" guard during the async gap -# between the guard check and actual agent creation. +# Sentinel placed into _running_agents immediately when a session starts processing, *before* any +# await. Prevents a second message for the same session from bypassing the "already running" guard +# during the async gap between the guard check and actual agent creation. _AGENT_PENDING_SENTINEL = object() # Conversation-scoped per-session state registry (legacy contract). @@ -3302,15 +2958,9 @@ _UNSET = object() def _resolve_runtime_agent_kwargs() -> dict: """Resolve provider credentials for gateway-created AIAgent instances. - Provider is read from ``config.yaml`` ``model.provider`` (the single - source of truth). ``resolve_runtime_provider()`` falls through to env - var lookups internally for legacy compatibility, but the gateway does - not consult environment variables for behavioral config — config.yaml - is authoritative. - - If the primary provider fails with an authentication error, attempt to - resolve credentials using the fallback provider chain from config.yaml - before giving up. + ``resolve_runtime_provider()`` falls through to env var lookups internally for legacy + compatibility, but the gateway does not consult environment variables for behavioral config — + config.yaml is authoritative. """ from hermes_cli.runtime_provider import ( resolve_runtime_provider, @@ -3322,10 +2972,9 @@ def _resolve_runtime_agent_kwargs() -> dict: try: runtime = resolve_runtime_provider() except AuthError as auth_exc: - # Distinguish a transient rate-limit/quota cap (credentials are fine, - # re-auth cannot help) from a genuine auth failure (expired/revoked - # token). Both fall through to the fallback chain, but the log message - # must not mislabel a quota exhaustion as an auth failure (#32790). + # Distinguish a transient rate-limit/quota cap (credentials are fine, re-auth cannot help) + # from a genuine auth failure (expired/revoked token). Both fall through to the fallback + # chain, but the log message must not mislabel a quota exhaustion as an auth failure. if is_rate_limited_auth_error(auth_exc): logger.warning("Primary provider rate-limited (429): %s — trying fallback", auth_exc) else: @@ -3580,10 +3229,9 @@ def _try_resolve_fallback_provider() -> dict | None: explicit_base_url=entry.get("base_url"), explicit_api_key=resolve_entry_api_key(entry), ) - # Log the literal `provider` key from config, not the resolved - # runtime category — an Ollama fallback resolves through the - # OpenAI-compatible path and would otherwise be logged as - # "openrouter", contradicting the operator's config (#32790). + # Log the literal `provider` key from config, not the resolved runtime category — an + # Ollama fallback resolves through the OpenAI-compatible path and would otherwise be + # logged as "openrouter", contradicting the operator's config. logger.info( "Fallback provider resolved: %s model=%s", entry.get("provider") or runtime.get("provider"), @@ -3623,11 +3271,9 @@ def _event_media_type_at(event, index: int) -> str: def _event_media_is_image(event, index: int) -> bool: """True if the attachment at *index* is an image. - Trust the per-attachment MIME when present. Only fall back to the - message-level ``PHOTO`` type when this attachment's MIME is unknown -- - otherwise a document (or any non-image) uploaded alongside an image in - the same message gets mis-routed as an image, base64'd into a vision - content part, and the provider 400s ("Could not process image"). + Trust the per-attachment MIME when present. Only fall back to the message-level ``PHOTO`` + type when this attachment's MIME is unknown — otherwise a document uploaded alongside an + image gets mis-routed as an image, base64'd into a vision part, and the provider 400s. """ mtype = _event_media_type_at(event, index) if mtype: @@ -3665,10 +3311,9 @@ def _event_media_is_video(event, index: int) -> bool: def _build_media_placeholder(event) -> str: """Build a text placeholder for media-only events so they aren't dropped. - When a photo/document is queued during active processing and later - dequeued, only .text is extracted. If the event has no caption, - the media would be silently lost. This builds a placeholder that - the vision enrichment pipeline will replace with a real description. + When a photo/document is queued during active processing and later dequeued, only .text is + extracted. If the event has no caption, the media would be silently lost. This builds a + placeholder that the vision enrichment pipeline will replace with a real description. """ parts = [] media_urls = getattr(event, "media_urls", None) or [] @@ -3693,15 +3338,10 @@ def _build_document_context_note( ) -> str: """Context note prepended to a user turn when they attach a document. - Text documents (``text/*``) are usually inlined upstream by the platform - adapter. ``content_inlined=False`` records adapters that cache the file - without injecting its content, so the note tells the agent to read it. - - Binary documents (PDF, DOCX, XLSX, …) cannot be inlined as text. The note - must tell the agent to *extract* the text itself before answering — earlier - wording ("Ask the user what they'd like you to do with it") steered the - model into punting back to the user, which is why attached PDFs/DOCX looked - "unreadable" to the agent even though it has the tools to read them. + ``content_inlined=False`` records adapters that cache the file without injecting its content, + so the note tells the agent to read it. Binary documents (PDF, DOCX, XLSX, …) cannot be + inlined as text; the note must tell the agent to *extract* the text itself — earlier "ask the + user what to do with it" wording made the model punt, so attachments looked "unreadable". """ if mtype.startswith("text/") and content_inlined: return ( @@ -3780,9 +3420,8 @@ async def _probe_audio_duration(path: str) -> Optional[str]: def _dequeue_pending_event(adapter, session_key: str) -> MessageEvent | None: """Consume and return the full pending event for a session. - Queued follow-ups must preserve their media metadata so they can re-enter - the normal image/STT/document preprocessing path instead of being reduced - to a placeholder string. + Queued follow-ups must preserve their media metadata so they can re-enter the normal + image/STT/document preprocessing path instead of being reduced to a placeholder string. """ return adapter.get_pending_message(session_key) @@ -3804,14 +3443,11 @@ def _reap_gateway_turn_processes( ) -> int: """Reap only background processes created by one abandoned turn. - ``task_id`` is session-scoped (task_id == session_id), not turn-scoped, - so a *replacement* turn on the same session can start and spawn its own - legitimate process while this reap is still in flight. ``is_still_current`` - — a closure over the run_generation captured when the reaping turn began - or was interrupted — lets the caller detect that a newer turn has since - claimed the session and bail out instead of killing that newer turn's - process. The newer turn snapshots its own baseline independently, so - skipping here does not leave anything permanently unreaped. + ``task_id`` is session-scoped (task_id == session_id), not turn-scoped, so a *replacement* + turn on the same session can start and spawn its own legitimate process while this reap is + still in flight. ``is_still_current`` (closure over the captured run_generation) lets the + caller bail out instead of killing the newer turn's process; that turn snapshots its own + baseline, so nothing stays permanently unreaped. """ if not task_id: # ProcessSession.task_id defaults to "" for sessionless callers, so a @@ -3845,10 +3481,9 @@ def _reap_gateway_turn_processes( source=source, ) except Exception: - # Runs on a detached daemon thread (interrupt and timeout call - # sites both fire-and-forget it) — an uncaught exception here - # would only surface via threading.excepthook, bypassing the - # app's logger. Swallow and log through the normal channel instead. + # Runs on a detached daemon thread (interrupt and timeout call sites both fire-and-forget + # it) — an uncaught exception here would only surface via threading.excepthook, bypassing + # the app's logger. Swallow and log through the normal channel instead. logger.warning( "Failed to reap background processes for turn %s (%s)", task_id, @@ -3879,18 +3514,11 @@ _TURN_STACK_DUMP_FRAME_MARKERS = ( def _dump_wedged_turn_stacks(task_id: str) -> None: """Log the stack of every thread that looks like turn work, at reap time. - When the inactivity reaper fires, the model loop is usually long done and - the worker thread is wedged somewhere in post-turn finalization — but the - reaper's hard interrupt frees it, so the blocked frame is gone before - anyone can attach a profiler. A live incident (Aug 2026, WhatsApp session - on a Relay-corrupted scope stack) wedged EVERY turn for exactly the - 1800s timeout between "Turn ended" and run_sync returning, and the wedge - point was unrecoverable post-mortem. Dumping the stacks here, BEFORE the - interrupt, names the frame. - - Best-effort and bounded: pure in-process frame walking (no signals, no - external tools), only threads whose stack mentions a turn-machinery - marker are logged, output capped per thread. Must never raise into the + When the inactivity reaper fires, the model loop is usually long done and the worker thread + is wedged somewhere in post-turn finalization — but the reaper's hard interrupt frees it, so + the blocked frame is gone before anyone can attach a profiler. Dumping BEFORE the interrupt + names the frame. Best-effort and bounded: in-process frame walking only, only threads whose + stack mentions a turn-machinery marker, output capped per thread. Must never raise into the reaper. """ try: @@ -4033,13 +3661,10 @@ def _is_control_interrupt_message(message: Optional[str]) -> bool: def _strip_response_attachments_for_direct_send(response: str, adapter) -> str: """Return the visible text portion of a response before direct send(). - Queued follow-up resends only replay explicit ``MEDIA:`` attachments in - this path. Keep bare local paths and ordinary image URLs visible because - the post-stream uploader intentionally ignores them (#20834). - - Do not apply a broad ``MEDIA:`` regex after ``extract_media()`` — the - extractor deliberately preserves protected code/inline spans and - unsupported or unvalidated tags in the cleaned text. + Queued follow-up resends only replay explicit ``MEDIA:`` attachments in this path. Keep bare + local paths and ordinary image URLs visible because the post-stream uploader intentionally + ignores them. Do not apply a broad ``MEDIA:`` regex after ``extract_media()`` — the extractor + deliberately preserves protected code/inline spans and unsupported/unvalidated tags. """ _, cleaned = adapter.extract_media(response) cleaned = cleaned.replace("[[audio_as_voice]]", "").strip() @@ -4050,18 +3675,10 @@ def _strip_response_attachments_for_direct_send(response: str, adapter) -> str: def _skill_slug_from_frontmatter(skill_md: Path) -> tuple[str | None, str | None]: """Derive the /command slug and declared frontmatter name from a SKILL.md. - Matches the exact normalization used by - :func:`agent.skill_commands.scan_skill_commands` so the slug here is the - same string a user types after the leading ``/`` (e.g. a skill with - frontmatter ``name: Stable Diffusion Image Generation`` resolves to - ``stable-diffusion-image-generation`` — NOT the parent directory name, - which is commonly shorter/different, e.g. ``stable-diffusion``). - - Using the directory name silently broke :func:`_check_unavailable_skill` - for every skill whose directory name drifted from its frontmatter name - (19 such skills on a standard install as of 2026-05), causing a generic - "unknown command" response where a "disabled — enable with …" or - "not installed — install with …" hint was expected. + Matches the normalization in :func:`agent.skill_commands.scan_skill_commands` so the slug is + the string a user types after ``/`` — derived from frontmatter ``name:``, NOT the directory + name (e.g. dir ``stable-diffusion`` vs slug ``stable-diffusion-image-generation``). Using the + directory name silently broke :func:`_check_unavailable_skill` for every drifted skill. Returns ``(slug, declared_name)`` or ``(None, None)`` when the file can't be read or lacks a ``name:`` in its frontmatter. @@ -4103,14 +3720,6 @@ def _check_unavailable_skill(command_name: str) -> str | None: Returns a helpful message if the skill exists but is disabled or only available as an optional install. Returns None if no match found. - - The slug for each on-disk skill is derived from its frontmatter ``name:`` - (via :func:`_skill_slug_from_frontmatter`), NOT from its containing - directory name — because the two can differ (e.g. directory - ``stable-diffusion`` + frontmatter ``Stable Diffusion Image Generation`` - yields slug ``stable-diffusion-image-generation``). Matching on - directory name would miss that slug entirely and fall through to the - generic "unknown command" path. """ # Normalize: command uses hyphens, skill names may use hyphens or underscores normalized = command_name.lower().replace("_", "-") @@ -4187,15 +3796,10 @@ def _gateway_config_home() -> Path: def _load_gateway_config(config_path: "Path | None" = None) -> dict: """Load and parse a gateway config.yaml, returning {} on any error. - Defaults to the active gateway home (so tests that monkeypatch - ``_hermes_home`` still see their fixture). Callers handling multiplexed - profile routes may pass that profile's explicit config path. The canonical - path shares the mtime-keyed raw-yaml cache from - ``hermes_cli.config.read_raw_config``. - - Managed scope is overlaid on the result (via the shared helper) so the - gateway honors administrator-pinned values — neither read_raw_config nor a - direct yaml.safe_load carries the managed merge on its own. Fail-open. + Defaults to the active gateway home (so tests that monkeypatch ``_hermes_home`` still see + their fixture). Callers handling multiplexed profile routes may pass that profile's explicit + config path. Managed scope is overlaid on the result so administrator-pinned values are + honored — neither read_raw_config nor yaml.safe_load carries the managed merge. Fail-open. """ if config_path is None: config_path = _gateway_config_home() / 'config.yaml' @@ -4203,9 +3807,8 @@ def _load_gateway_config(config_path: "Path | None" = None) -> dict: used_canonical = False try: from hermes_cli.config import get_config_path, read_raw_config - # Fast path: if _hermes_home agrees with the canonical config - # location, reuse the shared cache. Otherwise fall through to a - # direct read (keeps test fixtures with a monkeypatched + # Fast path: if _hermes_home agrees with the canonical config location, reuse the shared + # cache. Otherwise fall through to a direct read (keeps test fixtures with a monkeypatched # _hermes_home working). if config_path == get_config_path(): raw = read_raw_config() @@ -4223,10 +3826,9 @@ def _load_gateway_config(config_path: "Path | None" = None) -> dict: logger.debug("Could not load gateway config from %s", config_path) raw = {} - # Overlay managed scope. read_raw_config() returns the user's raw YAML - # WITHOUT the managed merge (that lives in load_config/_load_config_impl), - # so the overlay is required on both paths for the gateway to honor pinned - # values. Helper is fail-open and a no-op when no managed scope exists. + # Overlay managed scope. read_raw_config() returns the user's raw YAML WITHOUT the managed merge + # (that lives in load_config/_load_config_impl), so the overlay is required on both paths for + # the gateway to honor pinned values. try: from hermes_cli import managed_scope raw = managed_scope.apply_managed_overlay(raw if isinstance(raw, dict) else {}) @@ -4234,12 +3836,10 @@ def _load_gateway_config(config_path: "Path | None" = None) -> dict: pass if not isinstance(raw, dict): return {} - # Canonicalize model-id aliases (model.name / model.model → model.default) - # and migrate stale root-level provider/base_url into the model section. - # The gateway bypasses load_config() (it reads raw YAML for speed), so the - # normalization that load_config() applies must be replayed here or the - # gateway would resolve an empty model for ``model: {name: }`` configs - # while the CLI resolves it correctly. See issue #34500. Fail-open. + # Canonicalize model-id aliases (model.name / model.model → model.default) and migrate stale + # root-level provider/base_url into the model section. The gateway bypasses load_config() (raw + # YAML for speed), so its normalization must be replayed here or ``model: {name: }`` would + # resolve to an empty model while the CLI resolves it correctly. Fail-open. try: from hermes_cli.config import _normalize_root_model_keys raw = _normalize_root_model_keys(raw) @@ -4280,13 +3880,9 @@ def _checkpoint_agent_kwargs(config: dict | None) -> dict: def _load_gateway_runtime_config() -> dict: """Load gateway config for runtime reads, expanding supported ``${VAR}`` refs. - Runtime helpers should honor the same env-template expansion documented for - ``config.yaml`` while still respecting tests that monkeypatch - ``gateway.run._hermes_home``. Build on ``_load_gateway_config()`` rather - than calling the canonical loader directly so both behaviors stay aligned. - - Expansion failures are intentionally NOT swallowed — silently returning - the unexpanded dict would mask the very bug this helper exists to fix. + Build on ``_load_gateway_config()`` rather than calling the canonical loader directly so both + behaviors stay aligned. Expansion failures are intentionally NOT swallowed — silently + returning the unexpanded dict would mask the very bug this helper exists to fix. """ cfg = _load_gateway_config() if not isinstance(cfg, dict) or not cfg: @@ -4300,9 +3896,8 @@ def _load_gateway_runtime_config() -> dict: def _resolve_gateway_model(config: dict | None = None) -> str: """Read model from config.yaml — single source of truth. - Without this, temporary AIAgent instances (e.g. /compress) fall - back to the hardcoded default which fails when the active provider is - openai-codex. + Without this, temporary AIAgent instances (e.g. /compress) fall back to the hardcoded default + which fails when the active provider is openai-codex. """ cfg = config if config is not None else _load_gateway_config() model_cfg = cfg.get("model", {}) @@ -4369,10 +3964,9 @@ def _get_channel_override( def _resolve_hermes_bin() -> Optional[list[str]]: """Resolve the Hermes update command as argv parts. - Tries in order: - 1. ``shutil.which("hermes")`` — standard PATH lookup - 2. ``sys.executable -m hermes_cli.main`` — fallback when Hermes is running - from a venv/module invocation and the ``hermes`` shim is not on PATH + Tries in order: 1. ``shutil.which("hermes")`` — standard PATH lookup 2. ``sys.executable -m + hermes_cli.main`` — fallback when Hermes is running from a venv/module invocation and the + ``hermes`` shim is not on PATH Returns argv parts ready for quoting/joining, or ``None`` if neither works. """ @@ -4396,14 +3990,8 @@ def _resolve_hermes_bin() -> Optional[list[str]]: def _parse_session_key(session_key: str) -> "dict | None": """Parse a session key into its component parts. - Session keys follow the format - ``agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]``. - Returns a dict with ``platform``, ``chat_type``, ``chat_id``, and - optionally ``thread_id`` keys, or None if the key doesn't match. - - The 6th element is only returned as ``thread_id`` for chat types where - it is unambiguous (``dm`` and ``thread``). For group/channel sessions - the suffix may be a user_id (per-user isolation) rather than a + Session keys follow the format ``agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]``. + For group/channel sessions the suffix may be a user_id (per-user isolation) rather than a thread_id, so we leave ``thread_id`` out to avoid mis-routing. """ parts = session_key.split(":") @@ -4508,12 +4096,11 @@ def _format_gateway_process_notification(evt: dict) -> "str | None": def _drain_gateway_watch_events(completion_queue) -> "list[dict]": """Drain gateway-owned watch events without spinning on requeued events. - Watch events are handled by the post-turn gateway drain. Process - completions are owned by their per-process watcher task, and async - delegation completions are owned by ``_async_delegation_watcher``. - Requeueing async events inside ``while not queue.empty()`` would make the - loop non-terminating, so detach the current batch first, then requeue any - events this drain does not own after the queue is empty. + Watch events are handled by the post-turn gateway drain; process completions are owned by + their per-process watcher task and async delegation completions by + ``_async_delegation_watcher``. Requeueing async events inside + ``while not queue.empty()`` would make the loop non-terminating, so detach the current batch + first, then requeue any events this drain does not own after the queue is empty. """ watch_events: list[dict] = [] requeue: list[dict] = [] @@ -4553,30 +4140,23 @@ def _normalize_empty_agent_response( ) -> str: """Normalize empty/None agent responses into user-facing messages. - Consolidates the existing ``failed`` handler and adds a catch-all for - the case where the agent did work (api_calls > 0) but returned no text. - Fix for #18765. - - Also surfaces a retry hint when the agent never ran at all - (api_calls == 0) for a non-interrupted, non-failed turn -- this is the - silent-drop pattern observed after ``/stop`` where the next user - message hits a stale generation token and returns an empty result, - leaving the platform with nothing to send. (#31884) + Consolidates the existing ``failed`` handler and adds a catch-all for the case where the + agent did work (api_calls > 0) but returned no text. Also surfaces a retry hint when the + agent never ran (api_calls == 0, not interrupted/failed): the post-/stop silent-drop pattern + where a stale generation token returns an empty result. """ if response: return response if agent_result.get("failed"): - # None-safe: the gateway result dict is built with - # ``'error': holder.get('error')`` and can carry an EXPLICIT None, - # which bypasses dict.get's default and would render - # "The request failed: None". + # None-safe: the gateway result dict is built with ``'error': holder.get('error')`` and can + # carry an EXPLICIT None, which bypasses dict.get's default and would render "The request + # failed: None". error_detail = agent_result.get("error") or "unknown error" error_str = str(error_detail).lower() - # Session-persistence failures get a dedicated recovery message. - # Suggesting /reset here would be actively harmful: it destroys the - # user's conversation context and does nothing to fix the underlying - # storage problem (lock contention, disk exhaustion, ...). + # Session-persistence failures get a dedicated recovery message. Suggesting /reset here + # would be actively harmful: it destroys the user's conversation context and does nothing to + # fix the underlying storage problem (lock contention, disk exhaustion, ...). failure_reason = str(agent_result.get("failure_reason") or "") if failure_reason.startswith("session_persistence_failed") or ( "session storage" in error_str @@ -4611,14 +4191,11 @@ def _normalize_empty_agent_response( api_calls = int(agent_result.get("api_calls", 0) or 0) if agent_result.get("interrupted"): - # An interrupted run that did work (api_calls > 0) is the drain of a - # run the user deliberately stopped or steered — its silence is - # intentional, and any queued/interrupting message is delivered by - # the recursive drain inside _run_agent before this result is seen. - # An interrupted run with ZERO api_calls never processed the user's - # message at all: it was killed at the top of the tool loop by an - # interrupt flag left over from a recent /stop (#44212). Pure - # silence there swallows a real user message, so surface it. + # An interrupted run that did work (api_calls > 0) is the drain of a run the user + # deliberately stopped or steered — its silence is intentional, and any queued/interrupting + # message is delivered by the recursive drain inside _run_agent before this result is seen. + # An interrupted run with ZERO api_calls never processed the message at all (killed by an + # interrupt flag left over from a recent /stop) — silence would swallow it, so surface it. if api_calls == 0: return ( "⚠️ Your message was interrupted before processing started " @@ -4636,11 +4213,9 @@ def _normalize_empty_agent_response( "This may be a transient error — try sending your message again." ) - # api_calls == 0, not failed, not interrupted: the agent never ran for - # this turn. This is the post-/stop generation-race pattern where the - # gateway would otherwise silently drop the turn (response=0 chars) and - # the user sees no reply at all. Surface a short retry hint so the - # message isn't lost in silence. (#31884) + # api_calls == 0, not failed, not interrupted: the agent never ran for this turn. This is the + # post-/stop generation-race pattern where the gateway would otherwise silently drop the turn + # (response=0 chars) and the user sees no reply at all. if ( api_calls == 0 and not agent_result.get("interrupted") @@ -4658,13 +4233,11 @@ def _normalize_empty_agent_response( def _is_gateway_hidden_reasoning_incomplete_turn(agent_result: dict) -> bool: """Detect retry-exhausted turns with hidden reasoning but no visible answer. - The conversation loop returns the retry-exhaustion sentinel as BOTH - ``final_response`` and ``error`` ("Codex response remained incomplete - after 3 continuation attempts"), so ``final_response`` being non-empty - does not mean the model produced a visible answer. Treat the turn as - hidden when the error sentinel is present and ``final_response`` is - either empty or merely echoes that sentinel — any genuinely different - final text means the model DID answer and must be delivered. + The conversation loop returns the retry-exhaustion sentinel as BOTH ``final_response`` and + ``error`` ("Codex response remained incomplete after 3 continuation attempts"), so + ``final_response`` being non-empty does not mean the model produced a visible answer. Hidden + only when the sentinel is present and ``final_response`` is empty or merely echoes it — any + genuinely different text means the model DID answer and must be delivered. """ if not isinstance(agent_result, dict): return False @@ -4682,11 +4255,10 @@ def _is_gateway_hidden_reasoning_incomplete_turn(agent_result: dict) -> bool: def _should_clear_resume_pending_after_turn(agent_result: dict) -> bool: """Return True only when a gateway turn really completed successfully. - Restart recovery uses ``resume_pending`` as a durable marker for sessions - interrupted during gateway drain. A soft interrupt can still bubble out as - a syntactically normal agent result with an empty final response; clearing - the marker in that case loses the recovery signal and startup auto-resume - has nothing to schedule. + Restart recovery uses ``resume_pending`` as a durable marker for sessions interrupted during + gateway drain. A soft interrupt can surface as a normal-looking result with an empty final + response; clearing the marker then loses the recovery signal and startup auto-resume has + nothing to schedule. """ if not isinstance(agent_result, dict): return False @@ -4705,14 +4277,9 @@ def _preserve_queued_followup_history_offset( ) -> dict: """Carry the outer history offset through queued follow-up drains. - ``_process_message_background()`` persists transcript rows only once, after the - entire in-band queued-follow-up chain returns. Each recursive ``_run_agent()`` - call advances ``history_offset`` to the history it received, so without - correction the outermost persistence step sees only the *last* queued turn as + Each recursive ``_run_agent()`` call advances ``history_offset`` to the history it received, + so without correction the outermost persistence step sees only the *last* queued turn as "new" and silently drops earlier turns from the same drain chain. - - Preserve the earliest (outermost) history offset so the final transcript slice - still includes every queued turn that ran during the chain. """ if not isinstance(followup_result, dict): return followup_result @@ -4734,50 +4301,23 @@ def _preserve_queued_followup_history_offset( async def _dispose_unused_adapter(adapter: "BasePlatformAdapter | None") -> None: """Best-effort dispose for an adapter that never made it onto ``self.adapters``. - The reconnect watcher in ``GatewayRunner._platform_reconnect_watcher`` - constructs a fresh adapter on every retry attempt. When the connect - call fails — for any of the three reasons (non-retryable error, - retryable error, exception during connect) — the adapter is dropped - without ever being installed, so nothing else will call its - ``disconnect()``. Any resources the adapter opened in ``__init__`` - (e.g. ``APIServerAdapter`` opens a SQLite ``ResponseStore`` that - holds 2 fds — the db file and its WAL sidecar) stay open until - garbage collection sweeps the unreachable object, which Python's - cyclic GC does not do promptly for asyncio-bound objects with - native handles. The cumulative leak is 2 fds × every retry at the - 300s backoff cap ≈ 12 fds/hour, and the default 2560-fd ulimit - is exhausted in ~12h of continuous failure, after which every - open() call on the gateway raises ``OSError: [Errno 24] Too many - open files`` and the gateway becomes a zombie (#37011). - - This helper centralises the dispose-with-suppression so the three - failure paths in the reconnect watcher can all call it without - each one having to know that ``disconnect()`` may itself raise - on a half-constructed adapter. - - ``adapter`` may be ``None``: the reconnect watcher initialises - ``adapter = None`` before the ``try`` so the ``except Exception`` - arm can dispose a half-constructed object, and also early-returns - here when ``_create_adapter()`` returned ``None``. + When the connect call fails — for any of the three reasons (non-retryable error, retryable + error, exception during connect) — the adapter is dropped without ever being installed, so + nothing else will call its ``disconnect()``. Resources opened in ``__init__`` (e.g. + APIServerAdapter's SQLite store, 2 fds) then leak until GC, which is not prompt for + asyncio-bound objects; at the 300s backoff cap that exhausts the fd ulimit in ~12h and the + gateway becomes a zombie. ``adapter`` may be ``None`` (half-constructed / ``_create_adapter`` + returned None). """ if adapter is None: return try: await adapter.disconnect() except Exception: - # Half-constructed adapters (e.g. APIServerAdapter that - # crashed during aiohttp app setup) can raise from - # disconnect() on objects that never finished initializing. - # We must not let that escape and abort the watcher loop. - # - # On Python 3.8+, ``asyncio.CancelledError`` inherits from - # ``BaseException`` (not ``Exception``), so this ``except - # Exception`` does not swallow task cancellation. We don't - # re-raise explicitly because the watcher loop intentionally - # treats dispose failures as best-effort: a failed ``disconnect`` - # call should not take down the reconnect watcher that - # itself is what's keeping the gateway alive during a partial - # outage. + # Half-constructed adapters (e.g. APIServerAdapter that crashed during aiohttp app setup) + # can raise from disconnect() on objects that never finished initializing. We must not let + # that escape and abort the watcher loop. ``asyncio.CancelledError`` is a BaseException, so + # this does not swallow cancellation; dispose failures are best-effort by design. logger.debug( "Adapter dispose raised on unowned adapter %r", getattr(adapter, "name", type(adapter).__name__), @@ -4789,15 +4329,11 @@ async def _dispose_unused_adapter(adapter: "BasePlatformAdapter | None") -> None # secondary-profile reconnects share this policy — tune in one place). _RECONNECT_BACKOFF_CAP = 300 -# Seconds a platform may sit continuously in the reconnect queue before the -# watcher flags it NEEDS_ATTENTION in runtime status. Retrying never stops -# (auto-pause was deliberately removed — a transient outage must self-heal -# without operator action); this only makes a *long-lived* retry loop loud so -# owners and fleet monitoring can distinguish hour one from week three. -# A dead bot token, a revoked Discord intent, or a deterministically crashing -# sidecar all present as "retrying" forever without this signal. -# User-facing setting: agent.reconnect_attention_after in config.yaml -# (bridged to this env var above). 0 disables. +# Seconds a platform may sit continuously in the reconnect queue before the watcher flags it +# NEEDS_ATTENTION in runtime status. Retrying never stops (auto-pause was deliberately removed — a +# transient outage must self-heal without operator action); this only makes a long-lived retry +# loop loud. A dead bot token, a revoked Discord intent, or a deterministically crashing sidecar +# all present as "retrying" forever without this signal. 0 disables. _RECONNECT_ATTENTION_AFTER_SECONDS = _float_env( "HERMES_RECONNECT_ATTENTION_AFTER_SECONDS", 7200 ) @@ -4809,13 +4345,12 @@ def _reconnect_backoff(attempt: int) -> int: def _reconnect_needs_attention(info: dict, now: float) -> bool: - """Return True when a reconnect-queue entry has been continuously queued - long enough to warrant a NEEDS_ATTENTION signal. + """Return True when a reconnect-queue entry has been continuously queued long enough to warrant + a NEEDS_ATTENTION signal. - ``queued_at`` is (re)stamped whenever the platform (re)enters the queue, - so a platform that reconnects successfully and later fails again starts a - fresh clock — only *continuous* failure escalates. Entries queued before - this field existed (in-flight upgrade) are treated as newly queued. + ``queued_at`` is (re)stamped whenever the platform (re)enters the queue, so a platform that + reconnects successfully and later fails again starts a fresh clock — only *continuous* + failure escalates. """ if _RECONNECT_ATTENTION_AFTER_SECONDS <= 0: return False # escalation disabled @@ -4827,15 +4362,11 @@ def _reconnect_needs_attention(info: dict, now: float) -> bool: class TurnRunner: - """Per-turn collaborator carrying the tool-progress callbacks that used to - be nested closures inside ``GatewayRunner._run_agent_inner``. + """Per-turn collaborator carrying the tool-progress callbacks that used to be nested closures + inside ``GatewayRunner._run_agent_inner``. - The bodies are byte-identical to the original closures modulo - ``local_name`` -> ``ctx.field`` rewrites (closed-over locals now travel on - the shared :class:`gateway.turn_context.TurnContext`) and ``self`` -> - ``self._runner`` (the owning :class:`GatewayRunner`). Module-global - references (logger, cfg_get, BasePlatformAdapter, ...) resolve in this - same module exactly as before. + Module-global references (logger, cfg_get, BasePlatformAdapter, ...) resolve in this same + module exactly as before. """ def __init__(self, runner: "GatewayRunner", ctx: TurnContext) -> None: @@ -4845,13 +4376,10 @@ class TurnRunner: def progress_callback(self, event_type: str, tool_name: str = None, preview: str = None, args: dict = None, **kwargs): """Callback invoked by agent on tool lifecycle events.""" ctx = self._ctx - # Failed subagent → one clean user-facing notice. Handled FIRST, - # before every progress-queue gate: platforms that keep - # tool_progress off (Telegram, Slack, ...) must still hear about a - # delegation that died — a silently-vanishing subagent looks like - # the agent just dropped the task (community report, Aug 2026). - # Success/interrupt completions stay quiet; only terminal failure - # statuses render, via the same notice rail as credit warnings. + # Failed subagent → one clean user-facing notice. Handled FIRST, before every progress-queue + # gate: platforms with tool_progress off must still hear about a dead delegation, or it + # looks like the agent dropped the task. Success/interrupt completions stay quiet; + # only terminal failure statuses render, via the same notice rail as credit warnings. if event_type == "subagent.complete": _sub_status = kwargs.get("status") try: @@ -4875,13 +4403,9 @@ class TurnRunner: except Exception: logger.debug("subagent failure notice failed", exc_info=True) return - # Live status line (Slack's assistant status): stash the current - # tool phrase on the adapter; the _keep_typing refresh renders it - # within a couple of seconds. Handled before every other gate - # because it's independent of progress bubbles and queues (Slack - # keeps tool_progress off by default, but the ephemeral status - # line is always safe). Plain dict write — safe from the agent's - # sync worker thread, no event-loop hop needed. + # Live status line (Slack's assistant status): stash the current tool phrase on the adapter; + # the _keep_typing refresh renders it within a couple of seconds. Plain dict write — safe + # from the agent's sync worker thread, no event-loop hop needed. if ( ctx._live_status_adapter is not None and ctx._live_status_mode != "off" @@ -4914,12 +4438,9 @@ class TurnRunner: if not ctx.progress_queue or not ctx._run_still_current(): return - # First-touch onboarding: the first time a tool takes longer than - # _LONG_TOOL_THRESHOLD_S during a run that's streaming every tool - # (progress_mode == "all"), append a one-time hint suggesting - # /verbose. We only fire when (a) the user hasn't seen the hint - # before and (b) /verbose is actually usable on this platform - # (gateway gate must be open). The CLI has its own trigger. + # First-touch onboarding: the first time a tool takes longer than _LONG_TOOL_THRESHOLD_S + # during a run that's streaming every tool (progress_mode == "all"), append a one-time hint + # suggesting /verbose. The CLI has its own trigger. if event_type == "tool.completed" and not ctx.long_tool_hint_fired[0]: try: duration = kwargs.get("duration") or 0 @@ -4943,11 +4464,8 @@ class TurnRunner: logger.debug("tool-progress onboarding hint failed: %s", _hint_err) return - # "_thinking" is assistant scratch text between tool calls. It - # is never ordinary tool progress: only relay it when the platform - # explicitly opted into thinking_progress. Handle both legacy - # callback shapes: ("_thinking", text) and - # ("reasoning.available", "_thinking", text, ...). + # "_thinking" is assistant scratch text between tool calls. It is never ordinary tool + # progress: only relay it when the platform explicitly opted into thinking_progress. if event_type == "_thinking" or tool_name == "_thinking": if not ctx._thinking_enabled: return @@ -4957,9 +4475,8 @@ class TurnRunner: ctx.progress_queue.put(msg) return - # Native task cards consume the authoritative ID-bearing - # tool_start/tool_complete callbacks instead. Do not also enqueue - # name-correlated text events, which would duplicate cards and + # Native task cards consume the authoritative ID-bearing tool_start/tool_complete callbacks + # instead. Do not also enqueue name-correlated text events, which would duplicate cards and # mispair concurrent calls to the same tool. if ctx._native_slack_task_cards and event_type in { "tool.started", @@ -4976,14 +4493,10 @@ class TurnRunner: if event_type not in {"tool.started",}: return - # Never render a progress bubble for the clarify tool. The - # adapter's send_clarify IS the user-facing rendering (interactive - # buttons or the numbered-text fallback), so a progress bubble is - # pure duplication — and in verbose mode it dumps the raw - # tool-call args JSON ({"question": ..., "choices": [...]}) into - # the chat. Because the progress queue drains on a background - # task, that raw JSON typically lands right underneath the - # rendered prompt (#52374). + # Never render a progress bubble for the clarify tool: send_clarify IS the user-facing + # rendering, so a bubble is pure duplication — and verbose mode would dump the raw tool-call + # args JSON. Because the progress queue drains on a background task, that raw JSON typically + # lands right underneath the rendered prompt. if tool_name == "clarify": return @@ -5111,11 +4624,9 @@ class TurnRunner: preview = _progress_adapter.format_tool_preview(_prepared_preview) else: preview = _prepared_preview.text - # Friendly labels: render a human-phrased line for built-in - # tools ("🔍 Searching the web for ...") by prefixing the verb - # onto the preview the callback already computed (so the - # command/url/query is preserved). Custom/plugin/MCP tools - # have no verb and fall back to the raw "tool_name: ..." form. + # Friendly labels: render a human-phrased line for built-in tools ("🔍 Searching the web + # for ...") by prefixing the verb onto the preview the callback already computed (so the + # command/url/query is preserved). _verb = get_tool_verb(tool_name) if _verb: if verb_drops_preview(tool_name): @@ -5159,13 +4670,10 @@ class TurnRunner: ctx.progress_queue.put(msg) async def _send_native_task_card_progress(self, adapter) -> None: - """Drain the progress queue into Slack-native plan/task cards (#29483). + """Drain the progress queue into Slack-native plan/task cards. - Consumes the ID-bearing lifecycle dicts queued by - native_tool_start_callback / native_tool_complete_callback and renders - them through the adapter's chat.startStream plan/task-card stream. - On any native failure, falls back to an editable in-thread text - message so progress stays live for the rest of the turn. + On any native failure, falls back to an editable in-thread text message so progress stays + live for the rest of the turn. """ ctx = self._ctx tasks: Dict[str, Dict[str, str]] = {} @@ -5537,11 +5045,8 @@ class TurnRunner: raw = ctx.progress_queue.get_nowait() - # Drain silently when interrupted: events queued in the - # window between tool parse and interrupt processing - # should not render as bubbles. The "⚡ Interrupting - # current task" message is sent separately and is the - # last progress-flavored bubble the user should see. + # Drain silently when interrupted: events queued in the window between tool parse + # and interrupt processing should not render as bubbles. try: _agent_for_interrupt = ctx.agent_holder[0] if ctx.agent_holder else None if _agent_for_interrupt is not None and getattr( @@ -5560,14 +5065,11 @@ class TurnRunner: progress_lines[-1] = f"{base_msg} (×{count + 1})" msg = progress_lines[-1] if progress_lines else base_msg elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": - # Content bubble just landed on the platform — close off - # the current tool-progress bubble so the next tool - # starts a fresh bubble below the content. Without this, - # tool lines keep editing the ORIGINAL progress message - # above the new content, making the chat appear out of - # order. Mirrors GatewayStreamConsumer.on_segment_break - # on the content side. (Issue: tool + content - # linearization regression after PR #7885.) + # Content bubble just landed on the platform — close off the current tool- + # progress bubble so the next tool starts a fresh bubble below the content. + # Otherwise tool lines keep editing the ORIGINAL progress message above the new + # content and the chat reads out of order. + # Mirrors GatewayStreamConsumer.on_segment_break on the content side. progress_msg_id = None progress_lines = [] ctx.last_progress_msg[0] = None @@ -5584,10 +5086,9 @@ class TurnRunner: await adapter.send_typing(ctx.source.chat_id, metadata=ctx._progress_metadata) continue - # Throttle edits: batch rapid tool updates into fewer - # API calls to avoid hitting Telegram flood control. - # (grammY auto-retry pattern: proactively rate-limit - # instead of reacting to 429s.) + # Throttle edits: batch rapid tool updates into fewer API calls to avoid hitting + # Telegram flood control. (grammY auto-retry pattern: proactively rate-limit instead + # of reacting to 429s.) _now = time.monotonic() _remaining = _PROGRESS_EDIT_INTERVAL - (_now - _last_edit_ts) if _remaining > 0: @@ -5736,11 +5237,10 @@ class TurnRunner: except Exception as _ack_err: logger.debug("voice ack schedule failed: %s", _ack_err) - # ── Slack-native task cards: ID-bearing lifecycle callbacks (#29483) ── - # These ride agent.tool_start_callback / agent.tool_complete_callback so - # start/completion events correlate by the REAL tool-call id — the - # name-correlated text events in progress_callback would duplicate cards - # and mispair concurrent calls to the same tool. + # ── Slack-native task cards: ID-bearing lifecycle callbacks ── These ride + # agent.tool_start_callback / agent.tool_complete_callback so start/completion events correlate + # by the REAL tool-call id — the name-correlated text events in progress_callback would + # duplicate cards and mispair concurrent calls to the same tool. def native_tool_start_callback(self, call_id, tool_name, args): """Queue an ID-correlated native progress start from the agent thread.""" @@ -5838,17 +5338,14 @@ class TurnRunner: def _attach_session_title_callback(self, agent, ctx) -> None: """Wire the platform thread-rename lane onto the agent as `_on_session_title`. - The session titler runs inside the turn prologue now (it derives the - title from the user's first message, so it no longer needs the - response), which means the callback has to be attached before the run - rather than registered after it. The lane predicates and their - rationale are unchanged from the old post-response registration. + The session titler runs inside the turn prologue now (it derives the title from the + user's first message, so it no longer needs the response), which means the callback has + to be attached before the run rather than registered after it. """ try: - # Gateway auto-title failures must NOT be surfaced as user-visible - # messages (#23246) — they are not actionable to the end user. - # Overriding the failure sink here keeps CLI mode on the agent's - # _emit_auxiliary_failure path while the gateway logs at debug. + # Gateway auto-title failures must NOT be surfaced as user-visible messages — they are + # not actionable to the end user. Overriding the failure sink here keeps CLI mode on the + # agent's _emit_auxiliary_failure path while the gateway logs at debug. def _title_failure_cb(task: str, exc: BaseException) -> None: logger.debug( "Gateway auto-title failure suppressed (not user-visible): %s: %s", @@ -5860,11 +5357,9 @@ class TurnRunner: session_id = getattr(agent, "session_id", None) source = ctx.source - # Both lanes below spend a rate-limited platform call per title, so - # they take the model's title and skip the derived one — see - # TitleCallback. Renaming twice lands on the same name at twice the - # cost, and Discord's 2-per-10-minutes channel budget can spend - # itself on the throwaway and drop the one worth showing. + # Both lanes below spend a rate-limited platform call per title, so they take the + # model's title and skip the derived one — see TitleCallback. Renaming twice costs + # double and Discord's 2-per-10-min channel budget can burn on the throwaway title. if self._runner._is_telegram_topic_lane(source): agent._on_session_title = lambda title, title_source: ( title_source == "llm" @@ -5875,14 +5370,10 @@ class TurnRunner: elif self._runner._is_discord_auto_thread_lane(source) or ( self._runner._is_relay_discord_channel_lane(source) ): - # Relay note: the second predicate is shape-only (relay - # Discord channel event). Whether the connector actually - # auto-threaded our reply is only knowable AFTER delivery - # (send-result feedback), so the callback must be registered - # eagerly and the rename lane performs the cache lookup at - # fire time (staging repro 2026-07-31: gating registration on - # the cache read meant it never registered and no - # thread_rename op was ever sent). + # Relay note: the second predicate is shape-only (relay Discord channel event). + # Whether the connector auto-threaded our reply is only knowable AFTER delivery, so + # the callback must be registered eagerly and the rename lane does the cache lookup + # at fire time — gating registration on the cache read meant it never registered. agent._on_session_title = lambda title, title_source: ( title_source == "llm" and self._runner._schedule_discord_semantic_thread_rename( @@ -5930,14 +5421,9 @@ class TurnRunner: def run_sync(self): ctx = self._ctx - # Historical note: as a nested closure this body declared - # `nonlocal message` because the conditional re-assignments below - # (prepending model-switch / resume-recovery notes) would otherwise - # make `message` function-local and break the earlier read at - # `_resolve_turn_agent_config(message, …)`. As a method the turn - # message lives on the shared TurnContext instead: every rebind - # writes `ctx.message`, so the outer `_run_agent_inner` body observes - # the updated value exactly as it did through the closure cell. + # As a method the turn message lives on the shared TurnContext instead: every rebind writes + # `ctx.message`, so the outer `_run_agent_inner` body observes the updated value exactly as + # it did through the closure cell. # session_key is propagated via contextvars in _set_session_env() # (_SESSION_KEY) and via set_current_session_key() (_approval_session_key) @@ -5957,7 +5443,7 @@ class TurnRunner: # Map platform enum to the platform hint key the agent understands. # Platform.LOCAL ("local") maps to "cli"; others pass through as-is. platform_key = "cli" if ctx.source.platform == Platform.LOCAL else ctx.source.platform.value - + # Combine platform context, YAML channel_prompts hint for this chat, # channel_overrides system_prompt (or global ephemeral), and gateway # ephemeral prompt from _get_system_prompt_for_channel. @@ -6007,10 +5493,8 @@ class TurnRunner: # Set up stream consumer for token streaming or interim commentary. _stream_consumer = None _stream_delta_cb = None - # #60671 — streaming TTS consumer is created on the outer - # event-loop thread before run_sync launches. run_sync only - # reads it via ``streaming_tts_consumer_holder[0]`` for delta - # callback wiring. + # streaming TTS consumer is created on the outer event-loop thread before run_sync launches. + # run_sync only reads it via ``streaming_tts_consumer_holder[0]`` for delta callback wiring. _stts_consumer_ref = ctx.streaming_tts_consumer_holder[0] _scfg = getattr(getattr(self._runner, 'config', None), 'streaming', None) if _scfg is None: @@ -6101,17 +5585,13 @@ class TurnRunner: turn_route = self._runner._resolve_turn_agent_config(ctx.message, model, runtime_kwargs) - # Per-platform skip_context_files — messaging platforms can opt out - # of filesystem-heavy context-file discovery (SOUL.md, AGENTS.md, - # .cursorrules) to cut AIAgent construction latency. Especially - # impactful on Windows, where stat() + directory walks are 10-100x - # slower than Linux. Off by default; soul identity is preserved so - # the persona survives even with minimal context. + # Per-platform skip_context_files — messaging platforms can opt out of filesystem-heavy + # context-file discovery (SOUL.md, AGENTS.md, .cursorrules) to cut AIAgent construction + # latency. _platforms_gw_cfg = (ctx.user_config.get("gateway") or {}).get("platforms") or {} - # ``hermes gateway setup`` writes ``gateway.platforms`` as a LIST of - # enabled platform names (e.g. ``- telegram``), not a dict. Treat any - # non-dict shape as "no per-platform overrides" instead of crashing - # on ``.get()`` for every incoming turn (#83185). + # ``hermes gateway setup`` writes ``gateway.platforms`` as a LIST of enabled platform names + # (e.g. ``- telegram``), not a dict. Treat any non-dict shape as "no per-platform overrides" + # instead of crashing on ``.get()`` for every incoming turn. if not isinstance(_platforms_gw_cfg, dict): _platforms_gw_cfg = {} _plat_gw_cfg = _platforms_gw_cfg.get(platform_key) or {} @@ -6136,20 +5616,11 @@ class TurnRunner: _cache_lock = getattr(self._runner, "_agent_cache_lock", None) _cache = getattr(self._runner, "_agent_cache", None) - # Peek at the cached entry's snapshot session_id (if any) so we can - # check, OUTSIDE the cache lock, whether THAT session_id is a DEAD - # session in state.db. This closes a gap in the #54947 fix: that - # fix treats "cached session_id != current session_id" as an - # intentional /resume-style switch and reuses the agent unchanged. - # But the #54878 self-heal produces the exact same tuple shape - # when it recovers a routing key away from a session that was - # already ended — the cached AIAgent still belongs to the DEAD - # session, not a valid sibling conversation. Reusing it lets that - # turn's post-run "session split" sync write the routing key - # straight back onto the dead session_id, undoing the self-heal - # and looping every message until an interrupt happens to race in - # first (the #54878 x #54947 interaction — no existing upstream - # issue tracks this combination as of 2026-07-12). + # Peek at the cached entry's snapshot session_id (if any) so we can check, OUTSIDE the cache + # lock, whether THAT session_id is a DEAD session in state.db. The #54947 rule treats + # "cached sid != current sid" as an intentional switch and reuses the agent, but the #54878 + # self-heal yields the same shape with an agent bound to a DEAD session; reusing it writes + # the routing key back onto the dead sid and loops every message. _peek_cached_sid = None if _cache_lock and _cache is not None: with _cache_lock: @@ -6169,12 +5640,10 @@ class TurnRunner: except Exception: _cached_sid_is_dead = False - # Detect cross-process writes: when another process (e.g. hermes - # dashboard) appends to the same session in the shared SessionDB, - # the cached agent's in-memory transcript becomes stale. Compare - # the session's current message_count against the count recorded - # when the agent was cached; on mismatch, invalidate the cache - # so a fresh agent re-reads from disk. (#45966) + # Detect cross-process writes: when another process (e.g. hermes dashboard) appends to the + # same session in the shared SessionDB, the cached agent's in-memory transcript becomes + # stale. Compare current message_count against the count recorded at cache time; on + # mismatch invalidate so a fresh agent re-reads from disk. _current_msg_count = None if self._runner._session_db is not None and ctx.session_id: try: @@ -6190,51 +5659,35 @@ class TurnRunner: with _cache_lock: cached = _cache.get(ctx.session_key) if cached and cached[1] == _sig: - # cached[2] is the message_count at cache time; - # stale when a second process appended rows. - # cached[3] (when present) is the session_id the - # snapshot was taken for — used to skip the guard - # when the active session_id differs (#54947). + # cached[2] is the message_count at cache time; stale when a second process + # appended rows. cached[3] (when present) is the session_id the snapshot was + # taken for — used to skip the guard when the active session_id differs. _cached_mc = cached[2] if len(cached) > 2 else None _cached_sid = cached[3] if len(cached) > 3 else None - # If the snapshot belongs to a different session_id - # (same session_key, different conversation), the - # message_count comparison is meaningless — the - # counts track DIFFERENT DB rows. REUSE the cached - # agent rather than rebuild and bust the prompt cache - # on every session switch (#54947). + # If the snapshot belongs to a different session_id (same session_key, different + # conversation), the message_count comparison is meaningless — the counts track + # DIFFERENT DB rows. REUSE the cached agent rather than rebuild and bust the + # prompt cache on every session switch. _session_id_mismatch = ( _cached_sid is not None and ctx.session_id is not None and _cached_sid != ctx.session_id ) - # Re-validate the OUTSIDE-lock dead-session peek - # against the tuple actually read under THIS lock — - # the cache entry could have been replaced between - # the peek and this lock acquisition, and a stale - # "dead" verdict must never be applied to a - # different (possibly live) cached agent. + # Re-validate the OUTSIDE-lock dead-session peek against the tuple actually read + # under THIS lock — the cache entry could have been replaced between the peek + # and this lock acquisition, and a stale "dead" verdict must never be applied to + # a different (possibly live) cached agent. _stale_dead_sid_reuse = ( _session_id_mismatch and _cached_sid_is_dead and _cached_sid == _peek_cached_sid ) if _stale_dead_sid_reuse: - # #54878 x #54947 interaction: the routing key - # was just self-healed away from a session that - # state.db already marked ended, but the cached - # AIAgent here still belongs to that DEAD - # session_id. The #54947 "different session_id - # under the same key = intentional switch, reuse - # freely" rule does not hold here — this isn't a - # sibling conversation, it's a stale agent left - # over from before the self-heal. Reusing it lets - # this turn's post-run "session split" sync write - # the routing key straight back onto the dead - # session_id, undoing the self-heal and looping - # every message until an interrupt happens to - # race in first. Discard and rebuild fresh - # instead, same as a genuine cross-process write. + # #54878 x #54947 interaction: the routing key was just self-healed away + # from a session that state.db already marked ended, but the cached AIAgent + # here still belongs to that DEAD session_id. Not a sibling conversation — + # a stale agent; reusing it would write the routing key back onto the dead + # sid and undo the self-heal. Discard and rebuild fresh. logger.info( "Agent cache invalidated for session %s: " "cached agent's session_id %s is ended in " @@ -6246,10 +5699,9 @@ class TurnRunner: evicted = self._runner._agent_cache.pop(ctx.session_key, None) _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None if _ev_agent and _ev_agent is not _AGENT_PENDING_SENTINEL: - # Same deferred-cleanup rationale as the - # cross-process branch below (#52197): don't - # block the event loop / cache lock on - # memory-provider shutdown or socket teardown. + # Same deferred-cleanup rationale as the cross-process branch below: + # don't block the event loop / cache lock on memory-provider shutdown or + # socket teardown. _xproc_evicted_agent = _ev_agent elif ( not _session_id_mismatch @@ -6298,22 +5750,18 @@ class TurnRunner: logger.debug("Reusing cached agent for session %s", ctx.session_key) reused_cached_agent = True - # Lock released — refresh the fallback chain from disk for the - # reused agent OUTSIDE the cache lock (config.yaml read is disk - # I/O; the idle-sweep watcher contends on this lock and stalls - # Discord heartbeats — same reasoning as #52197). A chain - # configured after this agent was cached (or after gateway start) - # must reach the next turn (#60955). Per-session turn - # serialization (_running_agents) keeps this safe post-lock. + # Lock released — refresh the fallback chain from disk for the reused agent OUTSIDE the + # cache lock (config.yaml read is disk I/O; the idle-sweep watcher contends on this lock and + # stalls Discord heartbeats — same reasoning as #52197). A chain configured after this agent + # was cached must reach the next turn; per-session serialization keeps this safe post-lock. if reused_cached_agent and agent is not None: self._runner._apply_fallback_chain_to_agent( agent, self._runner._refresh_fallback_model(), ) - # Lock released — now schedule cleanup of any cross-process-evicted - # agent on a daemon thread so memory-provider shutdown / socket - # teardown never blocks the gateway event loop or the cache lock - # the session-expiry watcher needs (#52197). + # Lock released — now schedule cleanup of any cross-process-evicted agent on a daemon thread + # so memory-provider shutdown / socket teardown never blocks the gateway event loop or the + # cache lock the session-expiry watcher needs. if _xproc_evicted_agent is not None: try: threading.Thread( @@ -6372,32 +5820,20 @@ class TurnRunner: ) if _cache_lock and _cache is not None: with _cache_lock: - # Record the session_id the snapshot was taken for - # alongside the message_count, so the cross-process - # guard can skip the (meaningless) count comparison - # when the active session_id later switches under - # the same session_key (#54947). + # Record the session_id the snapshot was taken for alongside the message_count, + # so the cross-process guard can skip the (meaningless) count comparison when + # the active session_id later switches under the same session_key. _cache[ctx.session_key] = ( agent, _sig, _current_msg_count, ctx.session_id, ) self._runner._enforce_agent_cache_cap() logger.debug("Created new agent for session %s (sig=%s)", ctx.session_key, _sig) - # Per-message state — callbacks and reasoning config change every - # turn and must not be baked into the cached agent constructor. - # Gate on needs_progress_queue (tool_progress OR thinking_progress) - # rather than tool_progress alone: the progress_callback also relays - # _thinking assistant scratch text, which is gated on - # thinking_progress and is intentionally independent of tool - # progress. With the old `tool_progress_enabled`-only gate, a user - # who set thinking_progress:true but kept tool_progress:off got a - # None callback — so _thinking scratch bubbles never relayed even - # though the progress queue was created for them. - # Always attached (previously gated to None when no progress surface - # was active): the callback body gates each event class itself, and - # subagent-failure notices must fire even on platforms with - # tool_progress/thinking off — the None gate was exactly why a dead - # subagent vanished silently there. + # Per-message state — callbacks and reasoning config change every turn and must not be baked + # into the cached agent constructor. The progress callback is ALWAYS attached (never gated + # to None): its body gates each event class itself, and subagent-failure notices must fire + # even with tool_progress/thinking off — a None gate made dead subagents vanish silently. + # _thinking scratch relays on thinking_progress independently of tool_progress. agent.tool_progress_callback = ctx.progress_callback # Compose ID-bearing lifecycle consumers: Discord's one-time voice # ack and Slack's native task cards both ride the authoritative @@ -6421,18 +5857,11 @@ class TurnRunner: agent.stream_delta_callback = _stream_delta_cb agent.interim_assistant_callback = _interim_assistant_cb if _want_interim_messages else None agent.status_callback = ctx._status_callback_sync - # Credits / out-of-band notices (usage bands, depletion, restored). - # Messaging has no persistent status bar, so each notice is a - # standalone push: render to a single plaintext line and deliver via - # the shared _deliver_platform_notice rail (honors private/public + - # thread metadata). Fires from the agent's sync worker thread, so we - # hop onto the gateway loop with safe_schedule_threadsafe - same - # pattern as _status_callback_sync. The fired-once latch lives on the - # cached agent and persists across turns, so a band crosses -> one - # push (no per-turn re-nag). Recovery ("✓ Credit access restored") - # rides the same show path (it's emitted as a success notice, not a - # clear). The clear callback is a no-op: a sent platform message - # can't be cleanly retracted, and the band already fired once. + # Credits / out-of-band notices (usage bands, depletion, restored). Fires from the agent's + # sync worker thread, so we hop onto the gateway loop with safe_schedule_threadsafe - same + # pattern as _status_callback_sync. The fired-once latch lives on the cached agent, so a band + # crossing pushes once (no per-turn re-nag). The clear callback is a no-op: a sent platform + # message can't be retracted. def _notice_callback_sync(notice) -> None: if not ctx._status_adapter or not ctx._run_still_current(): return @@ -6455,12 +5884,9 @@ class TurnRunner: agent.event_callback = ctx._event_callback_sync agent.reasoning_config = reasoning_config agent.service_tier = self._runner._service_tier - # Merge, never overwrite: init-time request overrides (e.g. a custom - # provider's extra_body merged at agent construction) must survive - # every reused-agent turn. Drop only the PREVIOUS turn's routing - # overrides (fast-mode service_tier/speed) before layering this - # turn's route overrides on top, so stale per-turn values never - # linger while construction-time values persist. + # Merge, never overwrite: init-time request overrides (e.g. a custom provider's extra_body + # merged at agent construction) must survive every reused-agent turn. Drop only the PREVIOUS + # turn's routing overrides before layering this turn's, so stale per-turn values never linger. request_overrides = dict(getattr(agent, "request_overrides", {}) or {}) previous_turn_overrides = dict( getattr(agent, "_gateway_turn_request_overrides", {}) or {} @@ -6472,11 +5898,10 @@ class TurnRunner: request_overrides.update(turn_request_overrides) agent.request_overrides = request_overrides agent._gateway_turn_request_overrides = turn_request_overrides - # Must-deliver notes for THIS turn ride the current user message - # (api_content sidecar), never the system prompt: staged by - # _handle_message_with_agent (auto-reset note, first-contact - # intro, voice-channel change). Assigned unconditionally so a - # reused cached agent never replays a stale note. + # Must-deliver notes for THIS turn ride the current user message (api_content sidecar), + # never the system prompt: staged by _handle_message_with_agent (auto-reset note, first- + # contact intro, voice-channel change). Assigned unconditionally so a reused cached agent + # never replays a stale note. agent._gateway_turn_context_notes = "\n\n".join( self._runner._consume_pending_turn_sidecar_notes(ctx.session_key) ) @@ -6542,17 +5967,12 @@ class TurnRunner: agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on" # ------------------------------------------------------------------ - # Shared native-stream boundary close. For platforms with native - # streaming (e.g. WeCom msgtype:"stream"), an interaction that - # interrupts the stream — a dangerous-command approval prompt OR a - # clarify decision prompt — must finalize the current stream and - # disable native streaming first. Otherwise the agent's - # post-interaction output keeps flowing into send_stream_frame and - # updates the *old* bubble that preceded the prompt, instead of - # starting a fresh bubble below it (the "气泡割裂" symptom). After - # the boundary, post-prompt output goes through the reliable send() - # path as a new message. Runs on the agent thread; the consumer - # processes the boundary serially via its queue. + # Shared native-stream boundary close. For platforms with native streaming (e.g. WeCom + # msgtype:"stream"), an interaction that interrupts the stream — a dangerous-command + # approval prompt OR a clarify decision prompt — must finalize the current stream and + # disable native streaming first; otherwise post-interaction output keeps updating the OLD + # bubble above the prompt instead of starting a fresh one below it. Runs on the agent + # thread; the consumer processes the boundary serially via its queue. def _close_native_stream_boundary( _reason: str, _placeholder: str | None = None, _reopen: bool = False, ) -> bool: @@ -6613,38 +6033,28 @@ class TurnRunner: multi_select=bool(multi_select), ) - # For WeCom native streaming: finalize the current stream before - # showing the clarify prompt so the post-answer output opens a - # fresh bubble below the question instead of updating the bubble - # that preceded it — the "气泡割裂" symptom. Unlike the approval - # path, clarify passes reopen=True: native streaming stays - # enabled so the post-answer continuation re-opens a fresh - # native stream (typing bubble) rather than degrading to a - # one-shot send(). Clarify waits are short, so stream staleness - # is a low risk; if the re-seed fails the consumer degrades to - # send() automatically. The placeholder is only used in the - # narrow case where a frame was pushed but no text accumulated. + # For WeCom native streaming: finalize the current stream before showing the clarify + # prompt so the post-answer output opens a fresh bubble below the question instead of + # updating the bubble that preceded it — the "气泡割裂" symptom. Unlike approval, clarify + # passes reopen=True so the continuation re-opens a native stream rather than degrading + # to one-shot send(); if the re-seed fails the consumer degrades to send() automatically. _close_native_stream_boundary( "Clarify", "💬 等待你的选择...", _reopen=True, ) - # Pause typing — like approval, we don't want a "thinking..." - # status to obscure the prompt or block the user from typing - # an "Other" response on platforms that disable input while - # typing is active (Slack Assistant API). + # Pause typing — like approval, we don't want a "thinking..." status to obscure the + # prompt or block the user from typing an "Other" response on platforms that disable + # input while typing is active (Slack Assistant API). try: ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) except Exception: pass - # Ordering barrier (#clarify-ordering): flush any buffered - # assistant prose (interim commentary / streamed deltas) to the - # platform BEFORE sending the poll. The poll is delivered on a - # separate, agent-thread-blocking path; without this barrier it - # races ahead of prose still sitting in the stream consumer's - # queue, so the question renders ABOVE its own explanation. - # Best-effort + short timeout: never hang the agent thread if - # the consumer task isn't running. + # Ordering barrier (#clarify-ordering): flush any buffered assistant prose (interim + # commentary / streamed deltas) to the platform BEFORE sending the poll. The poll goes + # out on a separate agent-thread-blocking path and would otherwise render ABOVE its own + # explanation. Best-effort + short timeout: never hang the agent thread if the consumer + # task isn't running. try: _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None _flush = getattr(_sc, "flush_pending_sync", None) @@ -6669,10 +6079,9 @@ class TurnRunner: logger=logger, log_message="Clarify send failed to schedule", ) - # Boundary rule (see _approval_send_outcome): a send timeout is - # AMBIGUOUS — the card may have posted with a late ack. Only a - # definitive failure tears down the registration; ambiguous - # falls through to the bounded wait so a late reply resolves. + # Boundary rule (see _approval_send_outcome): a send timeout is AMBIGUOUS — the card may + # have posted with a late ack. Only a definitive failure tears down the registration; + # ambiguous falls through to the bounded wait so a late reply resolves. _clarify_response = _clarify_send_then_wait( fut, clarify_id=clarify_id, @@ -6686,15 +6095,10 @@ class TurnRunner: isinstance(_clarify_response, str) and _clarify_response.startswith("[") ): - # User answered. Reopen the typing indicator IMMEDIATELY — - # don't wait for the LLM's first post-answer token. On native - # streaming (WeCom) the typing bubble is driven by the stream - # seed frame, and the reopen path otherwise re-seeds lazily on - # the first delta (measured ~48s of dead air). request_reopen_seed - # is a no-op unless we're in the reopen-pending native state, so - # it's safe to call unconditionally here. resume_typing_for_chat - # covers non-native platforms (their pause was set before the - # prompt) — the WeCom send_typing is a no-op, harmless here. + # User answered. Reopen the typing indicator IMMEDIATELY — don't wait for the LLM's + # first post-answer token (native streaming otherwise re-seeds lazily on the first + # delta: ~48s of dead air). request_reopen_seed is a no-op outside the reopen-pending + # native state, so it is safe to call unconditionally. _sc_reopen = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None if _sc_reopen is not None: try: @@ -6732,7 +6136,7 @@ class TurnRunner: agent._gateway_turn_process_baseline = ctx.process_baseline # Capture the full tool definitions for transcript logging ctx.tools_holder[0] = agent.tools if hasattr(agent, 'tools') else None - + # Convert history to agent format. # Two cases: # 1. Normal path (from transcript): simple {role, content, timestamp} dicts @@ -6753,13 +6157,11 @@ class TurnRunner: inject_timestamps=_message_timestamps_enabled(ctx.user_config), ) - # FTS write-corruption guard (#50502): when message persistence - # fails silently through corrupt FTS triggers, the reloaded - # transcript above is stale/empty even though the SAME cached agent - # still holds the full live conversation in `_session_messages`. - # Replacing the live transcript with that shorter copy causes - # immediate same-session amnesia. Only applies when we reused a - # cached agent bound to this exact session_id. + # FTS write-corruption guard: when message persistence fails silently through corrupt FTS + # triggers, the reloaded transcript above is stale/empty even though the SAME cached agent + # still holds the full live conversation in `_session_messages`. Replacing it with the + # shorter copy causes immediate same-session amnesia. Only for a reused agent bound to + # this exact session_id. if reused_cached_agent and getattr(agent, "session_id", None) == ctx.session_id: _selected = _select_cached_agent_history( agent_history, getattr(agent, "_session_messages", None) @@ -6771,25 +6173,21 @@ class TurnRunner: "conversation context (possible FTS write corruption)", ctx.session_key, len(agent_history), len(_selected), ) - # The live in-memory history bypassed the - # _build_gateway_agent_history cleanup pipeline above — - # re-apply the stale-confirmation expiry (#59607) so a - # dangerous confirmation can't slip through this path - # either. Idempotent; messages without timestamps are - # untouched. + # The live in-memory history bypassed the _build_gateway_agent_history cleanup + # pipeline above — re-apply the stale-confirmation expiry so a dangerous + # confirmation can't slip through this path either. agent_history = strip_stale_dangerous_confirmations( _selected, now=time.time() ) - + # Collect MEDIA paths already in history so we can exclude them # from the current turn's extraction. This is compression-safe: # even if the message list shrinks, we know which paths are old. _history_media_paths: set = _collect_history_media_paths(agent_history) - - # Register per-session gateway approval callback so dangerous - # command approval blocks the agent thread (mirrors CLI input()). - # The callback bridges sync→async to send the approval request - # to the user immediately. + + # Register per-session gateway approval callback so dangerous command approval blocks the + # agent thread (mirrors CLI input()). The callback bridges sync→async to send the approval + # request to the user immediately. from tools.approval import ( register_gateway_notify, reset_current_session_key, @@ -6800,35 +6198,29 @@ class TurnRunner: def _approval_notify_sync(approval_data: dict) -> None: """Send the approval request to the user from the agent thread. - If the adapter supports interactive button-based approvals - (e.g. Discord's ``send_exec_approval``), use that for a richer - UX. Otherwise fall back to a plain text message with - ``/approve`` instructions. - """ - # Pause the typing indicator while the agent waits for - # user approval. Critical for Slack's Assistant API where - # assistant_threads_setStatus disables the compose box — the - # user literally cannot type /approve while "is thinking..." - # is active. The approval message send auto-clears the Slack - # status; pausing prevents _keep_typing from re-setting it. - # Typing resumes in _handle_approve_command/_handle_deny_command. + If the adapter supports interactive button-based approvals (e.g. Discord's + ``send_exec_approval``), use that for a richer UX. Otherwise fall back to a plain + text message with ``/approve`` instructions. + """ + # Pause the typing indicator while the agent waits for user approval. Critical for + # Slack's Assistant API where assistant_threads_setStatus disables the compose box — the + # user literally cannot type /approve while "is thinking..." is active. The approval send + # auto-clears the Slack status; pausing stops _keep_typing re-setting it. Typing resumes + # in _handle_approve_command/_handle_deny_command. ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) - # For WeCom native streaming: signal the stream consumer to close - # the current stream before showing the approval prompt. - # This goes through the consumer's queue for serial processing, - # avoiding race conditions with pending deltas. + # For WeCom native streaming: signal the stream consumer to close the current stream + # before showing the approval prompt. This goes through the consumer's queue for serial + # processing, avoiding race conditions with pending deltas. _close_native_stream_boundary("Approval") cmd = approval_data.get("command", "") desc = approval_data.get("description", "dangerous command") - # Redact credentials from the command before displaying it in - # the approval prompt — Tirith's findings are already redacted, - # but the raw command string still leaks secrets to the chat - # platform (#48456). Applied here so BOTH the button-based - # (send_exec_approval) and plain-text fallback paths below use - # the redacted value. + # Redact credentials from the command before displaying it in the approval prompt — + # Tirith's findings are already redacted, but the raw command string still leaks secrets + # to the chat platform. Applied here so BOTH the button-based and plain-text fallback + # paths below use the redacted value. cmd = _redact_approval_command(cmd) # Prefer button-based approval when the adapter supports it. @@ -6857,14 +6249,10 @@ class TurnRunner: if _outcome == "sent": return if _outcome == "ambiguous": - # Timeout ≠ failure: the card may have posted with a - # late ack (slow platform API call or transient - # connector backpressure). The prompt - # registration stays alive, so a tap on the rendered - # card still resolves; re-sending here is what - # produced duplicate cards and an orphaned - # "/approve: nothing pending" in live relay testing. - # Skip the text fallback. + # Timeout ≠ failure: the card may have posted with a late ack (slow platform + # API call or transient connector backpressure). The prompt registration stays + # alive so a tap on the rendered card still resolves; re-sending produced + # duplicate cards + an orphaned "/approve: nothing pending". Skip the fallback. logger.warning( "Button-based approval send timed out — treating " "as possibly-delivered (no re-send; the prompt " @@ -6879,9 +6267,8 @@ class TurnRunner: "Button-based approval failed, falling back to text: %s", _e ) - # Fallback: plain text approval prompt. Use the adapter's - # typed prefix so Slack/Matrix users are told the form they - # can actually type (`!approve`) — typed "/" is blocked in + # Fallback: plain text approval prompt. Use the adapter's typed prefix so Slack/Matrix + # users are told the form they can actually type (`!approve`) — typed "/" is blocked in # Slack threads and reserved by Matrix clients. _p = getattr(ctx._status_adapter, "typed_command_prefix", "/") msg = _format_exec_approval_fallback( @@ -6924,28 +6311,14 @@ class TurnRunner: if _msn: ctx.message = _msn + "\n\n" + ctx.message - # Auto-continue: if the loaded history ends with a tool result, - # the previous agent turn was interrupted mid-work (gateway - # restart, crash, SIGTERM). Prepend a system note so the model - # finishes processing the pending tool results before addressing - # the user's new message. (#4493) - # - # Session-level resume_pending (set on drain-timeout shutdown) - # escalates the wording — the transcript's last role may be - # anything (tool, assistant with unfinished work, etc.), so we - # give a stronger, reason-aware instruction that subsumes the - # tool-tail case. - # - # Freshness gate (#16802): both branches are gated on the age - # of the last persisted transcript row. That is the correct - # "when did we last do anything here" signal for both the - # resume_pending path (restart watchdog) and the tool-tail - # path (in-flight tool loop killed). We read ``history[-1]`` - # here because ``agent_history`` has already stripped the - # ``timestamp`` field off tool/tool_call rows for API purity - # (see the `k != "timestamp"` filter above). Rows without a - # timestamp (legacy transcripts) are treated as fresh so the - # historical auto-continue behaviour is preserved. + # Auto-continue: if the loaded history ends with a tool result, the previous agent turn was + # interrupted mid-work (gateway restart, crash, SIGTERM). Prepend a system note so the model + # finishes processing the pending tool results before addressing the user's new message. + # Session-level resume_pending (drain-timeout shutdown) escalates the wording: the last role + # may be anything, so a stronger reason-aware instruction subsumes the tool-tail case. + # Freshness gate: both branches are gated on the age of the last persisted transcript row. + # Read ``history[-1]`` (not agent_history, which stripped ``timestamp`` off tool rows for + # API purity). Rows without a timestamp (legacy) are treated as fresh. _freshness_window = _auto_continue_freshness_window() _interruption_is_fresh = _is_fresh_gateway_interruption( _last_transcript_timestamp(ctx.history), @@ -6991,14 +6364,11 @@ class TurnRunner: if _is_resume_pending: _reason = getattr(_resume_entry, "resume_reason", None) or "restart_timeout" - # The empty-message case is the auto-resume startup turn - # synthesized by _schedule_resume_pending_sessions — there is - # no NEW user message to address. Guidance is adapter-aware: - # interactive platforms report the restore and ask what next; - # non-interactive event platforms (webhook, API server) - # continue the interrupted work instead, because nobody is - # present to answer and an acknowledgement would silently - # abandon the task (#57056). + # The empty-message case is the auto-resume startup turn synthesized by + # _schedule_resume_pending_sessions — there is no NEW user message to address. Guidance + # is adapter-aware: interactive platforms report the restore and ask what next; event + # platforms (webhook, API server) continue the work — nobody is present to answer, and + # an acknowledgement would silently abandon the task. _resume_adapter = self._runner._adapter_for_source(ctx.source) _interactive_resume = bool( getattr(_resume_adapter, "interactive_resume", True) @@ -7016,25 +6386,20 @@ class TurnRunner: + ctx.message ) - # Consume one-shot /reload-skills note (if the user ran - # /reload-skills since their last turn in this session). Same - # queue pattern as CLI: prepend to the NEXT user message, then - # clear. Nothing was written to the transcript out-of-band, so - # message alternation stays intact. + # Consume one-shot /reload-skills note (if the user ran /reload-skills since their last turn + # in this session). Same queue pattern as CLI: prepend to the NEXT user message, then clear. + # Nothing was written to the transcript out-of-band, so message alternation stays intact. _pending_notes = getattr(self._runner, "_pending_skills_reload_notes", None) if _pending_notes and ctx.session_key and ctx.session_key in _pending_notes: _srn = _pending_notes.pop(ctx.session_key, None) if _srn: ctx.message = _srn + "\n\n" + ctx.message - # Safety net: a startup auto-resume event carries empty - # text and relies on the resume_pending branch above to supply the - # recovery note. If that branch did not fire for any reason (e.g. - # both freshness signals disagreed, or the marker was cleared - # between scheduling and dispatch) we must NOT hand the model a - # blank user turn — it responds with confused "the message came - # through blank" noise. Restricted to resume_pending sessions so - # legitimately empty user turns (e.g. an image with no caption, + # Safety net: a startup auto-resume event carries empty text and relies on the + # resume_pending branch above to supply the recovery note. If it did not fire (freshness + # signals disagreed, marker cleared between scheduling and dispatch) we must NOT hand the + # model a blank user turn — it replies with confused "message came through blank" noise. + # Restricted to resume_pending sessions so legitimately empty turns (image, no caption, # wrapped as native content below) are untouched. if ( isinstance(ctx.message, str) @@ -7058,10 +6423,9 @@ class TurnRunner: _approval_session_token = set_current_session_key(_approval_session_key) register_gateway_notify(_approval_session_key, _approval_notify_sync) try: - # If _prepare_inbound_message_text buffered image paths for native - # attachment, wrap the user turn as an OpenAI-style multimodal - # content list. Consume-and-clear so subsequent turns on the same - # runner instance don't re-attach stale images. + # If _prepare_inbound_message_text buffered image paths for native attachment, wrap the + # user turn as an OpenAI-style multimodal content list. Consume-and-clear so subsequent + # turns on the same runner instance don't re-attach stale images. _native_imgs = self._runner._consume_pending_native_image_paths(ctx.session_key) if _native_imgs: try: @@ -7102,10 +6466,9 @@ class TurnRunner: elif observed_group_context: _conversation_kwargs["persist_user_message"] = ctx.message if ctx.persist_user_display_kind: - # Internal self-injected turn (#82888): type the persisted user - # row at turn start so UIs render it as a timeline notice, not - # a user bubble. Role/content are untouched and the key is - # stripped from provider-bound payloads in conversation_loop. + # Internal self-injected turn: type the persisted user row at turn start so UIs + # render it as a timeline notice, not a user bubble. Role/content are untouched and + # the key is stripped from provider-bound payloads in conversation_loop. _conversation_kwargs["persist_user_display_kind"] = ( ctx.persist_user_display_kind ) @@ -7113,12 +6476,11 @@ class TurnRunner: _conversation_kwargs["moa_config"] = ctx.moa_config if _persist_user_timestamp_override is not None: _conversation_kwargs["persist_user_timestamp"] = _persist_user_timestamp_override - # Thread the platform-side inbound message id onto the persisted - # user turn so a turn interrupted by a gateway restart is durably - # recorded WITH its id — restart drain-window recovery dedups - # against has_platform_message_id, and without this the - # interrupted turn is invisible to that check. Uses the raw - # inbound id (NOT event_message_id, which is the reply anchor). + # Thread the platform-side inbound message id onto the persisted user turn so a turn + # interrupted by a gateway restart is durably recorded WITH its id — restart drain- + # window recovery dedups against has_platform_message_id, and without this the + # interrupted turn is invisible to that check. Uses the raw inbound id (NOT + # event_message_id, which is the reply anchor). if ctx.inbound_message_id is not None: _conversation_kwargs["persist_user_platform_id"] = str(ctx.inbound_message_id) result = agent.run_conversation(_api_run_message, **_conversation_kwargs) @@ -7133,10 +6495,9 @@ class TurnRunner: except Exception: pass reset_current_session_key(_approval_session_token) - # Canonicalize an explicitly emitted computer-use screenshot path at - # the common result boundary. The streaming finalizer below and the - # normal non-streaming delivery path must see the same response; - # repairing only during later media scanning leaves streaming with the + # Canonicalize an explicitly emitted computer-use screenshot path at the common result + # boundary: the streaming finalizer below and the non-streaming delivery path must see the + # same response; repairing only during later media scanning leaves streaming with the # model-mangled path and a rejected attachment. if isinstance(result, dict): _result_final = result.get("final_response") @@ -7149,14 +6510,11 @@ class TurnRunner: ctx.result_holder[0] = result - # Signal the stream consumer that the agent is done. Pass the - # completed final_response as the authoritative finalize payload: - # it includes post-stream augmentation (file-mutation verifier - # footer, turn-completion explainer) the consumer's accumulator - # never saw, so the seal/final edit delivers the TRUE final and no - # separate corrective send fires (live finding #11). Failed turns - # pass nothing — error text is delivered by the gateway's normal - # path, not baked into the stream. + # Signal the stream consumer that the agent is done. Pass the completed final_response as + # the authoritative finalize payload: it includes post-stream augmentation (verifier footer, + # turn-completion explainer) the consumer's accumulator never saw, so the seal delivers the + # TRUE final and no separate corrective send fires. Failed turns pass nothing — error text + # is delivered by the gateway's normal path, not baked into the stream. if _stream_consumer is not None: _final_for_stream = None # Adopt ONLY a genuinely completed final (review B6): interrupt @@ -7187,11 +6545,10 @@ class TurnRunner: else: _stream_consumer.finish() - # Signal the streaming-TTS consumer that the agent is done (#60671). - # finish() is called from the outer event-loop thread after the - # executor returns, so early returns from run_sync are also - # finalised. See the outer finally/completion section below. - + # Signal the streaming-TTS consumer that the agent is done. finish() is called from the + # outer event-loop thread after the executor returns, so early returns from run_sync are + # also finalised. See the outer finally/completion section below. + # Return final response, or a message if something went wrong final_response = result.get("final_response") @@ -7259,15 +6616,12 @@ class TurnRunner: ) _session_split_entry_persisted = True - # If this is a Telegram DM and source.thread_id was lost during - # the session split (synthetic / recovered event), restore it - # from the binding so _thread_metadata_for_source produces the - # correct message_thread_id instead of routing to the General - # thread. Failure here is non-fatal — we log and continue; - # worst case the message lands in General, which is the - # pre-fix behaviour. Only do this after this run successfully - # published its session split; a stale /stop→/new predecessor - # must not mutate routing/binding state for the fresh session. + # If this is a Telegram DM and source.thread_id was lost during the session split + # (synthetic / recovered event), restore it from the binding so + # _thread_metadata_for_source produces the correct message_thread_id instead of routing + # to the General thread. Non-fatal (worst case: lands in General). Only after this run + # published its session split — a stale /stop→/new predecessor must not mutate + # routing/binding state for the fresh session. if _session_split_entry_persisted and ( getattr(ctx.source, "platform", None) == Platform.TELEGRAM and getattr(ctx.source, "chat_type", None) == "dm" @@ -7299,10 +6653,9 @@ class TurnRunner: effective_session_id = agent_session_id self._runner._sync_session_model_from_agent(effective_session_id, agent) - # history_offset=0 whenever the agent's message list no longer has - # the original history prefix — i.e. on rotation (split) OR in-place - # compaction. In both cases the returned `messages` is the compacted - # set, so the gateway must persist all of it (offset 0), not slice + # history_offset=0 whenever the agent's message list no longer has the original history + # prefix — i.e. on rotation (split) OR in-place compaction. In both cases the returned + # `messages` is the compacted set, so persist all of it (offset 0) rather than slicing # past the pre-compaction length (which would drop everything). _effective_history_offset = ( 0 if (_session_was_split or _compacted_in_place) else len(agent_history) @@ -7345,26 +6698,14 @@ class TurnRunner: "context_length": _context_length, } - # Scan tool results for MEDIA: tags that need to be delivered - # as native audio/file attachments. The TTS tool embeds MEDIA: tags - # in its JSON response, but the model's final text reply usually - # doesn't include them. We collect unique tags from tool results and - # append any that aren't already present in the final response, so the - # adapter's extract_media() can find and deliver the files exactly once. - # - # Scope the scan to THIS turn's tool results only. ``agent_history`` - # was passed into run_conversation as ``conversation_history``, so the - # agent's returned ``messages`` list is ``agent_history`` followed by - # the messages produced this turn. Slicing at ``len(agent_history)`` - # isolates the current turn precisely, so a stale MEDIA: path emitted - # by a tool several turns earlier (still present in the full message - # list) can never leak onto a later text-only reply. (Fixes #34608) - # - # Path-based deduplication against _history_media_paths (collected - # before run_conversation) is retained as a secondary guard. It is - # also the sole guard on the fallback branch taken when mid-run - # context compression shrinks the message list below the original - # history length, preserving the compression-safe behaviour of #160. + # Scan tool results for MEDIA: tags that need to be delivered as native audio/file + # attachments. The TTS tool embeds MEDIA: tags in its JSON response, but the model's final + # text reply usually doesn't include them; append missing ones so extract_media() delivers + # each file exactly once. Scope to THIS turn only: returned ``messages`` is ``agent_history`` + # + this turn, so slicing at ``len(agent_history)`` keeps a stale MEDIA: path from an + # earlier turn off a later text-only reply. Path dedup against _history_media_paths is the + # secondary guard — and the sole guard on the fallback branch when mid-run compression + # shrank the message list below the original history length. if "MEDIA:" not in final_response: media_tags, has_voice_directive = _collect_auto_append_media_tags( result.get("messages", []), @@ -7383,13 +6724,10 @@ class TurnRunner: unique_tags.insert(0, "[[audio_as_voice]]") final_response = final_response + "\n" + "\n".join(unique_tags) - # Auto-titling runs at TURN START (agent/turn_context.py) from the - # user's message alone, so it no longer waits on final_response — a - # failed or interrupted turn still gets a titled session. The - # platform-specific thread-rename callbacks are attached to the agent - # as `_on_session_title` before the run starts (see - # _attach_session_title_callback), because the titler now fires from - # inside the turn prologue rather than from here. + # Auto-titling runs at TURN START (agent/turn_context.py) from the user's message alone, so + # it no longer waits on final_response — a failed or interrupted turn still gets a titled + # session. Thread-rename callbacks are attached as `_on_session_title` before the run + # (_attach_session_title_callback) because the titler fires from the turn prologue. return { "final_response": final_response, @@ -7435,21 +6773,17 @@ class TurnRunner: } - -# Sentinel for "no explicit session DB has been pinned on this runner", so the -# ``_session_db`` property can distinguish "resolve from the active profile -# scope" from a deliberate ``runner._session_db = None`` (which disables -# DB-backed commands and is how many suites construct a bare runner). A plain -# ``None`` cannot express both. Mirrors ``gateway.session._DB_UNPINNED``. +# Sentinel for "no explicit session DB has been pinned on this runner", so the ``_session_db`` +# property can distinguish "resolve from the active profile scope" from a deliberate +# ``runner._session_db = None`` (which disables DB-backed commands and is how many suites construct +# a bare runner). A plain ``None`` cannot express both. Mirrors ``gateway.session._DB_UNPINNED``. _SESSION_DB_UNPINNED = object() class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, GatewaySlashCommandsMixin): - """ - Main gateway controller. + """Main gateway controller. - Manages the lifecycle of all platform adapters and routes - messages to/from the agent. + Manages the lifecycle of all platform adapters and routes messages to/from the agent. """ # Class-level defaults so partial construction in tests doesn't @@ -7565,31 +6899,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def __init__(self, config: Optional[GatewayConfig] = None): global _gateway_runner_ref - # When multiplex_profiles is on, load under the default profile secret - # scope so bot tokens in that profile's .env resolve the same way - # secondary profiles do (#64674). Explicit config= injection (tests) - # is left untouched. + # When multiplex_profiles is on, load under the default profile secret scope so bot tokens + # in that profile's .env resolve the same way secondary profiles do. Explicit config= + # injection (tests) is left untouched. self.config = config if config is not None else load_gateway_config_for_runner() # Mark the process as a profile multiplexer when configured. This flips - # agent.secret_scope.get_secret() to fail-closed on any unscoped - # credential read, so a missed migration crashes loudly instead of - # leaking a cross-profile value (Workstream A). Inert when off. + # agent.secret_scope.get_secret() to fail-closed on any unscoped credential read, so a + # missed migration crashes loudly instead of leaking a cross-profile value (Workstream A). try: from agent.secret_scope import set_multiplex_active set_multiplex_active(bool(getattr(self.config, "multiplex_profiles", False))) except Exception: logger.debug("could not set multiplex-active flag", exc_info=True) self.adapters: Dict[Platform, BasePlatformAdapter] = {} - # When non-None, SessionDB init failed — the gateway broadcasts a - # one-time warning to the home channel(s) after connecting, so the - # user knows persistence is broken instead of discovering it later - # via a missing /resume or empty history (#88235). + # When non-None, SessionDB init failed — the gateway broadcasts a one-time warning to the + # home channel(s) after connecting, so the user knows persistence is broken instead of + # discovering it later via a missing /resume or empty history. self._session_db_init_error: Optional[str] = None - # Multi-profile multiplexing: adapters for NON-default profiles live - # here, keyed by profile name then Platform. self.adapters stays the - # default/active profile's map so the ~93 existing self.adapters[...] - # sites are untouched when multiplexing is off (this dict is empty). - # Populated by _start_secondary_profile_adapters(). + # Multi-profile multiplexing: adapters for NON-default profiles live here, keyed by profile + # name then Platform. self.adapters stays the default/active profile's map so the ~93 + # existing self.adapters[...] sites are untouched when multiplexing is off (this dict is + # empty). self._profile_adapters: Dict[str, Dict[Platform, BasePlatformAdapter]] = {} self._warn_if_docker_media_delivery_is_risky() _gateway_runner_ref = _weakref.ref(self) @@ -7616,11 +6946,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._provider_routing = self._load_provider_routing() self._fallback_model = self._load_fallback_model() - # Wire process registry into session store for reset protection. - # A background process older than the configured threshold (default 24h, - # session_reset.bg_process_max_age_hours) is treated as stale and no - # longer blocks session idle / daily reset — see #29177. The process is - # NOT killed, only ignored by the reset guard. + # Wire process registry into session store for reset protection. A background process older + # than session_reset.bg_process_max_age_hours (default 24h) is stale and no longer blocks + # idle/daily reset. The process is NOT killed, only ignored by the reset guard. from tools.process_registry import process_registry _bg_max_age_hours = getattr( self.config.default_reset_policy, "bg_process_max_age_hours", 24 @@ -7649,26 +6977,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._draining = False self._profile_failed_platforms: Dict[str, Dict[Platform, asyncio.Task]] = {} self._systemd_watchdog = None - # External (NAS-driven) drain state — distinct from the shutdown - # ``_draining`` flag above. Set by ``_drain_control_watcher`` when the - # ``.drain_request.json`` marker is present: the gateway flips - # ``gateway_state -> draining`` and refuses NEW turns, but the process - # does NOT exit (the whole point — quiesce-without-restart, D4a). It is - # fully reversible: removing the marker reverts to ``running`` and - # re-accepts turns. ``_draining`` (shutdown) is one-way and ends in - # process exit; this one is a steady state NAS polls during its - # request -> poll -> proceed loop. + # External (NAS-driven) drain state — distinct from the shutdown ``_draining`` flag above. + # Set by ``_drain_control_watcher`` when ``.drain_request.json`` is present: gateway_state + # -> draining, NEW turns refused, but the process does NOT exit (quiesce-without-restart). + # It is fully reversible: removing the marker reverts to ``running`` and re-accepts turns. + # ``_draining`` (shutdown) is one-way and ends in process exit. self._external_drain_active = False self._restart_requested = False - # Set by shutdown_signal_handler when a SIGTERM/SIGINT arrived - # WITHOUT a planned-stop / takeover marker — i.e. an unexpected - # external signal (container/s6 SIGTERM on `docker restart` or - # image upgrade, OOM-killer, bare `kill`). Distinct from an - # operator-requested stop, which writes a marker first. Used by - # _stop_impl to decide whether to persist gateway_state=stopped - # (see issue #42675): an unexpected signal must NOT persist - # "stopped", or container_boot refuses to auto-start the gateway - # on the next boot. + # Set by shutdown_signal_handler when a SIGTERM/SIGINT arrived WITHOUT a planned-stop / + # takeover marker — i.e. an unexpected external signal (container/s6 SIGTERM on `docker + # restart` or image upgrade, OOM-killer, bare `kill`). _stop_impl uses it to decide whether + # to persist gateway_state=stopped: an unexpected signal must NOT persist "stopped", or + # container_boot refuses to auto-start the gateway on the next boot. self._signal_initiated_shutdown = False self._restart_task_started = False self._restart_detached = False @@ -7679,11 +6999,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Used by the /restart redelivery guard to bound the window in which a # missing dedup marker is treated as a stale redelivery. self._startup_time: float = time.time() - # Set True at startup when this process booted as the result of a - # chat-originated /restart (i.e. .restart_notify.json existed on boot). - # A one-shot signal consumed by _is_stale_restart_redelivery so the - # marker-missing fallback only suppresses a /restart when we KNOW we - # just came out of a restart cycle — never on a genuine fresh boot. + # Set True at startup when this process booted as the result of a chat-originated /restart + # (i.e. .restart_notify.json existed on boot). One-shot signal consumed by + # _is_stale_restart_redelivery so the marker-missing fallback only suppresses a /restart + # when we KNOW we just came out of a restart cycle — never on a genuine fresh boot. self._booted_from_restart: bool = False self._stop_task: Optional[asyncio.Task] = None self._restart_task: Optional[asyncio.Task] = None @@ -7692,57 +7011,31 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Set on gateway stop so the recreate-on-shutdown path can't resurrect # the pool during a real shutdown. self._executor_closing = False - # ALL per-session state (turn / conversation / persistent scopes) - # lives in one container — see gateway/session_state.py. Access via - # self._session_state(key) (get-or-create) or + # ALL per-session state (turn / conversation / persistent scopes) lives in one container — + # see gateway/session_state.py. Access via self._session_state(key) (get-or-create) or # self._peek_session_state(key) (read-only). self._sessions: Dict[str, SessionState] = {} - # Per-SESSION_ID turn lease (#64934): serializes the - # [load history → run → flush] region when two ROUTING KEYS resolve - # to one session_id (switch_session's many-to-one mapping). The - # routing-key guards above cannot see that overlap. Acquired in - # _handle_message_with_agent after session resolution is final, - # released via _release_turn_lease in the same method's finally. + # Per-SESSION_ID turn lease: serializes the [load history → run → flush] region when two + # ROUTING KEYS resolve to one session_id (switch_session's many-to-one mapping). The + # routing-key guards above cannot see that overlap. self._turn_leases = SessionTurnLeaseRegistry() - # Tokens for held turn leases, keyed by (routing key, run generation) - # so release is granted per-turn and a stale unwind can never free a - # newer turn's lease (#28686 ownership lesson). - # Held turn-lease tokens live on SessionState.turn.lease_token / - # .lease_generation (the old dict was keyed (routing key, generation) - # so a stale unwind could never free a newer turn's lease — the - # generation field preserves that ownership check, #28686). - # Runner-level queued interrupt text lives on - # SessionState.persistent.pending_command_text (NOTE: distinct from - # the adapter-level _pending_messages Dict[str, MessageEvent] in - # gateway/platforms/base.py, which shares the legacy name). - # Last successfully-resolved (non-empty) model, keyed by session. Used - # as a fallback when a fresh config read transiently returns an empty - # model (e.g. an mtime-keyed config-cache miss during a post-interrupt - # recovery turn). Without this, the agent is built with model="" and - # every API call fails HTTP 400 "No models provided" — the session goes - # silent until the user manually re-sends. See #35314. The ``"*"`` - # session entry holds a process-wide last-known-good for sessions - # seen for the first time. Lives on - # SessionState.conversation.last_resolved_model. - # Overflow buffer for explicit /queue commands. The adapter-level - # _pending_messages dict is a single slot per session (designed for - # "next-turn" follow-ups where repeated sends collapse into one - # event). /queue has different semantics: each invocation must - # produce its own full agent turn, in FIFO order, with no merging. - # When the slot is occupied, additional /queue items land here and - # are promoted one-at-a-time after each run's drain. Cleared on - # /new and /reset. /model and other mid-session operations - # preserve the queue. Lives on SessionState.conversation.queued_events; - # native image paths, busy-ack debounce timestamps and the monotonic - # run-generation counter (#28686, NEVER reset) live on SessionState too. - # Session keys that already received a stall notification for the - # current stall episode (cleared when pending clears / activity resumes - # / conversation boundary). See gateway.session_stall. + # Held turn-lease tokens live on SessionState.turn.lease_token/.lease_generation; the + # generation check means a stale unwind can never free a newer turn's lease. + # SessionState.persistent.pending_command_text holds runner-level queued interrupt text + # (distinct from the adapter-level _pending_messages in gateway/platforms/base.py). + # SessionState.conversation.last_resolved_model: last non-empty model per session, used + # when a fresh config read transiently returns "" (config-cache miss) — otherwise the agent + # is built with model="" and every call fails HTTP 400; "*" holds a process-wide fallback. + # SessionState.conversation.queued_events: /queue overflow — the adapter slot is single; + # /queue needs one full turn per item in FIFO order, promoted one at a time after each + # drain. Cleared on /new and /reset (preserved across /model). The run-generation counter + # on SessionState is monotonic and NEVER reset. + # Session keys already notified for the current stall episode (cleared when pending clears + # / activity resumes / conversation boundary). See gateway.session_stall. self._session_stall_notified: Dict[str, bool] = {} - # Startup restore gate: while restart-interrupted sessions are being - # auto-resumed, real inbound messages are queued instead of competing - # with the synthetic resume turns for the same session. The queued - # events drain only after all startup resume tasks have finished. + # Startup restore gate: while restart-interrupted sessions are being auto-resumed, real + # inbound messages are queued instead of competing with the synthetic resume turns for the + # same session. The queued events drain only after all startup resume tasks have finished. self._startup_restore_in_progress = False # Set by start_gateway() only for an explicit ``--replace`` launch. # _connect_initial_adapter_with_timeout scopes it to each adapter's @@ -7750,18 +7043,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._platform_lock_takeover_on_start = False self._startup_restore_queue: List[MessageEvent] = [] self._startup_restore_tasks: List[asyncio.Task] = [] - # LRU cache of live SessionSources keyed by session_key. Used by - # fallback routing paths (shutdown notifications, synthetic - # background-process events) when the persisted origin is missing - # and _parse_session_key can't recover thread_id. Capped so it - # cannot grow unbounded over a long-running gateway lifetime. + # LRU cache of live SessionSources keyed by session_key. Used by fallback routing paths + # (shutdown notifications, synthetic background-process events) when the persisted origin is + # missing and _parse_session_key can't recover thread_id. Capped so it cannot grow unbounded. self._session_sources: "OrderedDict[str, SessionSource]" = OrderedDict() self._session_sources_max = 512 - # Completion delivery is intentionally lifecycle-scoped. This closes - # duplicate queue/watcher races inside one gateway without pretending - # the adapter call and a persistence write can be exactly-once across - # a process crash. Any durable async-delegation replay state remains - # owned by tools.async_delegation, not a parallel gateway ledger. + # Completion delivery is intentionally lifecycle-scoped: it closes duplicate queue/watcher + # races inside one gateway without pretending adapter send + persistence write are + # exactly-once across a crash. Any durable async-delegation replay state remains owned by + # tools.async_delegation, not a parallel gateway ledger. self._completion_delivery_lock = threading.Lock() self._completion_deliveries_inflight: set[tuple[str, str, object]] = set() self._completion_deliveries_delivered: "OrderedDict[tuple[str, str, object], None]" = OrderedDict() @@ -7775,24 +7065,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._completion_notification_batch_window = 0.1 self._completion_notification_batches_stopping = False - # Cache AIAgent instances per session to preserve prompt caching. - # Without this, a new AIAgent is created per message, rebuilding the - # system prompt (including memory) every turn — breaking prefix cache - # and costing ~10x more on providers with prompt caching (Anthropic). - # Key: session_key, Value: (AIAgent, config_signature_str) - # - # OrderedDict so _enforce_agent_cache_cap() can pop the least-recently- - # used entry (move_to_end() on cache hits, popitem(last=False) for - # eviction). Hard cap via _AGENT_CACHE_MAX_SIZE, idle TTL enforced - # from _session_expiry_watcher(). + # Cache AIAgent instances per session to preserve prompt caching: a fresh AIAgent per message + # rebuilds the system prompt (incl. memory) every turn, breaking the prefix cache (~10x cost + # on Anthropic). Key: session_key, Value: (AIAgent, config_signature_str). OrderedDict so + # _enforce_agent_cache_cap() can evict LRU (move_to_end on hit, popitem(last=False)); + # hard cap _AGENT_CACHE_MAX_SIZE, idle TTL enforced from _session_expiry_watcher(). import threading as _threading self._agent_cache: "OrderedDict[str, tuple]" = OrderedDict() self._agent_cache_lock = _threading.Lock() - # Conversation-scoped per-session state (/model, /model --once, - # /reasoning, /fast overrides; per-turn sidecar notes; ephemeral - # context pin; last-delivered voice-channel context) lives on - # SessionState.conversation — see gateway/session_state.py. + # Conversation-scoped per-session state (/model, /model --once, /reasoning, /fast overrides; + # per-turn sidecar notes; ephemeral context pin; last-delivered voice-channel context) lives + # on SessionState.conversation — see gateway/session_state.py. self._kanban_notifier_profile = self._active_profile_name() # Launch-time identity of the profile that owns ``self.adapters``; # ``_authorization_adapter`` compares against this rather than the @@ -7814,11 +7098,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Pending /update prompt flags live on # SessionState.persistent.update_prompt_pending. - # Slash-confirm state lives in tools.slash_confirm (module-level), - # so platform adapters can resolve callbacks without a backref to - # this runner. Keep a local counter for confirm_id generation so - # IDs stay compact (button callback_data has a 64-byte cap on - # some platforms). + # Slash-confirm state lives in tools.slash_confirm (module-level), so platform adapters can + # resolve callbacks without a backref to this runner. Keep a local counter for confirm_id + # generation so IDs stay compact (button callback_data has a 64-byte cap on some platforms). import itertools as _itertools self._slash_confirm_counter = _itertools.count(1) @@ -7826,8 +7108,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # This preserves write_frequency="session" semantics across short-lived # per-message AIAgent instances. - - # Ensure tirith security scanner is available (downloads if needed) try: from tools.tirith_security import ensure_installed @@ -7863,14 +7143,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("approvals.mode startup check skipped", exc_info=True) # Initialize session database for session_search tool support. - # - # Same frozen-handle class of bug as SessionStore._db (#88532): a - # handle bound here is pinned to the process's root home, but - # /resume, /title, /history and session search all run inside - # _profile_runtime_scope on a multiplexed gateway and must see that - # profile's own state.db. Resolve through a property that caches - # one AsyncSessionDB per resolved path; priming here keeps startup - # diagnostics (the #88235 broadcast) at construction time. + # A handle bound here would be pinned to the process root home, but /resume, /title, + # /history and session search run inside _profile_runtime_scope on a multiplexed gateway + # and must see that profile's state.db — so resolve via a property caching one + # AsyncSessionDB per path; priming here keeps startup diagnostics at construction time. self._session_db_pinned: Any = _SESSION_DB_UNPINNED self._session_db_handles: Dict[Path, Any] = {} self._session_db_handles_lock = threading.Lock() @@ -7883,18 +7159,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew try: self._open_session_db_for_active_scope(raise_on_error=True) except Exception as e: - # WARNING (not DEBUG) so the failure appears in errors.log — matches - # cli.py's handling of the same init path. Users hitting NFS-mounted - # HERMES_HOME silently lost /resume, /title, /history, /branch, and - # session search without this. The underlying cause (usually - # "locking protocol" from NFS) is now also captured by - # hermes_state.get_last_init_error() for slash-command error strings. + # WARNING (not DEBUG) so the failure appears in errors.log — matches cli.py's handling + # of the same init path. Users hitting NFS-mounted HERMES_HOME silently lost /resume, + # /title, /history, /branch, and session search without this. logger.warning("SQLite session store not available: %s", e) - # Surface the failure to the user via their home channel(s) once - # the gateway connects. Without this, state.db corruption or - # NFS/SMB lock failures silently degrade the entire gateway — - # messages may flow but nothing is persisted, and the user has - # no indication until they try /resume and find nothing (#88235). + # Surface the failure to the user via their home channel(s) once the gateway connects. + # Without this, state.db corruption or NFS/SMB lock failures silently degrade the whole + # gateway — messages flow but nothing is persisted until /resume finds nothing. self._session_db_init_error = str(e) # Opportunistic state.db maintenance: prune ended sessions inactive @@ -7935,12 +7206,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _ckpt_cfg = (_load_full_config().get("checkpoints") or {}) if _ckpt_cfg.get("auto_prune", False): from tools.checkpoint_manager import maybe_auto_prune_checkpoints - # delete_orphans is intentionally never honoured here: a - # missing workdir at startup is ambiguous (deleted project - # vs. an unmounted external volume / network share / VPN - # not yet up) and this sweep runs unattended. Orphan cleanup - # is only ever done via the explicit `hermes checkpoints - # prune` command, which the user has to invoke. + # delete_orphans is intentionally never honoured here: a missing workdir at startup + # is ambiguous (deleted project vs. an unmounted external volume / network share / + # VPN not yet up) and this sweep runs unattended. Orphan cleanup happens only via + # the explicit `hermes checkpoints prune` command. maybe_auto_prune_checkpoints( retention_days=int(_ckpt_cfg.get("retention_days", 7)), min_interval_hours=int(_ckpt_cfg.get("min_interval_hours", 24)), @@ -7950,16 +7219,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as exc: logger.debug("checkpoint auto-maintenance skipped: %s", exc) - # DM pairing store for code-based user authorization. - # ``pairing_store`` stays as the global/default store for the - # ``hermes pairing`` CLI and any caller without a profile context. - # ``pairing_stores`` is the per-profile map used by - # ``authz_mixin._is_user_authorized`` to route checks to the right - # whitelist (one per profile in multiplex mode). + # DM pairing store for code-based user authorization. ``pairing_store`` stays as the + # global/default store for the ``hermes pairing`` CLI and any caller without a profile + # context. ``pairing_stores`` is the per-profile map ``authz_mixin._is_user_authorized`` + # routes checks through (one whitelist per profile in multiplex mode). from gateway.pairing import PairingStore self.pairing_store = PairingStore() self.pairing_stores: Dict[str, "PairingStore"] = {} - + # Event hook system from gateway.hooks import HookRegistry self.hooks = HookRegistry() @@ -7982,12 +7249,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._loop_floor_timer_handle = None self._loop_liveness_watchdog = None - # scale-to-zero (Phase 0, F13): gateway-scoped "last inbound seen" clock. - # There is no such clock today (only a per-agent _last_activity_ts), so the - # idle predicate needs this. Stamped in _handle_message (the single inbound - # chokepoint all adapters call); seeded to "now" so a fresh gateway isn't - # considered idle from epoch. The scale-to-zero watcher (started only when - # the instance is opted in + relay-only + has a wakeUrl) reads it. + # scale-to-zero (Phase 0, F13): gateway-scoped "last inbound seen" clock. There is no such + # clock today (only a per-agent _last_activity_ts), so the idle predicate needs this. + # Stamped in _handle_message (the single inbound chokepoint); seeded to "now" so a fresh + # gateway isn't considered idle from epoch. Read by the scale-to-zero watcher. self._last_inbound_at: float = time.time() # Set after a wake (re-arm cooldown, 0.F) so we don't immediately re-go # dormant before the drained backlog has a chance to update the clock. @@ -7995,26 +7260,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # One-shot: log the "platform owns the suspend" notice once, not per tick. self._scale_to_zero_no_suspend_logged: bool = False - def _open_session_db_for_active_scope(self, raise_on_error: bool = False) -> Any: """Return the AsyncSessionDB for the profile scope active on this task. - Same per-path cache as ``SessionStore._open_session_db_for_active_scope`` - (#88532): ``SessionDB()`` resolves ``_default_db_path()`` at call time - through the context-local HERMES_HOME override installed by - ``_profile_runtime_scope``, so resolving per access — instead of once - in ``__init__`` — is what lets /resume, /title, /history and session - search on a multiplexed gateway read the *serving profile's* store - rather than the root one. + ``SessionDB()`` resolves ``_default_db_path()`` at call time through the context-local + HERMES_HOME override installed by ``_profile_runtime_scope``, so resolving per access + (not once in ``__init__``) is what lets a multiplexed profile read its own store. - One ``AsyncSessionDB`` is cached per resolved path, so - the wrapper identity is stable per profile (callers compare and stash - it) and two profiles never share a handle. A construction failure - enters bounded backoff; one caller retries after the deadline while - concurrent callers continue to see the unavailable fallback. - ``raise_on_error=True`` (construction-time priming) propagates the - failure after recording that recoverable state so ``__init__`` can - record ``_session_db_init_error`` for the #88235 broadcast. + One ``AsyncSessionDB`` is cached per resolved path, so the wrapper identity is stable per + profile (callers compare and stash it) and two profiles never share a handle. A + construction failure enters bounded backoff (one caller retries after the deadline). + ``raise_on_error=True`` (construction-time priming) propagates the failure after recording + that state so ``__init__`` can set ``_session_db_init_error``. """ from hermes_state import AsyncSessionDB, SessionDB, _default_db_path, get_shared_session_db from gateway.session_db_recovery import RecoverableHandleCache @@ -8031,18 +7288,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._session_db_handle_cache = cache def _open(): - # Borrow the SessionStore's handle for this path rather than - # opening a second one. Both caches resolve the SAME - # ``_default_db_path()``, so the process was holding two writer - # connections and two read pools against one state.db — the fd - # budget doubled for nothing, and doubled again per profile on a - # multiplexed gateway (#98573). The store owns the handle and - # sweeps it at shutdown; this cache holds only the async wrapper. - # - # A borrowed wrapper cannot go stale in practice: the store's - # cache only drops handles in close_all_db_handles() (shutdown), - # and while the store's own open is failing there is nothing to - # borrow, so nothing is cached here either. + # Borrow the SessionStore's handle for this path rather than opening a second one: both + # caches resolve the SAME path, so the process held two writer connections + two read + # pools per state.db (doubled again per profile). The store owns the handle and sweeps + # it at shutdown; this cache holds only the async wrapper. A borrowed wrapper cannot go + # stale: the store only drops handles in close_all_db_handles() (shutdown), and while + # its open is failing there is nothing to borrow, so nothing is cached here either. store = getattr(self, "session_store", None) borrowed = getattr(store, "_db", None) if store is not None else None if borrowed is not None: @@ -8052,10 +7303,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew wrapper.__dict__["_hermes_borrowed_handle"] = True return wrapper if store is not None: - # The store exists and its handle is unavailable (failed open - # or backoff). Opening our own here would resurrect exactly - # the duplicate this borrows away from, so report the same - # unavailability the store is already reporting. + # The store exists and its handle is unavailable (failed open or backoff). Opening + # our own here would resurrect exactly the duplicate this borrows away from, so + # report the same unavailability the store is already reporting. raise RuntimeError("SessionStore SQLite handle unavailable") try: return AsyncSessionDB(get_shared_session_db()) @@ -8078,11 +7328,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _session_db(self) -> Any: """The AsyncSessionDB for the active profile scope, or a pinned override. - Assigning ``runner._session_db`` pins that value for every subsequent - read — tests rely on installing fakes or ``None`` this way. Unpinned - (the production path), each read resolves the active scope so a - multiplexed profile's slash commands and session search hit its own - store. + Assigning ``runner._session_db`` pins that value for every subsequent read — tests rely + on installing fakes or ``None`` this way. Unpinned (the production path), each read + resolves the active scope so a multiplexed profile's slash commands and session search + hit its own store. """ if self._session_db_pinned is not _SESSION_DB_UNPINNED: return self._session_db_pinned @@ -8095,13 +7344,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def close_all_session_db_handles(self) -> None: """Close every per-profile AsyncSessionDB this runner opened. - Shutdown counterpart of the per-path cache above; mirrors - ``SessionStore.close_all_db_handles``. Handles are drained under the - lock and closed outside it; a pinned handle is the pinner's to close. - - Wrappers around a handle BORROWED from ``session_store`` (#98573) are - drained but not closed: the store owns that connection and its own - sweep — which runs first in the shutdown sequence — closes it. + Handles are drained under the lock and closed outside it; a pinned handle is the pinner's + to close. Wrappers around a handle BORROWED from ``session_store`` are drained but not + closed: the store owns that connection and its own sweep — which runs first in the + shutdown sequence — closes it. """ def _close(db) -> None: if getattr(db, "__dict__", {}).get("_hermes_borrowed_handle"): @@ -8147,15 +7393,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._teams_pipeline_runtime_error, ) - def _warn_if_docker_media_delivery_is_risky(self) -> None: """Warn when Docker-backed gateways lack an explicit export mount. - MEDIA delivery happens in the gateway process, so paths emitted by the model - must be readable from the host. A plain container-local path like - `/workspace/report.txt` or `/output/report.txt` often exists only inside - Docker, so users commonly need a dedicated export mount such as - `host-dir:/output`. + MEDIA delivery happens in the gateway process, so paths emitted by the model must be + readable from the host. A container-local path like `/output/report.txt` often exists + only inside Docker, so users need an export mount such as `host-dir:/output`. """ if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": return @@ -8195,8 +7438,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "for container-local paths like '/workspace/...' or '/output/...'." ) - - # -- Setup skill availability ---------------------------------------- def _has_setup_skill(self) -> bool: @@ -8216,11 +7457,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> str: """Return a platform-namespaced key for voice mode state. - Under multiplexing the key is additionally namespaced by the profile - whose bot speaks in the chat (``::``); the - default profile keeps the historical ``:`` shape so - persisted state stays valid. Two bots in one Discord channel otherwise - share a key and one profile's ``/voice`` flips the other's (#75198). + Under multiplexing the key is additionally namespaced by the profile whose bot speaks in + the chat (``::``); the default profile keeps the historical + ``:`` shape so persisted state stays valid. Two bots in one Discord + channel otherwise share a key and one profile's ``/voice`` flips the other's. """ base = f"{platform.value}:{chat_id}" profile = profile.strip() if isinstance(profile, str) else "" @@ -8231,10 +7471,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _voice_key_for_source(self, source: SessionSource) -> str: """Voice-state key for an inbound source, namespaced by its transport owner. - Voice mode belongs to the (bot, chat) pair, so the namespace is the - profile that OWNS the receiving adapter (``_adapter_profile_for_source``) - — the same profile ``_sync_voice_mode_state_to_adapter`` uses on - reconnect — not the routed runtime profile. + Voice mode belongs to the (bot, chat) pair, so the namespace is the profile that OWNS the + receiving adapter (``_adapter_profile_for_source``) — the same profile + ``_sync_voice_mode_state_to_adapter`` uses on reconnect — not the routed runtime profile. """ return self._voice_key( source.platform, @@ -8365,10 +7604,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> bool: """Wait for adapter cleanup without letting cancellation swallowing hang us. - ``asyncio.wait_for`` cancels an overdue child but then waits for it to - exit. An adapter close path that catches ``CancelledError`` can therefore - block recovery forever. Keep ownership of the old task through its done - callback, but release the runner at the deadline. + ``asyncio.wait_for`` cancels an overdue child but then waits for it to exit. An adapter + close path that catches ``CancelledError`` can therefore block recovery forever. Keep + ownership of the old task through its done callback, but release the runner at the deadline. """ if timeout <= 0: await awaitable @@ -8392,13 +7630,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _safe_adapter_disconnect(self, adapter, platform) -> None: """Call adapter.disconnect() defensively, swallowing any error. - Used when adapter.connect() failed or raised — the adapter may - have allocated partial resources (aiohttp.ClientSession, poll - tasks, child subprocesses) that would otherwise leak and surface - as "Unclosed client session" warnings at process exit. - - Must tolerate partial-init state and never raise, since callers - use it inside error-handling blocks. + Used when adapter.connect() failed or raised — the adapter may have allocated partial + resources (aiohttp.ClientSession, poll tasks, child subprocesses) that would otherwise + leak and surface as "Unclosed client session" warnings at process exit. Must tolerate + partial-init state and never raise — callers use it inside error-handling blocks. """ timeout = self._adapter_disconnect_timeout_secs() try: @@ -8423,18 +7658,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Tear down one adapter on the shutdown path with bounded awaits. - Both ``cancel_background_tasks()`` and ``disconnect()`` can block - indefinitely when a platform's network state is half-dead (e.g. a - wedged Feishu/Lark WebSocket thread waiting on I/O). An unbounded - await here stalls the entire shutdown sequence past systemd's - ``TimeoutStopSec``; the resulting SIGKILL skips ``atexit`` PID-file - cleanup, so the next start dies with "PID file race lost" (#14128). - - Each await uses the existing per-adapter timeout budget - (``HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT``). On timeout the old - task is cancelled and detached, then teardown forces forward progress; - the loop never hangs even if an adapter swallows cancellation. Never - raises. + ``cancel_background_tasks()`` and ``disconnect()`` can block indefinitely on half-dead + network state (e.g. a wedged Feishu/Lark WebSocket thread). An unbounded await here stalls + the entire shutdown sequence past systemd's ``TimeoutStopSec``; the resulting SIGKILL + skips ``atexit`` PID-file cleanup, so the next start dies with "PID file race lost". + Each await uses the per-adapter budget (``HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT``); on + timeout the task is cancelled and detached so the loop never hangs even if an adapter + swallows cancellation. Never raises. """ timeout = self._adapter_disconnect_timeout_secs() suffix = f" (profile: {profile})" if profile else "" @@ -8488,14 +7718,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _platform_connect_timeout_secs(self, platform=None, *, initial: bool = False) -> float: """Return the per-platform connect timeout used during startup/retry. - ``initial=True`` marks the cold-start connect awaited before the - gateway reaches ``running``. Telegram's full connect budget (180s, - raised for #67498 so cold polling can prove getUpdates readiness) is - deliberately NOT spent there: an unreachable Telegram would hold the - whole gateway out of the ``running`` state for the full budget - (#85993). The cold-start wait is capped and the platform is handed to - the reconnect watcher, which retries with the full budget (and - ``is_reconnect=True``, preserving the offline update queue — #46621). + Telegram's full connect budget (180s, raised for #67498 so cold polling can prove + getUpdates readiness) is deliberately NOT spent there: an unreachable Telegram would hold + the whole gateway out of the ``running`` state for the full budget. The cold-start wait is + capped and the platform handed to the reconnect watcher, which retries with the full + budget and ``is_reconnect=True`` (preserving the offline update queue). """ raw = os.getenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "").strip() if raw: @@ -8519,25 +7746,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> bool: """Connect an adapter without allowing one platform to block others. - ``is_reconnect`` is forwarded to ``adapter.connect()`` so platform - adapters can distinguish a cold first boot (drop any stale - server-side queue) from a watcher reconnect after a prolonged outage - (preserve the queue so messages sent during the outage are delivered - rather than silently dropped — #46621). + ``is_reconnect`` is forwarded to ``adapter.connect()`` so adapters can distinguish a cold + first boot (drop any stale server-side queue) from a watcher reconnect after an outage + (preserve the queue so messages sent meanwhile are delivered, not silently dropped). - ``initial`` selects the capped cold-start budget for platforms whose - full connect budget is too long to spend before the gateway reaches - ``running`` (#85993 — Telegram's 180s). + ``initial`` selects the capped cold-start budget for platforms whose full connect budget + is too long to spend before the gateway reaches ``running`` (#85993 — Telegram's 180s). """ timeout = self._platform_connect_timeout_secs(platform, initial=initial) if timeout <= 0: return await adapter.connect(is_reconnect=is_reconnect) - # Use the detach-on-timeout pattern instead of plain asyncio.wait_for: - # asyncio.wait_for cancels the overdue task but then waits for it to - # exit. An adapter connect() that catches CancelledError can therefore - # block recovery forever (the watcher never reaches the next retry). - # Keep ownership of the old task through its done callback, but - # release the runner at the deadline (#70344). + # Use the detach-on-timeout pattern instead of plain asyncio.wait_for: asyncio.wait_for + # cancels the overdue task but then waits for it to exit, so a connect() that catches + # CancelledError blocks recovery forever (the watcher never reaches the next retry). Keep + # ownership of the old task through its done callback, but release the runner at the deadline. task = asyncio.ensure_future( adapter.connect(is_reconnect=is_reconnect) ) @@ -8559,10 +7781,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _connect_initial_adapter_with_timeout(self, adapter, platform) -> bool: """Connect one cold-start adapter with tightly scoped replace intent. - The capability is visible only while this initial connect is awaited. - Reconnects call ``_connect_adapter_with_timeout`` directly and adapters - also default to deny, so a later network recovery can never evict a - healthy token holder. + The capability is visible only while this initial connect is awaited. Reconnects call + ``_connect_adapter_with_timeout`` directly and adapters also default to deny, so a later + network recovery can never evict a healthy token holder. """ adapter._platform_lock_takeover_allowed = bool( self._platform_lock_takeover_on_start @@ -8622,12 +7843,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew @staticmethod def _telegram_topic_profile_name(source: SessionSource) -> str: - """Profile namespace for Telegram topic-mode rows (issue #76423). + """Profile namespace for Telegram topic-mode rows. - Prefer the profile already stamped on the routed event - (``source.profile``). Do **not** fall back to the process-global - active profile here — under multiplex that can mis-attribute - topic state across bots sharing one ``state.db``. + Prefer the profile already stamped on the routed event (``source.profile``). Do **not** + fall back to the process-global active profile here — under multiplex that can mis- + attribute topic state across bots sharing one ``state.db``. """ name = str(getattr(source, "profile", None) or "").strip() return name if name else "default" @@ -8770,13 +7990,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Update the topic binding to point at ``session_entry.session_id``. - Telegram topic lanes persist a (chat_id, thread_id) -> session_id row - so reopening a topic in a fresh process resumes the right Hermes - session. When compression rotates ``session_entry.session_id`` mid-turn, - the binding goes stale and the next inbound message in that topic - reloads the oversized parent transcript instead of the compressed - child, retriggering preflight compression — sometimes in a loop - (#20470, #29712, #33414). + Telegram topic lanes persist a (chat_id, thread_id) -> session_id row so reopening a + topic in a fresh process resumes the right Hermes session. When compression rotates + ``session_entry.session_id`` mid-turn the binding goes stale and the next inbound message + reloads the oversized parent instead of the compressed child, retriggering preflight + compression — sometimes in a loop. """ if not self._is_telegram_topic_lane(source): return @@ -8793,14 +8011,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[str]: """Pin DM-topic routing to the user's last-active topic. - Telegram can omit ``message_thread_id`` or surface General (``1``) - for some topic-mode DM replies. In those lobby-shaped cases, keep the - conversation attached to the user's most-recent bound topic. - - Do not rewrite a non-lobby, previously-unbound thread id: a newly - created Telegram DM topic is also "unknown" until the first inbound - message is recorded, and rewriting it would send that brand-new topic's - answer into an older lane. Returns None to leave the source alone. + Telegram can omit ``message_thread_id`` or surface General (``1``) for some topic-mode DM + replies; in those lobby-shaped cases keep the conversation on the user's most-recent + bound topic. Do not rewrite a non-lobby, previously-unbound thread id: a newly created + Telegram DM topic is also "unknown" until the first inbound message is recorded, and + rewriting it would send that brand-new topic's answer into an older lane. Returns None to + leave the source alone. """ if ( source.platform != Platform.TELEGRAM @@ -8813,10 +8029,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew inbound = str(source.thread_id or "") is_lobby = not inbound or inbound in self._TELEGRAM_GENERAL_TOPIC_IDS if not is_lobby: - # A non-lobby, unknown thread_id is most likely the first message in - # a brand-new Telegram DM topic. Preserve it so it can be recorded - # as a new independent lane below instead of hijacking the latest - # existing topic binding. + # A non-lobby, unknown thread_id is most likely the first message in a brand-new + # Telegram DM topic. Preserve it so it can be recorded as a new independent lane below + # instead of hijacking the latest existing topic binding. return None session_db = getattr(self, "_session_db", None) if session_db is None: @@ -8849,14 +8064,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """Apply Telegram DM topic recovery to a source for session-key purposes. ``_handle_message_with_agent`` rewrites ``source.thread_id`` via - ``_recover_telegram_topic_thread_id`` *before* deriving the session - key for a normal message turn (a lobby/stripped reply gets pinned to - the user's last-active topic). Session-scoped command handlers like - ``/model`` and ``/reasoning`` derive their override key from the raw - inbound ``event.source``, which skips that recovery — so the override - is stored under a different key than the next message turn reads, - and the override is silently dropped on Telegram forum topics and - after compression session splits (#30479). + ``_recover_telegram_topic_thread_id`` *before* deriving the session key for a normal + message turn (a lobby/stripped reply gets pinned to the user's last-active topic). + Session-scoped handlers like ``/model`` deriving their key from the raw ``event.source`` + skip that recovery, so the override lands under a different key than the next turn reads + and is silently dropped on forum topics / after compression splits. Returns a recovery-normalized copy when a rewrite applies, otherwise the original source unchanged. Always derive the override storage key @@ -8879,9 +8091,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> tuple[str, dict]: """Resolve model/runtime for a session. - Priority (highest first): session ``/model`` → ``channel_overrides`` → - global config/env (``_resolve_gateway_model(user_config)`` and default - provider resolution). + Priority (highest first): session ``/model`` → ``channel_overrides`` → global config/env + (``_resolve_gateway_model(user_config)`` and default provider resolution). """ resolved_session_key = session_key if not resolved_session_key and source is not None: @@ -8988,10 +8199,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew resolved_session_key, model, runtime_kwargs ) - # When the config has no model.default but a provider was resolved - # (e.g. user ran `hermes auth add openai-codex` without `hermes model`), - # fall back to the provider's first catalog model so the API call - # doesn't fail with "model must be a non-empty string". + # When the config has no model.default but a provider was resolved (e.g. user ran `hermes + # auth add openai-codex` without `hermes model`), fall back to the provider's first catalog + # model so the API call doesn't fail with "model must be a non-empty string". if not model and runtime_kwargs.get("provider"): try: from hermes_cli.models import get_default_model_for_provider @@ -9044,16 +8254,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: """Build the effective model/runtime config for a single turn. - Always uses the session's primary model/provider. If `/fast` is - enabled and the model supports Priority Processing / Anthropic fast - mode, attach `request_overrides` so the API call is marked - accordingly. - - Per-provider ``request_overrides`` resolved by - ``resolve_runtime_provider`` (e.g. a ``custom_providers`` ``extra_body`` - carrying ``chat_template_kwargs``) are preserved here and merged *under* - the fast-mode overrides, so a provider's configured request body still - reaches the model on the gateway turn path. + Always uses the session's primary model/provider. If `/fast` is enabled and the model + supports Priority Processing / Anthropic fast mode, attach `request_overrides` so the API + call is marked accordingly. Per-provider ``request_overrides`` from + ``resolve_runtime_provider`` (e.g. a ``custom_providers`` ``extra_body``) are preserved and + merged *under* the fast-mode overrides so they still reach the model on the gateway path. """ from hermes_cli.models import resolve_fast_mode_overrides @@ -9084,11 +8289,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ), } - # Provider-level request_overrides (e.g. a custom_providers extra_body) - # resolved upstream by resolve_runtime_provider(). These were being - # dropped by the runtime whitelist above, so a custom provider's - # configured extra_body (chat_template_kwargs, etc.) never reached the - # model on the gateway path -- only /fast service-tier overrides did. + # Provider-level request_overrides (e.g. a custom_providers extra_body) resolved upstream by + # resolve_runtime_provider(). service_tier = getattr(self, "_service_tier", None) if service_tier != "priority": # None (normal) or auto/cold — the bounded window is applied per @@ -9115,14 +8317,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _sync_session_model_from_agent(self, session_id: str, agent: Any) -> None: """Persist the runtime model/provider actually used by a gateway turn. - Provider fallback can switch ``agent.model``/``agent.provider`` after the - session row was created. Keep the session DB metadata in sync so session - lists, desktop/dashboard details, and follow-up session tooling report the - backend that actually answered the latest turn. - - Called from the ``run_sync`` closure, which executes off the event loop - in the executor thread — so the synchronous ``SessionDB`` (``_db``) is - used directly rather than awaiting the AsyncSessionDB forwarder. + Provider fallback can switch ``agent.model``/``agent.provider`` after the session row was + created. Keep the session DB metadata in sync so session lists, desktop/dashboard details, + and follow-up session tooling report the backend that actually answered the latest turn. + Called from the ``run_sync`` closure (executor thread, off the event loop), so the sync + ``SessionDB`` (``_db``) is used directly rather than awaiting the AsyncSessionDB forwarder. """ if not session_id or agent is None or self._session_db is None: return @@ -9163,12 +8362,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _handle_reaction_event(self, ctx: Dict[str, Any]) -> None: """Fan a normalised platform reaction event out to the HookRegistry. - Adapters call this via ``set_reaction_handler`` for every - platform-native reaction event they surface. The adapter-supplied - ``event_name`` ("reaction:added" / "reaction:removed") becomes the - hook event so user hooks subscribe with the same name scheme as the - existing ``agent:*`` family. Errors never block the adapter's event - loop — the hook contract is non-blocking. + The adapter-supplied ``event_name`` ("reaction:added" / "reaction:removed") becomes the + hook event so user hooks subscribe with the same name scheme as the existing ``agent:*`` + family. Errors never block the adapter's event loop — the hook contract is non-blocking. """ event_name = str(ctx.get("event_name") or "reaction:added") try: @@ -9179,17 +8375,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None: """React to an adapter failure after startup. - If the error is retryable (e.g. network blip, DNS failure), queue the - platform for background reconnection instead of giving up permanently. + Retryable errors (network blip, DNS) queue the platform for background reconnection + instead of giving up permanently. - The notification arrives on the failing adapter's own polling task, - and the disconnect inside the handler can cancel that task mid-flight: - disconnect()'s current-task guard misses it because - _safe_adapter_disconnect runs the close in a wrapper task. A cancelled - handler dies between the fatal log and the reconnect queue, silently - stranding the platform (observed 2026-07-21: telegram popped from - adapters but never queued after a travel network outage). Run the real - work in a detached task that adapter teardown cannot cancel. + The notification arrives on the failing adapter's own polling task, and the disconnect + inside the handler can cancel that task mid-flight: disconnect()'s current-task guard + misses it because _safe_adapter_disconnect runs the close in a wrapper task. A cancelled + handler dies between the fatal log and the reconnect queue, silently stranding the + platform — so the real work runs in a detached task adapter teardown cannot cancel. """ tasks = getattr(self, "_fatal_handler_tasks", None) if tasks is None: @@ -9197,10 +8390,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew task = asyncio.create_task(self._handle_adapter_fatal_error_detached(adapter)) tasks.add(task) task.add_done_callback(tasks.discard) - # Await so callers that expect completion still get it — but through - # shield(): Task.cancel() on the caller also cancels the future it is - # awaiting (_fut_waiter), so a plain `await task` would tunnel the - # cancellation straight into the "detached" task. shield() absorbs + # Await so callers that expect completion still get it — but through shield(): Task.cancel() + # on the caller also cancels the future it is awaiting (_fut_waiter), so a plain `await + # task` would tunnel the cancellation straight into the "detached" task. shield() absorbs # it: the caller sees CancelledError, the handler runs to completion. await asyncio.shield(task) @@ -9217,23 +8409,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not platform_config: return False if adapter.platform in self._failed_platforms: - # Nothing to enqueue -- but "already queued" is precisely the state - # in which the watcher has had time to die, and the enqueue branch - # below holds the ONLY call to _ensure_reconnect_watcher_running(). - # - # _spawn_supervised auto-restarts the watcher after a crash (#71758), - # but only _MAX_SUPERVISED_RESTARTS times in rapid succession; past - # that it logs "giving up restarts" and the watcher stays dead - # forever. _ensure_reconnect_watcher_running is the documented - # backstop for exactly that budget exhaustion (#70344) -- and it was - # unreachable for a platform already in the queue, which is the only - # kind of platform the watcher can have been retrying long enough to - # exhaust it on. - # - # The result is a silent permanent outage: nothing retries, and the - # stranded check in _handle_adapter_fatal_error_detached deliberately - # treats a queued platform as safe, so the process never restarts - # either (#90386). + # Nothing to enqueue -- but "already queued" is precisely the state in which the watcher + # has had time to die, and the enqueue branch below holds the ONLY call to + # _ensure_reconnect_watcher_running(). _spawn_supervised restarts a crashed watcher only + # _MAX_SUPERVISED_RESTARTS times, then it stays dead; without this backstop a queued + # platform is a silent permanent outage (nothing retries, and the stranded check treats + # a queued platform as safe so the process never restarts either). self._ensure_reconnect_watcher_running() return False self._failed_platforms[adapter.platform] = { @@ -9266,10 +8447,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew gateway with failure so the service manager restarts it instead of leaving a silent partial outage.""" try: - # Outer hard deadline (#80598): even with queue-before-disconnect, - # a hang anywhere in the impl (status write side effects, detach - # races, etc.) must not leave this task wedged forever — the - # stranded check in ``finally`` only runs when we return. + # Outer hard deadline: even with queue-before-disconnect, a hang anywhere in the impl + # (status write side effects, detach races, etc.) must not leave this task wedged + # forever — the stranded check in ``finally`` only runs when we return. timeout = self._adapter_disconnect_timeout_secs() if timeout <= 0: await self._handle_adapter_fatal_error_impl(adapter) @@ -9339,14 +8519,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await self.stop() async def _handle_adapter_fatal_error_impl(self, adapter: BasePlatformAdapter) -> None: - # Snapshot the current owner of this platform slot before doing - # anything else. If it's neither this adapter nor empty, a different - # adapter has already taken over (e.g. this is a delayed notification - # from a background retry chain that raced with, and lost to, a - # reconnect that already succeeded). Acting on a stale notification - # would overwrite an already-healthy platform's runtime status and - # incorrectly re-queue it for reconnection, so bail out before any of - # that happens. + # Snapshot the current owner of this platform slot before doing anything else. Acting on a + # stale notification would overwrite an already-healthy platform's runtime status and + # incorrectly re-queue it for reconnection, so bail out before any of that happens. existing = self.adapters.get(adapter.platform) if existing is not None and existing is not adapter: logger.debug( @@ -9362,10 +8537,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew adapter.fatal_error_code or "unknown", adapter.fatal_error_message or "unknown error", ) - # Phase 7 Unit 7d-B: a relay credential revoked by opt-out is not an - # error to retry — render it as a clean "disabled" state, not red - # "fatal"/"retrying". (The code is set non-retryable, so it also drops - # out of the reconnect queue below.) + # Phase 7 Unit 7d-B: a relay credential revoked by opt-out is not an error to retry — render + # it as a clean "disabled" state, not red "fatal"/"retrying". (The code is set non- + # retryable, so it also drops out of the reconnect queue below.) if adapter.fatal_error_code == "relay_disabled": platform_state = "disabled" elif adapter.fatal_error_retryable: @@ -9380,19 +8554,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) if existing is adapter: - # Claim this adapter for teardown before awaiting disconnect() — - # a second fatal-error notification for the same adapter (e.g. - # from a concurrent recovery path) would otherwise still see - # itself as "existing" during the await below and disconnect() - # the same object twice. + # Claim this adapter for teardown before awaiting disconnect() — a second fatal-error + # notification for the same adapter (e.g. from a concurrent recovery path) would + # otherwise still see itself as "existing" during the await below and disconnect() the + # same object twice. self.adapters.pop(adapter.platform, None) self.delivery_router.adapters = self.adapters - # Queue retryable failures BEFORE any disconnect await (#80598). - # A half-dead transport can wedge native close() (or swallow - # CancelledError inside it) so the previous "disconnect then queue" - # order left platforms permanently deaf inside a live process even - # after the network recovered. Populate the queue first so the + # Queue retryable failures BEFORE any disconnect await: a half-dead transport can wedge + # native close() (or swallow CancelledError), so "disconnect then queue" left platforms + # permanently deaf in a live process after the network recovered. Populate the queue first so the # reconnect watcher always has work; teardown is best-effort after. self._queue_retryable_fatal_platform(adapter) @@ -9446,18 +8617,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) def _active_cron_job_count(self) -> int: - """Count of cron jobs currently executing, from the cron scheduler's - own in-flight tracking (``cron.scheduler._running_job_ids``). + """Count of cron jobs currently executing, from the cron scheduler's own in-flight tracking + (``cron.scheduler._running_job_ids``). - Cron jobs run through a standalone ``AIAgent`` on the scheduler's own - thread pool (``cron/scheduler.py::run_job``), entirely outside - ``self._running_agents`` — the dict every OTHER active-work check on - this class (``_running_agent_count``, ``_drain_active_agents``) reads. - Without this, the shutdown drain is structurally blind to in-flight - cron work: it can report ``active_at_start=0`` and proceed straight - to killing tool subprocesses while a cron job's terminal command is - still running (#60432). Best-effort: returns 0 if the cron module - can't be imported (e.g. a minimal test double for this class). + Cron jobs run through a standalone ``AIAgent`` on the scheduler's own thread pool, outside + ``self._running_agents`` — the dict every OTHER active-work check here reads. Without this + the shutdown drain is blind to in-flight cron work and can kill a cron job's tool + subprocess while it is still running. Best-effort: returns 0 if the cron module can't be + imported (e.g. a minimal test double for this class). """ try: from cron.scheduler import get_running_job_ids @@ -9482,11 +8649,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _interrupt_api_server_runs(self, reason: str) -> int: """Interrupt API-server agents that are not in ``_running_agents``. - Counterpart of ``_active_api_run_count()``: that method folds - adapter-owned API work into the shutdown drain, so this one must reach - the same agents when the drain times out. Duck-typed on the adapter so - an older adapter (or a minimal test double for this class) without the - hook is simply skipped rather than raising mid-shutdown. + Counterpart of ``_active_api_run_count()``: that method folds adapter-owned API work into + the shutdown drain, so this one must reach the same agents when the drain times out. + Duck-typed on the adapter so an older adapter (or minimal test double) without the hook is + skipped rather than raising mid-shutdown. """ try: adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) @@ -9557,27 +8723,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return interrupted # ── scale-to-zero idle detection / dormant-quiesce (Phase 0) ────────────── - # The gateway-side BEHAVIOUR that consumes the relay scale-to-zero primitives - # (gateway-gateway Phase 5). Pure logic lives in gateway/scale_to_zero.py; the - # methods here bind it to the live runner/transport. See ~/nous/specs/ - # scale-to-zero (decisions.md) for the design + the F12/F14 distinctions. + # The gateway-side BEHAVIOUR that consumes the relay scale-to-zero primitives (gateway-gateway + # Phase 5). Pure logic lives in gateway/scale_to_zero.py; the methods here bind it to the live + # runner/transport. def _scale_to_zero_has_live_background_work(self) -> bool: """Live background work that must block a suspend (D3/F7). - Backgrounded delegate_task / kanban / terminal(background=true) are NOT - counted by _running_agent_count(), but suspending mid-flight loses them. - Checks the runner's own tracked tasks + the process registry's running - processes + any pending process-completion watchers. - - PERMANENT supervised watchers (tagged _hermes_supervised_watcher by - _spawn_supervised) are excluded: they live for the whole process — - including the scale-to-zero watcher itself — so counting them would - make this predicate True forever and the gateway could never go - dormant. Verified live on staging (2026-08-12): an armed, fully idle - instance never logged "going dormant" because ~9 supervised watchers - sat in _background_tasks. Fly's coarse autostop used to mask this; - with the gateway owning the suspend it became load-bearing. + Backgrounded delegate_task / kanban / terminal(background=true) are NOT counted by + _running_agent_count(), but suspending mid-flight loses them. Checks tracked tasks + the + process registry + pending completion watchers. PERMANENT supervised watchers (tagged + _hermes_supervised_watcher) are excluded: they live for the whole process — including the + scale-to-zero watcher itself — so counting them would make this True forever and the + gateway could never go dormant. Fly's coarse autostop used to mask this; with the gateway + owning the suspend it became load-bearing. """ if any( not t.done() and not getattr(t, "_hermes_supervised_watcher", False) @@ -9617,14 +8776,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return parse_idle_timeout_seconds(raw) def _restart_loop_guard_config(self) -> tuple: - """Return ``(max_restarts, window_seconds, max_gap_seconds)`` for the - auto-resume restart-loop breaker (#30719, defense-3), read from - ``gateway.restart_loop_guard`` in config.yaml with the module defaults - as fallback. ``max_restarts <= 0`` disables the breaker. + """Return ``(max_restarts, window_seconds, max_gap_seconds)`` for the auto-resume restart- + loop breaker (#30719, defense-3), read from ``gateway.restart_loop_guard`` in config.yaml + with the module defaults as fallback. - ``max_gap_seconds`` is the longest spacing between two consecutive - restart-interrupted boots that still counts them as the same loop, so - a crash cycle slower than ``window_seconds`` stays visible (#81642). + ``max_restarts <= 0`` disables the breaker. ``max_gap_seconds`` is the longest spacing + between two consecutive restart-interrupted boots that still counts them as the same + loop, so a crash cycle slower than ``window_seconds`` stays visible. """ from gateway import restart_loop_guard as _rlg @@ -9653,17 +8811,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """ENABLED platforms that count for the relay-only arm gate (D1/F6). Two filters, both load-bearing: - - enabled only: config.platforms is pre-seeded with disabled - placeholders for the full platform catalog (the F25 bug). - - MESSAGING only: non-messaging surfaces must not disarm scale-to-zero. - The api_server is a loopback listener force-enabled by the presence - of API_SERVER_KEY (which the Docker stage2 hook now generates for - every container, so hosted instances ALWAYS have it enabled) — it - holds no outbound socket and Chronos fires through it already reset - the idle clock. Counting it made messaging_is_relay_only_or_absent - False on every hosted instance, silently disarming the feature. - Mirrors the non-messaging exclusion set used for handoff eligibility - (see the `messaging_platforms` computation in _connect_platforms). + - enabled only: config.platforms is pre-seeded with disabled placeholders for the full + platform catalog (the F25 bug). + - MESSAGING only: non-messaging surfaces must not disarm scale-to-zero. The api_server is + a loopback listener force-enabled by API_SERVER_KEY (present on every hosted container); + it holds no outbound socket and Chronos fires through it already reset the idle clock. + Counting it silently disarmed the feature on every hosted instance. Mirrors the + non-messaging exclusion used for handoff eligibility in _connect_platforms. """ if not self.config: return [] @@ -9700,10 +8854,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _log_scale_to_zero_not_armed_reason(self) -> None: """Log why the idle watcher did NOT arm — but only for an OPTED-IN instance. - A non-opted instance (no HERMES_SCALE_TO_ZERO stamp) not arming is the normal - case and must stay silent. When the Labs stamp IS set but the watcher still - didn't arm, that's the surprising case worth one INFO line so "why won't it - suspend/wake?" is a log grep, not a box-dive. + A non-opted instance (no HERMES_SCALE_TO_ZERO stamp) not arming is the normal case and + must stay silent. When the stamp IS set but the watcher didn't arm, that surprise earns + one INFO line so "why won't it suspend/wake?" is a log grep, not a box-dive. """ from gateway.relay import relay_wake_url from gateway.scale_to_zero import ( @@ -9763,13 +8916,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: # noqa: BLE001 - unreadable source => assume busy logger.debug("scale-to-zero: api work count unreadable — staying awake", exc_info=True) api_count = 1 - # An attached dashboard/desktop/TUI client is inbound activity too. It - # lives in the DASHBOARD process, so it reaches us as a file mtime that - # process refreshes on every WS frame (gateway/scale_to_zero.py). Fold - # it into the inbound clock rather than adding a conjunct: the client - # then gets the same idle_timeout grace after it disconnects as a chat - # message does, and a lingering marker cannot pin the box (an old mtime - # is outside idle_timeout just like an old _last_inbound_at). + # An attached dashboard/desktop/TUI client is inbound activity too. It lives in the + # DASHBOARD process, so it reaches us as a file mtime that process refreshes on every WS + # frame (gateway/scale_to_zero.py). Fold it into the inbound clock rather than adding a + # conjunct: the client gets the same idle_timeout grace after disconnect as a chat message, + # and a lingering marker cannot pin the box (an old mtime is outside idle_timeout). last_inbound = self._last_inbound_at try: from gateway.scale_to_zero import dashboard_client_last_seen @@ -9790,11 +8941,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _scale_to_zero_note_real_inbound(self) -> None: """Stamp real inbound and restore lifecycle after a dormant wake. - The watcher marks runtime status `draining` as it quiesces the relay, but - dormancy is not the stop/restart drain path: the process remains alive and - should present as running once real traffic wakes it and re-enters the - gateway. Internal completion/replay events intentionally do not call this - helper, so they do not keep an otherwise idle gateway awake. + The watcher marks runtime status `draining` as it quiesces the relay, but dormancy is not + the stop/restart drain path: the process remains alive and should present as running once + real traffic wakes it and re-enters the gateway. Internal completion/replay events + intentionally do not call this helper, so they do not keep an idle gateway awake. """ self._last_inbound_at = time.time() if getattr(self, "_scale_to_zero_cooldown_until", 0.0) > 0: @@ -9815,31 +8965,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _scale_to_zero_watcher(self, interval: float = 30.0) -> None: """Watch for idle, drive the relay dormant, then self-suspend the machine. - Started ONLY when _scale_to_zero_should_arm() (opted in via the Labs - HERMES_SCALE_TO_ZERO stamp + relay-only/absent messaging + a wakeUrl). - On a sustained idle window it runs the DORMANT sequence (D12/F12/F14): - - mark runtime status `draining` (composes with the existing state - machine, §3.4(6); does NOT set _running=False), - - relay adapter.go_dormant() — going_idle->ack + supervisor-preserving - socket close (NOT disconnect(), NOT the run.py stop path), - - deliberately NO mark_resume_pending (D13 — suspend preserves RAM), - - THEN suspend this machine through the local flaps socket - (gateway.scale_to_zero.suspend_self). The gateway owns the suspend - because Fly Proxy autostop judges idle on INBOUND connections only: - it cannot see an in-flight agent turn (outbound-only LLM traffic) - and, since the mid-2026 proxy change, an open outbound relay socket - no longer holds the machine awake — autostop:"suspend" would freeze - the machine mid-job or before the relay flip (the buffered-event - black hole). NAS therefore provisions scale-to-zero machines with - autostop:"off"; the suspend only ever happens HERE, strictly after - the idle predicate held and the dormant quiesce completed. - Autostart stays platform-side: the connector's wakeUrl poke (Fly-proxied) - wakes the machine, the preserved reconnect supervisor re-dials, and the - connector drains the buffered backlog. After driving dormant we set a - re-arm cooldown so a wake's drained backlog isn't immediately re-quiesced. - Off-Fly (no flaps socket / machine identity) the watcher does not quiesce - at all: the platform suspends on its own timer, so the gateway stays - connected and serving until the freeze lands. + Started ONLY when _scale_to_zero_should_arm() (Labs HERMES_SCALE_TO_ZERO stamp + + relay-only/absent messaging + a wakeUrl). On a sustained idle window it runs the DORMANT + sequence: mark status `draining` (does NOT set _running=False), relay adapter.go_dormant() + (supervisor-preserving socket close — NOT disconnect()/the stop path), deliberately NO + mark_resume_pending (suspend preserves RAM), THEN suspend the machine via the local flaps + socket. The gateway owns the suspend because Fly autostop judges idle on INBOUND + connections only: it cannot see an in-flight agent turn or an outbound relay socket, so + autostop:"suspend" would freeze mid-job or before the relay flip; NAS provisions these + machines with autostop:"off". Autostart stays platform-side (wakeUrl poke, supervisor + re-dials, connector drains the backlog). After driving dormant we set a re-arm cooldown so + a wake's drained backlog isn't immediately re-quiesced. Off-Fly (no flaps socket) the + watcher does not quiesce at all: the platform suspends on its own timer. """ await asyncio.sleep(min(interval, 30.0)) # let startup settle while self._running: @@ -9857,14 +8994,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew go_dormant = getattr(adapter, "go_dormant", None) if not callable(go_dormant): continue - # Quiesce only when a suspend can follow it. Off-Fly the platform - # owns the freeze on its own timer, so this does not bring it any - # closer, and go_dormant()'s socket close arms the reconnect - # supervisor: it re-dials ~1.4s later and the drain clears the - # flip, every cooldown. The destination is then unflipped when the - # freeze lands, and inbound is dropped instead of buffered. Stay - # connected and let the connector's orphan detection adopt the - # destination once the platform freezes us. + # Quiesce only when a suspend can follow it. Off-Fly the platform owns the freeze, + # and go_dormant()'s socket close arms the reconnect supervisor: it re-dials ~1.4s + # later and the drain clears the flip every cooldown. The destination is then + # unflipped when the freeze lands, and inbound is dropped instead of buffered. Stay + # connected and let the connector's orphan detection adopt the destination. from gateway.scale_to_zero import self_suspend_available if not self_suspend_available(): @@ -9917,10 +9051,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _scale_to_zero_self_suspend(self) -> None: """Suspend this Fly machine via the local flaps socket (fail-awake). - Runs the blocking unix-socket call in a worker thread so the event loop - stays live right up to the kernel freeze. On success the process is - frozen shortly after — nothing meaningful runs until the wake resume. - Off-Fly (self_suspend_available() False) this is a silent no-op. + Runs the blocking unix-socket call in a worker thread so the event loop stays live right + up to the kernel freeze. On success the process is frozen shortly after — nothing + meaningful runs until the wake resume. Off-Fly (self_suspend_available() False) this is a + silent no-op. """ from gateway.scale_to_zero import self_suspend_available, suspend_self @@ -9956,15 +9090,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return self._restart_requested and mode in {"queue", "steer"} # -------- /queue FIFO helpers -------------------------------------- - # /queue must produce one full agent turn per invocation, in FIFO - # order, with no merging. The adapter's _pending_messages dict is a - # single "next-up" slot (shared with photo-burst follow-ups), so we - # use it for the head of the queue and an overflow list for the - # tail. Enqueue puts new items in the slot when free, otherwise in - # the overflow. Promotion (called after each run's drain) moves the - # next overflow item into the slot so the following recursion picks - # it up. Clearing happens on /new and /reset via - # _handle_reset_command. + # /queue must produce one full agent turn per invocation, in FIFO order, with no merging. The + # adapter's _pending_messages dict is a single "next-up" slot (shared with photo-burst follow- + # ups), so we use it for the head of the queue and an overflow list for the tail. Enqueue fills + # the slot when free, else the overflow; promotion (after each run's drain) moves the next + # overflow item into the slot for the following recursion. Cleared on /new and /reset. def _enqueue_fifo(self, session_key: str, queued_event: "MessageEvent", adapter: Any) -> None: """Append a /queue event to the FIFO chain for a session.""" @@ -9988,14 +9118,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional["MessageEvent"]: """Promote the next overflow item after the slot was drained. - Called at the drain site after _dequeue_pending_event consumed - (or failed to consume) the slot. If there's an overflow item: - - When pending_event is None (slot was empty), return the - overflow head as the new pending_event. - - When pending_event already exists (slot was populated by an - interrupt follow-up or similar), stage the overflow head in - the slot so the NEXT recursion picks it up. - Returns the (possibly updated) pending_event for drain to use. + Called at the drain site after _dequeue_pending_event consumed (or failed to consume) the + slot. With an overflow item: if pending_event is None return the overflow head as the new + pending_event; if the slot is already populated (interrupt follow-up etc.), stage the head + in the slot so the NEXT recursion picks it up. Returns the (possibly updated) pending_event. """ _q_state = self._peek_session_state(session_key) overflow = _q_state.conversation.queued_events if _q_state else None @@ -10022,30 +9148,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _rescue_orphaned_overflow( self, session_key: str, adapter: Any ) -> Optional["MessageEvent"]: - """Pop the oldest orphaned FIFO overflow event for an idle session (#99882). + """Pop the oldest orphaned FIFO overflow event for an idle session. - The FIFO overflow (``queued_events``) drains only at the post-turn - promotion site (``_promote_queued_event`` inside the ``_run_agent`` - drain). When a busy window ends without that drain running — the - #99882 shape: a follow-up queued during compression-in-flight lands - in overflow, compression finishes, the slot event's turn runs, but - the drain recursion exits before promoting (or the busy window ends - through an exception / interrupt / generation-bump exit that never - reaches the promotion site) — the overflow entries are silently - orphaned: never dispatched, never persisted, never logged. + The FIFO overflow (``queued_events``) drains only at the post-turn promotion site inside + the ``_run_agent`` drain. When a busy window ends without that drain running (recursion + exits before promoting, or an exception / interrupt / generation-bump exit), the overflow + entries are silently orphaned: never dispatched, persisted, or logged. - This rescue runs at the point where a NEW event arrives for a - session that is NOT busy (the idle entry in - ``_process_message_priority``). If the session went idle with a - populated overflow, the oldest orphan is returned so the caller runs - it as THIS turn, and the next orphan (if any) is staged into the - slot so the post-turn drain continues the chain in arrival order - (#28503). The caller then enqueues the incoming event behind the - chain via ``_enqueue_fifo``. - - The returned event is REMOVED from both stores: leaving it in the - slot while it also runs as the current turn would make the post-turn - ``_dequeue_pending_event`` run it a second time. + This rescue runs at the point where a NEW event arrives for a session that is NOT busy + (the idle entry in ``_process_message_priority``). The oldest orphan is returned so the + caller runs it as THIS turn and the next one is staged into the slot so the post-turn + drain continues the chain in arrival order. The caller then enqueues the incoming event + behind the chain via ``_enqueue_fifo``. The returned event is REMOVED from both stores: + leaving it in the slot would make the post-turn ``_dequeue_pending_event`` run it twice. Returns the orphaned event to run now, or ``None`` when there is nothing to rescue (no overflow, slot occupied, or no slot storage). @@ -10061,10 +9176,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # this; do not fight it from the idle path. return None head = overflow.pop(0) - # Keep the slot occupied for the rest of the chain so the drain - # promotes in order and any mid-chain arrival routes to overflow - # instead of jumping the queue (same invariant as the drain's - # own _promote_queued_event). Only ONE event fits the slot. + # Keep the slot occupied for the rest of the chain so the drain promotes in order and + # any mid-chain arrival routes to overflow instead of jumping the queue (same invariant + # as the drain's own _promote_queued_event). Only ONE event fits the slot. if overflow: pending_slot[session_key] = overflow.pop(0) logger.warning( @@ -10089,9 +9203,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _is_goal_continuation_event(event_or_text: Any) -> bool: """Return True for synthetic /goal continuation turns. - Goal continuations are normal queued user-role events, so pause/clear - must distinguish them from real user /queue messages before removing or - suppressing them. + Goal continuations are normal queued user-role events, so pause/clear must distinguish + them from real user /queue messages before removing or suppressing them. """ text = getattr(event_or_text, "text", event_or_text) or "" return str(text).startswith("[Continuing toward your standing goal]\nGoal:") @@ -10149,18 +9262,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _persist_active_agents(self) -> None: """Persist the live in-flight agent count to ``gateway_state.json``. - Called at every turn boundary (a running-agent slot is claimed or - released) so the dashboard ``/api/status`` readout reflects in-flight - gateway turns in near-real-time. Without this the file is only - rewritten on lifecycle transitions, so any ``active_agents`` read - between transitions is stale (a turn could start and finish without the - file ever moving). - - Deliberately passes ONLY ``active_agents`` — ``gateway_state`` and the - other fields stay ``_UNSET`` so ``write_runtime_status``'s - read-merge-write preserves the current lifecycle state (``running`` / - ``draining`` / …). Passing ``gateway_state=None`` here would clobber it. - Best-effort: a failed status write must never disrupt a turn. + Called at every turn boundary (a running-agent slot is claimed or released) so the + dashboard ``/api/status`` readout reflects in-flight gateway turns in near-real-time + (otherwise the file only moves on lifecycle transitions and reads between them are stale). + Deliberately passes ONLY ``active_agents`` — the other fields stay ``_UNSET`` so + ``write_runtime_status``'s read-merge-write preserves the current lifecycle state; passing + ``gateway_state=None`` here would clobber it. Best-effort: a failed write must never + disrupt a turn. """ try: from gateway.status import write_runtime_status @@ -10225,26 +9333,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _drain_control_watcher(self, interval: float = 1.0) -> None: """Background task: reconcile gateway accept-state with the drain marker. - Polls ``.drain_request.json`` (presence-based contract, - gateway/drain_control.py). Marker present -> ``_enter_external_drain``; - marker absent -> ``_exit_external_drain``. The 1s cadence bounds the - observe-the-marker latency the live-validation gate checks (point a). - Reconciles once at startup. A marker stamped with a PRIOR - instantiation epoch (one that survived a machine restart on the durable - HERMES_HOME volume — NS-570) is treated as absent by ``drain_requested`` - and is NOT honoured; only a marker from the current instantiation flips - the gateway into drain. Best-effort: any tick error is logged and the - loop continues (a transient stat() failure must not wedge the gateway). + Polls ``.drain_request.json`` (presence-based contract, gateway/drain_control.py): marker + present -> ``_enter_external_drain``, absent -> ``_exit_external_drain``; the 1s cadence + bounds observe-the-marker latency. Reconciles once at startup. A marker stamped with a + PRIOR instantiation epoch (survived a machine restart on the durable volume) is treated as + absent by ``drain_requested`` and NOT honoured. Best-effort: any tick error is logged and + the loop continues (a transient stat() failure must not wedge the gateway). """ from gateway.drain_control import drain_requested while self._running: try: - # drain_requested() does a synchronous read_text() on the - # marker file. At this 1s cadence that puts a blocking disk - # read on the event loop ~86k times a day; when the host is - # under I/O pressure a single read can stall for 30s+ and - # take every platform heartbeat down with it. Off-thread it. + # drain_requested() does a synchronous read_text() on the marker file: at 1s cadence + # that is a blocking disk read on the event loop ~86k times/day, and under host I/O + # pressure one read can stall 30s+ and take every platform heartbeat down. Off-thread it. if await asyncio.to_thread(drain_requested): self._enter_external_drain() # API and cron work live outside messaging's @@ -10292,15 +9394,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # /platform pause|resume slash command for manual control. # ------------------------------------------------------------------ def _pause_failed_platform(self, platform, *, reason: str = "") -> None: - """Mark a queued platform as paused — keep it in ``_failed_platforms`` - but stop the reconnect watcher from hammering it. + """Mark a queued platform as paused — keep it in ``_failed_platforms`` but stop the + reconnect watcher from hammering it. - Used by ``/platform pause `` for manual operator intervention. - Paused platforms are surfaced in ``/platform list`` and resumed with - ``/platform resume ``. Note: the reconnect watcher does NOT - auto-pause — retryable (network/DNS) failures keep retrying at the - backoff cap indefinitely so a transient outage self-heals without - manual intervention. + Used by ``/platform pause `` for manual operator intervention. Note: the reconnect + watcher does NOT auto-pause — retryable (network/DNS) failures keep retrying at the + backoff cap indefinitely so a transient outage self-heals without manual intervention. """ info = getattr(self, "_failed_platforms", {}).get(platform) if info is None: @@ -10358,11 +9457,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew @staticmethod def _load_prefill_messages() -> List[Dict[str, Any]]: """Load ephemeral prefill messages from config or env var. - - Checks HERMES_PREFILL_MESSAGES_FILE env var first, then falls back to - the top-level prefill_messages_file key in ~/.hermes/config.yaml. - agent.prefill_messages_file is accepted as a legacy fallback. - Relative paths are resolved from ~/.hermes/. + + Checks HERMES_PREFILL_MESSAGES_FILE env var first, then falls back to the top-level + prefill_messages_file key in ~/.hermes/config.yaml. agent.prefill_messages_file is + accepted as a legacy fallback. Relative paths are resolved from ~/.hermes/. """ file_path = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "") if not file_path: @@ -10415,12 +9513,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> str: """Resolve model for this channel: channel_overrides else global default. - Delegates the precedence rule to - :func:`hermes_cli.model_switch.resolve_effective_model` (session - override > channel override > global default) — the single owner - shared with the API server, so the two surfaces cannot diverge - again (see 7dd00bb47d). This call site has no session tier: session - /model overrides are applied later by + Delegates the precedence rule to :func:`hermes_cli.model_switch.resolve_effective_model` + (session override > channel override > global default) — the single owner shared with the + API server, so the two surfaces cannot diverge again (see 7dd00bb47d). This call site has + no session tier: session /model overrides are applied later by ``_apply_session_model_override`` on the resolved runtime. """ from hermes_cli.model_switch import resolve_effective_model @@ -10451,16 +9547,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> str: """Ephemeral system prompt for this channel/thread. - Uses ``channel_overrides`` when set, else the gateway prompt resolved - from the CURRENT profile's config on every call. Callers run inside - ``_profile_runtime_scope`` (``run_sync`` under ``_run_agent``), so a - routed multiplex profile gets its own ``display.personality`` / - ``agent.system_prompt`` instead of a boot-time snapshot of the launch - profile's (#89161); ``/personality`` edits take effect on the next - turn for the same reason. - Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` - in ``run_sync`` (adapter ``resolve_channel_prompt``), so they are not - duplicated here. + Uses ``channel_overrides`` when set, else the gateway prompt resolved from the CURRENT + profile's config on every call: callers run inside ``_profile_runtime_scope``, so a routed + multiplex profile gets its own ``display.personality`` / ``agent.system_prompt`` rather + than a boot-time snapshot, and ``/personality`` edits apply on the next turn. + Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` in + ``run_sync`` (adapter ``resolve_channel_prompt``), so they are not duplicated here. """ config = getattr(self, "config", None) if config: @@ -10479,10 +9571,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _load_reasoning_config(model: str = "") -> dict | None: """Load reasoning effort from config.yaml, respecting per-model overrides. - Thin wrapper over the shared chokepoint - :func:`hermes_constants.resolve_reasoning_config` (per-model override > - global ``agent.reasoning_effort``; YAML boolean False = disabled). - Closes #21256. + Thin wrapper over the shared chokepoint :func:`hermes_constants.resolve_reasoning_config` + (per-model override > global ``agent.reasoning_effort``; YAML boolean False = disabled). Args: model: The effective model for the calling session. When empty, @@ -10527,12 +9617,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> dict | None: """Resolve reasoning effort for a session, honoring session overrides. - Priority: session-scoped ``/reasoning --session`` override > - per-model override (``agent.reasoning_overrides``) > global - ``agent.reasoning_effort``. ``model`` should be the session's - *effective* model (session ``/model`` override included) so - per-model overrides track what the session actually runs — when - empty, the config's ``model.default`` is used. + Priority: session-scoped ``/reasoning --session`` override > per-model override + (``agent.reasoning_overrides``) > global ``agent.reasoning_effort``. ``model`` should be + the session's *effective* model (session ``/model`` override included) so per-model + overrides track what the session actually runs — when empty, ``model.default`` is used. """ resolved_session_key = session_key if not resolved_session_key and source is not None: @@ -10603,10 +9691,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """ if not session_key: return - # Presence-sensitive: "priority" or None (explicit normal) both count - # as an override; the sentinel means "no override". Old code - # wholesale-replaced the dict on lazy init (cross-session race) — - # per-session field writes eliminate that class of bug. + # Presence-sensitive: "priority" or None (explicit normal) both count as an override; the + # sentinel means "no override". Old code wholesale-replaced the dict on lazy init (cross- + # session race) — per-session field writes eliminate that class of bug. self._session_state(session_key).conversation.service_tier_override = ( _SERVICE_TIER_UNSET if clear else service_tier ) @@ -10658,12 +9745,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _load_busy_text_mode() -> str: """Resolve normal busy TEXT follow-up behavior. - ``busy_input_mode`` is the single source of truth (default - ``interrupt``). The legacy ``busy_text_mode`` knob is honored only - when a user explicitly set it, so existing queue setups keep - working; new installs follow ``busy_input_mode``. Returns one of - ``interrupt`` | ``queue`` (``steer`` is handled upstream by - ``busy_input_mode`` and maps to non-queue text handling here). + ``busy_input_mode`` is the single source of truth (default ``interrupt``). The legacy + ``busy_text_mode`` knob is honored only when a user explicitly set it, so existing queue + setups keep working; new installs follow ``busy_input_mode``. """ # Legacy explicit override wins for backward compat. legacy = os.getenv("HERMES_GATEWAY_BUSY_TEXT_MODE", "").strip().lower() @@ -10855,16 +9939,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew @staticmethod def _load_background_notifications_mode() -> str: - """Load background process notification mode from config or env var. - - Modes: - - ``concise`` — one-line status message on completion (default); - failures append a short output tail - - ``all`` — running-output updates *and* the final raw-output message - - ``result`` — only the final raw-output completion message - - ``error`` — only the final raw-output message when exit code is non-zero - - ``off`` — no watcher messages at all - """ + """Load background process notification mode from config or env var.""" mode = os.getenv("HERMES_BACKGROUND_NOTIFICATIONS", "") if not mode: cfg = _load_gateway_runtime_config() @@ -10917,16 +9992,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _refresh_fallback_model(self) -> list | None: """Re-read fallback_providers from disk for the next agent create/reuse. - Cron already does this per job via ``get_fallback_chain``; the gateway - previously froze ``self._fallback_model`` at process start, so a chain - configured (or changed) after ``hermes gateway`` was running never - reached messaging sessions even though the same process's cron jobs - fell back correctly. Fixes #60955. - - A TRANSIENT read/parse failure (user mid-edit of config.yaml with a - non-atomic write) keeps the last known-good chain instead of wiping a - cached agent's working fallback for that turn. Only a successful read - that genuinely lacks the key clears the chain. + Cron already does this per job via ``get_fallback_chain``; the gateway previously froze + ``self._fallback_model`` at process start, so a chain changed after startup never reached + messaging sessions. A TRANSIENT read/parse failure (user mid-edit of config.yaml with a + non-atomic write) keeps the last known-good chain instead of wiping a cached agent's + working fallback for that turn. Only a successful read that genuinely lacks the key clears + the chain. """ try: from hermes_cli.config import read_user_config_raw @@ -10934,10 +10005,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not cfg_path.exists(): self._fallback_model = None return self._fallback_model - # Raw primitive (raises on parse failure) is required here: the - # canonical fail-open loader would return {} on a torn mid-edit - # write and WIPE the last known-good chain. The overlay/expansion - # below fixes the managed-scope/${VAR} drift without losing that. + # Raw primitive (raises on parse failure) is required here: the canonical fail-open + # loader would return {} on a torn mid-edit write and WIPE the last known-good chain. + # The overlay/expansion below fixes the managed-scope/${VAR} drift without losing that. cfg = read_user_config_raw(cfg_path) try: from hermes_cli import managed_scope @@ -10965,11 +10035,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _apply_fallback_chain_to_agent(agent: Any, chain: list | None) -> None: """Keep a cached agent's fallback chain aligned with current config. - Skips rewrite while a cooldown is holding the agent on an already- - activated fallback provider — ``restore_primary_runtime`` owns that - turn-scoped lifecycle. When primary is active (or cooldown expired), - replace the chain so mid-uptime ``fallback_providers`` edits take - effect without requiring a gateway restart (#60955). + Skips the rewrite while a cooldown holds the agent on an already-activated fallback + provider — ``restore_primary_runtime`` owns that turn-scoped lifecycle. When primary is + active (or cooldown expired), replace the chain so mid-uptime ``fallback_providers`` edits + take effect without requiring a gateway restart. """ if agent is None: return @@ -10985,13 +10054,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew agent._fallback_model = new_chain[0] if new_chain else None if not getattr(agent, "_fallback_activated", False): agent._fallback_index = 0 - # A config edit signals the user changed something — drop the - # session-scoped unavailability memo so re-configured entries - # (e.g. credentials added mid-uptime for a previously-failing - # provider) get retried instead of staying suppressed for the - # cached agent's lifetime. Only on actual content change, so - # the per-message no-op refresh keeps the memo's rate-limiting - # benefit (#60955). + # A config edit signals the user changed something — drop the session-scoped unavailability + # memo so re-configured entries (e.g. credentials added mid-uptime for a previously-failing + # provider) get retried instead of staying suppressed for the cached agent's lifetime. + # Only on actual content change, so the per-message no-op refresh keeps the memo's + # rate-limiting benefit. if new_chain != old_chain: unavailable = getattr(agent, "_unavailable_fallback_keys", None) if unavailable: @@ -11064,20 +10131,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew @staticmethod def _agent_has_active_subagents(running_agent: Any) -> bool: - """Return True when *running_agent* is currently driving subagents - via the ``delegate_task`` tool. + """Return True when *running_agent* is currently driving subagents via the ``delegate_task`` + tool. - Background (#30170): ``AIAgent.interrupt()`` cascades through the - parent's ``_active_children`` list and calls ``interrupt()`` on - every child synchronously, which aborts in-flight subagent work - and produces a fallback cascade with no actionable signal. - Demoting ``busy_input_mode='interrupt'`` to ``queue`` semantics - whenever this helper returns True protects subagent work from - conversational follow-ups while leaving the explicit ``/stop`` - path (which goes through ``_interrupt_and_clear_session``) - untouched. Safe-by-default: returns False on any attribute or - lock error so a missing/broken parent never blocks the existing - interrupt path. + ``AIAgent.interrupt()`` cascades through the parent's ``_active_children`` and interrupts + every child synchronously, aborting in-flight subagent work. Demoting + ``busy_input_mode='interrupt'`` to ``queue`` while this is True protects that work from + conversational follow-ups; explicit ``/stop`` (``_interrupt_and_clear_session``) is + untouched. Safe-by-default: returns False on any attribute or lock error so a + missing/broken parent never blocks the existing interrupt path. """ if running_agent is None or running_agent is _AGENT_PENDING_SENTINEL: return False @@ -11103,15 +10165,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _session_has_compression_in_flight(self, session_key: str) -> bool: """Return True when a compression lock is held for this session's id. - Context compression is interrupt-protected (#23975) but gateway - ``interrupt`` busy-input mode can still start a follow-up turn against - the pre-rotation parent while compression is mid-flight, producing - orphaned compression siblings (#56391). Callers demote interrupt to - queue when this returns True. - - Both blocking sources — the ``session_store`` lock + JSON load, and the - SQLite ``get_compression_lock_holder`` SELECT — are offloaded to a - worker thread so a large state.db never freezes the event loop (#5). + Context compression is interrupt-protected, but gateway ``interrupt`` busy mode can still + start a follow-up turn against the pre-rotation parent while compression is mid-flight, + producing orphaned compression siblings. Callers demote interrupt to queue when this + returns True. Both blocking sources — the ``session_store`` lock + JSON load, and the + SQLite ``get_compression_lock_holder`` SELECT — are offloaded to a worker thread so a + large state.db never freezes the event loop. """ session_store = getattr(self, "session_store", None) if not session_key or session_store is None: @@ -11166,26 +10225,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew entry = session_store._entries.get(session_key) # noqa: SLF001 return getattr(entry, "session_id", None) if entry is not None else None - # Hard cap on per-session pending follow-ups for busy_input_mode=queue - # (and the draining/steer-fallback/subagent-demotion paths that share - # this entry point). Without a cap, a stuck agent + a rapid-fire user - # could grow the overflow list unboundedly. 32 turns of queued - # follow-ups is far beyond any realistic conversational backlog while - # still small enough to never threaten memory. + # Hard cap on per-session pending follow-ups for busy_input_mode=queue (and the draining/steer- + # fallback/subagent-demotion paths that share this entry point). Without a cap, a stuck agent + + # a rapid-fire user could grow the overflow list unboundedly. _BUSY_QUEUE_MAX_PENDING = 32 def _queue_or_replace_pending_event(self, session_key: str, event: MessageEvent) -> None: adapter = self._adapter_for_source(event.source) if not adapter: return - # #28503 — Previously this called ``merge_pending_message_event`` - # with the default ``merge_text=False``, which silently OVERWROTE - # the single pending slot when consecutive text messages arrived - # in ``busy_input_mode: queue``. Route through the FIFO - # infrastructure shared with ``/queue`` so each follow-up gets - # its own turn in arrival order. Photo bursts still merge into - # the head slot via ``merge_pending_message_event`` (album - # semantics); everything else appends to the overflow tail. + # Route through the FIFO infrastructure shared with ``/queue`` so each follow-up gets its + # own turn in arrival order (the old merge_text=False call silently OVERWROTE the single + # pending slot). Photo bursts still merge into the head slot (album semantics); everything + # else appends to the overflow tail. pending_slot = getattr(adapter, "_pending_messages", None) existing = pending_slot.get(session_key) if isinstance(pending_slot, dict) else None security_metadata_keys = ( @@ -11233,23 +10285,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _prepare_busy_steer_text(self, event: MessageEvent) -> str: """Return steerable text for a busy follow-up, transcribing voice first. - Fresh and queued voice messages reach the normal inbound STT pipeline, - but successful steer messages intentionally bypass that queue. Without - preprocessing here, a media-only voice follow-up has an empty text - payload and steer mode silently degrades to queue mode. - - Audio file attachments remain files; only voice-message media follows - the automatic STT contract used by ``_prepare_inbound_message_text``. - If transcription fails, preserve any caption and let the existing - steer fallback handle an otherwise empty event without losing it. - - Routes through ``_transcribe_and_echo_pending_voice`` — the single - out-of-band transcription choke point shared with the interrupt - monitor and the pending-drain path — so the STT call is made at most - once per platform message (cached on the event) and the transcript - echo respects the count-based ledger. If steering later falls back - to queue mode, the drain path reuses the cached transcript instead of - paying for a second STT call or re-echoing the same line. + Fresh and queued voice messages reach the normal inbound STT pipeline, but successful + steer messages intentionally bypass that queue. Without preprocessing here, a media-only + voice follow-up has an empty text payload and steer mode silently degrades to queue mode. + Audio file attachments remain files; only voice-message media follows the automatic STT + contract. If transcription fails, preserve any caption and let the steer fallback handle + the event. Routes through ``_transcribe_and_echo_pending_voice`` — the single out-of-band + STT choke point — so STT runs at most once per message (cached on the event) and a later + queue fallback reuses the transcript instead of paying/echoing twice. """ text = (event.text or "").strip() if not self._pending_event_audio_paths(event): @@ -11269,10 +10312,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _handle_active_session_busy_message(self, event: MessageEvent, session_key: str) -> bool: # --- Authorization gate (#17775) --- - # The cold path (_handle_message) checks _is_user_authorized before - # creating a session. The busy path must enforce the same check; - # otherwise unauthorized users in shared threads (Slack/Telegram/Discord) - # can inject messages into an active session they don't own. + # The cold path (_handle_message) checks _is_user_authorized before creating a session. The + # busy path must enforce the same check; otherwise unauthorized users in shared threads + # (Slack/Telegram/Discord) can inject messages into an active session they don't own. if not self._is_user_authorized(event.source): logger.warning( "Dropping message from unauthorized user in active session: " @@ -11315,26 +10357,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return True # --- Approval response routing (#46866) --- - # When the agent is blocked waiting for a dangerous-command approval, - # plain-text responses like "yes" or "approve" must be routed to the - # approval handler instead of being steered/queued/interrupted. - # Otherwise approval via messaging platforms never succeeds — the - # reply is queued behind a turn that can't start until the approval - # resolves, so the approval times out and auto-denies (a deadlock). - # - # Slash forms (/approve, /deny) already bypass to the runner at the - # base-adapter guard. This handles the bare-word forms (Signal/SMS - # users naturally type "yes" rather than "/approve"). Gating on - # has_blocking_approval(session_key) is the disambiguator that keeps - # a conversational "yes" from triggering a dangerous command when no - # approval is actually pending (design intent — see run.py "Pending - # exec approvals are handled by /approve and /deny" note). - # - # We reuse the canonical /approve and /deny handlers rather than - # re-deriving the resolution + i18n messaging: they resolve the - # waiting thread, resume typing, AND return a localized confirmation - # string. The busy-handler path does not auto-send that return, so - # we deliver it ourselves (mirroring the draining-case send above). + # When the agent is blocked waiting for a dangerous-command approval, plain-text responses + # like "yes" or "approve" must be routed to the approval handler instead of being + # steered/queued/interrupted — otherwise the reply queues behind a turn that can't start + # until the approval resolves, so it times out and auto-denies (deadlock). Slash forms + # already bypass at the base-adapter guard; this handles bare words (Signal/SMS users type + # "yes"). Gating on has_blocking_approval(session_key) keeps a conversational "yes" from + # triggering a dangerous command when nothing is pending. Reuse the canonical /approve and + # /deny handlers (they resolve the thread, resume typing, return a localized confirmation); + # the busy path does not auto-send that return, so we deliver it ourselves. try: from tools.approval import has_blocking_approval if event.allow_gateway_control and has_blocking_approval(session_key): @@ -11354,12 +10385,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _approval_handler = self._handle_approve_command _normalized_args = "session" if _approval_handler is not None: - # Synthesize the canonical "/approve [args]" / "/deny" - # command text so the slash handlers parse modifiers via - # event.get_command_args(). Always use a literal "/" — - # MessageEvent.is_command()/get_command_args() only - # recognize the "/" prefix, not the per-platform display - # prefix ("!" on Slack/Matrix). + # Synthesize the canonical "/approve [args]" / "/deny" command text so the slash + # handlers parse modifiers via event.get_command_args(). Always a literal "/" — + # is_command()/get_command_args() only recognize "/", not the per-platform + # display prefix ("!" on Slack/Matrix). _verb = "approve" if _approval_handler is self._handle_approve_command else "deny" _synth = f"/{_verb}" if _normalized_args: @@ -11395,16 +10424,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False # let default path handle it # --- Internal synthetic events must never interrupt/steer --- - # Async-delegation completions (delegate_task(background=true)) and - # background-process completions (terminal notify_on_complete) re-enter - # the originating session as internal MessageEvents. When the session - # is busy, treating them like a user TEXT message means interrupt-mode - # (the default busy_text_mode) aborts the active turn AND sends a "⚡ - # Interrupting current task" ack — exactly the opposite of the design - # invariant that a completion surfaces as a NEW turn only when idle and - # never splices into a running turn. Plugin events carry untrusted - # payload text, so queue those through the gateway FIFO to keep their - # security metadata separate from pending user input. + # Async-delegation completions (delegate_task(background=true)) and background-process + # completions (terminal notify_on_complete) re-enter the originating session as internal + # MessageEvents. Treating them like user TEXT while busy would let interrupt mode abort the + # active turn and send an "Interrupting" ack — the opposite of the invariant that a + # completion surfaces as a NEW turn only when idle. Plugin events carry untrusted payload + # text, so queue those through the gateway FIFO to keep their security metadata separate. if getattr(event, "internal", False) and not event.allow_gateway_control: self._queue_or_replace_pending_event(session_key, event) return True @@ -11422,18 +10447,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): return False - # Steer mode: inject mid-run via running_agent.steer() instead of - # queueing + interrupting. If the agent isn't running yet - # (sentinel) or lacks steer(), or the payload is empty, fall back - # to queue semantics so nothing is lost. - # #30170 — Subagent protection. ``AIAgent.interrupt()`` cascades - # to every entry in the parent's ``_active_children`` list and - # aborts in-flight ``delegate_task`` work. Demote ``interrupt`` - # to ``queue`` when the parent is currently driving subagents so - # a conversational follow-up doesn't destroy minutes of subagent - # work. Explicit ``/stop`` and ``/new`` slash commands go through - # ``_interrupt_and_clear_session`` and are unaffected — the - # operator still has a way to force-cancel everything. + # Steer mode: inject mid-run via running_agent.steer() instead of queueing + interrupting. + # If the agent isn't running yet (sentinel) or lacks steer(), or the payload is empty, fall + # back to queue semantics so nothing is lost. Subagent protection: interrupt() cascades to + # ``_active_children`` and aborts delegate_task work, so demote ``interrupt`` to ``queue`` + # while the parent drives subagents. Explicit /stop and /new go through + # ``_interrupt_and_clear_session`` and still force-cancel everything. demoted_for_subagents = ( effective_mode == "interrupt" and self._agent_has_active_subagents(running_agent) @@ -11460,10 +10479,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew redirected = False if effective_mode == "steer": steer_text = await self._prepare_busy_steer_text(event) - # A follow-up qualifies for steering when it is plain text, OR - # when every attachment is STT-eligible voice media whose - # transcript was just folded into steer_text — otherwise a voice - # note in steer mode silently degrades to queue mode (#58780). + # A follow-up qualifies for steering when it is plain text, OR when every attachment is + # STT-eligible voice media whose transcript was just folded into steer_text — otherwise + # a voice note in steer mode silently degrades to queue mode. _steer_media_urls = getattr(event, "media_urls", None) or [] _steer_all_voice = bool(_steer_media_urls) and ( len(self._pending_event_audio_paths(event)) == len(_steer_media_urls) @@ -11507,21 +10525,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.warning("Gateway redirect failed for session %s: %s", session_key, exc) redirected = False - # Store the message so it's processed as the next turn after the - # current run finishes (or is interrupted). Skip this for a - # successful steer — the text already landed inside the run and - # must NOT also be replayed as a next-turn user message. - # - # Route through _queue_or_replace_pending_event (the same FIFO - # infrastructure used by busy queue-mode and /queue) rather than a - # raw merge_pending_message_event(merge_text=True). The raw merge - # newline-joins consecutive TEXT follow-ups into a SINGLE pending - # turn, destroying message boundaries — so two separate user - # messages sent while the agent was busy (interrupt mode, or a - # steer that fell back to queue) arrived as one mashed-together - # turn (#43066 sub-bug 2). The FIFO path gives each text its own - # turn in arrival order while still preserving photo-burst / album - # merge semantics for media. + # Store the message so it's processed as the next turn after the current run finishes (or is + # interrupted). Skip this for a successful steer — the text already landed inside the run + # and must NOT also be replayed as a next-turn user message. Route through + # _queue_or_replace_pending_event (FIFO) rather than a raw merge_pending_message_event + # (merge_text=True): the raw merge newline-joins consecutive TEXT follow-ups into ONE turn, + # destroying message boundaries. FIFO gives each text its own turn while keeping + # photo-burst / album merge semantics for media. if not steered and not redirected: self._queue_or_replace_pending_event(session_key, event) @@ -11575,10 +10585,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew from gateway.display_config import resolve_display_setting platform_key = _platform_config_key(event.source.platform) - # In steer mode the user's text has already been injected into the - # active run. Some mobile chat setups want that steering to be silent, - # like STT transcript echo suppression: keep the behavior, drop only - # the confirmation bubble. + # In steer mode the user's text has already been injected into the active run. Some mobile + # chat setups want that steering to be silent, like STT transcript echo suppression: keep + # the behavior, drop only the confirmation bubble. if is_steer_mode: steer_ack_env = os.environ.get("HERMES_GATEWAY_BUSY_STEER_ACK_ENABLED") if steer_ack_env is not None: @@ -11664,10 +10673,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"I'll respond to your message shortly." ) - # First-touch onboarding: the very first time a user sends a message - # while the agent is busy, append a one-time hint explaining the - # queue/interrupt knob. Flag is persisted to config.yaml so it never - # fires again on this install. + # First-touch onboarding: the very first time a user sends a message while the agent is + # busy, append a one-time hint explaining the queue/interrupt knob. Flag is persisted to + # config.yaml so it never fires again on this install. try: from agent.onboarding import ( BUSY_INPUT_FLAG, @@ -11746,12 +10754,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew last_deferred_count = deferred_count last_status_at = now - # Cron jobs run on the scheduler's own thread pool, outside - # ``self._running_agents`` — fold their in-flight count into the - # same wait/timeout this method already applies to chat sessions, - # or a cron job's tool work gets killed with zero warning the - # instant it's the only active thing running (#60432). - # API-server / desk sessions have the same structural gap (#63529). + # Cron jobs run on the scheduler's own thread pool, outside ``self._running_agents`` — fold + # their in-flight count into the same wait/timeout this method already applies to chat + # sessions, or a cron job's tool work gets killed with zero warning the instant it's the + # only active thing running. API-server / desk sessions and detached deferred workers + # have the same structural gap. if ( not self._running_agents and last_cron_count == 0 @@ -11785,10 +10792,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return True return bool(self._active_cron_job_count()) and now < cron_deadline - # Both budgets at 0 leave this loop unentered, which is the legacy - # "interrupt immediately" behaviour — expressed as an expired - # deadline rather than a special case, so the timed_out value below - # is always computed from real state instead of asserted up front. + # Both budgets at 0 leave this loop unentered, which is the legacy "interrupt immediately" + # behaviour — expressed as an expired deadline rather than a special case, so the timed_out + # value below is always computed from real state instead of asserted up front. while _still_draining(): _maybe_update_status() await asyncio.sleep(0.1) @@ -11826,21 +10832,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _notify_interrupted_cron_jobs(self, job_ids) -> int: """Tell the owner of each just-interrupted cron job that its run died. - The cron worker cannot do this itself. Its thread reaches - ``_deliver_result`` asynchronously, and by then - ``_bounded_adapter_teardown`` has closed the transport — so the notice - never leaves the process, and ``_consume_interrupted_flag`` discards - the resulting ``delivery_error`` along with it. The run's only trace is - a line in jobs.json nobody reads (#82232). - - Must therefore be called from the post-interrupt phase, while adapters - are still connected — the same window - ``_notify_active_sessions_of_shutdown`` relies on for chat sessions, - which is blind to cron work because cron runs on the scheduler's own - thread pool rather than ``self._running_agents`` (#60432). - - Best-effort by construction: every failure is swallowed so a wedged - adapter can never extend shutdown. Returns the number of notices sent. + The cron worker cannot do this itself: its thread reaches ``_deliver_result`` after + ``_bounded_adapter_teardown`` has closed the transport, so the notice never leaves the + process and the run's only trace is a line in jobs.json nobody reads. Must therefore run + in the post-interrupt phase while adapters are still connected — the same window + ``_notify_active_sessions_of_shutdown`` uses, which is blind to cron work because cron runs + on the scheduler's own pool rather than ``self._running_agents``. Best-effort by + construction: every failure is swallowed so a wedged adapter can never extend shutdown. + Returns the number of notices sent. """ if not job_ids: return 0 @@ -12065,11 +11064,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # fail toward the louder, more-visible behaviour. logger.debug("drain_notification_suppressed check failed: %s", e) - # Snapshot adapters up front: adapter.send() can hit a fatal error - # path that pops the adapter from self.adapters (see _handle_fatal - # elsewhere), which would otherwise trigger - # ``RuntimeError: dictionary changed size during iteration`` — - # observed in a user report during gateway shutdown. + # Snapshot adapters up front: adapter.send() can hit a fatal error path that pops the + # adapter from self.adapters (see _handle_fatal elsewhere), which would otherwise trigger + # ``RuntimeError: dictionary changed size during iteration`` — observed in a user report + # during gateway shutdown. for platform, adapter in list(self.adapters.items()): home = self.config.get_home_channel(platform) if not home or not home.chat_id: @@ -12123,29 +11121,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None: for agent in active_agents.values(): - # Persist any in-flight transcript to the SQLite session store - # before teardown (#13121). An agent forcibly interrupted by the - # drain-timeout escalation may never reach - # ``turn_finalizer.finalize_turn`` (the only place that flushes the - # turn to state.db) — e.g. it was blocked in a tool call that did - # not abort within the post-interrupt grace window. Its in-flight - # tool rounds live only in the in-memory ``_session_messages`` - # (refreshed per tool round in ``conversation_loop`` but never - # written to SQLite mid-turn), so the immediate pre-restart turn is - # silently dropped from ``load_transcript()`` on resume. Flushing - # here closes that gap; the resume_pending / fresh-tool-tail - # branches in ``_handle_message_with_agent`` already expect a - # transcript whose tail may be a pending tool result. The flush is - # idempotent (identity-tracked in ``_flush_messages_to_session_db``), - # so agents that DID finish gracefully re-flush nothing. + # Persist any in-flight transcript to the SQLite session store before teardown. An agent + # force-interrupted by the drain-timeout escalation may never reach finalize_turn (the + # only mid-turn flush to state.db); its tool rounds live only in in-memory + # ``_session_messages``, so the pre-restart turn would vanish from load_transcript() on + # resume. The resume branches already expect a tail that may be a pending tool result. + # The flush is idempotent (identity-tracked in ``_flush_messages_to_session_db``), so + # agents that DID finish gracefully re-flush nothing. try: _flush = getattr(agent, "_flush_messages_to_session_db", None) _session_messages = getattr(agent, "_session_messages", None) if callable(_flush) and isinstance(_session_messages, list) and _session_messages: - # Strip private empty-response retry scaffolding from the - # tail first, mirroring the graceful ``_persist_session`` - # path, so a resumed turn doesn't replay synthetic recovery - # nudges. + # Strip private empty-response retry scaffolding from the tail first, mirroring + # the graceful ``_persist_session`` path, so a resumed turn doesn't replay + # synthetic recovery nudges. _strip = getattr( agent, "_drop_trailing_empty_response_scaffolding", None ) @@ -12157,14 +11146,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew try: _flush(_session_messages) except Exception as _flush_err: - # The in-memory transcript could not be persisted - # (e.g. FTS/SQLite index corruption — #72680). A plain - # debug log loses the conversation permanently when the - # process exits. Dump the live agent history to an - # external JSON recovery snapshot so an operator can - # salvage it after repairing state.db. The flush is - # non-fatal; shutdown must never block on a best-effort - # backup. + # The in-memory transcript could not be persisted (e.g. FTS/SQLite index + # corruption). A debug log alone loses the conversation for good at exit, so + # dump the live agent history to an external JSON recovery snapshot an + # operator can salvage after repairing state.db. Non-fatal: shutdown must + # never block on a best-effort backup. logger.warning( "Shutdown transcript flush failed (%s); preserving " "%d in-memory message(s) to recovery snapshot", @@ -12178,14 +11164,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except Exception as _e: logger.debug("Shutdown transcript flush failed: %s", _e) - # Off-loop + bounded: finalize_session fans out to plugin - # on_session_finalize hooks that can do arbitrary synchronous - # work (e.g. an observability plugin serializing a full-session - # trace export). Running it inline on the event loop blocked the - # entire shutdown sequence past systemd's TimeoutStopSec on a - # multi-day 4.7G session — heartbeats froze and the process was - # SIGKILLed mid-export. Same class as the memory-provider hang - # below (#53175). + # Off-loop + bounded: finalize_session fans out to plugin on_session_finalize hooks that + # can do arbitrary synchronous work (e.g. an observability plugin serializing a full- + # session trace export). Same class as the memory-provider hang below. await self._finalize_session_off_loop( session_id=getattr(agent, "session_id", None), platform="gateway", @@ -12205,10 +11186,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> bool: """Only emit the heartbeat while this task still owns the live run. - Guards against a stale ``running: delegate_task`` heartbeat outliving the - run that started it: stop once the executor finishes, the agent is gone, - or the session key has been rebound to a different live agent (e.g. the - user sent ``/new`` and a fresh agent took the slot mid-run, #12029). + Stop once the executor finishes, the agent is gone, or the session key was rebound to a + different live agent (e.g. ``/new`` mid-run) — otherwise a stale ``running: + delegate_task`` heartbeat outlives the run that started it. """ if agent is None: return False @@ -12220,15 +11200,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False return True - # Upper bound on off-loop agent-resource cleanup invoked from coroutines - # running on the gateway's event loop (session-expiry sweep, in-turn - # cache-hygiene re-eviction). _cleanup_agent_resources is synchronous and - # can block for a long time (agent.close() does subprocess teardown; - # shutdown_memory_provider() may do network/SQLite IO via a memory plugin). - # Calling it inline wedges the whole loop — the bot goes silent, the - # runtime-status updated_at heartbeat freezes, and SIGTERM cannot be - # serviced (#53175). Offload to a worker thread under this timeout so the - # loop is never blocked; mirrors the /new reset path's fix (#35994). + # Upper bound on off-loop agent-resource cleanup invoked from coroutines running on the + # gateway's event loop (session-expiry sweep, in-turn cache-hygiene re-eviction). + # _cleanup_agent_resources is synchronous and can block for a long time (agent.close() does + # subprocess teardown; shutdown_memory_provider() may do network/SQLite IO). Inline it wedges + # the loop — bot silent, status heartbeat frozen, SIGTERM unserviced — so it is offloaded to a + # worker thread under this timeout (mirrors the /new reset path). _CLEANUP_TIMEOUT_S = 30.0 def _defer_agent_cleanup_until_future_done( @@ -12240,10 +11217,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Clean up ``agent`` only after its executor future has finished. - A timed-out executor call keeps running in its worker thread. Closing - the agent before that thread exits can tear down clients or providers - it is still using. Keep a strong task reference and wait for the real - future before invoking the normal bounded, off-loop cleanup path. + A timed-out executor call keeps running in its worker thread. Closing the agent before + that thread exits can tear down clients or providers it is still using, so keep a strong + task reference and wait for the real future before the normal bounded off-loop cleanup. """ async def _cleanup_when_done() -> None: @@ -12271,10 +11247,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew tasks.add(task) task.add_done_callback(tasks.discard) - # Bounded budget for one finalize_session() dispatch (plugin - # on_session_finalize hooks + core Relay conversation close). Generous - # enough for a normal trace-export flush, small enough that a wedged - # plugin can never eat the systemd stop window. + # Bounded budget for one finalize_session() dispatch (plugin on_session_finalize hooks + core + # Relay conversation close). Generous enough for a normal trace-export flush, small enough that + # a wedged plugin can never eat the systemd stop window. _FINALIZE_TIMEOUT_S = 10.0 async def _finalize_session_off_loop( @@ -12287,13 +11262,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Run hermes_cli.lifecycle.finalize_session off the event loop, bounded. - finalize_session() invokes plugin ``on_session_finalize`` hooks - synchronously; a hook doing heavy blocking work (observability trace - export, network flush) on the event loop freezes heartbeats, adapters, - and the shutdown drain itself. Off-loop + ``wait_for`` keeps the loop - live; on timeout the worker thread is left to finish (or leak) on its - own and the caller proceeds — mirroring - ``_cleanup_agent_resources_off_loop`` (#53175). + Off-loop + ``wait_for`` keeps the loop live; on timeout the worker thread is left to + finish (or leak) on its own and the caller proceeds — mirroring + ``_cleanup_agent_resources_off_loop``. """ def _call() -> None: @@ -12333,11 +11304,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Run _cleanup_agent_resources in a worker thread with a bounded wait. - Safe to await from coroutines on the gateway event loop: a slow or - wedged teardown (memory provider IO, subprocess close) can no longer - block message processing. On timeout the await is cancelled and the - worker thread is left to finish (or leak) on its own — the caller - proceeds regardless, exactly as the /new reset path does (#35994). + On timeout the await is cancelled and the worker thread is left to finish (or leak) on + its own — the caller proceeds regardless, exactly as the /new reset path does. """ if agent is None: return @@ -12374,34 +11342,23 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return try: if hasattr(agent, "shutdown_memory_provider"): - # Drain queued memory writes BEFORE tearing the provider down. - # The memory manager persists per-turn sync and end-of-session - # extraction on a single serialized background worker. - # shutdown_memory_provider() -> shutdown_all() only gives that - # worker a ~5s bounded drain and abandons (cancels) anything - # still queued past it, so a /reset — or any gateway session - # rotation that reaches this cleanup path — could silently drop - # writes the session had already handed off. The next session - # then loads stale memory (#73297). Give pending work a bounded - # head start through the manager's own barrier first, mirroring - # the CLI exit path (cli.py). Best-effort: a flush failure must - # never block teardown. + # Drain queued memory writes BEFORE tearing the provider down. The memory manager + # runs per-turn sync + end-of-session extraction on one serialized worker; + # shutdown_memory_provider() -> shutdown_all() gives it only a ~5s drain and cancels + # the rest, so a /reset or session rotation could drop writes already handed off and + # the next session loads stale memory. Give pending work a bounded head start via + # the manager's own barrier first (mirrors the CLI exit path). Best-effort: a flush + # failure must never block teardown. _mm = getattr(agent, "_memory_manager", None) if _mm is not None and hasattr(_mm, "flush_pending"): try: _mm.flush_pending(timeout=10) except Exception: pass - # Pass the agent's own conversation transcript so memory - # providers' ``on_session_end`` hooks see the real messages - # instead of the empty default (#15165). ``_session_messages`` - # is set on ``AIAgent`` (run_agent.py:1518) and refreshed at - # the end of every ``run_conversation`` turn via - # ``_persist_session``; on an agent built through - # ``object.__new__`` (test stubs) the attribute may be - # absent, so ``getattr`` with a ``None`` default keeps the - # call signature-compatible with the pre-fix behaviour - # (``shutdown_memory_provider(messages=None)``). + # Pass the agent's own conversation transcript so memory providers' + # ``on_session_end`` hooks see the real messages instead of the empty default. + # ``_session_messages`` may be absent on agents built via ``object.__new__`` (test + # stubs), so getattr with a ``None`` default keeps the call signature-compatible. session_messages = getattr(agent, "_session_messages", None) if isinstance(session_messages, list): agent.shutdown_memory_provider(session_messages) @@ -12417,10 +11374,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew agent.close() except Exception: pass - # Auxiliary async clients (session_search/web/vision/etc.) live in a - # process-global cache and are created inside worker threads. Clean up - # any entries whose event loop is now dead so their httpx transports do - # not accumulate across gateway turns. + # Auxiliary async clients (session_search/web/vision/etc.) live in a process-global cache + # and are created inside worker threads. Clean up any entries whose event loop is now dead + # so their httpx transports do not accumulate across gateway turns. try: from agent.auxiliary_client import cleanup_stale_async_clients cleanup_stale_async_clients() @@ -12433,9 +11389,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _increment_restart_failure_counts(self, active_session_keys: set) -> None: """Increment restart-failure counters for sessions active at shutdown. - Persists to a JSON file so counters survive across restarts. - Sessions NOT in active_session_keys are removed (they completed - successfully, so the loop is broken). + Persists to a JSON file so counters survive across restarts. Sessions NOT in + active_session_keys are removed (they completed successfully, so the loop is broken). """ import json @@ -12544,11 +11499,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew current_pid = os.getpid() restart_after_s = max(float(getattr(self, "_restart_drain_timeout", 0.0) or 0.0) + 5.0, 5.0) - # On Windows there's no bash/setsid chain — spawn a tiny Python - # watcher directly via sys.executable instead. The watcher polls - # current_pid, waits for our exit, then runs `hermes gateway - # restart` with detach flags so the respawn survives the CLI - # that triggered the /restart command closing its console. + # On Windows there's no bash/setsid chain — spawn a tiny Python watcher directly via + # sys.executable instead. if sys.platform == "win32": import textwrap from hermes_cli._subprocess_compat import ( @@ -12612,12 +11564,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # run as a self-restart loop guard and the gateway stays stopped. watcher_env.pop("_HERMES_GATEWAY", None) project_root = Path(__file__).resolve().parent.parent - # The watcher runs sys.executable (console python) under the - # CREATE_NO_WINDOW detach kwargs below: it owns one hidden - # console, inherited by the `hermes gateway restart` child, so - # nothing flashes. Do NOT swap in GUI-subsystem pythonw.exe — - # a console-less watcher forces every console-subsystem - # descendant to allocate a visible conhost (#54220/#56747). + # The watcher runs sys.executable (console python) under the CREATE_NO_WINDOW detach + # kwargs below: it owns one hidden console, inherited by the `hermes gateway restart` + # child, so nothing flashes. Do NOT swap in GUI-subsystem pythonw.exe — a console-less + # watcher forces every console-subsystem descendant to allocate a visible conhost. watcher_python = sys.executable venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv") site_packages = venv_dir / "Lib" / "site-packages" @@ -12663,14 +11613,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew creationflags=windows_detach_flags_without_breakaway(), ) except OSError as exc: - # Both spawn attempts failed (a breakaway-denying job object - # is the common cause, but OSError covers others too). - # Record a minimal, path-safe diagnostic and return without - # crashing the caller: state plainly that no watcher was - # started, and log only the interpreter basename and a - # numeric error code — never argv, env, watcher source, or - # str(exc) (which can carry a full interpreter path for a - # FileNotFoundError). + # Both spawn attempts failed (a breakaway-denying job object is the common + # cause, but OSError covers others too). Record a minimal, path-safe diagnostic + # and return without crashing the caller: log only the interpreter basename and + # a numeric error code — never argv, env, watcher source, or str(exc) (which can + # carry a full interpreter path). winerror = getattr(exc, "winerror", None) error_code = winerror if winerror is not None else exc.errno error_field = "winerror" if winerror is not None else "errno" @@ -12690,10 +11637,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"while kill -0 {current_pid} 2>/dev/null && [ $(date +%s) -lt $deadline ]; do sleep 0.2; done; " f"{cmd} gateway restart" ) - # Same marker scrub as the Windows watcher above: this watcher runs - # `hermes gateway restart` from outside the gateway, but it inherits - # _HERMES_GATEWAY=1 from us, and the CLI's self-restart loop guard - # refuses to run when that marker is set — silently (DEVNULL), so the + # Same marker scrub as the Windows watcher above: this watcher runs `hermes gateway restart` + # from outside the gateway, but it inherits _HERMES_GATEWAY=1 from us, and the CLI's self- + # restart loop guard refuses to run when that marker is set — silently (DEVNULL), so the # gateway stops and never comes back. from tools.environments.local import build_subprocess_env watcher_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=True) @@ -12719,14 +11665,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _wedged_agent_count(self) -> int: """Count running chat agents already past the inactivity timeout. - A turn whose agent has recorded no activity (no API bytes, no tool - progress) for longer than ``agent.gateway_timeout`` is wedged — the - same threshold at which the turn reaper gives up on it. The restart - after-turn wait must not treat such turns as work worth waiting for: - a wedged agent pinned ``hermes update`` in "draining" for the full - ``restart_after_turn_timeout`` cap because the drain counted it as - active while its own inactivity watchdog had already declared it dead - (Aug 2026, WhatsApp turn idle 30+ min, drain waited on it anyway). + A turn whose agent has recorded no activity (no API bytes, no tool progress) for longer + than ``agent.gateway_timeout`` is wedged — the same threshold at which the turn reaper + gives up on it. Returns 0 when the inactivity timeout is disabled (``gateway_timeout`` 0/unset ⇒ the operator opted into unbounded turns; the after-turn cap @@ -12763,17 +11704,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _await_active_work_before_restart(self) -> bool: """Wait for in-flight work to finish before entering ``stop()``. - In-band restart used to call ``stop()`` immediately, which folded the - requesting turn into the drain wait set and force-interrupted it at - ``restart_drain_timeout`` (#77184). Instead we refuse new turns and - wait here for active agents/cron/api work to reach zero, then let + In-band restart used to call ``stop()`` immediately, which folded the requesting turn + into the drain wait set and force-interrupted it at ``restart_drain_timeout``. Instead we + refuse new turns, wait here for active agents/cron/api work to reach zero, then let ``stop()`` run against an idle gateway (drain is instant). Turns already past the inactivity timeout are excluded from the wait - (``_wedged_agent_count``): restart is usually the *remedy* for a - wedged turn, so deferring it behind one inverts the point of the - graceful path. ``stop()``'s drain interrupts them under - ``restart_drain_timeout`` instead. + (``_wedged_agent_count``): restart is usually the *remedy* for a wedged turn, so + deferring it behind one inverts the point of the graceful path. ``stop()``'s drain + interrupts them under ``restart_drain_timeout`` instead. Returns True when work drained to zero, False when the safety cap elapsed with work still active — or when only wedged work remains — @@ -12872,10 +11811,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _run_restart() -> None: await self._await_active_work_before_restart() - # Launch the detached helper only AFTER the after-turn wait. - # Its deadline is drain_timeout+5 and covers stop() teardown — - # launching earlier would fire `hermes gateway restart` while - # the requesting turn was still running. + # Launch the detached helper only AFTER the after-turn wait. Its deadline is + # drain_timeout+5 and covers stop() teardown — launching earlier would fire `hermes + # gateway restart` while the requesting turn was still running. if detached: try: await self._launch_detached_restart_command() @@ -12884,26 +11822,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await asyncio.sleep(0.05) await self.stop(restart=True, detached_restart=detached, service_restart=via_service) - # _run_restart is a short-lived self-terminating task (calls stop() - # then returns). Don't add it to _background_tasks — _stop_impl - # cancels all entries in that set, which would cancel _run_restart - # while it's awaiting _stop_task, propagating CancelledError into - # _stop_impl and preventing _shutdown_event.set() / _exit_code = 75. - # See #12875. - # - # We still hold a strong reference in self._restart_task: a bare - # asyncio.create_task() keeps only a weak reference, so the event - # loop may garbage-collect a still-pending task mid-flight. The - # cancel loop in _stop_impl explicitly skips _restart_task for the - # same reason it skips _stop_task. + # _run_restart is a short-lived self-terminating task (calls stop() then returns). Do NOT + # add it to _background_tasks: _stop_impl cancels every entry there, which would cancel + # _run_restart while it awaits _stop_task and propagate CancelledError into _stop_impl, + # preventing _shutdown_event.set() / _exit_code = 75. Keep a strong ref in + # self._restart_task (bare create_task holds only a weak ref, so a pending task can be + # GC'd mid-flight); _stop_impl's cancel loop skips it, as it skips _stop_task. self._restart_task = asyncio.create_task(_run_restart()) return True - # Drain-timeout reasons set by _stop_impl() when a still-running turn is - # force-interrupted; "restart_interrupted" is set by - # SessionStore.suspend_recently_active() on crash recovery (no - # .clean_shutdown marker). All three mean "the agent was mid-turn and - # we killed it" — eligible for startup auto-resume. + # Drain-timeout reasons set by _stop_impl() when a still-running turn is force-interrupted; + # "restart_interrupted" is set by SessionStore.suspend_recently_active() on crash recovery (no + # .clean_shutdown marker). All mean "the agent was mid-turn and we killed it" — eligible for + # startup auto-resume. _AUTO_RESUME_REASONS = frozenset( {"restart_timeout", "shutdown_timeout", "restart_interrupted"} ) @@ -12916,12 +11847,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Dispatch one synthetic startup resume and wait for its agent turn. - ``BasePlatformAdapter.handle_message()`` returns after it installs the - adapter-level guard and spawns the background processing task. Startup - restore needs a stronger boundary: inbound messages must stay queued - until the resumed agent turn itself has finished, otherwise a user - message can race the restore turn immediately after ``handle_message`` - returns. + Startup restore needs a stronger boundary: inbound messages must stay queued until the + resumed agent turn itself has finished, otherwise a user message can race the restore + turn immediately after ``handle_message`` returns. """ try: await adapter.handle_message(event) @@ -12930,10 +11858,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if task is not None: await asyncio.shield(task) finally: - # _schedule_resume_pending_sessions pre-claims the runner slot - # before spawning this task. If adapter.handle_message raises - # before _handle_message takes ownership, release that pre-claim; - # otherwise the real run's normal cleanup owns the slot. + # _schedule_resume_pending_sessions pre-claims the runner slot before spawning this + # task. If adapter.handle_message raises before _handle_message takes ownership, release + # that pre-claim; otherwise the real run's normal cleanup owns the slot. _pre_state = self._peek_session_state(session_key) if (_pre_state.turn.agent if _pre_state else None) is _AGENT_PENDING_SENTINEL: self._release_running_agent_state(session_key) @@ -12981,12 +11908,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return drained def _start_startup_warmup(self) -> None: - """Kick off the boot turn-machinery warm-up in the background (#99373). + """Kick off the boot turn-machinery warm-up in the background. - Called from ``start()`` right after the startup-restore gate closes, - so the warm-up overlaps the (slow, network-bound) platform connects - instead of adding boot latency. ``_finish_startup_restore`` awaits - it (bounded) before opening the inbound gate. + Called from ``start()`` right after the startup-restore gate closes, so the warm-up + overlaps the (slow, network-bound) platform connects instead of adding boot latency. + ``_finish_startup_restore`` awaits it (bounded) before opening the inbound gate. """ timeout = _startup_warmup_timeout_secs() if timeout <= 0: @@ -12997,13 +11923,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) async def _warm_turn_prerequisites(self) -> None: - """Initialize turn machinery off-loop before the gate opens (#99373). + """Initialize turn machinery off-loop before the gate opens. - Runs ``_warm_turn_machinery_sync`` (run_agent import graph, tool - schemas + check_fn probe cache, context-file tier) on an executor - thread so the event loop — platform heartbeats, connects — stays - responsive. Never raises: a failed warm-up degrades to the - historical lazy init, it must not block startup. + Runs ``_warm_turn_machinery_sync`` (run_agent import graph, tool schemas + check_fn probe + cache, context-file tier) on an executor thread so the event loop — platform heartbeats, + connects — stays responsive. Never raises: a failed warm-up degrades to the historical + lazy init; it must not block startup. """ try: loop = asyncio.get_running_loop() @@ -13024,10 +11949,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _await_startup_warmup(self) -> None: """Bounded wait for the boot warm-up before the inbound gate opens. - On timeout the gate opens anyway (availability outranks prompt - completeness for a WEDGED init — same principle as the bounded - restore-drain wait above) and the warm-up continues in the - background; a late failure is still logged. + On timeout the gate opens anyway (availability outranks prompt completeness for a WEDGED + init — same principle as the bounded restore-drain wait above) and the warm-up continues + in the background; a late failure is still logged. """ task = getattr(self, "_startup_warmup_task", None) if task is None or task.done(): @@ -13055,15 +11979,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _finish_startup_restore(self) -> None: """Wait (BOUNDED) for startup auto-resume, then release + drain inbound. - The wait is bounded by ``_startup_restore_drain_timeout_secs`` so that - a single pathologically long boot-resume turn cannot hold the inbound - gate shut for every channel. On timeout we release the gate and let - the still-running resume turn(s) finish in the background — they are - NOT cancelled. This is safe because duplicate-agent protection does - not depend on the wait: ``_schedule_resume_pending_sessions`` claims - each session's ``_running_agents`` slot SYNCHRONOUSLY before this gate - runs, so any inbound message drained while a resume turn is still in - flight queues behind that slot instead of spawning a second agent. + The wait is bounded by ``_startup_restore_drain_timeout_secs`` so that a single + pathologically long boot-resume turn cannot hold the inbound gate shut for every channel. + On timeout the gate is released and the resume turn(s) finish in the background — NOT + cancelled. Safe because duplicate-agent protection does not depend on the wait: + ``_schedule_resume_pending_sessions`` claims each session's ``_running_agents`` slot + SYNCHRONOUSLY before this gate runs, so a drained inbound message queues behind that slot + instead of spawning a second agent. """ tasks = list(getattr(self, "_startup_restore_tasks", []) or []) if tasks: @@ -13083,11 +12005,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew timeout, len(pending), ) - # These tasks outlive the gate. Their normal done-callback - # only discards them from _background_tasks, so a LATER - # failure would be silently swallowed. Attach a logging - # callback so a background resume turn that fails after the - # timeout is still recorded. + # These tasks outlive the gate. Their normal done-callback only discards them + # from _background_tasks, so a LATER failure would be silently swallowed. for task in pending: task.add_done_callback(self._log_background_resume_result) else: @@ -13150,24 +12069,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Run boot-path sends without letting them pin the inbound restore gate. - ``_send_restart_notification`` and ``_redeliver_pending_obligations`` - used to be awaited inline *before* ``_finish_startup_restore`` - released the gate. A single Telegram flood-control sleep on either - send froze inbound on every platform for the full ``retry_after`` - (#91969). - - This uses the same bounded ``asyncio.wait`` the resume gate already - uses: on timeout we return and let the sends finish in the + ``_send_restart_notification`` and ``_redeliver_pending_obligations`` used to be awaited + inline *before* ``_finish_startup_restore`` released the gate; one Telegram flood-control + sleep froze inbound on every platform for the full ``retry_after``. Uses the same bounded + ``asyncio.wait`` as the resume gate: on timeout return and let the sends finish in the background. Tasks are not cancelled. - The ledger claim + ``resume_pending`` clear happen INLINE here, - before the send task exists: they are pure DB work (no network, - bounded by claimed-row count), and deferring them into the send - task left a window where a hung restart notification ahead of the - redelivery step let the gate expire with zero rows claimed — the - resume scheduler then replayed turns whose answers were already in - the ledger, and the background task later redelivered them too - (duplicate delivery + re-paid turn). + The ledger claim + ``resume_pending`` clear happen INLINE here, before the send task + exists: they are pure bounded DB work, and deferring them into the send task let a hung + restart notification expire the gate with zero rows claimed — the resume scheduler then + replayed turns already answered in the ledger and the background task redelivered them + too (duplicate delivery + re-paid turn). """ claimed = await self._claim_pending_obligations() @@ -13216,10 +12128,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> list: """Clear resume flags and return rows safe to redeliver. - Startup recovery preserves its historical best-effort behavior. Runtime - reconnect recovery is stricter: if the session-store write fails, the - corresponding response must not be sent because the same agent turn - could otherwise be resumed immediately afterward. + Startup recovery preserves its historical best-effort behavior. Runtime reconnect + recovery is stricter: if the session-store write fails, the corresponding response must + not be sent because the same agent turn could otherwise be resumed immediately afterward. """ sendable = [] for row in claimed: @@ -13241,21 +12152,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return sendable async def _claim_pending_obligations(self) -> list: - """Claim recoverable delivery-ledger rows and clear their - ``resume_pending`` flags. Pure DB work — no network sends. + """Claim recoverable delivery-ledger rows and clear their ``resume_pending`` flags. - Runs INLINE at startup BEFORE ``_schedule_resume_pending_sessions`` - and before the (bounded, abandonable) boot-send task exists. A - session with a recoverable obligation already produced its answer — - the turn completed and only delivery is owed — so clearing - ``resume_pending`` here prevents the resume path from re-running - (and re-paying for) a turn whose output we hold, regardless of how - long the sends ahead of redelivery take (#91969). - - Crash-ambiguity contract (see gateway/delivery_ledger.py): - rows that were mid-send or previously rejected carry a visible - recovered-reply marker so a possible duplicate is labeled, never - silent. Returns the claimed rows for redelivery. + Pure DB work — no network sends. Must run INLINE at startup BEFORE + ``_schedule_resume_pending_sessions`` and before the abandonable boot-send task exists: + these sessions already produced their answer (only delivery is owed), so clearing + ``resume_pending`` here stops the resume path from re-running (and re-paying for) the + turn, however long the sends ahead of redelivery take. Crash-ambiguity contract (see + gateway/delivery_ledger.py): rows that were mid-send or previously rejected carry a + visible recovered-reply marker so a possible duplicate is labeled, never silent. + Returns the claimed rows for redelivery. """ try: from gateway.delivery_ledger import ( @@ -13265,10 +12171,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not await asyncio.to_thread(ledger_enabled): return [] - # Only claim rows whose exact transport owner is connected this - # boot. A multiplexed gateway can host several bot identities for - # one platform; platform-only filtering would spend a disconnected - # bot's retry budget merely because another bot is online. + # Only claim rows whose exact transport owner is connected this boot. A multiplexed + # gateway can host several bot identities for one platform; platform-only filtering + # would spend a disconnected bot's retry budget merely because another bot is online. _profile_adapters = getattr(self, "_profile_adapters", None) or {} _deliverable_targets = { (getattr(p, "value", str(p)), "default") for p in self.adapters @@ -13297,10 +12202,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not claimed: return [] - # Clear resume_pending for EVERY claimed row up front, before any - # send. Claiming already spent one of the row's redelivery attempts — - # the answer is in the ledger, so the resume path must never re-run - # these turns (#91969). + # Clear resume_pending for EVERY claimed row up front, before any send. Claiming already + # spent one of the row's redelivery attempts — the answer is in the ledger, so the resume + # path must never re-run these turns. await self._clear_resume_pending_for_claimed_obligations(claimed) return claimed @@ -13402,13 +12306,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return redelivered async def _redeliver_pending_obligations(self) -> int: - """Claim + redeliver in one call — composition of - :meth:`_claim_pending_obligations` and + """Claim + redeliver in one call — composition of :meth:`_claim_pending_obligations` and :meth:`_redeliver_claimed_obligations`. - Kept as the stable public shape (tests and any external callers - drive this name); the startup path calls the two halves separately - so the DB half can run inline before the abandonable send task. + Kept as the stable public shape (tests and any external callers drive this name); the + startup path calls the two halves separately so the DB half can run inline before the + abandonable send task. """ return await self._redeliver_claimed_obligations( await self._claim_pending_obligations() @@ -13422,11 +12325,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> int: """Replay one adapter identity's transient failures after reconnect. - The startup sweep cannot claim live-owner rows by design. A platform - adapter can reconnect without the gateway process exiting, however, so - ``send_path_degraded`` responses otherwise remain failed until the next - process restart. Claiming, resume clearing, and sending stay best-effort - and reuse the startup redelivery path's attempt and ambiguity contract. + The startup sweep cannot claim live-owner rows by design. A platform adapter can + reconnect without the gateway process exiting, however, so ``send_path_degraded`` + responses otherwise remain failed until the next process restart. Claim/clear/send stay + best-effort and reuse the startup redelivery path's attempt and ambiguity contract. """ try: from gateway.delivery_ledger import ( @@ -13478,25 +12380,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _schedule_resume_pending_sessions(self, platform=None) -> int: """Auto-continue fresh restart-interrupted sessions after startup. - ``resume_pending`` already preserves the transcript AND the existing - ``_is_resume_pending`` branch in ``_handle_message_with_agent`` - injects a reason-aware recovery system note on the next turn. This - method closes the UX gap by synthesizing that next turn once - adapters are back online — the event text is empty so the existing - injection path owns the wording and we never double up. - - Adapters that are not yet ready (adapter missing from - ``self.adapters``) are skipped silently; their sessions stay - ``resume_pending`` and will auto-resume on the next real user - message, or when the platform reconnects — the reconnect watcher - calls this again scoped to that ``platform``. - - ``platform`` (a ``Platform``) restricts the pass to sessions that - originated on that platform. The reconnect path passes it so a - platform coming back online retries only its own sessions and never - re-touches another platform's in-flight recoveries. Sessions whose - agent is already running are skipped regardless, so a session - scheduled at startup is never resumed a second time. + ``resume_pending`` preserves the transcript and ``_is_resume_pending`` in + ``_handle_message_with_agent`` injects the recovery note on the next turn. This method + closes the UX gap by synthesizing that next turn once adapters are back online — the + event text is empty so the existing injection path owns the wording and we never + double up. Sessions whose adapter is not in ``self.adapters`` are skipped silently + (they stay ``resume_pending``; the reconnect watcher calls this again scoped to that + ``platform``). ``platform`` restricts the pass so a reconnecting platform never + re-touches another platform's in-flight recoveries; sessions whose agent is already + running are skipped, so a startup-scheduled session is never resumed twice. """ window = _auto_continue_freshness_window() try: @@ -13514,16 +12406,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.warning("Failed to enumerate resume-pending sessions: %s", exc) return 0 - # Defense-3 (#30719): break the SIGTERM-respawn loop. Only count this - # boot when there are restart-interrupted sessions to resume — a clean - # boot must not accrue toward the breaker. If too many such boots have - # happened in the configured window, skip auto-resume for THIS boot: - # the gateway still comes up and serves real inbound messages, it just - # stops replaying the session that keeps killing it. The session stays - # resume_pending, so a real user message can still continue it (a human - # is now in the loop). Defenses 1-2 cover the cron/CLI/terminal paths; - # this catches every other SIGTERM source (e.g. a raw `terminal( - # "launchctl kickstart ai.hermes.gateway")`). + # Defense-3: break the SIGTERM-respawn loop. Only count this boot when there are restart- + # interrupted sessions to resume — a clean boot must not accrue toward the breaker. If too + # many such boots hit the window, skip auto-resume for THIS boot only: the gateway still + # serves inbound; the session stays resume_pending so a real user message can continue it. if candidates: try: from gateway import restart_loop_guard as _rlg @@ -13558,12 +12444,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) continue - # Validate the session owner against the current allowlist - # before auto-resuming. A session created before - # TELEGRAM_ALLOWED_USERS (or equivalent) was configured, or - # before the owner was removed from it, must not silently - # receive a full agent response on gateway restart just - # because it has a resume-pending marker (issue #23778). + # Validate the session owner against the current allowlist before auto-resuming: a + # session created before the allowlist existed (or whose owner was since removed) must + # not silently receive a full agent response just because it carries a resume marker. try: if not self._is_user_authorized(source): logger.warning( @@ -13579,11 +12462,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) continue - # Claim the session slot *before* spawning the task so that an - # inbound message arriving between task creation and the task's - # first await (where _process_message_background sets the real - # sentinel) sees the slot as occupied and queues behind it - # instead of spinning up a duplicate AIAgent (#45456). + # Claim the session slot *before* spawning the task so that an inbound message arriving + # between task creation and the task's first await (where _process_message_background + # sets the real sentinel) sees the slot as occupied and queues behind it instead of + # spinning up a duplicate AIAgent. _resume_state = self._session_state(entry.session_key) _resume_state.turn.agent = _AGENT_PENDING_SENTINEL _resume_state.turn.started_ts = time.time() @@ -13713,12 +12595,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: logger.debug("Failed to cancel gateway loop floor timer", exc_info=True) - # Also disarm the heartbeat writer task itself (ported from #95808): - # once shutdown starts loading the loop, a heartbeat that keeps - # refreshing the file can make a draining gateway look healthy to - # external probes. Cancel is idempotent; the task is also in - # _background_tasks so this is belt-and-braces ordering, not new - # lifecycle. + # Also disarm the heartbeat writer task itself (ported from #95808): once shutdown starts + # loading the loop, a heartbeat that keeps refreshing the file can make a draining gateway + # look healthy to external probes. heartbeat = getattr(self, "_loop_heartbeat_task", None) self._loop_heartbeat_task = None if heartbeat is not None: @@ -13796,12 +12675,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) def _start_loop_heartbeat_task(self) -> None: - """Start the loop-liveness heartbeat task (#66892), idempotent. + """Start the loop-liveness heartbeat task, idempotent. - An asyncio task so a frozen loop stops refreshing - ``state/gateway.heartbeat``. Cancelled with the other background - tasks during stop(). Best-effort — a liveness probe must never be - able to abort startup. + An asyncio task so a frozen loop stops refreshing ``state/gateway.heartbeat``. Cancelled + with the other background tasks during stop(). Best-effort — a liveness probe must never + be able to abort startup. """ try: _existing_hb = getattr(self, "_loop_heartbeat_task", None) @@ -13813,10 +12691,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew start_time=getattr(self, "_gateway_started_at", 0.0), ) ) - # PERMANENT for the process lifetime, same as a - # _spawn_supervised watcher — tag it so - # _scale_to_zero_has_live_background_work() doesn't treat an - # armed, otherwise-idle gateway as busy forever. + # PERMANENT for the process lifetime, same as a _spawn_supervised watcher — tag it so + # _scale_to_zero_has_live_background_work() doesn't treat an armed, otherwise-idle + # gateway as busy forever. self._loop_heartbeat_task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: @@ -13826,16 +12703,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("Failed to start gateway loop heartbeat", exc_info=True) async def start(self) -> bool: - """ - Start the gateway and all configured platform adapters. - + """Start the gateway and all configured platform adapters. + Returns True if at least one adapter connected successfully. """ logger.info("Starting Hermes Gateway...") - # Enable faulthandler for stack dumps on freezes/crashes (#70344). - # Falls back to a log file when sys.stderr is None (Windows VBS / - # pythonw / detached service) — otherwise the gateway would die - # here and take every adapter offline. See #71671. + # Enable faulthandler for stack dumps on freezes/crashes. Falls back to a log file when + # sys.stderr is None (Windows VBS / pythonw / detached service) — otherwise the gateway + # would die here and take every adapter offline. try: faulthandler.enable() except (RuntimeError, ValueError, OSError): @@ -13850,11 +12725,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew faulthandler.enable(file=_fh_enable_file, all_threads=True) except Exception: logger.debug("faulthandler.enable() unavailable", exc_info=True) - # Also dump stacks to a rotating file for off-line analysis when - # the gateway is running under a service manager that doesn't - # capture stderr. - # faulthandler.register() and SIGUSR2 are POSIX-only; skip the - # signal-triggered file dump on Windows (faulthandler.enable() + # Also dump stacks to a rotating file for off-line analysis when the gateway is running + # under a service manager that doesn't capture stderr. faulthandler.register() and SIGUSR2 + # are POSIX-only; skip the signal-triggered file dump on Windows (faulthandler.enable() # above still covers fatal-error dumps there). _sigusr2 = getattr(signal, "SIGUSR2", None) if _sigusr2 is not None and hasattr(faulthandler, "register"): @@ -13985,11 +12858,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: logger.debug("gateway health OTLP export startup failed", exc_info=True) - # Log any active supply-chain security advisories. Operators see this - # in gateway.log and `hermes status` surfaces it; we do NOT block - # startup or surface it inline to user messages, since the gateway - # operator is the one who can act on it (uninstall the package, - # rotate credentials). See hermes_cli/security_advisories.py. + # Log any active supply-chain security advisories. Deliberately does NOT block startup or + # surface inline to users — only the operator can act (uninstall, rotate credentials). + # See hermes_cli/security_advisories.py. try: from hermes_cli.security_advisories import ( detect_compromised, @@ -14010,7 +12881,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) if await self._abort_startup_if_shutdown_requested(): return True - + # Warn if no user allowlists are configured and open access is not opted in _builtin_allowed_vars = ( "TELEGRAM_ALLOWED_USERS", "DISCORD_ALLOWED_USERS", @@ -14100,13 +12971,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew pass self._request_clean_exit(reason) return True - - # Discover Python plugins before shell hooks so plugin block - # decisions take precedence in tie cases. The CLI startup path - # does this via an explicit call in hermes_cli/main.py; the - # gateway lazily imports run_agent inside per-request handlers, - # so the discover_plugins() side-effect in model_tools.py is NOT - # guaranteed to have run by the time we reach this point. + + # Discover Python plugins before shell hooks so plugin block decisions take precedence in + # tie cases. Explicit here because the gateway lazily imports run_agent per request, so + # the discover_plugins() side-effect in model_tools.py is NOT guaranteed to have run yet. try: from hermes_cli.plugins import discover_plugins discover_plugins() @@ -14115,11 +12983,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "plugin discovery failed at gateway startup", exc_info=True, ) - # Register the generic relay adapter when a connector relay URL is - # configured (GATEWAY_RELAY_URL / gateway.relay_url). No URL -> no-op, so - # direct/single-tenant deployments are unaffected. When configured, the - # adapter dials the connector over a WebSocket, negotiates its capability - # descriptor at handshake, and bridges inbound/outbound like any platform. + # Register the generic relay adapter when a connector relay URL is configured + # (GATEWAY_RELAY_URL / gateway.relay_url). No URL -> no-op, so direct/single-tenant + # deployments are unaffected. try: from gateway.relay import ( register_relay_adapter, @@ -14128,33 +12994,25 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew send_relay_policy, ) - # Boot-time relay self-provision: resolve the agent's NAS token -> - # POST /relay/provision -> set GATEWAY_RELAY_* in os.environ BEFORE - # registration reads them. No-op when relay is unconfigured, a secret - # is already pinned, or no NAS token resolves (self-hosted, unenrolled). - # Never raises. + # Boot-time relay self-provision: resolve the agent's NAS token -> POST /relay/provision + # -> set GATEWAY_RELAY_* in os.environ BEFORE registration reads them. Never raises. self_provision_relay() if register_relay_adapter(): logger.info("relay adapter registered (connector at %s)", relay_url()) - # Declare this gateway's relevance policy (mention-gating / - # free-response / allow-bots) to the connector so the SAME - # behavior governs relay delivery (Phase 6 Unit ζ). Runs after - # the secret is resolved; never raises, never blocks boot. + # Declare this gateway's relevance policy (mention-gating / free-response / allow- + # bots) to the connector so the SAME behavior governs relay delivery (Phase 6 Unit + # ζ). Runs after the secret is resolved; never raises, never blocks boot. send_relay_policy() except Exception: logger.warning( "relay adapter registration failed at gateway startup", exc_info=True, ) - # Register declarative shell hooks from cli-config.yaml. Gateway - # has no TTY, so consent has to come from one of the three opt-in - # channels (--accept-hooks on launch, HERMES_ACCEPT_HOOKS env var, - # or hooks_auto_accept: true in config.yaml). We pass - # accept_hooks=False here and let register_from_config resolve - # the effective value from env + config itself — the CLI-side - # registration already honored --accept-hooks, and re-reading - # hooks_auto_accept here would just duplicate that lookup. + # Register declarative shell hooks from cli-config.yaml. Gateway has no TTY, so consent has + # to come from one of the three opt-in channels (--accept-hooks on launch, + # HERMES_ACCEPT_HOOKS env var, or hooks_auto_accept: true in config.yaml). We pass + # accept_hooks=False and let register_from_config resolve env + config itself. # Failures are logged but must never block gateway startup. try: from hermes_cli.config import load_config @@ -14175,7 +13033,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Discover and load event hooks self.hooks.discover_and_load() - # Recover background processes from checkpoint (crash recovery) try: from tools.process_registry import process_registry @@ -14185,15 +13042,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as e: logger.warning("Process checkpoint recovery: %s", e) - # Recover sessions that were active when the gateway last exited. - # Exact durable turn markers cover long-running work; the 120-second - # recency heuristic remains as an upgrade fallback for turns started by - # older Hermes versions that did not write exact markers. - # - # SKIP suspension after a clean (graceful) shutdown — the previous - # process already drained active agents, so sessions aren't stuck. - # This prevents unwanted auto-resets after `hermes update`, - # `hermes gateway restart`, or `/restart`. + # Recover sessions that were active when the gateway last exited. Exact durable turn markers + # cover long-running work; the 120-second recency heuristic remains as an upgrade fallback + # for turns started by older Hermes versions that did not write exact markers. + # SKIP suspension after a clean (graceful) shutdown — the previous process already drained + # active agents, so this must not auto-reset sessions after update/restart. _clean_marker = _hermes_home / ".clean_shutdown" if _clean_marker.exists(): logger.info("Previous gateway exited cleanly — skipping session suspension") @@ -14223,10 +13076,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew fallback, ) - # Stuck-loop detection (#7536): if a session has been active across - # 3+ consecutive restarts, it's probably stuck in a loop (the same - # history keeps causing the agent to hang). Auto-suspend it so the - # user gets a clean slate on the next message. + # Stuck-loop detection: if a session has been active across 3+ consecutive restarts, it's + # probably stuck in a loop (the same history keeps causing the agent to hang). Auto-suspend + # it so the user gets a clean slate on the next message. try: stuck = self._suspend_stuck_loop_sessions() if stuck: @@ -14234,20 +13086,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as e: logger.debug("Stuck-loop detection failed: %s", e) - # Serialize startup restore against inbound dispatch. Platform - # adapters can begin receiving messages as soon as they connect, but - # restart-interrupted sessions are not auto-resumed until all startup - # wiring below completes. Queue inbound messages until the resume - # pass runs and every synthetic resume turn has finished. + # Serialize startup restore against inbound dispatch. Platform adapters can begin receiving + # messages as soon as they connect, but restart-interrupted sessions are not auto-resumed + # until all startup wiring below completes. Inbound messages queue until the resume pass + # runs and every synthetic resume turn has finished. self._startup_restore_in_progress = True self._startup_restore_queue = [] self._startup_restore_tasks = [] - # Fresh-boot readiness (#99373): with no resume_pending sessions the - # gate above opens almost immediately, while the agent-side turn - # machinery (run_agent import graph, tool schemas, check_fn probes, - # context tier) is still cold — a message in that window was served - # with a skeleton system prompt. Start warming NOW so the work - # overlaps the network-bound platform connects below; + # Fresh-boot readiness: with no resume_pending sessions the gate above opens almost + # immediately, while the agent-side turn machinery (run_agent import graph, tool schemas, + # check_fn probes, context tier) is still cold — a message in that window was served with a + # skeleton system prompt. Warm NOW so the work overlaps the network-bound connects below; # _finish_startup_restore awaits it (bounded) before opening the gate. self._start_startup_warmup() @@ -14257,30 +13106,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew startup_retryable_errors: list[str] = [] _multiplex_on = bool(getattr(self.config, "multiplex_profiles", False)) _multiplex_skipped_platforms: list[Platform] = [] - # Initialize and connect each configured platform. - # - # Parallel startup connect (#83791): the original code ran a serial for-loop, - # so every platform's connect() (with its own timeout) had to finish before - # the next began. A single slow/failing platform (e.g. Telegram behind a dead - # proxy) therefore delayed every other platform's connect by a full timeout - # window, cascading one platform's failure onto WeChat/QQ/etc. We now launch - # all platform connects concurrently and let each resolve on its own timeline; - # per-platform timeouts and error handling are unchanged. - # The serial pre-filter (cheap checks, adapter creation, handler wiring) stays - # sequential -- only the (slow) connect() calls run in parallel. + # Initialize and connect each configured platform. connect() calls run concurrently so one + # slow/failing platform (e.g. Telegram behind a dead proxy) cannot delay every other + # platform by a full timeout window; the cheap pre-filter (checks, adapter creation, + # handler wiring) stays serial. Per-platform timeouts/error handling are unchanged. _pending_connects = [] # (platform, platform_config, adapter) for platform, platform_config in self.config.platforms.items(): if await self._abort_startup_if_shutdown_requested(): return True if not platform_config.enabled: continue - # Under multiplexing, a platform may be enabled on the default - # profile's config.yaml while its bot token lives only in a - # secondary profile's .env. Starting that primary adapter with an - # empty token fails immediately and queues an infinite reconnect - # loop that can never heal (#64674). Secondary profiles still - # start their own adapters under _profile_runtime_scope with the - # real token -- skip the empty primary instead of failing loudly. + # Under multiplexing, a platform may be enabled on the default profile's config.yaml + # while its bot token lives only in a secondary profile's .env. Starting the primary with + # an empty token fails at once and queues a reconnect loop that can never heal; the + # secondary starts its own adapter with the real token, so skip the empty primary. if _multiplex_on and not _platform_has_bot_credential(platform, platform_config): logger.info( "Skipping %s on default profile: no bot credential in this " @@ -14307,10 +13146,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.warning("No adapter available for %s", _pval) continue - # Set up message + fatal error handlers. Under multiplexing the - # default profile needs the same whole-handler runtime scope as a - # secondary profile: authorization and prompt rendering both run - # before the narrower agent-turn scope is installed. + # Set up message + fatal error handlers. Under multiplexing the default profile needs + # the same whole-handler runtime scope as a secondary profile: authorization and prompt + # rendering both run before the narrower agent-turn scope is installed. adapter.set_message_handler(self._primary_message_handler()) adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) adapter.set_session_store(self.session_store) @@ -14399,10 +13237,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else: _raw = [] - # Aggregate results single-threaded so shared state (self.adapters, - # self._failed_platforms, the error lists, connected_count) is mutated - # exactly as the original serial loop did -- only the connect() wall-clock - # overlap changed. + # Aggregate results single-threaded so shared state (self.adapters, self._failed_platforms, + # the error lists, connected_count) is mutated exactly as the original serial loop did -- + # only the connect() wall-clock overlap changed. for _item in _raw: if isinstance(_item, Exception): # Unexpected escape from _connect_one_startup (shouldn't happen); @@ -14446,9 +13283,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else: # outcome == "failed" logger.warning("\u2717 %s failed to connect", platform.value) # Defensive cleanup: a failed connect() may have allocated resources - # (aiohttp.ClientSession, poll tasks, bridge subprocesses) before - # giving up. Without this call, those resources are orphaned and - # Python logs "Unclosed client session" at process exit. + # (aiohttp.ClientSession, poll tasks, bridge subprocesses) before giving up. await self._safe_adapter_disconnect(adapter, platform) if adapter.has_fatal_error: # A live foreign holder of this bot token / identity is @@ -14499,10 +13334,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if await self._abort_startup_if_shutdown_requested(): return True - # Multi-profile multiplexing: bring up adapters for every OTHER profile - # this gateway serves. Each profile's adapters connect under that - # profile's home + credential scope and stamp their inbound events with - # the profile so the agent turn resolves correctly. No-op when off. + # Multi-profile multiplexing: bring up adapters for every OTHER profile this gateway serves. + # Each profile's adapters connect under that profile's home + credential scope and stamp + # their inbound events with the profile so the agent turn resolves correctly. try: _secondary_connected = await self._start_secondary_profile_adapters() connected_count += _secondary_connected @@ -14560,20 +13394,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._startup_restore_in_progress = False return True if startup_nonretryable_errors: - # Mixed failure mode (NS-609): some platforms are fatally - # misconfigured (e.g. WhatsApp enabled but never paired) while - # others hit merely transient errors (e.g. Telegram TimedOut - # during polling startup). Exiting with - # GATEWAY_FATAL_CONFIG_EXIT_CODE here is wrong in both - # supervision worlds: under supervisors that honor the - # exit-78 contract (systemd RestartPreventExitStatus, s6 - # finish→125 since #51228) the gateway goes PERMANENTLY down - # over a network blip; under anything else it crash-loops. - # Either way the retryable platforms never get their retry. - # Log the fatal side loudly, then fall through to the - # degraded/retry path below: the reconnect watcher recovers - # the retryable platforms; the non-retryable ones remain - # fatal-parked and visible in runtime status. + # Mixed failure mode (NS-609): some platforms are fatally misconfigured (e.g. + # WhatsApp enabled but never paired) while others hit merely transient errors (e.g. + # Telegram TimedOut). Exiting with GATEWAY_FATAL_CONFIG_EXIT_CODE is wrong here: + # supervisors honoring the exit-78 contract take the gateway PERMANENTLY down over + # a network blip, others crash-loop. Either way the retryable platforms never get + # their retry. Log the fatal side loudly, then fall through to the degraded/retry + # path: the reconnect watcher recovers the retryable ones; the rest stay fatal-parked. logger.error( "%d platform(s) fatally misconfigured and parked: %s. " "Staying alive so retryable platforms can recover.", @@ -14606,12 +13433,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except Exception: pass - # Fall through to the normal "running" state — reconnect - # watcher takes it from here. - # All enabled platforms had no adapter (missing library or credentials). - # In fleet deployments the same config.yaml is shared across nodes that - # may only have credentials for a subset of platforms. Rather than - # failing hard, degrade gracefully and allow cron jobs to run (#5196). + # Fall through to the normal "running" state — reconnect watcher takes it from + # here. + # All enabled platforms had no adapter (missing library or credentials). Fleet + # deployments share one config.yaml across nodes holding credentials for only a + # subset of platforms, so degrade gracefully and allow cron jobs to run. logger.warning( "No adapter could be created for any of the %d configured platform(s). " "Check that required dependencies are installed and credentials are set. " @@ -14621,7 +13447,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else: logger.warning("No messaging platforms enabled.") logger.info("Gateway will continue running for cron job execution.") - + # Update delivery router with adapters if await self._abort_startup_if_shutdown_requested(): return True @@ -14654,10 +13480,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await self.hooks.emit("gateway:startup", { "platforms": [p.value for p in self.adapters.keys()], }) - + if connected_count > 0: logger.info("Gateway running with %s platform(s)", connected_count) - + # Build initial channel directory for send_message name resolution try: from gateway.channel_directory import build_channel_directory @@ -14666,7 +13492,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.info("Channel directory built: %d target(s)", ch_count) except Exception as e: logger.warning("Channel directory build failed: %s", e) - + # Check if we're restarting after a /update command. If the update is # still running, keep watching so we notify once it actually finishes. notified = await self._send_update_notification() @@ -14688,32 +13514,24 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Notify the chat that initiated /restart that the gateway is back. chat_restart_notification_pending = _restart_notification_pending() planned_restart_notification_pending = _planned_restart_notification_pending() - # Capture, before _send_restart_notification() unlinks the marker, - # whether this process booted from a chat-originated /restart. Used as - # a one-shot signal by the /restart redelivery guard so a missing - # dedup marker only suppresses a /restart when we KNOW we just came out - # of a restart cycle (see _is_stale_restart_redelivery). + # Capture, before _send_restart_notification() unlinks the marker, whether this process + # booted from a chat-originated /restart. One-shot signal for the /restart redelivery + # guard (_is_stale_restart_redelivery): a missing dedup marker only suppresses a /restart + # when we KNOW we just came out of a restart cycle. if chat_restart_notification_pending: self._booted_from_restart = True - # Restart notification, home-channel startup notice, and obligation - # redelivery all call adapter.send(). Those sends must not pin the - # inbound restore gate — a Telegram flood-control sleep on this path - # froze every platform for the full penalty (#91969). Bound them the - # same way _finish_startup_restore bounds resume turns. + # Restart notification, home-channel startup notice, and obligation redelivery all call + # adapter.send(). Those sends must not pin the inbound restore gate — a Telegram flood- + # control sleep on this path froze every platform for the full penalty. await self._await_startup_boot_sends( planned_restart_notification_pending=planned_restart_notification_pending, ) - # Automatically continue fresh sessions that were interrupted by the - # previous gateway restart/shutdown. The resume_pending flag is cleared - # by the normal successful-turn path, so a failed auto-resume remains - # visible for manual recovery on the next user message. - # - # Delivery-obligation redelivery already ran inside - # _await_startup_boot_sends (and clears resume_pending before send): - # a session whose final response was generated but never - # confirmed-delivered has its answer in the ledger — redelivering it - # is strictly cheaper and more correct than re-running the whole turn. + # Automatically continue fresh sessions that were interrupted by the previous gateway + # restart/shutdown. The resume_pending flag is cleared by the normal successful-turn path, + # so a failed auto-resume remains visible for manual recovery on the next user message. + # Obligation redelivery already ran in _await_startup_boot_sends and cleared resume_pending + # for sessions whose answer is in the ledger — redelivering beats re-running the turn. self._schedule_resume_pending_sessions() await self._finish_startup_restore() @@ -14724,10 +13542,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Drain any recovered process watchers (from crash recovery checkpoint) try: from tools.process_registry import process_registry - # Detach the current batch atomically: reassigning to a fresh list - # takes ownership of exactly the watchers present now, so any watcher - # appended concurrently during the yield below isn't silently dropped - # by a clear() on the shared list. + # Detach the current batch atomically: reassigning to a fresh list takes ownership of + # exactly the watchers present now, so any watcher appended concurrently during the + # yield below isn't silently dropped by a clear() on the shared list. watchers = process_registry.pending_watchers process_registry.pending_watchers = [] # Process in batches of 100 with event-loop yield points to avoid @@ -14747,10 +13564,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Start background session expiry watcher to finalize expired sessions self._spawn_supervised(self._session_expiry_watcher, "session_expiry_watcher") - # Keep the /model picker's remote catalogs (curated manifest, - # OpenRouter live list, Nous Portal recommendations) warm on disk so a - # delisted or newly-published model reaches the picker within one TTL - # window (model_catalog.ttl_minutes, default 20) without waiting for a + # Keep the /model picker's remote catalogs (curated manifest, OpenRouter live list, Nous + # Portal recommendations) warm on disk so a delisted or newly-published model reaches the + # picker within one TTL window (model_catalog.ttl_minutes, default 20) without waiting for a # cold /model open to trigger the refresh. self._spawn_supervised(self._model_catalog_refresh_watcher, "model_catalog_refresh_watcher") @@ -14763,10 +13579,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # when another gateway owns the single dispatcher. self._spawn_supervised(self._kanban_notifier_watcher, "kanban_notifier_watcher") - # Start background kanban dispatcher — spawns workers for ready - # tasks. Gated by `kanban.dispatch_in_gateway` (default True). - # When false, users run `hermes kanban daemon` externally or - # simply don't use kanban; this loop becomes a no-op. + # Start background kanban dispatcher — spawns workers for ready tasks. Gated by + # `kanban.dispatch_in_gateway` (default True). When false, users run `hermes kanban daemon` + # externally or simply don't use kanban; this loop becomes a no-op. self._spawn_supervised(self._kanban_dispatcher_watcher, "kanban_dispatcher_watcher") # Start background reconnection watcher for platforms that failed at startup @@ -14794,10 +13609,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # for a dead watcher and spawns a duplicate. self._spawn_reconnect_watcher() - # Start background handoff watcher — picks up CLI sessions marked - # handoff_state='pending' in state.db and re-binds them to the - # destination platform's home channel, then forges a synthetic user - # turn so the agent kicks off the new chat. + # Start background handoff watcher — picks up CLI sessions marked handoff_state='pending' in + # state.db and re-binds them to the destination platform's home channel, then forges a + # synthetic user turn so the agent kicks off the new chat. self._spawn_supervised(self._handoff_watcher, "handoff_watcher") # Start background async-delegation watcher — drains completion events @@ -14811,14 +13625,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # originating chats while the session is idle. self._spawn_supervised(self._loop_wakeup_watcher, "loop_wakeup_watcher") - # Start the scale-to-zero idle watcher ONLY when this instance is opted - # in (the NAS "Labs" HERMES_SCALE_TO_ZERO stamp), messaging is - # relay-only/absent, and a wakeUrl is registered (decisions.md D1/D11/ - # §3.4(1)). A non-opted instance never starts it, so behaviour is exactly - # as today. When armed, the watcher drives the relay dormant on sustained - # idle and then suspends the machine itself via the local flaps socket - # (Fly Proxy autostop is inbound-only and job-blind, so the gateway owns - # the suspend decision; NAS provisions these machines autostop:"off"). + # Start the scale-to-zero idle watcher ONLY when this instance is opted in (the NAS "Labs" + # HERMES_SCALE_TO_ZERO stamp), messaging is relay-only/absent, and a wakeUrl is registered + # (decisions.md D1/D11/ §3.4(1)). Non-opted instances are unchanged. When armed it drives + # the relay dormant on sustained idle, then suspends the machine via the local flaps socket + # — Fly Proxy autostop is inbound-only and job-blind, so the gateway owns the decision. try: if self._scale_to_zero_should_arm(): logger.info( @@ -14834,38 +13645,31 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: # noqa: BLE001 - arming must never block startup logger.debug("scale-to-zero: arm check failed at startup", exc_info=True) - # Start background drain-control watcher — reconciles the gateway's - # new-turn accept-state with the external ``.drain_request.json`` marker - # the dashboard begin/cancel-drain endpoint writes (Phase 2). A marker - # left behind by a prior instantiation (durable-volume restart, NS-570) - # is ignored via its instantiation epoch; only a current-epoch marker - # engages drain on the first tick. + # Start background drain-control watcher — reconciles the gateway's new-turn accept-state + # with the external ``.drain_request.json`` marker the dashboard begin/cancel-drain endpoint + # writes (Phase 2). A marker left by a prior instantiation (durable-volume restart) is + # ignored via its instantiation epoch; only a current-epoch marker engages drain. self._spawn_supervised(self._drain_control_watcher, "drain_control_watcher") logger.info("Press Ctrl+C to stop") - + return True _MAX_SUPERVISED_RESTARTS = 5 - # A task that ran at least this long before crashing is treated as having - # been HEALTHY — its crash is a fresh, isolated failure rather than part of - # a rapid crash-loop, so the consecutive-restart counter resets to 0. Only - # crashes that happen within this window of a (re)spawn accumulate toward - # ``_MAX_SUPERVISED_RESTARTS``. Without this, a long-lived launchd daemon - # whose watcher crashes a handful of times over days would hit the cap and - # be permanently abandoned (NS: silent loss of platform-reconnect / kanban / - # handoff for the rest of the process life). + # A task that ran at least this long before crashing is treated as having been HEALTHY — its + # crash is a fresh, isolated failure rather than part of a rapid crash-loop, so the consecutive- + # restart counter resets to 0. Only crashes within this window of a (re)spawn count toward + # _MAX_SUPERVISED_RESTARTS; otherwise a long-lived daemon whose watcher crashed a few times + # over days would hit the cap and be permanently abandoned (silent loss of reconnect/kanban). _SUPERVISED_HEALTHY_SECS = 300 @staticmethod def _supervised_backoff(attempt: int) -> float: """Delay before the supervisor's next respawn, in seconds. - Capped exponential. A method rather than an inline expression so the - schedule has one name, and so a test can collapse it -- the ordering - of crash / give-up / slow-tier is what the exhaustion tests assert, - and sleeping through the real curve to observe it would make them - take minutes. + Capped exponential. A method (not an inline expression) so the schedule has one name and + tests can collapse it — exhaustion tests assert crash/give-up/slow-tier ordering and would + take minutes sleeping through the real curve. """ return min(60, 2 ** min(attempt, 6)) @@ -14875,43 +13679,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): """Launch a long-lived background task with task-level supervision. - Complements upstream's per-iteration inner-loop try/except (which only - guards a single loop-body) by covering what that CANNOT: an exception - raised in the watcher's OUTER ``while self._running:`` loop or its - pre-try setup region, plus task-level death generally. A bare - ``asyncio.create_task`` drops such an exception on the floor — no log, - no restart, the watcher silently gone. This retains the handle in - ``self._background_tasks``, logs any crash, and restarts with capped - exponential backoff up to ``_MAX_SUPERVISED_RESTARTS`` failures in rapid - succession (each within ``_SUPERVISED_HEALTHY_SECS`` of its restart). - The counter resets after any run that stayed healthy for at least - ``_SUPERVISED_HEALTHY_SECS`` — so a long-lived daemon that crashes - occasionally over days is never permanently abandoned. + Covers what a per-iteration try/except cannot: exceptions in the watcher's OUTER loop or + pre-try setup, and task-level death. A bare ``asyncio.create_task`` drops such an exception + on the floor — no log, no restart, the watcher silently gone. Restarts with capped backoff + up to ``_MAX_SUPERVISED_RESTARTS`` rapid failures; the counter resets after any run healthy + for ``_SUPERVISED_HEALTHY_SECS``. - Each watcher starts in a fresh ``Context``. These are process-level - services, not continuations of whichever message turn happened to - spawn them; inheriting a delegated-child marker would make the Kanban - dispatcher reject its own writes when ``asyncio.to_thread`` copies the - watcher's context. + Each watcher starts in a fresh ``Context``: these are process-level services, and an + inherited delegated-child marker would make the Kanban dispatcher reject its own writes. - ``on_spawn`` (optional) is invoked with the freshly-created task on - every spawn, INCLUDING internal backoff respawns. Callers that also - track the live handle elsewhere (e.g. ``self._reconnect_watcher_task`` - for ``_ensure_reconnect_watcher_running``) MUST pass it — otherwise the - supervisor's own respawn creates a new task without updating that - external handle, so ``_ensure_...`` later sees the stale/done handle - and spawns a SECOND concurrent watcher (double reconnect attempts). + ``on_spawn`` fires on EVERY spawn, including backoff respawns. Callers tracking the live + handle elsewhere (e.g. ``_reconnect_watcher_task``) MUST pass it, or a respawn leaves a + stale handle and ``_ensure_...`` spawns a SECOND concurrent watcher. - ``on_give_up`` (optional) is invoked with ``name`` when supervision is - abandoned — the restart budget is spent and this task will never be - respawned by the supervisor again. Supervision being finite is correct; - having no owner of the invariant afterwards is not. A task that still - has queued work depending on it needs somewhere to hand that fact to, - and before this hook existed the only thing standing between budget - exhaustion and a permanent silent outage was a *later, unrelated - event* happening to call ``_ensure_...`` (#90386). This is the - supervisor telling its caller "I am done; the invariant is yours now", - which is a thing only the supervisor knows. + ``on_give_up`` fires with ``name`` when the restart budget is spent and this task will + never be respawned. Supervision being finite is correct; having no owner of the invariant + afterwards is not — only the supervisor knows it is done. """ if getattr(self, "_background_tasks", None) is None: self._background_tasks = set() @@ -14920,17 +13703,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # uses it to distinguish a rapid crash-loop from a healthy-run-then-crash. _started = time.monotonic() - # Deliberately do NOT pass kwargs to create_task — some test doubles - # mock it with a narrow signature. Calling it from a fresh Context has - # the same isolation semantics as create_task(..., context=Context()) - # while preserving that compatibility. + # Deliberately do NOT pass kwargs to create_task — some test doubles mock it with a narrow + # signature. Calling it from a fresh Context has the same isolation semantics as + # create_task(..., context=Context()) while preserving that compatibility. task = Context().run(lambda: asyncio.create_task(coro_factory())) - # Mark this as a PERMANENT supervised watcher, not transient background - # WORK. The scale-to-zero idle check must ignore these: supervised - # watchers (session-expiry, kanban, reconnect, the scale-to-zero watcher - # itself, ...) live for the whole process, so counting them as "live - # background work" would make the gateway consider itself busy forever - # and never go dormant/suspend. Transient tasks added to + # Mark this as a PERMANENT supervised watcher, not transient background WORK: the + # scale-to-zero idle check must ignore process-lifetime watchers or the gateway would + # count itself busy forever and never go dormant. Transient tasks added to # _background_tasks elsewhere (startup-resume events etc.) stay counted. task._hermes_supervised_watcher = True # type: ignore[attr-defined] self._background_tasks.add(task) @@ -14957,10 +13736,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if restart and self._running: ran_for = time.monotonic() - _started if ran_for >= self._SUPERVISED_HEALTHY_SECS: - # Ran healthily for a while before crashing — this is a - # FRESH failure, not part of a rapid crash-loop. Reset the - # consecutive counter so a daemon that crashes a handful of - # times over days is never permanently abandoned. + # Ran healthily for a while before crashing — this is a FRESH failure, not part + # of a rapid crash-loop. Reset the consecutive counter so a daemon that crashes + # a handful of times over days is never permanently abandoned. effective_attempt = 0 else: effective_attempt = _attempt @@ -15015,21 +13793,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Background task that processes pending CLI→gateway session handoffs. - Polls ``state.db`` for sessions in ``handoff_state='pending'`` and, - for each one: - - 1. Atomically claims it (pending → running). - 2. Resolves the destination platform's configured home channel. - 3. Re-binds the gateway's session_key for that home channel to the - CLI's existing session_id via ``session_store.switch_session`` so - the full role-aware transcript replays on the next agent turn. - 4. Forges a synthetic ``MessageEvent`` (``internal=True``) with a - handoff-notice text and dispatches through the normal gateway - message pipeline so the agent runs and replies on the platform. - 5. Marks the row ``completed`` (or ``failed`` with ``handoff_error``). - - The CLI process is poll-blocked on the row's terminal state and - prints the result to the user. + Polls ``state.db`` for sessions in ``handoff_state='pending'`` and, for each one: + atomically claims it (pending → running); resolves the destination platform's home + channel; re-binds that channel's session_key to the CLI session_id via + ``session_store.switch_session`` so the full transcript replays; dispatches a synthetic + ``MessageEvent`` (``internal=True``) through the normal pipeline; marks the row + ``completed`` (or ``failed`` with ``handoff_error``). The CLI polls the terminal state. """ # Initial delay so the gateway is fully connected to its platforms # before we try to dispatch handoffs through them. @@ -15046,12 +13815,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: _process_takes_profile = False - # In-flight dispatches, keyed by session id. A handoff runs a FULL - # agent turn plus delivery, which can take far longer than the CLI's - # 60s wait. Processing them inline would make one slow handoff block - # every other profile's poll — a legitimate handoff could then time - # out purely because another profile was ahead of it in the queue. - # Dispatch is therefore fire-and-forget; the poll loop only claims. + # In-flight dispatches, keyed by session id. A handoff runs a FULL agent turn plus delivery + # (far longer than the CLI's 60s wait); processing them inline would make one slow handoff + # block every other profile's poll — a legitimate handoff could then time out purely because + # another profile was ahead of it. Dispatch is fire-and-forget; the poll loop only claims. inflight: Dict[str, "asyncio.Task"] = {} async def _dispatch(row, session_id, session_db, profile_name) -> None: @@ -15081,19 +13848,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _tick(profile_name: Optional[str] = None) -> None: """One poll of the CURRENTLY-SCOPED session store. - Deliberately a closure over ``self`` rather than a method: the - watcher's unit tests bind ``_handoff_watcher`` onto a minimal - ``SimpleNamespace`` stand-in that exposes only ``_session_db``, - ``_running`` and ``_process_handoff``. Any ``self.`` - call here would raise AttributeError on that stand-in, get - swallowed by the loop's ``except Exception``, and silently turn - the whole watcher into a no-op — which is exactly what the - mutation-survivable assertions caught. Touch only attributes the - stand-in already provides. - - ``profile_name`` is threaded to ``_process_handoff`` so delivery - uses that profile's OWN adapter/home channel; see the docstring - there. ``None`` means the root/default profile. + Deliberately a closure over ``self`` rather than a method: the watcher's unit tests + bind ``_handoff_watcher`` onto a minimal ``SimpleNamespace`` stand-in that exposes + only ``_session_db``, ``_running`` and ``_process_handoff``. Any ``self.`` + call would raise AttributeError, be swallowed by the loop's ``except Exception``, and + silently turn the watcher into a no-op — touch only what the stand-in provides. + ``profile_name`` (``None`` = root) is threaded to ``_process_handoff`` so delivery + uses that profile's OWN adapter/home channel. """ session_db = getattr(self, "_session_db", None) if session_db is None: @@ -15106,31 +13867,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not await session_db.claim_handoff(session_id): # Another tick or another gateway already claimed it. continue - # Positional, not keyword: the watcher's existing unit tests - # bind a stand-in ``_process_handoff(row)`` with no second - # parameter, and a keyword call would TypeError into the - # failure branch — turning a passing suite into a silent - # no-op watcher. Arity is probed above. - # - # INVARIANT (do not weaken): this task is created inside - # ``_profile_runtime_scope(profile_home)`` but typically RUNS - # after the scope exits. It still sees the profile's home and - # secret scope only because ``set_hermes_home_override`` and - # ``set_secret_scope`` are ContextVar-based — ensure_future - # copies the current Context into the Task. If either seam is - # ever migrated to a thread-local or module global, secondary- - # profile handoffs silently regress to primary-config delivery - # (the exact bug fixed in #91217) while still recording - # handoff_state='completed'. + # Positional, not keyword: the watcher's existing unit tests bind a stand-in + # ``_process_handoff(row)`` with no second parameter, and a keyword call would + # TypeError into the failure branch — turning a passing suite into a silent no-op + # watcher. Arity is probed above. + # INVARIANT (do not weaken): this task is created inside _profile_runtime_scope but + # typically RUNS after the scope exits; it sees the profile's home/secret scope only + # because those seams are ContextVar-based and ensure_future copies the Context. A + # thread-local/global there silently regresses secondary handoffs to primary delivery. inflight[session_id] = asyncio.ensure_future( _dispatch(row, session_id, session_db, profile_name) ) - # A row still in 'running' at startup belongs to a gateway that died - # mid-dispatch. It can never reach a terminal state on its own, and - # request_handoff refuses new requests while it sits there — so the - # session would be permanently unable to hand off again. Reclaim once, - # per store, before the first poll. + # A row still in 'running' at startup belongs to a gateway that died mid-dispatch. It can + # never reach a terminal state on its own, and request_handoff refuses new requests while it + # sits there — so the session would be permanently unable to hand off again. for _pname, _phome in _handoff_watch_scopes(self): try: if _phome is None: @@ -15156,11 +13907,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("Handoff watcher tick error: %s", exc, exc_info=True) await asyncio.sleep(interval) finally: - # Drain in-flight dispatches before returning. Cancelling them - # outright would strand their rows in 'running'; giving them a - # bounded grace period lets an almost-done handoff record its own - # terminal state. Whatever is still running after that is - # cancelled and reclaimed at the next startup. + # Drain in-flight dispatches before returning. Cancelling them outright would strand + # their rows in 'running'; giving them a bounded grace period lets an almost-done + # handoff record its own terminal state. pending_tasks = [t for t in inflight.values() if not t.done()] if pending_tasks: try: @@ -15174,24 +13923,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _process_handoff( self, row: Dict[str, Any], profile_name: Optional[str] = None, ) -> None: - """Execute one handoff row. Raises on failure (caller marks failed). + """Execute one handoff row. - ``profile_name`` is the profile whose store queued this handoff - (``None`` = root/default). On a multiplexed gateway it is load-bearing - for THREE things that otherwise silently resolve to the primary - profile and deliver through the wrong bot: - - - ``self.adapters`` only ever holds the DEFAULT profile's adapters; - secondary profiles live in ``self._profile_adapters[name]``. - - ``self.config`` is the primary's config, so ``get_home_channel()`` - returns the primary's chat — a medicina handoff would be delivered - by the default bot, to the default's home channel. - - the session key must be namespaced ``agent::...`` to match - the key that profile's own adapter uses for organic inbound - messages; otherwise the handoff binds a key nobody reads. - - Passing the name explicitly beats re-deriving it from the active - contextvar: the caller already knows which store the row came from. + Raises on failure (caller marks failed). ``profile_name`` (``None`` = root) is the profile + whose store queued this handoff. On a multiplexed gateway it is load-bearing for THREE + things that otherwise silently resolve to the primary profile and deliver through the + wrong bot: ``self.adapters`` holds only the DEFAULT profile's adapters (secondaries live + in ``self._profile_adapters[name]``); ``self.config`` is the primary's, so + ``get_home_channel()`` would return the primary's chat; and the session key must be + namespaced ``agent::...`` to match that profile's adapter or it binds a key + nobody reads. Passing the name beats re-deriving it from the contextvar. """ from gateway.config import Platform from gateway.session import SessionSource, build_session_key @@ -15208,12 +13949,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except (ValueError, KeyError): raise RuntimeError(f"unknown platform '{platform_name}'") - # Resolve the config + adapter map for the profile that queued this - # handoff. On a single-profile gateway (or a default-profile handoff) - # both fall back to self.config/self.adapters, so behaviour is - # byte-identical to before. On a multiplexed gateway a secondary - # profile MUST use its own map — self.adapters holds only the primary's - # adapters, and self.config only the primary's home channel. + # Resolve the config + adapter map for the profile that queued this handoff. On a single- + # profile gateway (or a default-profile handoff) both fall back to + # self.config/self.adapters, so behaviour is byte-identical to before. handoff_config = self.config handoff_adapters = self.adapters if profile_name and profile_name != "default": @@ -15223,13 +13961,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"profile '{profile_name}' has no live adapters in this gateway" ) handoff_adapters = secondary - # The watcher already entered _profile_runtime_scope for this - # profile, so a fresh load resolves that profile's config.yaml - # and .env (home channel, tokens) rather than the primary's. - # Fail closed on a load error: self.config is the primary's, so - # falling back would deliver through the right bot to the - # WRONG chat and report completed. A failed row the CLI can - # retry beats a wrong delivery. + # The watcher already entered _profile_runtime_scope for this profile, so a fresh load + # resolves that profile's config.yaml and .env (home channel, tokens) rather than the + # primary's. Fail closed on a load error: falling back to self.config would deliver + # through the right bot to the WRONG chat and report completed. A failed row the CLI + # can retry beats a wrong delivery. try: handoff_config = load_gateway_config() except Exception as exc: @@ -15243,12 +13979,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"could not load config for profile '{profile_name}': {exc}" ) from exc - # Adapter must be live. A relay-fronted gateway registers ONE adapter - # under Platform.RELAY that fronts N logical platforms — so a literal - # adapters.get(discord) misses even though "discord" is deliverable. - # resolve_delivery_transport is the shared alias-aware resolver (native - # adapter wins; relay eligible only when its authenticated transport - # advertises it fronts the logical platform). + # Adapter must be live. A relay-fronted gateway registers ONE adapter under Platform.RELAY + # fronting N logical platforms, so a literal adapters.get(discord) misses a deliverable + # platform; resolve_delivery_transport is the alias-aware resolver (native adapter wins). transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters) if not transport: raise RuntimeError( @@ -15266,12 +13999,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew cli_title = row.get("title") or cli_session_id[:8] - # Try to create a fresh thread on the destination so the handoff - # has its own scrollback. Adapter returns None if threading isn't - # supported (Matrix/WhatsApp/Signal/SMS) or if creation failed - # (no permission, topics-mode off, parent is a DM, etc.). When - # None we fall through to using the home channel directly — the - # synthetic turn still lands; just without thread isolation. + # Try to create a fresh thread on the destination so the handoff has its own scrollback. + # Adapter returns None if threading isn't supported (Matrix/WhatsApp/Signal/SMS) or if + # creation failed (no permission, topics-mode off, parent is a DM, etc.). thread_name = f"Hermes — {cli_title}" try: new_thread_id = await adapter.create_handoff_thread( @@ -15291,14 +14021,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew str(home.thread_id) if home.thread_id else None ) - # Determine chat_type/user_id for the destination source. - # - # Telegram private-chat DM topics are represented differently from - # group/forum threads by the inbound adapter. A handoff-created topic - # in a positive Telegram chat_id must therefore use the same DM-topic - # source shape as the user's next real message; otherwise the synthetic - # handoff turn binds a generic `thread` session key while real replies - # arrive on a `dm` session key. + # Determine chat_type/user_id for the destination source. Telegram private-chat DM topics + # are shaped differently from group/forum threads by the inbound adapter, so a handoff- + # created topic in a positive chat_id must use the same DM-topic source shape as the + # user's next real message — otherwise the synthetic turn binds a `thread` key while + # real replies arrive on a `dm` key. home_chat_id = str(home.chat_id) is_telegram_private_chat = ( platform == Platform.TELEGRAM @@ -15309,26 +14036,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew dest_chat_type = "thread" dest_user_id = "system:handoff" else: - # No thread — assume DM-style for the home channel. For Telegram - # private-chat topics, use the real user id (same as chat_id) so - # topic-mode checks and binding persistence see the same identity as - # subsequent inbound user messages. + # No thread — assume DM-style for the home channel. For Telegram private-chat topics, + # use the real user id (same as chat_id) so topic-mode checks and binding persistence + # see the same identity as subsequent inbound user messages. dest_chat_type = "dm" dest_user_id = home_chat_id if is_telegram_private_chat else "system:handoff" - # Discord thread destinations must key on the thread's OWN id, not the - # parent channel's, because the Discord adapter builds organic in-thread - # messages with ``chat_id == thread id`` — so ``build_session_key`` - # yields ``…:thread:{thread}:{thread}``. If the handoff keys on the - # parent channel (``…:thread:{parent}:{thread}``) the next real user - # reply in the thread resolves to a DIFFERENT session_key and spawns a - # fresh session instead of continuing the handed-off one. - # - # This is Discord-specific: Slack and Telegram adapters key organic - # thread messages with ``chat_id == parent_channel`` and the thread - #/topic id only in ``thread_id``, so for those platforms the parent - # channel is correct (and the deeper chat_type normalization — handoff - # uses "thread" but Slack organic uses "group" — is a separate issue). + # Discord thread destinations must key on the thread's OWN id, not the parent channel's, + # because the Discord adapter builds organic in-thread messages with ``chat_id == thread + # id`` — so ``build_session_key`` yields ``…:thread:{thread}:{thread}``; keying on the + # parent would make the next real reply spawn a fresh session. Discord-specific: Slack and + # Telegram key organic thread messages with ``chat_id == parent_channel``. if platform == Platform.DISCORD and dest_chat_type == "thread" and effective_thread_id: dest_chat_id = str(effective_thread_id) else: @@ -15344,29 +14062,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew profile=profile_name, ) - # Compute the gateway's session_key for that destination using the - # same rules its adapters use, so switch_session targets the right - # entry. For thread destinations build_session_key keys without - # user_id (thread_sessions_per_user defaults to False) — so the - # next real user message in the thread shares this same session. + # Compute the gateway's session_key for that destination using the same rules its adapters + # use, so switch_session targets the right entry. Thread destinations key without user_id + # (thread_sessions_per_user defaults False) so the next real message shares this session. platform_cfg = handoff_config.platforms.get(platform) extra = platform_cfg.extra if platform_cfg else {} - # Namespace the key to the profile that queued this handoff. Without - # it, a multiplexed gateway builds ``agent:main:...`` here while the - # profile's own adapter routes real inbound messages on - # ``agent::...`` — the handoff would bind a key nobody reads. - # - # ``profile_name`` comes straight from the watcher, which knows which - # store the row was claimed from; prefer it over re-deriving from the - # ambient contextvar. The resolver stays as the fallback for the - # root/default case (and returns None when multiplexing is off, which - # reproduces the previous key byte-for-byte). - # - # The isinstance check is load-bearing, not defensive noise: a - # duck-typed/Mock session store returns a truthy MagicMock from the - # resolver, which would be interpolated straight into the key as - # ``agent::``. Same pitfall guarded in - # ``BasePlatformAdapter._session_key_profile``. + # Namespace the key to the profile that queued this handoff. Without it, a multiplexed + # gateway builds ``agent:main:...`` here while the profile's own adapter routes real inbound + # messages on ``agent::...`` — the handoff would bind a key nobody reads. The + # resolver is only the fallback for the root case (None when multiplexing is off, keeping + # the old key byte-for-byte). The isinstance check below is load-bearing: a Mock session + # store returns a truthy MagicMock that would be interpolated straight into the key. handoff_profile = profile_name if (profile_name and profile_name != "default") else None if handoff_profile is None: try: @@ -15390,10 +14096,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # creates one; switch_session then re-points it. await self.async_session_store.get_or_create_session(dest_source) - # Re-bind the destination key to the CLI session_id. switch_session - # ends the prior session in SQLite and reopens the CLI session under - # the new key. The CLI's transcript becomes the active one for the - # gateway from this moment on. + # Re-bind the destination key to the CLI session_id. switch_session ends the prior session + # in SQLite and reopens the CLI session under the new key. The CLI's transcript becomes the + # active one for the gateway from this moment on. switched = await self.async_session_store.switch_session(session_key, cli_session_id) if switched is None: raise RuntimeError( @@ -15428,9 +14133,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_key, ) - # Dispatch through the runner directly. Going through - # adapter.handle_message would spawn a background task and we'd - # lose synchronous error visibility; calling _handle_message inline + # Dispatch through the runner directly. Going through adapter.handle_message would spawn a + # background task and we'd lose synchronous error visibility; calling _handle_message inline # keeps the success/failure path observable for the watcher. response_text = await self._handle_message(synthetic_event) if not response_text: @@ -15438,11 +14142,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Either way, agent ran without raising — count as success. return - # Send the agent's reply to the destination. Route to the new - # thread if we created one; otherwise the configured home channel - # (which may itself carry a thread_id). Send through the resolved - # transport (not adapter.send directly) so a relay-fronted logical - # platform is stamped on the outbound frame (send_for_platform). + # Send the agent's reply to the destination. Route to the new thread if we created one; + # otherwise the configured home channel (which may itself carry a thread_id). Send via the + # resolved transport (not adapter.send) so a relay-fronted logical platform is stamped on + # the outbound frame (send_for_platform). send_metadata: Dict[str, Any] = {} if effective_thread_id: send_metadata["thread_id"] = effective_thread_id @@ -15463,11 +14166,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _session_expiry_watcher(self, interval: int = 300): """Background task that finalizes expired sessions. - Runs every ``interval`` seconds (default 5 min). For each session - whose reset policy has expired, invokes ``on_session_finalize`` - hooks, cleans up the cached AIAgent's tool resources, evicts the - cache entry so it can be garbage-collected, and marks the session - so it won't be finalized again. + For each session whose reset policy has expired, invokes ``on_session_finalize`` hooks, + cleans up the cached AIAgent's tool resources, evicts the cache entry so it can be + garbage-collected, and marks the session so it won't be finalized again. """ await asyncio.sleep(60) # initial delay — let the gateway fully start _finalize_failures: dict[str, int] = {} # session_id -> consecutive failure count @@ -15533,28 +14234,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await self._cleanup_agent_resources_off_loop( _cached_agent, context="session expiry" ) - # Drop the cache entry so the AIAgent (and its LLM - # clients, tool schemas, memory provider refs) can - # be garbage-collected. Otherwise the cache grows + # Drop the cache entry so the AIAgent (and its LLM clients, tool schemas, + # memory provider refs) can be garbage-collected. Otherwise the cache grows # unbounded across the gateway's lifetime. self._evict_cached_agent(key) - # Permanently finalizing this session — one funnel - # call drops every conversation-scoped dict AND the - # boundary security state (approvals, update - # prompts, slash-confirm) so the dicts don't grow - # unbounded across the gateway's lifetime. (Idle - # agent-cache eviction must NOT do this: the - # session is still alive and a resumed turn rebuilds - # its agent from these overrides. Only true session - # finalization, /new, and /reset clear them.) See - # _CONVERSATION_SCOPED_STATE. + # Permanently finalizing this session — one funnel call drops every + # conversation-scoped dict AND the boundary security state (approvals, + # update prompts, slash-confirm) so the dicts don't grow unbounded across + # the gateway's lifetime. Idle agent-cache eviction must NOT do this: that + # session is still alive and a resumed turn rebuilds its agent from these + # overrides. Only finalization, /new and /reset clear them. self._clear_conversation_scope( key, reason="expiry_finalized" ) - # Persist the finalized flag to sessions.json AND - # state.db (single write-path, #9006) — also drops - # the persisted /model override, since finalization - # is a conversation boundary. + # Persist the finalized flag to sessions.json AND state.db (single write- + # path, #9006) — also drops the persisted /model override, since + # finalization is a conversation boundary. await self.async_session_store.set_expiry_finalized(entry) logger.debug( "Session expiry finalized for %s", @@ -15595,10 +14290,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Session expiry done: %d finalized", _done, ) - # Sweep agents that have been idle beyond the TTL regardless - # of session reset policy. This catches sessions with very - # long / "never" reset windows, whose cached AIAgents would - # otherwise pin memory for the gateway's entire lifetime. + # Sweep agents that have been idle beyond the TTL regardless of session reset + # policy. This catches sessions with very long / "never" reset windows, whose cached + # AIAgents would otherwise pin memory for the gateway's entire lifetime. try: _idle_evicted = self._sweep_idle_cached_agents() if _idle_evicted: @@ -15609,23 +14303,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _e: logger.debug("Idle agent sweep failed: %s", _e) - # Neither the LRU cap nor the idle TTL is aware of how much - # memory a cached transcript costs, so a busy gateway keeps - # every warm session's tool output resident until RSS hits the - # cgroup limit (#80764). Shed LRU transcripts once the heap is - # over budget; they reload from the persisted session on the - # next turn. + # Neither the LRU cap nor the idle TTL is aware of how much memory a cached + # transcript costs, so a busy gateway keeps every warm session's tool output + # resident until RSS hits the cgroup limit. try: self._sweep_agent_cache_under_pressure() except Exception as _e: logger.debug("Agent cache pressure sweep failed: %s", _e) - # Periodically prune stale SessionStore entries. The - # in-memory dict (and sessions.json) would otherwise grow - # unbounded in gateways serving many rotating chats / - # threads / users over long time windows. Pruning is - # invisible to users — a resumed session just gets a - # fresh session_id, exactly as if the reset policy fired. + # Periodically prune stale SessionStore entries. The in-memory dict (and + # sessions.json) would otherwise grow unbounded in gateways serving many rotating + # chats / threads / users over long time windows. _last_prune_ts = getattr(self, "_last_session_store_prune_ts", 0.0) _prune_interval = 3600.0 # once per hour if time.time() - _last_prune_ts > _prune_interval: @@ -15790,12 +14478,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Cannot deliver; latch to avoid log spam every tick. notified_map[session_key] = True continue - # #76354 review S2: re-read pending state + activity timestamp - # IMMEDIATELY before delivery. The snapshot above ages while - # earlier candidates in this pass await their sends; an agent - # that made progress (or drained its queue) in that window must - # not receive a false stall notice. Abort and leave the latch - # un-set so the next tick re-evaluates from scratch. + # #76354 review S2: re-read pending state + activity timestamp IMMEDIATELY before + # delivery. The snapshot above ages while earlier candidates await their sends; an + # agent that made progress (or drained its queue) meanwhile must not get a false stall + # notice. Abort and leave the latch un-set so the next tick re-evaluates from scratch. still_pending = ( (getattr(adapter, "_pending_messages", None) or {}).get( session_key @@ -15831,10 +14517,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if source is not None and hasattr(self, "_thread_metadata_for_source") else None ) - # Round-2 #2: bound the send. A wedged adapter transport - # (network hang, dead websocket) must not block the whole - # watcher pass — sibling candidates in this loop would never - # be evaluated and the watcher itself would stop ticking. + # Round-2 #2: bound the send. A wedged adapter transport (network hang, dead + # websocket) must not block the whole watcher pass — sibling candidates in this loop + # would never be evaluated and the watcher itself would stop ticking. try: result = await asyncio.wait_for( adapter.send( @@ -15880,12 +14565,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _model_catalog_refresh_watcher(self) -> None: """Refresh the /model picker's remote catalogs every TTL window. - The picker itself only refreshes on a cold or stale open, so a - gateway that nobody opens ``/model`` in keeps serving whatever was - cached. This loop calls ``model_catalog.refresh_catalogs()`` (manifest - + OpenRouter live filter + Nous Portal recommendations) off-thread on - the configured cadence (``model_catalog.ttl_minutes``, default 20) so - the on-disk caches every surface reads are never older than one window. + The picker itself only refreshes on a cold or stale open, so a gateway that nobody opens + ``/model`` in keeps serving whatever was cached. """ from hermes_cli.model_catalog import refresh_catalogs, refresh_interval_seconds @@ -15904,12 +14585,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await asyncio.sleep(min(30.0, max(0.0, deadline - time.monotonic()))) async def _session_stall_watcher(self, interval: float = 30.0): - """Periodic pending-inbound + stale-activity stall watchdog (#72016). + """Periodic pending-inbound + stale-activity stall watchdog. - Progress comes only from ``get_activity_summary()`` (#72039). - Pending inbound is a notify policy gate, not a progress clock. - Notify-only: does not kill the turn (contrast ``gateway_timeout`` / - ``shutdown_watchdog``). + Progress comes only from ``get_activity_summary()``. Pending inbound is a notify policy + gate, not a progress clock. Notify-only: does not kill the turn (contrast + ``gateway_timeout`` / ``shutdown_watchdog``). """ # Short initial delay so startup reconnect noise does not false-fire. await asyncio.sleep(min(30.0, max(1.0, float(interval)))) @@ -15937,48 +14617,31 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # ── Kanban board watchers ─────────────────────────────────────────── # The kanban notifier/dispatcher watcher loops + their helpers live in - # GatewayKanbanWatchersMixin (gateway/kanban_watchers.py). They use only - # self state, so inheriting the mixin keeps every self._kanban_* call site - # working unchanged while lifting ~1,000 LOC out of this file. + # GatewayKanbanWatchersMixin (gateway/kanban_watchers.py). - #: Interval of the slow respawn tier that takes over once the reconnect - #: watcher has exhausted its supervised restart budget. Long on purpose: - #: the budget is spent precisely when the watcher is crashing on contact, - #: so the useful cadence is "check back later", not "try again now". A - #: tight loop here would be worse than the outage it is healing. + #: Interval of the slow respawn tier that takes over once the reconnect watcher has exhausted + #: its supervised restart budget. Long on purpose: the budget is spent precisely when the + #: watcher is crashing on contact, so the useful cadence is "check back later". A tight loop + #: here would be worse than the outage it is healing. _RECONNECT_WATCHER_SLOW_RETRY_SECS = 300 - #: How many slow-tier respawns to attempt while work is still queued. - #: Bounded, not infinite: if half an hour of five-minute retries cannot - #: keep a watcher alive, the fault is not transient and a louder failure - #: is more useful than a quieter one that never stops. + #: How many slow-tier respawns to attempt while work is still queued. Bounded, not infinite: + #: if half an hour of five-minute retries cannot keep a watcher alive, the fault is not + #: transient and a louder failure is more useful than a quieter one that never stops. _MAX_SLOW_WATCHER_RESPAWNS = 6 def _on_reconnect_watcher_gave_up(self, name: str = "") -> None: """Own the reconnect invariant once supervision has abandoned it. - The invariant this closes: **while the gateway is running and - ``_failed_platforms`` is non-empty, either a reconnect watcher is live - or a bounded respawn is scheduled.** - - Before this, the only thing that noticed a dead watcher was a *later - fatal error from some other platform* reaching - ``_queue_retryable_fatal_platform``. That is event-coupled recovery: it - needs an event that, by construction, may never come. #81036 moved - queue publication ahead of disconnect and drops the failed adapter from - the live map, so once the watcher's budget is spent there may be no - adapter left that can emit the event recovery was waiting on. The - platform stays queued, nothing retries it, and the stranded check in - ``_handle_adapter_fatal_error_detached`` treats a queued platform as - safe — so the process is never restarted either. - - Deliberately NOT done here: requesting a supervisor/process restart - when the slow tier is also exhausted. That is a policy decision about - blast radius (a gateway serving healthy platforms would be taken down - to heal a sick one) and it belongs to a maintainer, not to this patch. - What happens instead is a single loud error naming the still-queued - platforms, which is the state an operator or an external supervisor can - act on. + Invariant: while the gateway is running and ``_failed_platforms`` is non-empty, either a + reconnect watcher is live or a bounded respawn is scheduled. Previously only a *later + fatal error from another platform* noticed a dead watcher — event-coupled recovery that + needs an event which, by construction, may never come (the failed adapter is dropped from + the live map, so nothing is left to emit it; the platform stays queued and the stranded + check treats a queued platform as safe, so the process is never restarted either). + Deliberately NOT done here: requesting a supervisor/process restart when the slow tier is + also exhausted — that is a blast-radius policy decision for a maintainer. Instead a single + loud error names the still-queued platforms for an operator/external supervisor. """ if not getattr(self, "_running", False): return @@ -16039,10 +14702,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _spawn_reconnect_watcher(self, *, on_give_up=None): """Single place that knows how to launch the reconnect watcher. - Three call sites used to repeat this triple (factory, name, on_spawn), - and the ``on_spawn`` half of it is load-bearing: without it the - supervisor's own respawn leaves ``_reconnect_watcher_task`` pointing at - a dead handle and ``_ensure_...`` spawns a second concurrent watcher. + The ``on_spawn`` half is load-bearing: without it the supervisor's own respawn leaves + ``_reconnect_watcher_task`` pointing at a dead handle and ``_ensure_...`` spawns a second + concurrent watcher. """ self._reconnect_watcher_task = self._spawn_supervised( self._platform_reconnect_watcher, @@ -16055,14 +14717,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _ensure_reconnect_watcher_running(self) -> None: """Ensure the platform reconnect watcher background task is alive. - If the tracked reconnect watcher task has died (e.g. from exhausting - its restart budget, or a terminal exception that _spawn_supervised - could not recover), respawns it so platforms queued for reconnection - are not permanently stranded. Called from - _queue_retryable_fatal_platform on BOTH paths (#70344, #90386): after a - new enqueue, and after a re-fatal for a platform that is already queued - -- the latter being the only case in which the watcher can have been - retrying long enough to exhaust its supervised restart budget. + If the tracked reconnect watcher task has died (e.g. from exhausting its restart budget, + or a terminal exception that _spawn_supervised could not recover), respawns it so + platforms queued for reconnection are not permanently stranded. Called from + _queue_retryable_fatal_platform on BOTH paths: after a new enqueue, and after a re-fatal + for an already-queued platform — the only case in which the watcher can have been + retrying long enough to exhaust its budget. """ if not getattr(self, "_running", False): return @@ -16078,16 +14738,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _platform_reconnect_watcher(self) -> None: """Background task that periodically retries connecting failed platforms. - Uses exponential backoff: 30s → 60s → 120s → 240s → 300s (cap). - Retryable failures (network/DNS blips) keep retrying at the backoff - cap indefinitely — they self-heal once connectivity returns, so a - transient outage never requires manual intervention. Non-retryable - failures (bad auth, etc.) drop out of the queue immediately. The - circuit breaker (``_pause_failed_platform`` / ``/platform pause``) - remains available for manual operator control via ``/platform list`` - and ``/platform resume ``, but is no longer triggered - automatically — auto-pausing a recovered platform was the cause of - bots silently staying dead after a transient DNS failure. + Uses exponential backoff: 30s → 60s → 120s → 240s → 300s (cap). Retryable failures + (network/DNS blips) keep retrying at the backoff cap indefinitely — they self-heal once + connectivity returns, so a transient outage never requires manual intervention. + Non-retryable failures (bad auth) drop out of the queue immediately. The circuit breaker + (``/platform pause``) stays available manually but is never triggered automatically — + auto-pausing left bots silently dead after a transient DNS failure. """ await asyncio.sleep(10) # initial delay — let startup finish while self._running: @@ -16107,10 +14763,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return info = self._failed_platforms.get(platform) if info is None: - # Removed concurrently (e.g. a manual /platform resume, - # or a reconnect that succeeded via a different path) - # between the snapshot above and this lookup. Not an - # error -- just nothing to do for it this pass. + # Removed concurrently (e.g. a manual /platform resume, or a reconnect that + # succeeded via a different path) between the snapshot above and this lookup. + # Not an error -- just nothing to do for it this pass. continue # Skip paused platforms entirely — they need explicit # /platform resume to come back. @@ -16213,10 +14868,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) logger.info("✓ %s reconnected successfully", platform.value) - # Final responses rejected while this adapter was down - # are still owned by this live process, so startup - # recovery cannot claim them. Replay the explicitly - # transient subset now that the platform is usable. + # Final responses rejected while this adapter was down are still owned by + # this live process, so startup recovery cannot claim them. Replay the + # explicitly transient subset now that the platform is usable. try: await self._redeliver_failed_obligations_for_platform( platform @@ -16235,12 +14889,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: pass - # A platform that was offline at gateway startup never - # got its restart-interrupted sessions auto-resumed — - # the startup pass skips sessions whose adapter isn't - # connected yet. Now that it's back, retry the - # auto-resume scoped to this platform so recovery - # doesn't silently wait for a manual user message. + # A platform that was offline at gateway startup never got its restart- + # interrupted sessions auto-resumed — the startup pass skips sessions whose + # adapter isn't connected yet. try: self._schedule_resume_pending_sessions(platform=platform) except Exception: @@ -16261,14 +14912,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Reconnect %s: non-retryable error (%s), removing from retry queue", platform.value, adapter.fatal_error_message, ) - # The adapter is about to be dropped from the queue - # without ever being installed on self.adapters, so - # nothing else will call disconnect() on it. We must - # dispose it here, otherwise the resource owners it - # constructed in __init__ (ResponseStore for - # APIServerAdapter, etc.) leak 2 fds each. The - # gateway hits the 2560-fd limit after ~12h of - # failed reconnects at the 300s backoff cap (#37011). + # The adapter is about to be dropped from the queue without ever being + # installed on self.adapters, so nothing else will call disconnect() on it. + # Dispose here or the resource owners built in __init__ (ResponseStore etc.) + # leak ~2 fds each; at the 300s cap the gateway hits the fd limit in ~12h. await _dispose_unused_adapter(adapter) del self._failed_platforms[platform] else: @@ -16285,30 +14932,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Reconnect %s failed, next retry in %ds", platform.value, backoff, ) - # Same fd-leak concern as the non-retryable branch - # above: the adapter failed to connect and is being - # thrown away. Without an explicit dispose call, the - # resources it opened in __init__ stay open until - # the next GC pass — and aiohttp/SQLite handles - # don't get GC'd promptly, so 2 fds/retry leak at - # 300s backoff cap = ~12 fds/hour (#37011). + # Same fd-leak concern as the non-retryable branch above: the adapter failed + # to connect and is being thrown away. await _dispose_unused_adapter(adapter) - # Retryable failures (network/DNS blips) keep retrying - # at the backoff cap indefinitely — they self-heal once - # connectivity returns. We do NOT auto-pause them: a - # transient outage must never require manual `/platform - # resume` to recover. Non-retryable failures (bad auth, - # etc.) already drop out of the queue via the - # `not fatal_error_retryable` branch above, so anything - # reaching here is by definition retryable. + # Retryable failures (network/DNS blips) keep retrying at the backoff cap + # indefinitely — they self-heal once connectivity returns. We do NOT + # auto-pause them: a transient outage must never require a manual + # `/platform resume`. Anything reaching here is retryable by construction. except Exception as e: if adapter is not None: - # An exception escaping the connect call path - # (DNS timeout, aiohttp server.start() crash, etc.) - # leaves the adapter in the same unowned state as - # the two branches above. Dispose so __init__ - # resources don't accumulate while the watcher - # keeps retrying. + # An exception escaping the connect call path (DNS timeout, aiohttp + # server.start() crash, etc.) leaves the adapter in the same unowned state + # as the two branches above. await _dispose_unused_adapter(adapter) self._update_platform_runtime_status( platform.value, @@ -16336,11 +14971,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _cancel_secondary_profile_reconnect_tasks(self) -> None: """Cancel profile-scoped reconnects before tearing down their registry. - A reconnect can be waiting in adapter setup while shutdown begins. It - must not republish an adapter after the secondary registry is drained. - Waiting is bounded by the same adapter-cleanup budget; if a task does - not finish in time, the stopped runner state still prevents it from - installing an adapter when it eventually resumes. + A reconnect can be waiting in adapter setup while shutdown begins. It must not republish + an adapter after the secondary registry is drained. Waiting is bounded by the adapter- + cleanup budget; a task that overruns is still blocked by the stopped runner state. """ pending = self._profile_failed_platforms if not isinstance(pending, dict): @@ -16418,15 +15051,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew caller can notify their owners while adapters are still up (#82232). Empty list when no cron work was in flight. - Called twice in the shutdown path: once eagerly after a - drain timeout forces agent interrupt (so we reclaim bash/ - sleep children before systemd TimeoutStopSec escalates to - SIGKILL on the cgroup — #8202), and once as a final - catch-all at the end of _stop_impl() for the graceful - path or anything respawned mid-teardown. - - All steps are best-effort; exceptions are swallowed so - one subsystem's failure doesn't block the rest. + Called twice in shutdown: eagerly after a drain timeout forces agent interrupt + (reclaim bash/sleep children before systemd TimeoutStopSec escalates to SIGKILL), + and as a final catch-all at the end of _stop_impl(). All steps best-effort; + exceptions swallowed so one subsystem cannot block the rest. """ try: from tools.process_registry import process_registry @@ -16440,14 +15068,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("process_registry.kill_all (%s) error: %s", phase, _e) _marked_cron_jobs: list = [] try: - # Any cron job still dispatched at this instant just had - # its tool subprocess killed above (kill_all() has no - # per-job-ID targeting — it's a global sweep). Its agent - # thread is still alive in this process and may go on to - # produce a plausible-looking final response from the - # now-truncated tool output; mark the run interrupted so - # the scheduler can never report that as success (#60432). - # No-op when no cron job is in flight. + # Any cron job still dispatched at this instant just had its tool subprocess + # killed above (kill_all() has no per-job-ID targeting — it's a global sweep). + # Its agent thread may still produce a plausible final response from the + # truncated output; mark the run interrupted so it can never be reported as + # success. No-op when no cron job is in flight. from cron.scheduler import mark_running_jobs_interrupted _interrupted = _marked_cron_jobs = mark_running_jobs_interrupted( f"Gateway shutdown ({phase}) killed the job's tool " @@ -16482,12 +15107,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("cleanup_all_browsers (%s) error: %s", phase, _e) return _marked_cron_jobs - # Thread-based shutdown watchdog (#66892): asyncio timeouts cannot - # recover a frozen loop. Arm a plain OS thread at the start of - # stop(); if teardown never finishes within drain+grace it dumps - # faulthandler stacks and os._exit so KeepAlive/systemd can revive. - # Skip under pytest so stop()-driving unit tests don't get a - # delayed hard-exit in the worker. + # Thread-based shutdown watchdog: asyncio timeouts cannot recover a frozen loop. Arm a + # plain OS thread at the start of stop(); if teardown never finishes within drain+grace + # it dumps faulthandler stacks and os._exit so KeepAlive/systemd can revive. Skipped + # under pytest so stop()-driving tests don't get a delayed hard-exit in the worker. _watchdog_done = threading.Event() self._shutdown_watchdog_done = _watchdog_done _stop_started_at_box: dict[str, float] = {} @@ -16585,10 +15208,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew timeout = self._restart_drain_timeout - # Pre-mark sessions as resume_pending BEFORE the drain wait. - # If the process is killed by the service manager during the - # drain, the durable marker is already written so the next - # gateway boot can recover in-flight sessions (#27856). + # Pre-mark sessions as resume_pending BEFORE the drain wait. If the process is killed by + # the service manager during the drain, the durable marker is already written so the + # next gateway boot can recover in-flight sessions. _pre_drain_keys: list[str] = [] for _sk, _agent in list(self._running_agents.items()): if _agent is _AGENT_PENDING_SENTINEL: @@ -16605,12 +15227,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _cron_at_start = self._active_cron_job_count() _api_at_start = self._active_api_run_count() _deferred_at_start = _deferred_worker_count() - # In-flight cron work gets its own floor, clamped to the watchdog - # leash we're already running under so the extra wait can never - # cost us the post-drain cleanup window (#82161). - # getattr-guard: shutdown-path tests drive _stop_impl_body from - # bare doubles that aren't GatewayRunner instances, so they don't - # pick up the class-level default. + # In-flight cron work gets its own floor, clamped to the watchdog leash we're already + # running under so the extra wait can never cost us the post-drain cleanup window. + # getattr-guard: shutdown-path tests drive _stop_impl_body from bare doubles that aren't + # GatewayRunner instances, so they don't pick up the class-level default. _cron_drain_cfg = getattr( self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT ) @@ -16764,23 +15384,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Re-signaled interrupt for work still live at settle-window exit" ) - # Kill lingering tool subprocesses NOW, before we spend more - # budget on adapter disconnect / session DB close. Under - # systemd (TimeoutStopSec bounded by drain_timeout+headroom), - # deferring this to the end of stop() risks systemd escalating - # to SIGKILL on the cgroup first — at which point bash/sleep - # children left behind by an interrupted terminal tool get - # killed by systemd instead of us (issue #8202). The final - # catch-all cleanup below still runs for the graceful path. + # Kill lingering tool subprocesses NOW, before we spend more budget on adapter + # disconnect / session DB close: under systemd (TimeoutStopSec ≈ drain_timeout + + # headroom) deferring risks SIGKILL on the cgroup first, so orphaned bash/sleep + # children die by systemd instead of us. The final catch-all below still runs for + # the graceful path. _interrupted_cron_jobs = _kill_tool_subprocesses("post-interrupt") logger.info( "Shutdown phase: post-interrupt tool kill done at +%.2fs", _phase_elapsed(), ) - # Last window where the transport is still up. The cron worker - # whose run we just killed will try to deliver its own - # "interrupted" notice, but it gets there after the adapter - # teardown below and the message is lost (#82232). + # Last window where the transport is still up. The cron worker whose run we just + # killed will try to deliver its own "interrupted" notice, but it gets there after + # the adapter teardown below and the message is lost. try: await self._notify_interrupted_cron_jobs(_interrupted_cron_jobs) except Exception as _e: @@ -16798,11 +15414,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await self._finalize_shutdown_agents(active_agents) - # Also shut down memory providers on idle cached agents. - # _finalize_shutdown_agents only handles agents that were - # mid-turn at drain time; the _agent_cache may still hold - # idle agents whose MemoryProviders never received - # on_session_end(). + # Also shut down memory providers on idle cached agents. _finalize_shutdown_agents only + # handles agents that were mid-turn at drain time; the _agent_cache may still hold idle + # agents whose MemoryProviders never received on_session_end(). _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock is not None and _cache is not None: @@ -16820,10 +15434,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _agent, context="shutdown idle-cache" ) - # Completion flush tasks can be sleeping in their fan-in window or - # blocked in adapter delivery. Cancel and await them while adapters - # are still alive so every watcher receives a retryable result - # before platform teardown begins. + # Completion flush tasks can be sleeping in their fan-in window or blocked in adapter + # delivery. Cancel and await them while adapters are still alive so every watcher + # receives a retryable result before platform teardown begins. cancel_completion_batches = getattr( self, "_cancel_process_completion_batch_tasks", None ) @@ -16851,10 +15464,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if _task is self._stop_task: continue if _task is self._restart_task: - # The restart orchestration task is awaiting _stop_task - # right now; cancelling it would propagate CancelledError - # into this _stop_impl and skip _shutdown_event.set() / - # _exit_code = 75 (#12875). It self-terminates anyway. + # The restart orchestration task is awaiting _stop_task right now; cancelling it + # would propagate CancelledError into this _stop_impl and skip + # _shutdown_event.set() / _exit_code = 75. It self-terminates anyway. continue _task.cancel() self._background_tasks.clear() @@ -16862,10 +15474,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self.adapters.clear() for _session_key in list(self._running_agents): self._release_running_agent_state(_session_key) - # Flush pending messages to disk before clearing (#72680). - # When FTS5 corruption prevents message persistence, the - # in-memory pending text is the only surviving copy. Clearing - # without flushing causes permanent data loss. + # Flush pending messages to disk before clearing. When FTS5 corruption prevents message + # persistence, the in-memory pending text is the only surviving copy. Clearing without + # flushing causes permanent data loss. try: from gateway.shutdown_flush import flush_pending_to_file flush_pending_to_file(dict(self._pending_messages), reason="shutdown") @@ -16886,10 +15497,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except Exception: pass - # On the real runner these are live SessionState views whose - # clear() resets one field per session — never a wholesale dict - # swap, so a concurrent writer on another session can't lose its - # entry. Test fakes borrowing _stop_impl keep plain dicts. + # On the real runner these are live SessionState views whose clear() resets one field + # per session — never a wholesale dict swap, so a concurrent writer on another session + # can't lose its entry. Test fakes borrowing _stop_impl keep plain dicts. self._running_agents.clear() self._running_agents_ts.clear() if hasattr(self, "_active_session_leases"): @@ -16926,29 +15536,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _e: logger.debug("shutdown_cached_clients error: %s", _e) - # Quiesce the gateway thread pool BEFORE the session databases - # are closed. This used to run *after* the close block below, - # which left two holes: - # - # (a) `_executor_closing` was still False during the close, so - # any coroutine reaching `_run_in_executor_with_context` - # minted a brand-new pool and ran more blocking DB work - # against handles that had just been closed; - # (b) cancelling `self._background_tasks` above does not stop a - # `run_in_executor` future that already started — the task - # dies, the worker thread keeps writing. - # - # Either way a write lands after `SessionDB.close()`, which has - # already checkpointed the WAL and let SQLite unlink the sidecar. - # The late write silently reopens the handle (#94736) and mints a - # fresh WAL generation behind that checkpoint, so teardown - # checkpoints the same file a second time from a connection the - # shutdown log never accounts for — the close-time page-write - # damage in #101093 and the split WAL generation in #101064. - # - # The wait is bounded and clamped to what is left of the shutdown - # watchdog leash (minus a second for the close itself), so a stuck - # worker can never cost us the post-close cleanup window (#82161). + # Quiesce the gateway thread pool BEFORE the session databases are closed. Running it + # after the close left two holes: (a) ``_executor_closing`` was still False, so any + # coroutine reaching ``_run_in_executor_with_context`` minted a fresh pool and ran more + # blocking DB work against just-closed handles; (b) cancelling ``self._background_tasks`` + # does not stop a ``run_in_executor`` future that already started — the task dies, the + # worker keeps writing. Either way a write lands after ``SessionDB.close()`` has + # checkpointed the WAL and let SQLite unlink the sidecar; the late write silently + # reopens the handle and mints a fresh WAL generation behind that checkpoint, so + # teardown checkpoints the same file twice from an unaccounted connection + # (close-time page-write corruption / split WAL generation). + # The wait is bounded and clamped to what is left of the shutdown watchdog leash + # (minus a second for the close itself), so a stuck worker can never cost us the + # post-close cleanup window. _exec_quiesce_budget = max( 0.0, min( @@ -16984,13 +15584,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _phase_elapsed(), ) - # Close SQLite session DBs so the WAL write lock is released. - # Without this, --replace and similar restart flows leave the - # old gateway's connection holding the WAL lock until Python - # actually exits — causing 'database is locked' errors when - # the new gateway tries to open the same file. - # ``self`` holds the DB at ``_session_db`` (an AsyncSessionDB facade); - # unwrap to the sync handle. ``session_store`` holds it at ``_db``. + # Close SQLite session DBs so the WAL write lock is released. Otherwise --replace and + # similar restart flows leave the old connection holding the lock until Python exits, + # so the new gateway gets 'database is locked'. ``self`` holds the DB at + # ``_session_db`` (an AsyncSessionDB facade); unwrap to the sync handle. + # ``session_store`` holds it at ``_db``. _self_db = getattr(self, "_session_db", None) _self_db = getattr(_self_db, "_db", _self_db) for _db in (_self_db, getattr(getattr(self, "session_store", None), "_db", None)): @@ -17019,11 +15617,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew GatewayRunner.close_all_session_db_handles(self) except Exception as _e: logger.debug("Runner SessionDB handle sweep error: %s", _e) - # Final sweep: close any shared SessionDB instances still held by - # the process-wide registry (in-process tools, cron, mirror, etc. - # that opened via get_shared_session_db but weren't released by - # the sweeps above). This is the safety net that guarantees no - # WAL write lock survives past gateway shutdown (#90837). + # Final sweep: close any shared SessionDB instances still held by the process-wide + # registry (in-process tools, cron, mirror, etc. that opened via get_shared_session_db + # but weren't released by the sweeps above). try: from hermes_state import close_shared_session_dbs closed = close_shared_session_dbs() @@ -17040,14 +15636,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew remove_pid_file() release_gateway_runtime_lock() - # Write a clean-shutdown marker so the next startup knows this - # wasn't a crash. suspend_recently_active() only needs to run - # after unexpected exits. However, if the drain timed out and - # agents were force-interrupted, their sessions may be in an - # incomplete state (trailing tool response, no final assistant - # message). Skip the marker in that case so the next startup - # suspends those sessions — giving users a clean slate instead - # of resuming a half-finished tool loop. + # Write a clean-shutdown marker so the next startup knows this wasn't a crash. + # suspend_recently_active() only needs to run after unexpected exits. If the drain + # timed out and agents were force-interrupted, their sessions may be half-finished + # (trailing tool response, no final assistant message) — skip the marker so the next + # startup suspends them and users get a clean slate. if not timed_out: try: (_hermes_home / ".clean_shutdown").touch() @@ -17083,37 +15676,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("Failed to write planned restart notification marker: %s", e) if self._restart_requested and self._restart_via_service: - # The service manager is the sole restart owner. Exit 75 - # paired with ``RestartForceExitStatus=75`` asks systemd to - # replace this process without a second helper racing the - # unit's stop/start job. The generated launchd plist's - # unconditional ``KeepAlive`` likewise replaces the process - # after this planned exit. + # The service manager is the sole restart owner. Exit 75 paired with + # ``RestartForceExitStatus=75`` asks systemd to replace this process without a + # second helper racing the unit's stop/start job. self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" self._draining = False - # Persist the terminal gateway_state. The default is "stopped", - # but when this teardown was triggered by an UNEXPECTED external - # signal (container/s6 SIGTERM on `docker restart` or image - # upgrade, OOM-killer, bare `kill`) we instead persist "running" - # to preserve the operator's run-intent across the restart. - # - # On Docker (s6-overlay), container_boot.py reads gateway_state - # on the next boot and only auto-starts gateways whose last - # state was "running" (_AUTOSTART_STATES). Persisting "stopped" - # — or leaving the mid-shutdown "draining" marker in place — for - # a routine `docker compose up --force-recreate` permanently - # suppresses auto-start, so the messaging channels silently stay - # dark until the operator manually restarts (issue #42675). - # - # An operator-initiated stop (`hermes gateway stop`, - # systemd/launchd ExecStop, the s6 stop path, Ctrl+C) writes a - # planned-stop marker BEFORE signalling, so it is classified as - # a planned stop (not signal-initiated) and correctly persists - # "stopped" — respecting the explicit intent. A restart also - # persists "stopped" here; the restarting process brings the - # gateway back up itself. + # Persist the terminal gateway_state. Default "stopped", but an UNEXPECTED external + # signal (container/s6 SIGTERM on docker restart, OOM-kill, bare kill) persists + # "running" to preserve run-intent: container_boot.py only auto-starts gateways whose + # last state was "running", so writing "stopped" (or leaving "draining") for a routine + # `docker compose up --force-recreate` would leave channels dark until a manual + # restart. Operator-initiated stops write a planned-stop marker BEFORE signalling, so + # they persist "stopped"; a restart also persists "stopped" (the new process brings + # the gateway back up itself). if getattr(self, "_signal_initiated_shutdown", False) and not self._restart_requested: logger.info( "Gateway stopped by an unexpected signal — persisting " @@ -17139,14 +15716,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew Returns the number of secondary adapters that connected. No-op (returns 0) unless ``gateway.multiplex_profiles`` is on. - Each profile's adapters are created and connected under that profile's - HERMES_HOME + secret scope (``_profile_runtime_scope``), stored in - ``self._profile_adapters[profile]``, and given a message handler that - stamps ``source.profile`` before delegating to the shared - ``_handle_message`` — so the agent turn resolves that profile's config, - skills, and credentials. Same-platform credential collisions (two - profiles polling the same bot token) are detected and refused here, the - only point that sees every profile's resolved credentials together. + Each profile's adapters connect under that profile's HERMES_HOME + secret scope, live in + ``self._profile_adapters[profile]``, and get a handler that stamps ``source.profile`` so + the turn resolves that profile's config/skills/credentials. Same-platform credential + collisions (two profiles polling one bot token) are refused here — the only point that + sees every profile's resolved credentials together. """ if not getattr(self.config, "multiplex_profiles", False): return 0 @@ -17200,20 +15774,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew profile_name, e, exc_info=True, ) - # Record the authoritative served set in runtime status for `hermes status`. - # "Served" means eligible for shared routing, HTTP prefixes, cron, and - # profile runtime scope; it is intentionally broader than profiles with a - # successfully connected secondary adapter (or any adapter configured). + # Record the authoritative served set in runtime status for `hermes status`. "Served" + # means eligible for shared routing, HTTP prefixes, cron, and profile runtime scope — + # intentionally broader than profiles with a connected (or any) secondary adapter. try: from gateway.status import write_runtime_status from gateway.pairing import PairingStore served = [active] + sorted( name for name, _home in profile_homes if name != active ) - # Per-profile PairingStores so authz_mixin can route pairing - # checks to the right whitelist. The active profile gets a store - # at its HERMES_HOME; additional served profiles resolve from - # their own profile homes. See gateway.pairing.PairingStore. + # Per-profile PairingStores so authz_mixin can route pairing checks to the right + # whitelist. The active profile gets a store at its HERMES_HOME; additional served + # profiles resolve from their own profile homes. See gateway.pairing.PairingStore. for name in served: if name and name not in self.pairing_stores: self.pairing_stores[name] = ( @@ -17234,10 +15806,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew from gateway.config import load_gateway_config from hermes_cli.env_loader import hydrate_profile_secret_sources - # Hydrate external secret sources (1Password/vault/...) off-loop ONCE, - # then enter the scope without re-hydrating: the sync hydration is - # network-bound and would otherwise stall every other profile's - # heartbeat while this one boots (same class as the reconnect path). + # Hydrate external secret sources (1Password/vault/...) off-loop ONCE, then enter the scope + # without re-hydrating: the sync hydration is network-bound and would otherwise stall every + # other profile's heartbeat while this one boots (same class as the reconnect path). await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) with _profile_runtime_scope(profile_home, hydrate_secrets=False): @@ -17246,14 +15817,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew discover_plugins() - # Register this profile's own declarative shell hooks and - # outbound webhooks. The startup-time registration in - # start() only ever sees the root/default profile's config - # (it runs before any profile scope exists), so without this - # a secondary profile's `hooks:` block is silently inert — - # its turns run under this profile's own plugin manager - # (hermes_cli.plugins.get_plugin_manager keys by resolved - # home), which never received the callbacks. + # Register this profile's own declarative shell hooks and outbound webhooks. The + # registration in start() runs before any profile scope exists and only sees the root + # profile's config, so without this a secondary profile's `hooks:` block is silently + # inert (its turns use a plugin manager keyed by resolved home). try: from hermes_cli.config import load_config as _load_profile_config from agent.shell_hooks import ( @@ -17306,14 +15873,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew for platform, platform_config in profile_cfg.platforms.items(): if not platform_config.enabled: continue - # A platform enabled in a secondary profile's config.yaml may - # have no credential in that profile's secret scope — the shared - # YAML enables it for the default profile only (#84079). Building - # an adapter here would treat every credential-less profile as - # configured for the platform and one inbound message would fan - # out across all of them. Mirror the primary startup loop's - # credential gate and skip instead; profiles with their own - # credential still connect below. + # A platform enabled in a secondary profile's config.yaml may have no credential in that + # profile's secret scope — the shared YAML enables it for the default profile only. + # Building an adapter anyway would fan one inbound message out across every + # credential-less profile; mirror the primary loop's credential gate and skip. if ( getattr(self.config, "multiplex_profiles", False) and not _platform_has_bot_credential(platform, platform_config) @@ -17325,14 +15888,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew platform.value, ) continue - # Relay and WhatsApp are shared process-level ingress in multiplex - # mode: one connection owned by the active profile, with - # route-stamped source.profile fanning inbound turns out to - # secondary profiles. The WhatsApp bridge is a single authenticated - # session tied to one phone number -- a secondary profile has no - # credential of its own to bring, so constructing an adapter for it - # only yields a connect/retry loop that stalls startup for every - # profile queued behind it. + # Relay and WhatsApp are shared process-level ingress in multiplex mode: one connection + # owned by the active profile, with route-stamped source.profile fanning inbound turns + # out to secondary profiles. The WhatsApp bridge is one authenticated session tied to + # one phone number; a secondary has no credential to bring, so an adapter for it would + # only connect/retry-loop and stall startup for every profile queued behind it. if ( getattr(self.config, "multiplex_profiles", False) and platform in (Platform.RELAY, Platform.WHATSAPP) @@ -17381,10 +15941,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew error_code="duplicate_credential", error_message=message, ) - # This adapter has not connected and therefore owns no - # resources to clean up. Calling disconnect here can mutate - # the shared platform state and, for a same-credential Photon - # adapter, shut down the primary profile's live sidecar. + # This adapter has not connected and therefore owns no resources to clean up. + # Calling disconnect here can mutate the shared platform state and, for a same- + # credential Photon adapter, shut down the primary profile's live sidecar. continue listener_claim = self._adapter_listener_claim(platform, adapter) @@ -17468,12 +16027,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._make_profile_fatal_error_handler(profile_name, platform) ) adapter.set_session_store(self.session_store) - # Declare credential ownership BEFORE any inbound event can be handled. - # Adapter-level session keys (text/media batching, _active_sessions, the - # busy guard) are derived at ingress, before _make_profile_message_handler - # stamps source.profile — without this every secondary bot would key into - # the default profile's `agent:main:` lane and share it (see - # BasePlatformAdapter._session_key_profile). + # Declare credential ownership BEFORE any inbound event can be handled: adapter-level + # session keys (batching, _active_sessions, busy guard) are derived at ingress, before the + # handler stamps source.profile — without this every secondary bot would key into the + # default profile's `agent:main:` lane (see BasePlatformAdapter._session_key_profile). _set_owner = getattr(adapter, "set_owner_profile", None) if callable(_set_owner): _set_owner(profile_name) @@ -17626,25 +16183,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Queue a cold-start reconnect for a secondary adapter. - Startup failure branches run BEFORE ``self._running`` flips True - (``_start_secondary_profile_adapters()`` is called mid-``start()``, - while ``self._running`` is still False), so the regular scheduler's - ``not self._running`` guard would silently drop the request and the - runner's ``while self._running`` loop would exit immediately. This - bridge parks a background task across the remainder of startup and - hands off to the regular scheduler once the gateway is live (the - scheduler's own ``_profile_failed_platforms`` slot dedupes at - handoff); if shutdown begins first, the request is released. - Non-retryable failures are dropped here exactly as the regular - scheduler would. + Startup failure branches run BEFORE ``self._running`` flips True, so the regular + scheduler's ``not self._running`` guard would drop the request and the runner loop would + exit immediately. This bridge parks a task across the rest of startup and hands off to + the regular scheduler once live (its ``_profile_failed_platforms`` slot dedupes); if + shutdown begins first, the request is released. Non-retryable failures are dropped here + exactly as the regular scheduler would. """ if not getattr(adapter, "fatal_error_retryable", True): return if is_global_startup_conflict(getattr(adapter, "fatal_error_code", None)): - # Same startup contract as the primary path: a live foreign holder - # of this profile's token/identity is an ownership conflict, not - # a transient blip. Park it fatal (like ``duplicate_credential``) - # instead of retry-storming the token every backoff (#83183). + # Same startup contract as the primary path: a live foreign holder of this profile's + # token/identity is an ownership conflict, not a transient blip. Park it fatal (like + # ``duplicate_credential``) instead of retry-storming the token every backoff. logger.error( "[MULTIPLEX] Profile '%s': %s credential is held by another " "gateway (%s) — parked, not retried. %s", @@ -17688,10 +16239,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew profile_name, platform, adapter ) except Exception: - # The handoff touches live registries; if it raises, the - # parked task would otherwise die as an unretrieved-task - # exception logged only at GC time. Surface it where - # operators look. + # The handoff touches live registries; if it raises, the parked task would + # otherwise die as an unretrieved-task exception logged only at GC time. Surface + # it where operators look. logger.exception( "secondary-startup-reconnect handoff failed " "(profile=%s platform=%s)", @@ -17752,10 +16302,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Remove a failed multiplexed adapter without touching the primary slot. - Secondary adapters are owned by ``_profile_adapters`` rather than - ``self.adapters``. The primary-only fatal handler intentionally ignores - them; without this route, a fatal secondary Discord client stayed live - forever after its liveness sampler stopped. + Secondary adapters are owned by ``_profile_adapters`` rather than ``self.adapters``. The + primary-only fatal handler intentionally ignores them; without this route, a fatal + secondary Discord client stayed live forever after its liveness sampler stopped. """ profile_map = getattr(self, "_profile_adapters", {}).get(profile_name) if not isinstance(profile_map, dict) or profile_map.get(platform) is not adapter: @@ -17782,10 +16331,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _make_profile_message_handler(self, profile_name: str): """Return a message handler that stamps source.profile then delegates. - Auth runs inside ``_handle_message`` *before* the agent-turn scope is - installed. For secondary profiles under multiplex, wrap the whole - handler in ``_profile_runtime_scope`` so allowlists/tokens from that - profile's ``.env`` are visible to ``get_secret`` / authz. + Auth runs inside ``_handle_message`` *before* the agent-turn scope is installed. For + secondary profiles under multiplex, wrap the whole handler in ``_profile_runtime_scope`` + so allowlists/tokens from that profile's ``.env`` are visible to ``get_secret`` / authz. """ from hermes_cli.profiles import get_profile_dir @@ -17825,16 +16373,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _make_default_profile_message_handler(self): """Scope primary-adapter messages to their routed multiplex profile. - Profile routes are normally stamped on ``event.source`` before this - handler runs. Resolve the home per event so session lookup and transcript - loading use the same profile store as the later agent run and persistence - path. Authorization still belongs to the primary transport profile: a - shared Discord/Telegram adapter can route the turn to a profile that - intentionally has no bot credential or platform allowlist. Preserve that - transport home on the live source so the auth gate does not re-check the - sender against the routed runtime's unrelated secret scope. Sources that - bypass adapter routing are resolved here; genuinely unrouted events retain - the gateway's launch/default home. + Profile routes are normally stamped on ``event.source`` before this handler runs. Resolve + the home per event so session lookup and transcript loading use the same profile store as + the later agent run and persistence path. Authorization still belongs to the primary + transport profile (a shared adapter may route to a profile that intentionally has no bot + credential or allowlist), so the transport home is preserved on the live source and the + auth gate never re-checks the sender against the routed profile's secret scope. + Genuinely unrouted events retain the gateway's launch/default home. """ default_home = Path(get_hermes_home()) @@ -17853,11 +16398,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew try: source.profile = self._profile_name_for_source(source) except ProfileRouteRejected: - # NOT write-only: ``_handle_message``'s ingress gate reads - # this exact marker and drops the message fail-closed - # ("explicit profile route targets an unserved profile"). - # Setting it here also stops that gate from re-running - # routing for the same source. + # NOT write-only: ``_handle_message``'s ingress gate reads this exact marker and + # drops the message fail-closed ("explicit profile route targets an unserved + # profile"). source.profile_route_rejected = True profile_home = ( @@ -17928,12 +16471,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> bool: """Authorize under the live transport's profile, not the routed runtime. - A primary adapter may route one chat into another profile's agent/session - namespace. That runtime profile need not (and normally should not) copy the - shared bot token or allowlist. The primary message/platform-event handlers - stamp the transport home as an in-process-only attribute before entering - the routed scope; consult it here for the narrow authorization read, then - restore the routed scope for the remainder of the turn. + A primary adapter may route one chat into another profile's agent/session namespace. That + runtime profile need not (and normally should not) copy the shared bot token or + allowlist. The primary handlers stamp the transport home as an in-process-only attribute + before entering the routed scope; consult it here for the narrow authorization read, + then restore the routed scope for the rest of the turn. """ def _check() -> bool: # Preserve the historical one-argument seam used by plugins/tests; @@ -17970,11 +16512,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _adapter_listener_claim(platform: Platform, adapter: Any) -> Optional[tuple]: """Return the exclusive listener resource claimed by an adapter. - Photon sidecars are per-profile processes. Even when two profiles use - different project credentials, their sidecars cannot share a bind and - port. Represent that endpoint as a claim so multiplex startup rejects - the later adapter before either ``connect()`` or ``disconnect()`` can - disturb the first profile. + Even when two profiles use different project credentials, their sidecars cannot share a + bind and port. Represent that endpoint as a claim so multiplex startup rejects the later + adapter before either ``connect()`` or ``disconnect()`` can disturb the first profile. """ if getattr(platform, "value", None) != "photon": return None @@ -17992,10 +16532,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _adapter_credential_fingerprint(adapter: Any) -> Optional[str]: """Return a stable, log-safe fingerprint of an adapter's credential. - Used only to detect two profiles claiming the same platform credential. - Returns a salted hash (never the credential itself) of the adapter's - primary credential, or None when no credential is discoverable (in - which case we don't attempt conflict detection for it). + Used only to detect two profiles claiming the same platform credential. Returns a salted + hash (never the credential itself) of the adapter's primary credential, or None when no + credential is discoverable (in which case we don't attempt conflict detection for it). """ token = None for attr in ( @@ -18113,7 +16652,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return None return WhatsAppCloudAdapter(config) - + elif platform == Platform.SIGNAL: from gateway.platforms.signal import ( SignalAdapter, @@ -18189,30 +16728,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Callable[[str, Optional[str], Optional[str]], bool]: """Build a platform-bound auth callback for adapter use. - Adapters that fetch external context (e.g. Slack - ``conversations.replies``) call this through - ``BasePlatformAdapter._is_sender_authorized`` to mark non-allowlisted - senders as unverified in LLM context, mitigating indirect prompt - injection from third parties in shared threads/channels. - - The returned callback delegates to :meth:`_is_user_authorized` so the - full auth chain — platform allowlists, group allowlists, pairing - store, allow-all flags — stays the single source of truth. - - ``profile_name`` binds the callback to the secondary adapter's own - multiplex profile, so its ``SessionSource`` resolves that profile's - secret scope instead of falling back to the active profile. - - For the shared primary adapter under ``multiplex_profiles`` - (``profile_name`` is None) the callback mirrors the inbound message - path exactly: the chat's ``profile_routes`` match is stamped on the - source so the routed profile's pairing store is consulted, while the - allowlist/gate reads stay under the transport (launch) home via - ``_is_user_authorized_for_source`` — the same split - ``_make_default_profile_message_handler`` applies. Without this an - inline-button caller approved only in the routed profile's pairing - store was denied (#86296), because the adapter's callback source was - never route-stamped. + Adapters that fetch external context (e.g. Slack ``conversations.replies``) call this via + ``BasePlatformAdapter._is_sender_authorized`` to mark non-allowlisted senders as + unverified in LLM context (indirect prompt-injection mitigation). The returned callback + delegates to :meth:`_is_user_authorized` so the full auth chain — platform allowlists, + group allowlists, pairing store, allow-all flags — stays the single source of truth. + ``profile_name`` binds the callback to a secondary adapter's own profile so its + ``SessionSource`` resolves that secret scope. For the shared primary adapter under + multiplex (``profile_name`` None) the callback mirrors the inbound path: the + ``profile_routes`` match is stamped on the source so the routed profile's pairing store + is consulted, while allowlist reads stay under the transport home — without this a + caller approved only in the routed profile's pairing store was denied. """ multiplex = bool(getattr(self.config, "multiplex_profiles", False)) transport_home = ( @@ -18238,10 +16764,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew is_bot=bool(is_bot), profile=profile_name, ) - # Same in-process transport provenance ``build_source`` retains, so - # adapter-level policy reads (config.yaml group_allowed_chats, - # allow_from) resolve the receiving adapter even once the routed - # profile is stamped below. + # Same in-process transport provenance ``build_source`` retains, so adapter-level policy + # reads (config.yaml group_allowed_chats, allow_from) resolve the receiving adapter even + # once the routed profile is stamped below. registry = ( (getattr(self, "_profile_adapters", None) or {}).get(profile_name) if profile_name @@ -18264,11 +16789,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return self._is_user_authorized_for_source(source) return check - - - - - async def _deliver_platform_notice(self, source, content: str) -> None: """Deliver a setup/operational notice using platform-specific privacy rules.""" adapter = self._adapter_for_source(source) @@ -18318,10 +16838,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[SessionEntry]: """Resolve an async completion to its verified owning gateway session. - A compression rotation ends the physical parent row while continuing - the same logical conversation in a child. Follow that lineage, but - never let a late completion override an unrelated /new or restored - route. Unknown ownership remains fail-closed; the result is still + A compression rotation ends the physical parent row while continuing the same logical + conversation in a child. Follow that lineage, but never let a late completion override an + unrelated /new or restored route. Unknown ownership stays fail-closed; the result remains available in the delegation records. """ session_db = cast(Any, self._session_db) @@ -18364,16 +16883,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return None if _end_reason != "compression": - # Idle/timeout/lifecycle end (scale-to-zero norm): the chat - # route remains valid and ``session_entry`` IS the routing - # key's current session for this same chat, so deliver the - # finished work there instead of dropping it. This is the - # delivery leg _classify_completion_target promises when it - # returns "deliver" for non-boundary ends — without it the - # pre-flight verdict and this resolver disagree, and the - # durable row is acked at adapter acceptance then silently - # dropped here (falsely-acknowledged permanent loss; - # staging incident 2026-08-09 defect #2). + # Idle/timeout/lifecycle end (scale-to-zero norm): the chat route remains valid and + # ``session_entry`` IS the routing key's current session for this same chat, so + # deliver the finished work there instead of dropping it. This is the delivery leg + # _classify_completion_target promises for non-boundary ends; without it the row is + # acked at adapter acceptance then silently dropped (falsely-acknowledged loss). logger.info( "Async-delegation completion pinned to %s-ended session %s; " "retargeting to the chat's current session %s.", @@ -18532,18 +17046,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): """Dispatch a recognized slash command while an agent is running. - Resolution order: - 1. ``busy_handler`` — special mid-run variant (e.g. /goal's - control-verb whitelist, /queue's FIFO enqueue, /model's - custom reject text). - 2. ``busy_policy == "dispatch"`` — the command's normal handler. - 3. Catch-all busy-reject text. Rejecting is required rather than - falling through to interrupt + discard: commands like /model, - /reasoning, /voice, /insights, /title, /resume, /retry, - /undo, /compress, /usage, /reload-mcp, /sethome, /reset (all - registered as Discord slash commands) would interrupt the - agent AND get silently discarded by the slash-command safety - net, producing a zero-char response. See #5057, #6252, #10370. + Resolution order: 1. ``busy_handler`` — special mid-run variant (e.g. /goal's control- + verb whitelist, /queue's FIFO enqueue, /model's custom reject text). 2. ``busy_policy == + "dispatch"`` — the command's normal handler. 3. Catch-all busy-reject text. Rejecting is + required rather than falling through to interrupt + discard: Discord-registered slash + commands (/model, /reasoning, /reset, ...) would interrupt the agent AND be discarded by + the slash-command safety net, producing a zero-char response. """ name = cmd_def.name policy = getattr(cmd_def, "busy_policy", "reject") @@ -18626,11 +17134,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return format_status_text() async def _busy_stop_command(self, event: MessageEvent, quick_key: str, source): - # /stop must hard-kill the session when an agent is running. - # A soft interrupt (agent.interrupt()) doesn't help when the agent - # is truly hung — the executor thread is blocked and never checks - # _interrupt_requested. Force-clean _running_agents so the session - # is unlocked and subsequent messages are processed normally. + # /stop must hard-kill the session when an agent is running. A soft interrupt + # (agent.interrupt()) doesn't help when the agent is truly hung — the executor thread is + # blocked and never checks _interrupt_requested. await self._interrupt_and_clear_session( quick_key, source, @@ -18641,14 +17147,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return EphemeralReply(t("gateway.stop.stopped")) async def _busy_new_command(self, event: MessageEvent, quick_key: str, source): - # /reset and /new must bypass the running-agent guard so they - # actually dispatch as commands instead of being queued as user - # text (which would be fed back to the agent with the same - # broken history — #2170). Interrupt the agent first, then - # clear the adapter's pending queue so the stale "/reset" text - # doesn't get re-processed as a user message after the - # interrupt completes. - # Clear any pending messages so the old text doesn't replay + # /reset and /new must bypass the running-agent guard so they actually dispatch as commands + # instead of being queued as user text (which would be fed back to the agent with the same + # broken history — #2170). Clear any pending messages so the old text doesn't replay await self._interrupt_and_clear_session( quick_key, source, @@ -18660,14 +17161,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return await self._handle_reset_command(event) async def _busy_queue_command(self, event: MessageEvent, quick_key: str, source): - # /queue — queue without interrupting. - # Semantics: each /queue invocation produces its own full agent - # turn, processed in FIFO order after the current run (and any - # earlier /queue items) finishes. Messages are NOT merged. + # /queue — queue without interrupting. Semantics: each /queue invocation produces + # its own full agent turn, processed in FIFO order after the current run (and any earlier + # /queue items) finishes. Messages are NOT merged. queued_text = event.get_command_args().strip() - # Preserve media/reply payloads: a /queue carrying a photo, - # document, or reply context is valid even with no prompt text - # (e.g. "/queue" as the caption of an image). Dropping these + # Preserve media/reply payloads: a /queue carrying a photo, document, or reply context is + # valid even with no prompt text (e.g. "/queue" as the caption of an image). Dropping these # fields silently lost the attachment when the queued turn ran. has_media = bool(getattr(event, "media_urls", None)) if not queued_text and not has_media: @@ -18701,11 +17200,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return f"Queued for the next turn. ({depth} queued)" async def _busy_steer_command(self, event: MessageEvent, quick_key: str, source): - # /steer — inject mid-run after the next tool call. - # Unlike /queue (turn boundary), /steer lands BETWEEN tool-call - # iterations inside the same agent run, by appending to the - # last tool result's content. No interrupt, no new user turn, - # no role-alternation violation. + # /steer — inject mid-run after the next tool call. Unlike /queue (turn boundary), + # /steer lands BETWEEN tool-call iterations inside the same agent run, by appending to the + # last tool result's content. No interrupt, no new user turn, no role-alternation violation. steer_text = event.get_command_args().strip() if not steer_text: return "Usage: /steer " @@ -18750,17 +17247,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "No active agent — /steer queued for the next turn." async def _busy_goal_command(self, event: MessageEvent, quick_key: str, source): - # /goal is safe mid-run for status/pause/clear/wait (inspection - # and control-plane only — doesn't interrupt the running turn). - # Setting a new goal text mid-run is rejected with the same - # "wait or /stop" message as /model so we don't race a second - # continuation prompt against the current turn. + # /goal is safe mid-run for status/pause/clear/wait (inspection and control-plane only — + # doesn't interrupt the running turn). Setting new goal text mid-run is rejected like + # /model so we don't race a second continuation prompt against the current turn. _goal_arg = (event.get_command_args() or "").strip().lower() _goal_verb = _goal_arg.split(None, 1)[0] if _goal_arg else "" - # Exact-match control verbs (unchanged semantics), plus the - # wait/unwait barrier verbs which take a pid argument and the - # gate management verb (inspection/mutation of the gate list only — - # gates run at turn boundary, so editing them mid-run is safe). + # Exact-match control verbs (unchanged semantics), plus the wait/unwait barrier verbs which + # take a pid argument and the gate management verb (inspection/mutation of the gate list + # only — gates run at turn boundary, so editing them mid-run is safe). _is_control = ( not _goal_arg or _goal_arg in {"status", "pause", "resume", "clear", "stop", "done", "unwait"} @@ -18780,17 +17274,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "Agent is running — use /loop status / pause / stop mid-run, or /stop before setting a new loop." async def _handle_message(self, event: MessageEvent) -> Optional[str]: - """ - Handle an incoming message from any platform. - - This is the core message processing pipeline: - 1. Check user authorization - 2. Check for commands (/new, /reset, etc.) - 3. Check for running agent and interrupt if needed - 4. Get or create session - 5. Build context for agent - 6. Run agent conversation - 7. Return response + """Handle an incoming message from any platform. + + This is the core message processing pipeline: 1. Check user authorization 2. Check for + commands (/new, /reset, etc.) 3. Check for running agent and interrupt if needed 4. Get + or create session 5. Build context for agent 6. Run agent conversation 7. Return response """ source = event.source @@ -18811,10 +17299,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: logger.debug("reset_session_vars failed at handler entry", exc_info=True) - # Most adapters resolve profile routes in build_source(), before they - # hand us the event. A few internal/voice paths construct SessionSource - # directly, so resolve those here as the shared fail-closed ingress gate - # before authorization, hooks, or session side effects. + # Most adapters resolve profile routes in build_source(), before they hand us the event. A + # few internal/voice paths construct SessionSource directly, so resolve those here as the + # shared fail-closed ingress gate before authorization, hooks, or session side effects. if ( getattr(getattr(self, "config", None), "multiplex_profiles", False) and not getattr(source, "profile", None) @@ -18841,12 +17328,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # are system-generated and must skip user authorization. is_internal = bool(getattr(event, "internal", False)) - # Ignored-channel guard runs FIRST — before startup-restore queueing, - # plugin hooks, auth, and session setup — so a configured ignored - # channel can never reach pairing/auth/session state (#51899). - # getattr: bare test runners construct GatewayRunner via - # object.__new__ without config (see AGENTS.md pitfall on - # object.__new__ test pattern). + # Ignored-channel guard runs FIRST — before startup-restore queueing, plugin hooks, auth, + # and session setup — so a configured ignored channel can never reach pairing/auth/session + # state. getattr: bare test runners construct GatewayRunner via object.__new__ without + # config (see AGENTS.md pitfall on object.__new__ test pattern). if ( not is_internal and getattr(source, "platform", None) == Platform.SLACK @@ -18923,12 +17408,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if is_internal: pass elif source.user_id is None: - # Messages with no user identity (Telegram service messages, - # channel forwards, anonymous admin posts, sender_chat) can't - # be paired, but they can still be authorized via a - # chat-scoped allowlist (e.g. TELEGRAM_GROUP_ALLOWED_CHATS - # authorizes every member of the listed chat regardless of - # sender). Defer to _is_user_authorized so that path runs. + # Messages with no user identity (Telegram service messages, channel forwards, anonymous + # admin posts, sender_chat) can't be paired, but they can still be authorized via a + # chat-scoped allowlist (e.g. TELEGRAM_GROUP_ALLOWED_CHATS authorizes every member of + # the listed chat). Defer to _is_user_authorized so that path runs. if not self._is_user_authorized_for_source(source): logger.debug("Ignoring message with no user_id from %s", source.platform.value) return None @@ -19061,14 +17544,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return _paused_notice - # Intercept messages that are responses to a pending /update prompt. - # The update process (detached) wrote .update_prompt.json; the watcher - # forwarded it to the user; now the user's reply goes back via - # .update_response so the update process can continue. - # - # IMPORTANT: recognized slash commands must bypass this interception. - # Otherwise control/session commands like /new or /help get silently - # consumed as update answers instead of being dispatched normally. + # Intercept messages that are responses to a pending /update prompt. The update process + # (detached) wrote .update_prompt.json; the watcher forwarded it to the user; now the user's + # reply goes back via .update_response so the update process can continue. IMPORTANT: + # recognized slash commands must bypass this interception or /new, /help etc. get silently + # consumed as update answers. _quick_key = self._session_key_for_source(source) allow_gateway_control = event.allow_gateway_control _up_state = self._peek_session_state(_quick_key) @@ -19115,12 +17595,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _up_state.persistent.update_prompt_pending = False label = response_text if len(response_text) <= 20 else response_text[:20] + "…" return f"✓ Sent `{label}` to the update process." - # Recognized slash command during a pending update prompt: - # unblock the detached update subprocess by writing a blank - # response so ``_gateway_prompt`` returns the prompt's default - # (typically a safe "n" / skip) and exits cleanly instead of - # blocking on stdin until the 30-minute watcher timeout. - # The slash command then falls through to normal dispatch. + # Recognized slash command during a pending update prompt: unblock the detached update + # subprocess by writing a blank response so ``_gateway_prompt`` returns the prompt's + # default (typically a safe "n" / skip) and exits cleanly instead of blocking on stdin + # until the 30-minute watcher timeout. if _recognized_cmd: response_path = _hermes_home / ".update_response" prompt_path = _hermes_home / ".update_prompt.json" @@ -19142,11 +17620,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) _up_state.persistent.update_prompt_pending = False - # Intercept messages that are responses to a pending clarify. - # Open-ended prompts and "Other" responses are captured as free text; - # direct replies to multi-choice prompts are accepted too ("2" maps - # to the second option). Slash - # commands still bypass this path so /stop and friends keep working. + # Intercept messages that are responses to a pending clarify. Open-ended prompts and "Other" + # responses are captured as free text; direct replies to multi-choice prompts are accepted + # too ("2" maps to the second option). _clarify_mod = None try: from tools import clarify_gateway as _clarify_mod @@ -19170,10 +17646,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _pending_clarify.clarify_id, ) return "" - # Skip slash commands — the user clearly wanted to issue a - # command, not answer the clarify. Leave the clarify pending - # so the user can retry; if it times out, the agent unblocks - # with an empty response. + # Skip slash commands — the user clearly wanted to issue a command, not answer the + # clarify. Leave the clarify pending so the user can retry; if it times out, the agent + # unblocks with an empty response. if _raw_clarify_reply and not _raw_clarify_reply.startswith("/"): _text_outcome = _clarify_mod.attempt_text_response_for_session( _quick_key, _raw_clarify_reply, @@ -19183,12 +17658,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Gateway intercepted clarify text response (session=%s, id=%s)", _quick_key, _pending_clarify.clarify_id, ) - # The clarify callback pauses the platform typing/status - # indicator while waiting so Slack users can type their - # answer. The active agent resumes as soon as this reply - # resolves the wait, so re-enable its indicator here too. - # Without this, Slack stays silent until the independent - # long-running heartbeat fires (three minutes by default). + # The clarify callback pauses the platform typing/status indicator while waiting + # so Slack users can type their answer. The active agent resumes as soon as this + # reply resolves the wait, so re-enable its indicator here too. _clarify_adapter = self._adapter_for_source(source) if _clarify_adapter: try: @@ -19203,9 +17675,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # itself will produce the next user-facing message. return "" if _text_outcome == _clarify_mod.TEXT_REJECTED_SELECTION: - # Selection-shaped but invalid (out-of-range number, - # unrecognised comma-list). Keep the clarify armed so - # the user can retry — do not cancel and do not treat + # Selection-shaped but invalid (out-of-range number, unrecognised comma-list). + # Keep the clarify armed so the user can retry — do not cancel and do not treat # this as an unrelated follow-up turn. logger.info( "Gateway retained pending clarify after invalid " @@ -19214,11 +17685,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return "" if _text_outcome == _clarify_mod.TEXT_REJECTED_PROSE: - # Native-choice prompts deliberately reject unmatched - # prose so it can continue through normal busy-message - # routing. Release this clarify first: redirect() - # degrades to steer() while tools are executing, and - # that steer cannot drain until the clarify tool returns. + # Native-choice prompts deliberately reject unmatched prose so it can continue + # through normal busy-message routing. Release this clarify first: redirect() + # degrades to steer() while tools execute, and that steer cannot drain until + # the clarify tool returns. _clarify_mod.resolve_gateway_clarify( _pending_clarify.clarify_id, "", @@ -19244,11 +17714,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _tool_approval_live = False if allow_gateway_control and _pending_confirm and not _tool_approval_live: _raw_reply = (event.text or "").strip() - # Accept bang-prefixed replies (`!always`, `!cancel`) verbatim. - # Slack/Matrix instruction text shows the `!` prefix (typed `/` - # is blocked in Slack threads), but the adapters only rewrite - # `!` — `always`/`cancel` are confirm keywords, - # not registered commands, so the `!` survives to here. + # Accept bang-prefixed replies (`!always`, `!cancel`) verbatim: Slack/Matrix show the + # `!` prefix (typed `/` is blocked in Slack threads) and adapters only rewrite + # `!` — confirm keywords aren't commands, so the `!` survives to here. _norm_reply = _raw_reply.lstrip("!/").lower() _cmd_reply = event.get_command() _confirm_choice = None @@ -19274,30 +17742,25 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # clearly moved on. _slash_confirm_mod.clear_if_stale(_quick_key) - # PRIORITY handling when an agent is already running for this session. - # Default behavior is to interrupt immediately so user text/stop messages - # are handled with minimal latency. - # - # Special case: Telegram/photo bursts often arrive as multiple near- - # simultaneous updates. Do NOT interrupt for photo-only follow-ups here; - # let the adapter-level batching/queueing logic absorb them. + # PRIORITY handling when an agent is already running for this session. Default behavior is + # to interrupt immediately so user text/stop messages are handled with minimal latency. + # Exception: Telegram photo bursts arrive as near-simultaneous updates — do NOT interrupt + # for photo-only follow-ups; adapter-level batching absorbs them. - # Staleness eviction: detect leaked locks from hung/crashed handlers. - # With inactivity-based timeout, active tasks can run for hours, so - # wall-clock age alone isn't sufficient. Evict only when the agent - # has been *idle* beyond the inactivity threshold (or when the agent - # object has no activity tracker and wall-clock age is extreme). + # Staleness eviction: detect leaked locks from hung/crashed handlers. With inactivity-based + # timeout, active tasks can run for hours, so wall-clock age alone isn't sufficient. Evict + # only when the agent has been *idle* past the threshold (or has no activity tracker and + # wall-clock age is extreme). _raw_stale_timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) _quick_state = self._peek_session_state(_quick_key) _stale_ts = _quick_state.turn.started_ts if _quick_state else 0 if _quick_state is not None and _quick_state.turn.agent is not None and _stale_ts: _stale_age = time.time() - _stale_ts _stale_agent = _quick_state.turn.agent - # Never evict the pending sentinel — it was just placed moments - # ago during the async setup phase before the real agent is - # created. Sentinels have no get_activity_summary(), so the - # idle check below would always evaluate to inf >= timeout and - # immediately evict them, racing with the setup path. + # Never evict the pending sentinel — it was just placed moments ago during the async + # setup phase before the real agent is created. Sentinels have no + # get_activity_summary(), so the idle check would read inf >= timeout and evict them + # immediately, racing the setup path. _stale_idle = float("inf") # assume idle if we can't check _stale_detail = "" _activity_summary_valid = False @@ -19353,25 +17816,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) self._release_running_agent_state(_quick_key) - # #99106: durable-reaped guard. A session whose routing row was - # ended in state.db (e.g. ``ws_orphan_reap`` / ``agent_close``) while - # the gateway stayed alive keeps its in-memory turn slot alive - # (``_is_session_running`` stays True). The priority fast-path would - # then queue every next user message into the dead runtime instead of - # healing the routing via ``get_or_create_session`` → ``reopen``. - # This is the live-gateway variant of #54878 and the #632 detached/ - # 405 suppressions in production. Evict the stale slot so the next - # message falls through to the cold path and re-attaches or creates a - # fresh session; /status then correctly shows 代理运行中: 否 before the - # heal and a live turn after. + # Durable-reaped guard. A session whose routing row was ended in state.db (e.g. + # ``ws_orphan_reap`` / ``agent_close``) while the gateway stayed alive keeps its in-memory + # turn slot (``_is_session_running`` stays True). The priority fast-path would then queue + # every next user message into the dead runtime instead of healing the routing via + # ``get_or_create_session`` → ``reopen``. Evict the stale slot so the next message falls + # through to the cold path and re-attaches or creates a fresh session. if self._is_session_running(_quick_key): try: _reap_store = getattr(self, "session_store", None) - # Use the public, lock-held accessors: peek_session_id resolves - # key -> session_id under the store lock, and returns a - # non-str on stubbed stores in bare test runners — both the - # isinstance() gate and the ``is True`` gate below keep this - # guard inert unless a real SessionStore answers. + # Use the public, lock-held accessors: peek_session_id resolves key -> session_id + # under the store lock, and returns a non-str on stubbed stores in bare test runners + # — both the isinstance() gate and the ``is True`` gate below keep this guard inert + # unless a real SessionStore answers. _reap_peek = getattr(_reap_store, "peek_session_id", None) _is_ended = getattr(_reap_store, "_is_session_ended_in_db", None) _reap_sid = _reap_peek(_quick_key) if callable(_reap_peek) else None @@ -19397,11 +17854,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("reaped-session staleness check failed", exc_info=True) if self._is_session_running(_quick_key): - # Resolve the command once; every command's mid-run behavior is - # declared on its CommandDef (busy_policy / busy_handler in - # hermes_cli/commands.py) and dispatched through the single - # resolver _dispatch_busy_slash_command below — no per-command - # if-chain here. + # Resolve the command once; every command's mid-run behavior is declared on its + # CommandDef (busy_policy / busy_handler in hermes_cli/commands.py) and dispatched + # through the single resolver _dispatch_busy_slash_command below — no per-command if- + # chain here. from hermes_cli.commands import resolve_command as _resolve_cmd_inner _evt_cmd = event.get_command() _cmd_def_inner = _resolve_cmd_inner(_evt_cmd) if _evt_cmd else None @@ -19413,20 +17869,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if _cmd_def_inner and _cmd_def_inner.name == "context": return await self._handle_context_command(event) - # Slash command access control on the running-agent fast-path. - # Mirrors the cold-path gate further below so non-admin users - # can't bypass gating just because an agent happens to be busy. - # /status above is intentionally pre-gate so users always see - # session state. /help and /whoami fall under the always-allowed - # floor inside _check_slash_access. + # Slash command access control on the running-agent fast-path. Mirrors the cold-path + # gate below so non-admins can't bypass gating just because an agent is busy. /status + # above is intentionally pre-gate; /help and /whoami are the always-allowed floor. if _evt_cmd and _cmd_def_inner is not None: _denied = self._check_slash_access(source, _cmd_def_inner.name) if _denied is not None: return _denied - # Any recognized slash command: dispatch according to its - # declared busy_policy (dispatch / interrupt_then_dispatch / - # reject). Unrecognized commands and plain text fall through + # Any recognized slash command: dispatch according to its declared busy_policy (dispatch + # / interrupt_then_dispatch / reject). Unrecognized commands and plain text fall through # to the interrupt/queue logic below. if _cmd_def_inner: return await self._dispatch_busy_slash_command( @@ -19530,14 +17982,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("PRIORITY steer-fallback-to-queue for session %s", _quick_key) self._queue_or_replace_pending_event(_quick_key, event) return None - # #30170 — Subagent protection (PRIORITY path). Same rationale - # as ``_handle_active_session_busy_message``: an interrupt - # cascades through ``_active_children`` and aborts in-flight - # delegate_task work. Demote to queue semantics when the - # parent is currently driving subagents so a conversational - # follow-up doesn't destroy minutes of subagent progress. - # /stop reaches its dedicated handler above, so the operator - # still has a clean escape hatch. + # Subagent protection (PRIORITY path). Same rationale as + # ``_handle_active_session_busy_message``: an interrupt cascades through + # ``_active_children`` and aborts in-flight delegate_task work, so demote to queue + # semantics while subagents run. /stop reached its handler above — still an escape hatch. if self._agent_has_active_subagents(running_agent): logger.info( "PRIORITY interrupt demoted to queue for session %s " @@ -19616,10 +18064,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command - # Expand alias quick commands before built-in dispatch so targets like - # /model openai/gpt-5.5 --provider openrouter reach the /model handler. - # Preserve built-in precedence; aliases only need early handling when - # the typed command is not already known. + # Expand alias quick commands before built-in dispatch so targets like /model openai/gpt-5.5 + # --provider openrouter reach the /model handler. Preserve built-in precedence; aliases only + # need early handling when the typed command is not already known. if command and _cmd_def is None: if isinstance(self.config, dict): quick_commands = self.config.get("quick_commands", {}) or {} @@ -19638,29 +18085,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command - # Per-platform slash command access control. Only kicks in when the - # operator has set ``allow_admin_from`` for the source's scope (DM - # vs group). When unset → backward-compat: every allowed user can - # run every command. When set → non-admins can run only commands in - # ``user_allowed_commands`` (plus the always-allowed floor: /help, - # /whoami). Plain chat is unaffected — only slash commands gate. + # Per-platform slash command access control. Only kicks in when the operator has set + # ``allow_admin_from`` for the source's scope (DM vs group). When unset → backward-compat: + # every allowed user can run every command. When set → non-admins get only + # ``user_allowed_commands`` plus the /help, /whoami floor. Plain chat is never gated. if command and canonical and is_gateway_known_command(canonical): _denied = self._check_slash_access(source, canonical) if _denied is not None: return _denied - # pre_command observer hook (#64204): fires for every recognized - # slash command BEFORE core handling, mirroring the CLI fire-site in - # cli.py process_command. Observer-only in v1 (returns ignored). - # - # Placement matters: this cold-path dispatch is only reached when NO - # agent is running for the session. The running-agent intercept path - # above (/stop, /approve, busy_policy dispatch via - # _dispatch_busy_slash_command) deliberately does NOT fire this hook — - # those are control-plane operations on an in-flight run, and giving - # plugins an observation (and eventually veto) point there would let - # a slow or hostile plugin interfere with the operator's escape - # hatches for a live agent. + # pre_command observer hook: fires for every recognized slash command BEFORE core handling, + # mirroring the CLI fire-site in cli.py process_command. Observer-only in v1 (returns + # ignored). Placement matters: the running-agent intercept path above (/stop, /approve, + # busy_policy dispatch) deliberately does NOT fire it — a slow or hostile plugin must not + # be able to interfere with the operator's escape hatches for a live agent. if command and is_gateway_known_command(canonical): try: from hermes_cli.plugins import fire_pre_command_hook @@ -19755,7 +18193,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if canonical == "topic": return await self._handle_topic_command(event) - + if canonical == "start": logger.info("Ignoring /start platform ping for session %s", _quick_key) return "" @@ -19773,7 +18211,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if canonical == "stop": return await self._handle_stop_command(event) - + if canonical == "reasoning": return await self._handle_reasoning_command(event) @@ -19784,12 +18222,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return await self._handle_skills_command(event) if canonical == "learn": - # Open-ended: rewrite the turn to a standards-guided prompt and fall - # through to normal agent processing. The live agent gathers the - # sources the user described (dirs via read_file, URLs via - # web_extract, this conversation, pasted text) and authors the skill - # via skill_manage. Mirrors the /blueprint fall-through so role - # alternation is preserved. No engine, works on any backend. + # Open-ended: rewrite the turn to a standards-guided prompt and fall through to normal + # agent processing. Mirrors the /blueprint fall-through so role alternation is + # preserved. No engine, works on any backend. from agent.learn_prompt import build_learn_prompt _learn_req = event.get_command_args().strip() @@ -19812,12 +18247,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "Could not start /learn — please try again." if canonical == "plan": - # /plan: rewrite the turn to the plan-mode prompt and fall - # through to normal agent processing (same fall-through as /learn - # so role alternation is preserved). The live agent inspects the - # workspace with read-only tools and saves the markdown plan - # under .hermes/plans/ via write_file. No engine, works on any - # backend. + # /plan: rewrite the turn to the plan-mode prompt and fall through to normal agent + # processing (same fall-through as /learn so role alternation is preserved). No engine, + # works on any backend. from agent.plan_prompt import build_plan_prompt _plan_task = event.get_command_args().strip() @@ -19840,11 +18272,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "Could not start /plan — please try again." if canonical == "init": - # /init: rewrite the turn to a guidance-laden prompt and fall - # through to normal agent processing (same fall-through as /learn - # so role alternation is preserved). The live agent scans the - # project with its own read-only tools and writes/updates - # AGENTS.md via write_file. No engine, works on any backend. + # /init: rewrite the turn to a guidance-laden prompt and fall through to normal agent + # processing (same fall-through as /learn so role alternation is preserved). No engine, + # works on any backend. from hermes_cli.init_command import build_init_prompt_for_cwd _init_notes = event.get_command_args().strip() @@ -19889,14 +18319,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _blueprint_result = await self._handle_blueprint_command(event) _blueprint_seed = getattr(_blueprint_result, "agent_seed", None) if _blueprint_seed: - # Blueprint matched — rewrite the turn to the seed and fall - # through to _handle_message_with_agent so the agent asks the - # user for each slot value conversationally and then calls the - # cronjob tool (the /steer fall-through pattern). The seed - # enters as a normal user turn, preserving role alternation. - # Send the "Setting up X…" ack first so the user gets the same - # immediate feedback CLI users see, instead of silence until - # the agent's first question. + # Blueprint matched — rewrite the turn to the seed and fall through to + # _handle_message_with_agent so the agent asks the user for each slot value + # conversationally and then calls the cronjob tool (the /steer fall-through + # pattern). _ack = getattr(_blueprint_result, "text", "") or "" if _ack: try: @@ -19918,7 +18344,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if canonical == "retry": return await self._handle_retry_command(event) - + if canonical == "undo": async def _do_undo(): return await self._handle_undo_command(event) @@ -19941,7 +18367,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew detail=_undo_detail, execute=_do_undo, ) - + if canonical == "sethome": return await self._handle_set_home_command(event) @@ -20023,10 +18449,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return await self._handle_review_command(event) if canonical == "moa": - # /moa is one-shot sugar only: run a single prompt through the - # default MoA preset, then restore the prior model. To *switch* to a - # MoA preset for the session, pick it from the model picker (MoA - # presets surface as a virtual "Mixture of Agents" provider). + # /moa is one-shot sugar only: run a single prompt through the default MoA preset, then + # restore the prior model. To *switch* to a MoA preset for the session, pick it from the + # model picker (MoA presets surface as a virtual "Mixture of Agents" provider). from hermes_cli.moa_config import ( moa_usage, normalize_moa_config, @@ -20073,13 +18498,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not isinstance(quick_commands, dict): quick_commands = {} if command in quick_commands: - # Quick commands are slash capabilities too — and type:exec - # ones run a shell command in the gateway process. The early - # gate above only fires for registry-known commands, so quick - # commands (never in the registry) would otherwise reach this - # dispatch sink unchecked. Apply the same admin/user policy to - # the raw typed name here so non-admins can't invoke admin-only - # quick commands. (#44727) + # Quick commands are slash capabilities too — and type:exec ones run a shell command + # in the gateway process. The early gate only fires for registry-known commands and + # quick commands are never in the registry, so apply the same admin/user policy to + # the raw typed name here so non-admins can't invoke admin-only quick commands. _denied = self._check_slash_access(source, command) if _denied is not None: return _denied @@ -20144,9 +18566,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.warning("Plugin command dispatch failed: %s", e) # Skill slash commands: /skill-name loads the skill and sends to agent. - # resolve_skill_command_key() handles the Telegram underscore/hyphen - # round-trip so /claude_code from Telegram autocomplete still resolves - # to the claude-code skill. + # resolve_skill_command_key() handles the Telegram underscore/hyphen round-trip so + # /claude_code from Telegram autocomplete still resolves to the claude-code skill. if command: # Skill bundles take precedence over individual skill commands — # / loads multiple skills at once. Mirrors CLI dispatch. @@ -20159,11 +18580,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew bundle_key = resolve_bundle_command_key(command) if bundle_key is not None: user_instruction = event.get_command_args().strip() - # Pass the platform explicitly: bundle skill loading - # bypasses get_skill_commands()' scan-time disabled - # filter, and the gateway serves multiple platforms in - # one process, so env-var platform resolution can't be - # trusted here. Mirrors the stacked-skill gate (#58888). + # Pass the platform explicitly: bundle skill loading bypasses + # get_skill_commands()' scan-time disabled filter, and the gateway serves + # multiple platforms in one process, so env-var platform resolution can't be + # trusted here. _bundle_plat = source.platform.value if source.platform else None bundle_result = build_bundle_invocation_message( bundle_key, user_instruction, task_id=_quick_key, @@ -20192,10 +18612,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew skill_cmds = get_skill_commands() cmd_key = resolve_skill_command_key(command) if cmd_key is not None: - # Check per-platform disabled status before executing. - # get_skill_commands() only applies the *global* disabled - # list at scan time; per-platform overrides need checking - # here because the cache is process-global across platforms. + # Check per-platform disabled status before executing. get_skill_commands() only + # applies the *global* disabled list at scan time; per-platform overrides need + # checking here because the cache is process-global across platforms. _skill_name = skill_cmds[cmd_key].get("name", "") _plat = source.platform.value if source.platform else None if _plat and _skill_name: @@ -20221,13 +18640,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _build_stacked = None extra_keys, stacked_instruction = [], user_instruction if extra_keys and _plat: - # split_stacked_skill_commands() only resolves that - # each extra token is a KNOWN skill command — like - # get_skill_commands() itself, it has no per-platform - # view. Re-check every stacked skill (not just the - # leading one above) against the same disabled list, - # or a skill an operator disabled for this platform - # still gets its full content loaded via the stack. + # split_stacked_skill_commands() only resolves that each extra token is a + # KNOWN skill command — like get_skill_commands() itself, it has no per- + # platform view. Re-check every stacked skill against the same disabled list, + # or a skill disabled for this platform still gets loaded via the stack. from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled _plat_disabled = _get_plat_disabled(platform=_plat) _disabled_extra = [ @@ -20266,14 +18682,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _unavail_msg = _check_unavailable_skill(command) if _unavail_msg: return _unavail_msg - # Genuinely unrecognized /command: not a built-in, not a - # plugin, not a skill, not a known-inactive skill. Warn - # the user instead of silently forwarding it to the LLM - # as free text (which leads to silent-failure behavior - # like the model inventing a delegate_task call). - # Normalize to hyphenated form before checking known - # built-ins (command may be an alias target set by the - # quick-command block above, so _cmd_def can be stale). + # Genuinely unrecognized /command: not a built-in, not a plugin, not a skill, + # not a known-inactive skill. Warn instead of silently forwarding it to the LLM as + # free text (which leads to the model inventing e.g. a delegate_task call). + # Normalize to hyphenated form first: command may be an alias target set by the + # quick-command block above, so _cmd_def can be stale. if command.replace("_", "-") not in GATEWAY_KNOWN_COMMANDS: logger.warning( "Unrecognized slash command /%s from %s — " @@ -20289,7 +18702,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except Exception as e: logger.debug("Skill command check failed (non-fatal): %s", e) - + # Pending exec approvals are handled by /approve and /deny commands above. # No bare text matching — "yes" in normal conversation must not trigger # execution of a dangerous command. @@ -20304,15 +18717,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return None # ── External-drain new-turn gate (Phase 2) ──────────────────── - # When NAS has engaged an external drain (.drain_request.json present, - # observed by _drain_control_watcher), refuse to START a new turn so - # the in-flight set can only fall to zero — eliminating the TOCTOU race - # (D4a: stop accepting new turns FIRST, then NAS polls until - # active_agents==0). In-flight turns are untouched; this only blocks the - # claim of a NEW session slot. Internal/system events (restart-recovery - # replays, background-process completions) bypass the gate — they are - # not user-initiated new work and must still flow during a drain. - # Reversible: once the marker is removed the gate opens again. + # When NAS has engaged an external drain (.drain_request.json present, observed by + # _drain_control_watcher), refuse to START a new turn so the in-flight set can only fall to + # zero — eliminating the TOCTOU race (D4a: stop accepting new turns FIRST, then NAS polls + # until active_agents==0). In-flight turns are untouched; internal/system events bypass the + # gate (not user-initiated, must still flow during a drain). Reversible once the marker goes. if self._external_drain_active and not is_internal: logger.info( "Refusing new turn for session %s — external drain active.", @@ -20325,12 +18734,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) # ── Claim this session before any await ─────────────────────── - # Between here and _run_agent registering the real AIAgent, there - # are numerous await points (hooks, vision enrichment, STT, - # session hygiene compression). Without this sentinel a second - # message arriving during any of those yields would pass the - # "already running" guard and spin up a duplicate agent for the - # same session — corrupting the transcript. + # Between here and _run_agent registering the real AIAgent, there are numerous await points + # (hooks, vision enrichment, STT, session hygiene compression). Without this sentinel a + # second message arriving during any of those yields would pass the "already running" guard + # and spin up a duplicate agent for the same session — corrupting the transcript. _active_session_lease, _limit_message = self._claim_active_session_slot( _quick_key, source, @@ -20343,16 +18750,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return _limit_message # ── FIFO orphan rescue (#99882) ──────────────────────────────── - # If this session went idle with a populated overflow (queued - # during a busy window whose post-turn drain never promoted — - # e.g. a compression-demoted follow-up after the compression - # window ended through an exit that skipped the promotion site), - # those events were silently orphaned. We are starting the next - # turn for this session NOW: re-stage the orphans in FIFO order - # and enqueue the incoming event behind them, so arrival order - # (#28503) holds: oldest orphan runs as this turn, the rest drain - # in order, the new message last. Skipped for control commands - # (/stop etc. own their own semantics) and internal events. + # If this session went idle with a populated overflow (queued during a busy window whose + # post-turn drain never promoted — e.g. a compression-demoted follow-up after the + # compression window ended through an exit that skipped the promotion site), those events + # were silently orphaned. Re-stage them in FIFO order and enqueue the incoming event behind + # them so arrival order holds. Skipped for control commands (/stop etc.) and internal events. try: _orphan_adapter = self._adapter_for_source(source) if ( @@ -20364,11 +18766,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _quick_key, _orphan_adapter ) if _rescued is not None: - # The oldest orphan runs as THIS turn. Park the - # incoming event behind the rest of the chain: into the - # slot when the chain was a single orphan (so the - # post-turn drain picks it up), otherwise into overflow - # behind the already-staged next orphan (FIFO). + # The oldest orphan runs as THIS turn. Park the incoming event behind the rest of + # the chain: into the slot when the chain was a single orphan (post-turn drain + # picks it up), otherwise into overflow behind the already-staged next orphan. self._enqueue_fifo(_quick_key, event, _orphan_adapter) event = _rescued # Same session key by construction; carry the orphan's @@ -20425,27 +18825,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("post-turn hook failed: %s", _goal_exc) return _agent_result finally: - # MoA one-shot restore must run on EVERY exit path, not just - # success. The restore data lives on the per-turn event object - # (_moa_restore_override), which is discarded once the event goes - # out of scope — so if _handle_message_with_agent raises, a restore - # in the try block would be skipped and the MoA override would leak - # permanently (every later message silently fans out through MoA). - # Putting it in finally guarantees the revert on success, exception, - # and interrupt alike. + # MoA one-shot restore must run on EVERY exit path, not just success. The restore data + # lives on the per-turn event object; if the handler raised, a restore in the try block + # would be skipped and the MoA override would leak permanently. Putting it in + # finally guarantees the revert on success, exception, and interrupt alike. self._restore_moa_one_shot(event, _quick_key) self._restore_pending_one_turn_model_override(_quick_key) # Normal completion/exception/interrupt owns and clears this exact # durable marker. SIGKILL/OOM skips finally, leaving the marker for # the next unclean startup's recovery pass. await self._clear_durable_active_turn(event) - # Unconditional release covers every exit path. _release_running_agent_state - # is idempotent (pop-on-absent is harmless) and, called without a - # run_generation guard, always clears the slot regardless of which - # generation it holds. This evicts the zombie left when session_reset - # bumps the generation (N -> N+1) mid-flight: gen-N's guarded release - # inside _run_agent returns False, and the old sentinel-only check here - # missed the leftover real agent — locking the session out forever (#28686). + # Unconditional release covers every exit path. _release_running_agent_state is + # idempotent (pop-on-absent is harmless) and, called without a run_generation guard, + # always clears the slot regardless of which generation it holds. This evicts the zombie + # left when session_reset bumps the generation mid-flight: gen-N's guarded release in + # _run_agent returns False, and a sentinel-only check here would lock the session forever. self._release_running_agent_state(_quick_key) # Turn lease (#64934): release THIS turn's lease token — keyed by # (routing key, run generation) so this unwind can only ever free @@ -20455,11 +18849,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _restore_moa_one_shot(self, event: "MessageEvent", quick_key: str) -> None: """Revert a ``/moa `` one-shot model override after its turn. - Called from the ``finally`` of the message-handling path so the revert - fires whether the turn succeeded, raised, or was interrupted. A no-op - unless ``event._moa_disable_after_turn`` is set. ``_moa_restore_override`` - carries the prior per-session override (``None`` means the user had no - override, so the MoA override is cleared outright). + Called from the ``finally`` of the message-handling path so the revert fires whether the + turn succeeded, raised, or was interrupted. A no-op unless + ``event._moa_disable_after_turn`` is set. ``_moa_restore_override`` carries the prior + per-session override (``None`` = no override, so the MoA override is cleared outright). """ if not getattr(event, "_moa_disable_after_turn", False): return @@ -20495,16 +18888,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[str]: """Prepare inbound event text for the agent. - Keep the normal inbound path and the queued follow-up path on the same - preprocessing pipeline so sender attribution, image enrichment, STT, - document notes, reply context, and @ references all behave the same. + Keep the normal inbound path and the queued follow-up path on the same preprocessing + pipeline so sender attribution, image enrichment, STT, document notes, reply context, and + @ references all behave the same. - Side effect: buffers per-session native image paths when the active - model supports native vision AND the user has images attached. The - caller consumes and clears that session-scoped buffer at the - ``run_conversation`` site to build a multimodal user turn. When the - list is empty, the ``_enrich_message_with_vision`` text path has - already run and images are represented in-text. + Side effect: buffers per-session native image paths when the model supports native vision + and images are attached; the caller consumes/clears that buffer at the ``run_conversation`` + site. When the list is empty, the ``_enrich_message_with_vision`` text path already ran. """ history = history or [] _pending_stt_prepared = hasattr(event, "_gateway_pending_stt_text") @@ -20529,20 +18919,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew thread_sessions_per_user=_thread_sessions_per_user, ) if _is_shared_multi_user and source.user_name: - # source.user_name is the platform display name — attacker- - # influenceable on any platform that lets participants set their - # own name. Neutralize embedded newlines/control chars before - # interpolating it into every message in the shared session, or - # a hostile name can masquerade as a fake markdown section - # (mirrors the same field's treatment in - # build_session_context_prompt via _format_untrusted_prompt_value). + # source.user_name is the platform display name — attacker-influenceable on any + # platform that lets participants set their own name. Neutralize newlines/control chars + # before interpolating it into every message, or a hostile name can masquerade as a + # fake markdown section (mirrors build_session_context_prompt's treatment). _safe_user_name = neutralize_untrusted_inline_text(source.user_name) - # On Slack, expose the current author's verifiable user ID next to - # the display name (#17916): "mention me again" requests need a - # trusted `<@U...>` target for the CURRENT speaker — display names - # are ambiguous and historical mentions may point at someone else. - # The user_id comes from the Slack event envelope (not - # user-editable text), so it does not need neutralization. + # On Slack, expose the current author's verifiable user ID next to the display name: + # "mention me again" requests need a trusted `<@U...>` target for the CURRENT speaker — + # display names are ambiguous and historical mentions may point at someone else. The + # user_id comes from the Slack event envelope (not user-editable), so no neutralization. if source.platform == Platform.SLACK and source.user_id: _safe_user_name = ( f"{_safe_user_name} | Slack user <@{source.user_id}>" @@ -20565,11 +18950,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew audio_paths = [] for i, path in enumerate(event.media_urls): mtype = event.media_types[i] if i < len(event.media_types) else "" - # Classify images per-attachment: trust this attachment's own - # MIME, and only honour the message-level PHOTO type when the - # per-attachment MIME is unknown. Otherwise a document (or any - # non-image) sent alongside an image in the same message gets - # mis-routed here as an image and the provider 400s. + # Classify images per-attachment: trust this attachment's own MIME, and only honour + # the message-level PHOTO type when the per-attachment MIME is unknown. Otherwise a + # document sent alongside an image gets mis-routed as an image and the provider 400s. if _event_media_is_image(event, i): image_paths.append(path) # MessageType.AUDIO = audio file attachment (e.g. .mp3, .m4a) — never STT. @@ -20584,13 +18967,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew video_paths.append(path) if image_paths: - # Decide routing: native (attach pixels) vs text (vision_analyze - # pre-run + prepend description). See agent/image_routing.py. - # Offload to a worker thread: the decision does blocking network - # I/O — a models.dev fetch on cache miss, and the Ollama - # ``/api/show`` capability probe for local servers — whose - # request timeout would otherwise stall the whole gateway event - # loop (every session) while a single image is routed. + # Decide routing: native (attach pixels) vs text (vision_analyze pre-run + prepend + # description). See agent/image_routing.py. Offloaded to a thread: the decision does + # blocking network I/O (models.dev fetch on cache miss, Ollama /api/show probe) whose + # timeout would otherwise stall the whole gateway event loop. _img_mode = await asyncio.to_thread( self._decide_image_input_mode, source=source, @@ -20640,10 +19020,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew message_text, audio_paths, ) - # Echo each successful transcript back to the user immediately - # when configured. Lets users verify STT quality in real-time, - # while allowing quiet STT for users who only want the agent to - # receive the transcription. + # Echo each successful transcript back to the user immediately when configured. Lets + # users verify STT quality in real-time, while allowing quiet STT for users who only + # want the agent to receive the transcription. if _successful_transcripts and self._should_echo_stt_transcripts(): _echo_adapter = self._adapter_for_source(source) _echo_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) @@ -20659,15 +19038,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug( "Transcript echo failed (non-fatal): %s", _echo_exc, ) - # NOTE: Previously, when transcription failed (e.g. no STT - # provider configured), the gateway also emitted a hardcoded - # English notice via `_stt_adapter.send()`. That bypassed the - # LLM and produced two replies — one pre-canned English clip - # (which TTS then spoke aloud, in the wrong language) and one - # correct, localized LLM reply from the enriched message text. - # The enrichment step now leaves a single neutral marker in the - # prompt, so the LLM produces one coherent reply in the user's - # language. The hardcoded send has therefore been removed. + # NOTE: Previously, when transcription failed (e.g. no STT provider configured), the + # gateway also emitted a hardcoded English notice via `_stt_adapter.send()`. That + # bypassed the LLM and produced two replies (one pre-canned, spoken by TTS in the + # wrong language). Enrichment now leaves a single neutral marker in the prompt so the + # LLM produces one localized reply; the hardcoded send has therefore been removed. if audio_file_paths: from tools.credential_files import to_agent_visible_cache_path as _to_agent_path @@ -20761,12 +19136,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) message_text = f"{context_note}\n\n{message_text}" - # Discord: surface the triggering message id per-turn on the user - # message rather than in the cached system prompt. message_id changes - # every turn, so baking it into build_session_context_prompt() would - # bust the agent-cache signature and rebuild the AIAgent every message - # (destroying prompt caching). The static IDs block points the agent - # here; the volatile id rides the per-turn user content. + # Discord: surface the triggering message id per-turn on the user message rather than in the + # cached system prompt. message_id changes every turn, so baking it into + # build_session_context_prompt() would bust the agent-cache signature and rebuild the + # AIAgent every message (destroying prompt caching). if ( source is not None and getattr(source, "platform", None) == Platform.DISCORD @@ -20781,12 +19154,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) if getattr(event, "reply_to_text", None) and event.reply_to_message_id: - # Always inject the reply-to pointer — even when the quoted text - # already appears in history. The prefix isn't deduplication, it's - # disambiguation: it tells the agent *which* prior message the user - # is referencing. History can contain the same or similar text - # multiple times, and without an explicit pointer the agent has to - # guess (or answer for both subjects). Token overhead is minimal. + # Always inject the reply-to pointer — even when the quoted text already appears in + # history. The prefix isn't deduplication, it's disambiguation: it tells the agent + # *which* prior message the user is referencing. Token overhead is minimal. reply_snippet = event.reply_to_text[:500] if getattr(event, "reply_to_is_own_message", False): message_text = ( @@ -20826,13 +19196,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _msg_custom_providers = _msg_cfg.get("custom_providers") or [] except Exception: pass - # Resolve the session's actual model/provider/base_url the - # same way the hygiene compression block does (~11080). - # GatewayRunner has no self._model/self._base_url attrs - # (that was copy-pasted from HermesCLI, which does carry - # self.model/self.base_url), so using them here always raised - # AttributeError, silently caught below, meaning this feature - # never ran. + # Resolve the session's actual model/provider/base_url the same way the hygiene + # compression block does. GatewayRunner has no self._model/self._base_url (that was + # copy-pasted from HermesCLI); using them raised AttributeError, silently caught + # below, so this feature never ran. _msg_model, _msg_runtime = self._resolve_session_agent_runtime( source=source, session_key=session_key, @@ -21228,10 +19595,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew source.chat_id or "unknown", _msg_preview, _reply_id, _reply_txt, ) - # Get or create session - # Topic-mode DMs: rewrite a stale/foreign thread_id to the user's - # last-active topic so a cross-topic Reply or stripped plain reply - # doesn't fragment the conversation across sessions. + # Get or create session Topic-mode DMs: rewrite a stale/foreign thread_id to the user's + # last-active topic so a cross-topic Reply or stripped plain reply doesn't fragment the + # conversation across sessions. recovered = await asyncio.to_thread(self._recover_telegram_topic_thread_id, source) if recovered is not None: logger.info( @@ -21278,10 +19644,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return else: - # Internal wakes must observe reset policy without becoming user - # activity themselves. Otherwise periodic Kanban/process - # notifications keep the stable routing key alive across every - # daily/idle boundary. + # Internal wakes must observe reset policy without becoming user activity themselves. + # Otherwise periodic Kanban/process notifications keep the stable routing key alive + # across every daily/idle boundary. session_entry = await self.async_session_store.get_or_create_session( source, touch_activity=not bool(getattr(event, "internal", False)), @@ -21308,12 +19673,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew binding = None if binding: bound_session_id = str(binding.get("session_id") or "") - # Heal bindings that point at a pre-compression parent: walk - # the compression-continuation chain forward to its tip so the - # next message resumes the compressed child instead of - # reloading the oversized parent transcript (#20470/#29712/ - # #33414). Returns the input unchanged when the session isn't - # a compression parent, so this is cheap and safe. + # Heal bindings that point at a pre-compression parent: walk the compression- + # continuation chain forward to its tip so the next message resumes the compressed + # child instead of reloading the oversized parent transcript. if bound_session_id and self._session_db is not None: try: canonical_session_id = await self._session_db.get_compression_tip( @@ -21331,11 +19693,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): bound_session_id = canonical_session_id if bound_session_id and bound_session_id != session_entry.session_id: - # Route the override through SessionStore so the session_key - # → session_id mapping is persisted to disk and the previous - # lane session is ended cleanly. Mutating session_entry in - # place here created a split-brain state where the JSON - # index pointed at one id but code downstream used another. + # Route the override through SessionStore so the session_key → session_id + # mapping is persisted to disk and the previous lane session is ended cleanly. + # Mutating session_entry in place created a split-brain: the JSON index pointed + # at one id while downstream code used another. switched = await self.async_session_store.switch_session(session_key, bound_session_id) if switched is not None: session_entry = switched @@ -21359,22 +19720,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # wiping model/reasoning overrides set between turns (Closes #48031). _was_auto_reset = getattr(session_entry, "was_auto_reset", False) if _was_auto_reset: - # Treat auto-reset as a full conversation boundary — clear every - # conversation-scoped per-session dict in one funnel call so the - # fresh session does not inherit the previous conversation's - # model/reasoning overrides, a queued "/model switched" note, or - # a stale resolved-model cache (#48031, #58403). See - # _CONVERSATION_SCOPED_STATE. + # Treat auto-reset as a full conversation boundary — clear every conversation-scoped + # per-session dict in one funnel call so the fresh session does not inherit the previous + # conversation's model/reasoning overrides, a queued "/model switched" note, or a stale + # resolved-model cache. self._clear_conversation_scope(session_key, reason="auto_reset") - # Evict the cached agent so the fresh session does not inherit the - # previous conversation's context_compressor._previous_summary — - # the cache is keyed on the stable session_key, so an auto-reset - # otherwise reuses the old agent and leaks prior history into new - # compaction summaries. Mirrors /reset and the compression-exhausted - # path (#9893). Covers daily/idle/suspended auto-reset. + # Evict the cached agent so the fresh session does not inherit the previous + # conversation's context_compressor._previous_summary — the cache is keyed on the stable + # session_key, so an auto-reset otherwise reuses the old agent and leaks prior history + # into new compaction summaries. self._evict_cached_agent(session_key) session_entry.was_auto_reset = False - + # Emit session:start for new or auto-reset sessions _is_new_session = ( session_entry.created_at == session_entry.updated_at @@ -21392,25 +19749,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "session_id": session_entry.session_id, "session_key": session_key, }) - + # Build session context context = build_session_context(source, self.config, session_entry) - + # Set session context variables for tools (task-local, concurrency-safe) _session_env_tokens = self._set_session_env(context) - + # Read privacy.redact_pii from config (re-read per message) _redact_pii = False persist_user_message = None persist_user_timestamp = None - # Synthetic self-injected turns (async-delegation batch completions, - # background watch notifications, resume wake-ups) arrive as - # MessageEvent(internal=True). Persist their user row typed with - # display_kind="internal_notification" so transcripts/UIs can render - # them as timeline notices instead of user bubbles (#82888). Role and - # content are untouched — display_kind is a DB-only sidecar stripped - # from every provider-bound payload (see conversation_loop's - # api_msg.pop("display_kind")). + # Synthetic self-injected turns (async-delegation batch completions, background watch + # notifications, resume wake-ups) arrive as MessageEvent(internal=True). Persist their user + # row with display_kind="internal_notification" so UIs render them as timeline notices, not + # user bubbles. Role/content untouched — display_kind is a DB-only sidecar stripped from + # every provider-bound payload (conversation_loop's api_msg.pop("display_kind")). persist_user_display_kind = ( "internal_notification" if getattr(event, "internal", False) else None ) @@ -21430,11 +19784,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew context, _redact_pii, session_key ) - # Per-turn must-deliver notes. These used to be appended to - # context_prompt (the ephemeral system prompt), which guaranteed a - # turn1→turn2 system-prompt diff and a full agent rebuild. They now - # ride the current user message via the api_content sidecar instead - # (staged below, consumed in run_sync → build_turn_context). + # Per-turn must-deliver notes. These used to be appended to context_prompt (the ephemeral + # system prompt), which guaranteed a turn1→turn2 system-prompt diff and a full agent + # rebuild. They now ride the current user message via the api_content sidecar (staged + # below, consumed in run_sync → build_turn_context). turn_sidecar_notes: List[str] = [] # If the previous session expired and was auto-reset, deliver a notice @@ -21449,11 +19802,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew context_note = "[System note: The previous gateway session could not be recovered after a restart (API recovery timed out). This is a fresh conversation — use /resume to restore history if needed.]" else: context_note = "[System note: The user's previous session expired due to inactivity. This is a fresh conversation with no prior context.]" - # Slack/Discord channels/threads are long-lived: point the agent at - # the specific prior same-channel session so it recalls that context - # via session_search instead of an unrelated recent session. Returns - # None (appends nothing) for other platforms or when there's no prior - # activity to recall. Deterministic — no extra API/DB calls (#36220). + # Slack/Discord channels/threads are long-lived: point the agent at the specific prior + # same-channel session so it recalls that context via session_search instead of an + # unrelated recent session. Deterministic — no extra API/DB calls. try: continuity_note = build_channel_continuity_note(session_entry, source) except Exception: @@ -21473,10 +19824,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) platform_name = source.platform.value if source.platform else "" had_activity = getattr(session_entry, 'reset_had_activity', False) - # Suspended and restart-recovery-expired sessions always notify - # regardless of policy.notify — the user had an active session - # that was silently replaced, so they need to know they can - # /resume it. Idle/daily resets respect the policy flag. + # Suspended and restart-recovery-expired sessions always notify regardless of + # policy.notify — the user had an active session that was silently replaced, so they + # need to know they can /resume it. Idle/daily resets respect the policy flag. should_notify = reset_reason in {"suspended", "resume_pending_expired"} or ( policy.notify and had_activity @@ -21521,10 +19871,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # (single source of truth); only the reset reason needs clearing here. session_entry.auto_reset_reason = None - # Auto-load skill(s) for topic/channel bindings (Telegram DM Topics, - # Discord channel_skill_bindings). Supports a single name or ordered list. - # Only inject on NEW sessions — ongoing conversations already have the - # skill content in their conversation history from the first message. + # Auto-load skill(s) for topic/channel bindings (Telegram DM Topics, Discord + # channel_skill_bindings). Supports a single name or ordered list. Only inject on NEW + # sessions — ongoing conversations already carry the skill content in their history. _auto = getattr(event, "auto_skill", None) if _is_new_session and _auto: _skill_names = [_auto] if isinstance(_auto, str) else list(_auto) @@ -21594,17 +19943,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _lease_state.lease_token = _lease_token _lease_state.lease_generation = run_generation - # A turn only becomes durable recovery work after it owns (or has - # explicitly degraded past) the per-session lease. Marking before the - # await above would falsely recover an alias-routed message that never - # began processing if the gateway died while it was still waiting. + # A turn only becomes durable recovery work after it owns (or has explicitly degraded past) + # the per-session lease. Marking before the await above would falsely recover an alias- + # routed message that never began processing if the gateway died while it was still waiting. await self._mark_durable_active_turn(event, session_entry.session_key) - # Load conversation history from transcript. An unreadable canonical - # store is not an empty conversation: stop before the agent can invent - # continuity from a plausible-looking []. This return happens before - # the broad cleanup finally below, so restore task-local context here; - # the outer dispatch still clears the durable marker and turn lease. + # Load conversation history from transcript. An unreadable canonical store is not an empty + # conversation: stop before the agent can invent continuity from a plausible-looking []. + # This return happens before the broad cleanup finally below, so restore task-local context + # here; the outer dispatch still clears the durable marker and turn lease. try: history = await self.async_session_store.load_transcript( session_entry.session_id @@ -21617,7 +19964,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "state.db, then resend after it is healthy. Use /reset only " "if you intentionally want to start a new conversation." ) - + # ----------------------------------------------------------------- # Session hygiene: auto-compress pathologically large transcripts # @@ -21639,27 +19986,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew get_model_context_length_async, ) - # Read model + compression config from config.yaml. - # NOTE: hygiene threshold is intentionally HIGHER than the agent's - # own compressor (0.85 vs 0.50). Hygiene is a safety net for - # sessions that grew too large between turns — it fires pre-agent - # to prevent API failures. The agent's own compressor handles - # normal context management during its tool loop with accurate - # real token counts. Having hygiene at 0.50 caused premature - # compression on every turn in long gateway sessions. + # Read model + compression config from config.yaml. NOTE: hygiene threshold is + # intentionally HIGHER than the agent's own compressor (0.85 vs 0.50). Hygiene is a + # pre-agent safety net for sessions that grew too large between turns; the agent's own + # compressor handles normal context management with real token counts. Having hygiene at + # 0.50 caused premature compression on every turn in long gateway sessions. _hyg_model = "anthropic/claude-sonnet-4.6" _hyg_threshold_pct = 0.85 _hyg_compression_enabled = True _hyg_hard_msg_limit = 5000 _hyg_timeout_seconds = 30.0 _hyg_total_ceiling_seconds = 600.0 - # Max wall-clock the user's TURN is held waiting on hygiene - # compression before the gateway stops waiting and proceeds on the - # uncompressed transcript (#TKT-0029). The compressor keeps running - # detached; its commit is fenced (revoke_commit_admission) so a - # stale result can never clobber turns appended after the wait was - # abandoned. Capped well below typical transport idle-timeouts - # (Telegram ~30s) so the wire never goes silent long enough to sever. + # Max wall-clock the user's TURN is held waiting on hygiene compression before the + # gateway stops waiting and proceeds on the uncompressed transcript. The compressor keeps + # running detached; its commit is fenced (revoke_commit_admission) so a stale result can + # never clobber later turns. Kept well below transport idle-timeouts (Telegram ~30s). _hyg_max_turn_hold_seconds = 10.0 _hyg_failure_cooldown_seconds = 300.0 _hyg_config_context_length = None @@ -21828,26 +20169,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else: _approx_tokens = estimate_messages_tokens_rough(history) _token_source = "estimated" - # Note: rough estimates overestimate by 30-50% for code/JSON-heavy - # sessions, but that just means hygiene fires a bit early — which - # is safe and harmless. The 85% threshold already provides ample - # headroom (agent's own compressor runs at 50%). A previous 1.4x - # multiplier tried to compensate by inflating the threshold, but - # 85% * 1.4 = 119% of context — which exceeds the model's limit - # and prevented hygiene from ever firing for ~200K models (GLM-5). + # Note: rough estimates overestimate by 30-50% for code/JSON-heavy sessions, but + # that just means hygiene fires a bit early — which is safe and harmless. Do NOT + # compensate with a threshold multiplier: 85% * 1.4 = 119% of context, which + # kept hygiene from ever firing for ~200K models. - # Hard safety valve: force compression if message count is - # extreme, regardless of token estimates. This breaks the - # death spiral where API disconnects prevent token data - # collection, which prevents compression, which causes more - # disconnects. 5000 messages is far above any normal session - # but catches truly runaway growth before it becomes - # unrecoverable. Set well clear of legitimate large-context - # (1M+) sessions doing thousands of short turns — those - # compress on the token threshold, not this count-based floor. - # Threshold is configurable via + # Hard safety valve: force compression if message count is extreme, regardless of + # token estimates. This breaks the death spiral where API disconnects prevent token + # data collection, which prevents compression, which causes more disconnects. The + # default (5000) sits well clear of legitimate 1M+ context sessions doing thousands + # of short turns — those compress on the token threshold. Configurable via # compression.hygiene_hard_message_limit. - # (#2153) _HARD_MSG_LIMIT = _hyg_hard_msg_limit _needs_compress = ( _approx_tokens >= _compress_token_threshold @@ -21855,11 +20187,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) if _needs_compress: - # Use the persistent DB-backed cooldown (same as the - # in-conversation compression path in context_compressor.py) - # so the cooldown survives gateway restarts. The in-memory - # dict was reset on every restart, re-triggering the same - # failing compression and wedging session storage (#74136). + # Use the persistent DB-backed cooldown (same as the in-conversation compression + # path in context_compressor.py) so the cooldown survives gateway restarts. The + # in-memory dict reset on every restart, re-triggering the same failing + # compression and wedging session storage. _session_db = getattr(self, "_session_db", None) if _session_db is not None: _session_db = getattr(_session_db, "_db", _session_db) @@ -21881,12 +20212,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if _needs_compress and await self._session_has_compression_in_flight( session_key ): - # A prior hygiene/agent compression still holds the - # durable lock (typically a shielded worker left behind - # by /stop or /restart). Starting another attempt waits - # up to the 600s ceiling behind a commit that the fence - # will refuse, while inbound messages demote to queue - # (#96953). + # A prior hygiene/agent compression still holds the durable lock (typically a + # shielded worker left behind by /stop or /restart). Starting another attempt + # would wait up to the 600s ceiling behind a commit the fence will refuse, while + # inbound messages demote to queue. logger.info( "Session hygiene: skipping compression for %s; " "another compression is already in flight", @@ -21960,14 +20289,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"{_approx_tokens:,}", ) elif _hyg_runtime.get("api_key"): - # Pass the FULL transcript (tool results included). - # Filtering to user/assistant-only starved the - # compressor: tool results are usually the bulk of - # the context, _prune_old_tool_results never saw - # them, and short filtered histories tripped the - # protect-first/last early-return so nothing was - # compressed at all (#3854). The agent loop passes - # its full message list to _compress_context — the + # Pass the FULL transcript (tool results included). Filtering to + # user/assistant starved the compressor: tool results are the bulk of + # context, _prune_old_tool_results never saw them, and short histories + # tripped the protect-first/last early-return so nothing compressed. The + # agent loop passes its full message list to _compress_context — the # gateway now matches. _hyg_msgs = [ m for m in history @@ -22028,15 +20354,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_agent.platform = _GATEWAY_HYGIENE_PLATFORM _hyg_cleanup_deferred = False try: - # Gateway hygiene runs before the user turn - # starts and already owns the session binding. - # Prefer in-place compaction here: it archives - # old rows under the same session id instead of - # minting a continuation child that then has to - # be published back to SessionStore/topic - # bindings. If no SessionDB is available, - # compress_context leaves this flag false and - # the guard below preserves the transcript. + # Gateway hygiene runs before the user turn starts and already + # owns the session binding, so prefer in-place compaction: it + # archives old rows under the same session id instead of minting + # a continuation child that must be published back to + # SessionStore/topic bindings. Without a SessionDB, + # compress_context leaves this False and the guard below + # preserves the transcript. _hyg_agent.compression_in_place = True _bind_hyg_state = getattr( getattr(_hyg_agent, "context_compressor", None), @@ -22057,19 +20381,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_commit_fence = CompressionCommitFence( total_ceiling_seconds=_hyg_total_ceiling_seconds ) - # Default executor (NOT self._get_executor): - # a fence-cancelled hung summary must never - # occupy one of the gateway's agent-work - # slots. But it MUST run inside the caller's - # contextvars: under multiplex_profiles the - # profile secret scope / HERMES_HOME override - # live in ContextVars, and a bare - # run_in_executor worker starts with an empty - # Context — the summary model's - # get_secret(_API_KEY) then fails - # closed (UnscopedSecretError) and every - # hygiene compaction silently degrades to a - # lossy truncation (#100849 bundle). + # Default executor (NOT self._get_executor): a fence-cancelled + # hung summary must never occupy one of the gateway's agent-work + # slots. But it MUST run inside the caller's contextvars: under + # multiplex_profiles the profile secret scope / HERMES_HOME live + # in ContextVars; a bare run_in_executor worker starts with an + # empty Context, get_secret() fails closed, and every hygiene + # compaction silently degrades to lossy truncation. _hyg_future = loop.run_in_executor( None, copy_context().run, @@ -22080,26 +20398,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ), ) try: - # Progress-aware wait: the timeout is an - # INACTIVITY budget, not a total one. The - # compression worker streams its summary - # call and ticks the fence per token - # (CompressionCommitFence.touch_progress), - # so a slow reasoning model that is still - # generating keeps extending the deadline; - # only a genuinely silent worker times out. - # A hard ceiling bounds the total wait so - # a degenerate trickle stream can't hold + # Progress-aware wait: the timeout is an INACTIVITY budget, + # not a total one — the worker ticks the fence per streamed + # token (touch_progress), so a slow reasoning model that is + # still generating keeps extending the deadline; only a + # genuinely silent worker times out. A hard ceiling bounds + # the total wait so a degenerate trickle stream can't hold # the turn forever. _hyg_wait_started = time.monotonic() while True: if _hyg_commit_fence.is_cancelled: raise asyncio.TimeoutError - # #76354 S3: charge the idle budget - # from the LAST PROGRESS event, not - # from the start of this wait slice — - # otherwise silence can approach 2x - # the configured timeout. + # #76354 S3: charge the idle budget from the LAST + # PROGRESS event, not from the start of this wait slice + # — otherwise silence can approach 2x the configured + # timeout. _hyg_waited = ( time.monotonic() - _hyg_wait_started ) @@ -22331,15 +20644,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _rs_err, ) else: - # Nothing to adopt (summary failed, - # fence refused the commit, or the - # attempt was superseded). Restore - # the pre-#97963 spacing so - # sustained traffic does not spawn - # and abandon a fresh compressor - # every turn. Flat and - # non-escalating: the streak must - # not advance for a deferral. + # Nothing to adopt (summary failed, fence + # refused the commit, or the attempt was + # superseded). Record flat spacing so sustained + # traffic does not spawn and abandon a fresh + # compressor every turn; non-escalating — the + # failure streak must not advance for a deferral. _record_hygiene_cooldown( _gw, _sid, _HYGIENE_TURNHOLD_RETRY_SECONDS, @@ -22400,15 +20710,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if _cancelled is None: await asyncio.sleep(0.025) if not _cancelled: - # NOTE: bounded overshoot by design. - # The turn can be held past - # _hyg_max_turn_hold_seconds by up to the - # commit duration (summary apply + storage - # write). Aborting mid-commit would corrupt - # the message-store transaction — the - # overshoot is the cheaper failure mode. - # Do NOT "fix" this into a mid-commit - # cancellation. + # NOTE: bounded overshoot by design: the turn can be held + # past _hyg_max_turn_hold_seconds by up to the commit + # duration. Aborting mid-commit would corrupt the + # message-store transaction — the overshoot is the + # cheaper failure mode. Do NOT "fix" this into a + # mid-commit cancellation. _compressed, _ = await _hyg_future else: _hyg_commit_fence.release_cancelled_compression_lock() @@ -22418,16 +20725,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew context="session hygiene turn-hold", ) _hyg_cleanup_deferred = True - # Short, NON-escalating retry-after. Without - # it, every subsequent turn re-spawns a fresh - # compressor, holds it for the turn-hold - # budget, and cancels it again — a per-turn - # summary-model token burn that never commits - # under sustained traffic. This is deliberately - # NOT _hygiene_cooldown_for_failure: the - # compressor is healthy, so the failure streak - # must not advance (behavior witness below); - # only the flat retry spacing is recorded. + # Short, NON-escalating retry-after. Without it every + # turn re-spawns a compressor, holds it for the turn-hold + # budget and cancels it — token burn that never commits. + # Deliberately NOT _hygiene_cooldown_for_failure: the + # compressor is healthy, so the failure streak must not + # advance; only flat retry spacing is recorded. _record_hygiene_cooldown( self, session_entry.session_id, _HYGIENE_TURNHOLD_RETRY_SECONDS, @@ -22477,26 +20780,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew or _hyg_commit_fence.deadline_exceeded ) if _hyg_total_exhausted: - # The worker cooperatively checks this - # deadline between digest calls. Keep - # its lease until it exits so an - # unchanged session cannot overlap a - # retry. Inactivity timeouts retain the - # established release behavior for a - # provider call that may never return. + # The worker cooperatively checks this deadline between + # digest calls. Keep its lease until it exits so an + # unchanged session cannot overlap a retry. _hyg_commit_fence.retain_compression_lock_until_worker_done() - # Capture fence state BEFORE try_cancel - # — that call itself sets is_cancelled, - # which would mis-label a genuine idle - # timeout as a fence cancel (#96953). + # Capture fence state BEFORE try_cancel — that call itself + # sets is_cancelled, which would mis-label a genuine idle + # timeout as a fence cancel. _hyg_fence_cancelled = ( _hyg_commit_fence.is_cancelled ) _cancelled = None while _cancelled is None: - # #76354 F1: a hung commit retains the - # fence lock; the lock-free phase - # marker keeps this loop from spinning + # #76354 F1: a hung commit retains the fence lock; the + # lock-free phase marker keeps this loop from spinning # forever while the commit blocks. if _hyg_commit_fence.commit_in_flight: _cancelled = False @@ -22505,25 +20802,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_commit_fence.try_cancel_before_commit() ) if _cancelled is None: - # Round-2 #5: transient - # lock-setup windows ride - # write patience for seconds; - # 25ms keeps sub-tick latency - # without 1kHz spin. + # Round-2 #5: transient lock-setup windows ride + # write patience for seconds; 25ms keeps sub-tick + # latency without 1kHz spin. await asyncio.sleep(0.025) if not _cancelled: - # The worker crossed the commit boundary just - # before the timeout. The fence poll waited for - # that boundary to finish, so consume the - # completed result instead of treating a - # successful compaction as a timeout. + # The worker crossed the commit boundary just before the + # timeout; the fence poll waited for it to finish, so + # consume the result instead of treating a successful + # compaction as a timeout. _compressed, _ = await _hyg_future else: - # Release an inactivity-timed-out - # worker's holder-qualified lease - # promptly. Total-ceiling attempts - # retained it above, so this is a - # no-op until worker cleanup there. + # Release an inactivity-timed-out worker's holder- + # qualified lease promptly. Total-ceiling attempts + # retained it above, so this is a no-op for them. _hyg_commit_fence.release_cancelled_compression_lock() self._defer_agent_cleanup_until_future_done( _hyg_future, @@ -22639,14 +20931,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) raise except BaseException: - # #76354 F2: non-timeout unwind while the - # detached hygiene worker may still run — - # KeyboardInterrupt, task cancellation, or - # any unexpected error. Revoke commit - # admission (and release the worker's - # durable lease via the holder-qualified - # hook) BEFORE the host unwinds so the - # worker can never commit later. + # #76354 F2: non-timeout unwind while the detached hygiene + # worker may still run — KeyboardInterrupt, task + # cancellation, or any unexpected error. Revoke commit + # admission (and release the worker's durable lease) BEFORE + # the host unwinds so the worker can never commit later. _hyg_commit_fence.revoke_commit_admission() if not _hyg_cleanup_deferred: self._defer_agent_cleanup_until_future_done( @@ -22655,11 +20944,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew context="session hygiene unwind", ) _hyg_cleanup_deferred = True - # #96953: restart drain / task cancel used - # to re-raise with no cooldown, so the - # next turn immediately re-armed hygiene - # and waited up to 600s behind a fence - # that would refuse the commit again. + # restart drain / task cancel used to re-raise with no + # cooldown, so the next turn immediately re-armed hygiene + # and waited up to 600s behind a fence that would refuse the + # commit again. if _hyg_failure_cooldown_seconds >= 0: try: _hyg_cooldown = _hygiene_cooldown_for_failure( @@ -22681,19 +20969,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) raise - # _compress_context ends the old session and creates - # a new session_id. Write compressed messages into - # the NEW session so the old transcript stays intact - # and searchable via session_search. + # _compress_context ends the old session and creates a new + # session_id. Write compressed messages into the NEW session so + # the old transcript stays intact and searchable. _hyg_new_sid = _hyg_agent.session_id _hyg_rotated = _hyg_new_sid != session_entry.session_id _hyg_in_place = bool( getattr(_hyg_agent, "_last_compaction_in_place", False) ) - # Anti-growth guard: refuse a compression - # that did not shrink the transcript - # (observed: 427K -> 598K). Compare - # like-for-like rough estimates. + # Anti-growth guard: refuse a compression that did not shrink + # the transcript (observed: 427K -> 598K). Compare like-for-like + # rough estimates. _hyg_in_toks = estimate_messages_tokens_rough(history) _hyg_out_toks = estimate_messages_tokens_rough(_compressed) if _hyg_rotated and _hyg_out_toks > _hyg_in_toks: @@ -22753,10 +21039,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_in_place = False else: session_entry.session_id = _hyg_new_sid - # The held turn lease follows the - # rotation so an alias key resolving - # the fresh child still serializes - # against this turn (#64934). + # The held turn lease follows the rotation so an alias + # key resolving the fresh child still serializes against + # this turn. self._rebind_turn_lease( _quick_key, run_generation, _hyg_new_sid ) @@ -22827,16 +21112,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_aborted = _comp is not None and getattr( _comp, "_last_compress_aborted", False ) - # Fence-cancelled _compress_context returns - # the original transcript with - # _last_compress_aborted still False - # (failure_class=commit_fence_cancelled, - # chunk_count=0). Treat that no-op as an - # abort so hygiene records a cooldown - # instead of retrying into the 600s wait - # (#96953). A successful rotate/in-place - # commit is not an abort even if a later - # invalidation flipped the fence. + # Fence-cancelled _compress_context returns the original + # transcript with _last_compress_aborted still False + # (failure_class=commit_fence_cancelled, chunk_count=0). Treat + # that no-op as an abort so hygiene records a cooldown instead + # of retrying into the 600s wait. A successful rotate/in-place + # commit is not an abort even if a later invalidation flipped + # the fence. _hyg_fence_cancelled = bool( _hyg_commit_fence.is_cancelled and not _hyg_rotated @@ -22921,12 +21203,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Failed to deliver compression-failure warning to user: %s", _werr, ) - # Separately: if the user's CONFIGURED aux - # model failed and we recovered by falling - # back to the main model, tell them — a - # misconfigured auxiliary.compression.model - # is something only they can fix, and - # silent recovery would hide it. + # Separately: if the user's CONFIGURED aux model failed and we + # recovered by falling back to the main model, tell them — a + # misconfigured auxiliary.compression.model is something only + # they can fix, and silent recovery would hide it. elif _comp is not None and getattr(_comp, "_last_aux_model_failure_model", None): _aux_model = getattr(_comp, "_last_aux_model_failure_model", "") _aux_err = getattr(_comp, "_last_aux_model_failure_error", None) or "unknown error" @@ -22956,22 +21236,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except HygieneTurnHoldExceeded: - # Availability boundary, not a failure — already logged - # at INFO by the turn-hold handler. Must not hit the - # generic "auto-compress failed" warning below: that - # log is how thinking-model deployments read as - # permanently broken (#97963; surfaced by @686f6c61 - # in PR #99657). + # Availability boundary, not a failure — already logged at INFO by the turn- + # hold handler. Must not hit the generic "auto-compress failed" warning + # below: that log made thinking-model deployments read as permanently broken. pass except Exception as e: logger.warning( "Session hygiene auto-compress failed: %s", e ) - # First-message onboarding -- only on the very first interaction ever. - # Delivered on the current user message (sidecar), NOT the ephemeral - # system prompt: present-on-turn-1/absent-on-turn-2 was a guaranteed - # system-prompt diff and agent rebuild. + # First-message onboarding -- only on the very first interaction ever. Delivered on the + # current user message (sidecar), NOT the ephemeral system prompt: present-on-turn-1/absent- + # on-turn-2 was a guaranteed system-prompt diff and agent rebuild. if not history and not await self.async_session_store.has_any_sessions(): # Default first-contact note: a brief self-introduction. _intro_note = ( @@ -23008,7 +21284,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _pb_err, ) turn_sidecar_notes.append(_intro_note) - + # One-time prompt if no home channel is set for this platform # Skip for webhooks - they deliver directly to configured targets (github_comment, etc.) if not history and source.platform and source.platform != Platform.LOCAL and source.platform != Platform.WEBHOOK: @@ -23062,7 +21338,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"or ignore to skip." ) await self._deliver_platform_notice(source, notice) - + # ----------------------------------------------------------------- # Voice channel awareness — deliver current voice channel state so # the agent knows who is in the channel and who is speaking, without @@ -23098,12 +21374,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if message_text is None: return - # Capture the platform event time as message metadata and keep the - # persisted transcript clean (strip any leading timestamp prefix). - # This runs regardless of the toggle so storage stays clean and the - # send-time is preserved. Only the in-context RENDER (prepending the - # human-readable prefix the model sees) is gated behind - # gateway.message_timestamps.enabled — default OFF. + # Capture the platform event time as message metadata and keep the persisted transcript + # clean (strip any leading timestamp prefix). This runs regardless of the toggle so storage + # stays clean and the send-time is preserved. Only the in-context RENDER (the prefix the + # model sees) is gated behind gateway.message_timestamps.enabled — default OFF. try: from hermes_time import get_timezone as _get_evt_tz from gateway.message_timestamps import ( @@ -23134,10 +21408,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _ts_err: logger.debug("Message timestamp injection failed (non-fatal): %s", _ts_err) - # Stage the collected must-deliver notes for this turn's agent run - # (one-shot; consumed in run_sync). Staged AFTER the message_text - # early-out above so an aborted turn cannot leak its notes into the - # next turn's user message. + # Stage the collected must-deliver notes for this turn's agent run (one-shot; consumed in + # run_sync). Staged AFTER the message_text early-out above so an aborted turn cannot leak + # its notes into the next turn's user message. if turn_sidecar_notes and session_key: self._set_pending_turn_sidecar_notes(session_key, turn_sidecar_notes) @@ -23163,10 +21436,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew } await self.hooks.emit("agent:start", hook_ctx) - # Run the agent. Capture the session id that this run was launched - # against so post-run compression publication can be identity-guarded - # below; a /new or another lifecycle transition may move - # session_entry.session_id while the old run is still unwinding. + # Run the agent. Capture the session id that this run was launched against so post-run + # compression publication can be identity-guarded below; a /new or another lifecycle + # transition may move session_entry.session_id while the old run is still unwinding. _run_start_session_id = session_entry.session_id _turn_started_monotonic = time.monotonic() agent_result = await self._run_agent( @@ -23228,12 +21500,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return None response = agent_result.get("final_response") or "" - # Hidden-reasoning-only retry exhaustion: the loop's sentinel text - # ("Codex response remained incomplete after 3 continuation - # attempts") doubles as final_response, so it would be delivered - # verbatim into the channel — where peer agents can ingest it as a - # completed assistant turn (#51628). Blank it here so the normal - # empty-response handling (and the suppression below) applies. + # Hidden-reasoning-only retry exhaustion: the loop's sentinel text ("Codex response + # remained incomplete after 3 continuation attempts") doubles as final_response, so it + # would be delivered verbatim into the channel — where peer agents can ingest it as a + # completed assistant turn. if _is_gateway_hidden_reasoning_incomplete_turn(agent_result): response = "" try: @@ -23244,11 +21514,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: _intentional_silence = False - # Convert the agent's internal "(empty)" sentinel into a - # user-friendly message. "(empty)" means the model failed to - # produce visible content after exhausting all retries (nudge, - # prefill, empty-retry, fallback). Sending the raw sentinel - # looks like a bug; a short explanation is more helpful. + # Convert the agent's internal "(empty)" sentinel into a user-friendly message. + # "(empty)" means the model failed to produce visible content after exhausting all + # retries (nudge, prefill, empty-retry, fallback). if response == "(empty)" and not _intentional_silence: response = ( "⚠️ The model returned no response after processing tool " @@ -23266,20 +21534,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) # NOTE: the cross-process cache-coherence re-baseline - # (_refresh_agent_cache_message_count) is intentionally deferred - # until AFTER this turn's transcript persistence block below — it - # must include the first-turn `session_meta` marker row and the - # compression session_id swap, both of which happen later. See - # the call site after the `update_session(...)` write. + # (_refresh_agent_cache_message_count) is intentionally deferred until AFTER this turn's + # transcript persistence block below — it must include the first-turn `session_meta` + # marker row and the compression session_id swap, both of which happen later. - # Successful turn — clear any stuck-loop counter for this session. - # This ensures the counter only accumulates across CONSECUTIVE - # restarts where the session was active (never completed). - # - # Also clear the resume_pending flag (set by drain-timeout - # shutdown) — the turn ran to completion, so recovery - # succeeded and subsequent messages should no longer receive - # the restart-interruption system note. + # Successful turn — clear any stuck-loop counter for this session. This ensures the + # counter only accumulates across CONSECUTIVE restarts where the session was active + # (never completed). Also clear resume_pending (set by drain-timeout shutdown): the turn + # completed, so later messages must not get the restart-interruption system note. if session_key and _should_clear_resume_pending_after_turn(agent_result): await self._clear_restart_failure_count(session_key) try: @@ -23298,17 +21560,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) response = _sanitize_gateway_final_response(source.platform, response) - # Ordering contract: the agent thread already updated the contextvar - # in conversation_compression.py; propagate to SessionEntry + _save(). - # If the agent's session_id changed during compression, update - # session_entry so transcript writes below go to the right session. + # Ordering contract: the agent thread already updated the contextvar in + # conversation_compression.py; propagate to SessionEntry + _save(). if agent_result.get("session_id") and agent_result["session_id"] != session_entry.session_id: if session_entry.session_id == _run_start_session_id: session_entry.session_id = agent_result["session_id"] - # The held turn lease follows the rotation: the transcript - # persistence below writes to the NEW id, so the - # serialization boundary must move with it or an alias - # key resolving the fresh child could interleave (#64934). + # The held turn lease follows the rotation: the transcript persistence below + # writes to the NEW id, so the serialization boundary must move with it or an + # alias key resolving the fresh child could interleave. self._rebind_turn_lease( _quick_key, run_generation, session_entry.session_id ) @@ -23390,10 +21649,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew display_reasoning = escape_code_fences_for_display(display_reasoning) response = f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}" - # Runtime-metadata footer — only on the FINAL message of the turn. - # Off by default (display.runtime_footer.enabled=false). When - # streaming already delivered the body, we can't mutate the sent - # text, so we fire a separate trailing send below. + # Runtime-metadata footer — only on the FINAL message of the turn. Off by default + # (display.runtime_footer.enabled=false). When streaming already delivered the body, we + # can't mutate the sent text, so we fire a separate trailing send below. _footer_line = "" try: from gateway.runtime_footer import build_footer_line as _bfl @@ -23419,7 +21677,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "model": agent_result.get("model", ""), "provider": agent_result.get("provider", ""), }) - + # Check for pending process watchers (check_interval on background processes) try: from tools.process_registry import process_registry @@ -23435,28 +21693,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as e: logger.error("Process watcher setup error: %s", e) - # Drain watch pattern notifications that arrived during the agent run. - # Watch events and completions share the same queue; process - # completions are already handled by the per-process watcher task - # above, so we only inject watch-type events here. - # - # Async-delegation completions ALSO ride this shared queue but are - # owned by the dedicated _async_delegation_watcher (started at - # boot), which covers both the idle and post-turn cases with a - # single consumer — so we leave them on the queue here. + # Drain watch pattern notifications that arrived during the agent run. Watch events and + # completions share the same queue; process completions are already handled by the per- + # process watcher task above, so we only inject watch-type events here. Async-delegation + # completions also ride this queue but are owned by _async_delegation_watcher (single + # consumer for idle and post-turn cases) — leave them on the queue. try: from tools.process_registry import process_registry as _pr await self._drain_watch_notifications(_pr.completion_queue) except Exception as e: logger.debug("Watch queue drain error: %s", e) - # NOTE: Dangerous command approvals are now handled inline by the - # blocking gateway approval mechanism in tools/approval.py. The agent - # thread blocks until the user responds with /approve or /deny, so by - # the time we reach here the approval has already been resolved. The - # old post-loop pop_pending + approval_hint code was removed in favour - # of the blocking approach that mirrors CLI's synchronous input(). - + # NOTE: Dangerous command approvals are now handled inline by the blocking gateway + # approval mechanism in tools/approval.py. + # Save the full conversation to the transcript, including tool calls. # This preserves the complete agent loop (tool_calls, tool results, # intermediate reasoning) so sessions can be resumed with full context @@ -23477,10 +21727,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew agent_result ) _err_str_for_classify = str(agent_result.get("error", "")).lower() - # Use specific multi-word phrases (not bare "exceed" or "token") - # to avoid false positives on transient errors like "rate limit - # exceeded" or "invalid auth token". Matches run_agent.py's - # own context-length classifier. + # Use specific multi-word phrases (not bare "exceed" or "token") to avoid false + # positives on transient errors like "rate limit exceeded" or "invalid auth token". + # Matches run_agent.py's own context-length classifier. is_context_overflow_failure = agent_failed_early and ( bool(agent_result.get("compression_exhausted")) or any(p in _err_str_for_classify for p in ( @@ -23512,16 +21761,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew agent_result.get("error", "processing incomplete"), ) - # When compression is exhausted, the session is permanently too - # large to process. Auto-reset it so the next message starts - # fresh instead of replaying the same oversized context in an - # infinite fail loop. (#9893) - # - # A lock-contended defer is the OPPOSITE case: the session is - # temporarily uncompressible only because a concurrent path holds - # the compression lock and is actively shrinking it. Never wipe - # the session for that — retry-next-message semantics apply - # (#69870 lock-skip consumer; salvaged from #49874). + # When compression is exhausted, the session is permanently too large to process. Auto- + # reset it so the next message starts fresh instead of replaying the same oversized + # context in an infinite fail loop. A lock-contended defer is the OPPOSITE case: a + # concurrent path holds the compression lock and is shrinking it — never wipe the + # session for that; retry-next-message semantics apply. if agent_result.get("compression_deferred"): logger.info( "Compression deferred for session %s — the compression " @@ -23567,7 +21811,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) ts = time.time() # Unix epoch float — consistent with DB storage - + # If this is a fresh session (no history), write the full tool # definitions as the first entry so the transcript is self-describing # -- the same list of dicts sent as tools=[...] in the API request. @@ -23585,7 +21829,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "timestamp": ts, } ) - + # The agent already persisted these messages to SQLite via # _flush_messages_to_session_db(), so skip the DB write here # to prevent the duplicate-write bug (#860 / #42039). This holds @@ -23598,20 +21842,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # runtime opt into a gateway-side write by returning False. agent_persisted = agent_result.get("agent_persisted", self._session_db is not None) - # Find only the NEW messages from this turn (skip history we loaded). - # Use the filtered history length (history_offset) that was actually - # passed to the agent, not len(history) which includes session_meta - # entries that were stripped before the agent saw them. + # Find only the NEW messages from this turn (skip history we loaded). Use history_offset + # (what the agent actually saw), not len(history), which includes session_meta entries + # stripped before the agent saw them. if is_context_overflow_failure: pass # handled above — skip all transcript writes elif agent_failed_early or hidden_reasoning_incomplete: - # Transient failure (429/timeout/5xx): persist only the user - # message so the next message can load a transcript that - # reflects what was said. Skip the assistant error text since - # it's a gateway-generated hint, not model output. Hidden- - # reasoning-only incomplete turns follow the same persistence - # rule so peer-agent channels don't ingest them as completed - # assistant turns. (#7100, #51628) + # Transient failure (429/timeout/5xx): persist only the user message so the next + # message can load a transcript that reflects what was said. Skip the assistant + # error text since it's a gateway-generated hint, not model output. Hidden-reasoning + # incomplete turns follow the same rule so peer-agent channels don't ingest them. _user_entry = { "role": "user", "content": ( @@ -23685,10 +21925,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew skip_db=agent_persisted, ) else: - # Attach the inbound platform message_id to the first user - # entry written this turn so platform-level quote-resolution - # (e.g. Yuanbao QuoteContextMiddleware's transcript fallback) - # can find earlier @bot messages by their original message_id. + # Attach the inbound platform message_id to the first user entry written this + # turn so platform-level quote-resolution (e.g. Yuanbao QuoteContextMiddleware's + # transcript fallback) can find earlier @bot messages by their original id. _user_msg_id_attached = False for msg in new_messages: # Skip system messages (they're rebuilt each run) @@ -23708,7 +21947,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_entry.session_id, entry, skip_db=agent_persisted, ) - + # Token counts and model are now persisted by the agent directly. # Keep only last_prompt_tokens here for context-window tracking and # compression decisions. @@ -23718,33 +21957,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew touch_activity=not bool(getattr(event, "internal", False)), ) - # Re-baseline the cached agent's message_count snapshot now that - # ALL of this turn's transcript writes are done — the agent's - # flushed user/assistant/tool rows AND the first-turn `session_meta` - # marker appended above. The cross-process coherence guard (#45966) - # snapshots the count at agent-BUILD time (before this turn's own - # writes) and never refreshes it on reuse, so without this the - # process's own turn grows message_count and the next turn sees a - # mismatch and rebuilds the agent — destroying prompt caching. - # - # This MUST run after the `session_meta` append: that row also - # increments message_count, so re-baselining before it (the old - # position) left the snapshot one short and the guard mis-fired on - # turn 2 of EVERY fresh gateway conversation, rebuilding the cached - # agent and busting the prompt cache. Running here also uses the - # compaction-updated session_id (the agent_result session_id swap - # above), matching this function's documented contract. Refreshing - # here makes the guard fire only on a DIFFERENT process's writes. - # Fail-safe inside the helper. + # Re-baseline the cached agent's message_count snapshot now that ALL of this turn's + # transcript writes are done — the agent's flushed user/assistant/tool rows AND the + # first-turn `session_meta` marker appended above. The cross-process coherence guard + # snapshots at agent-BUILD time and never refreshes on reuse, so without this our own + # writes trigger a rebuild next turn (destroying prompt caching). MUST run after the + # session_meta append (that row also bumps the count; before it, the snapshot was one + # short and turn 2 of every fresh conversation rebuilt). Fail-safe inside the helper. await self._refresh_agent_cache_message_count( session_key, session_entry.session_id ) - # Intentional silence is a delivery decision, not a transcript - # mutation. The agent's [SILENT]/NO_REPLY assistant turn above is - # still persisted in session history so later turns keep normal - # user/assistant alternation; only the outbound chat delivery is - # suppressed. + # Intentional silence is a delivery decision, not a transcript mutation: the [SILENT] + # assistant turn stays persisted so later turns keep user/assistant alternation; only + # the outbound delivery is suppressed. if _intentional_silence: logger.info( "Suppressing intentional silence marker for session %s", @@ -23766,17 +21992,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): await self._send_voice_reply(event, response) - # If streaming already delivered the response, extract and - # deliver any MEDIA: files before returning None. Streaming - # sends raw text chunks that include MEDIA: tags — the normal - # post-processing in _process_message_background is skipped - # when already_sent is True, so media files would never be - # delivered without this. - # - # Never skip when the agent failed — the error message is new - # content the user hasn't seen (streaming only sent earlier - # partial output before the failure). Without this guard, - # users see the agent "stop responding without explanation." + # If streaming already delivered the response, extract and deliver any MEDIA: files + # before returning None — streamed chunks carry MEDIA: tags verbatim and the normal + # post-processing is skipped when already_sent, so media would never be delivered. + # Never skip when the agent failed: the error text is new content the user hasn't + # seen (streaming only sent partial output before the failure). if agent_result.get("already_sent") and not agent_result.get("failed"): if response: _media_adapter = self._adapter_for_source(source) @@ -23784,10 +22004,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await self._deliver_media_from_response( response, event, _media_adapter, ) - # Streaming already delivered the body text, but the footer was - # intentionally held back (see the `not already_sent` gate above). - # Send it now as a small trailing message so Telegram/Discord/etc. - # still surface the runtime metadata on the final reply. + # Streaming already delivered the body text, but the footer was intentionally held + # back (see the `not already_sent` gate above). if _footer_line: try: _foot_adapter = self._adapter_for_source(source) @@ -23799,10 +22017,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except Exception as _e: logger.debug("trailing footer send failed: %s", _e) - # This branch returns None so the adapter does not send the - # body twice. /loop and /goal hooks in _handle_message read - # the return value, so stash the delivered text on the event - # or those hooks never run and a /loop tick stays awaiting. + # This branch returns None so the adapter does not send the body twice. /loop and + # /goal hooks in _handle_message read the return value, so stash the delivered text + # on the event or those hooks never run and a /loop tick stays awaiting. try: event._streamed_final_response = str(response or "") except Exception: @@ -23810,7 +22027,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return None return response - + except Exception as e: # Stop typing indicator on error too, retaining Slack thread/workspace # routing so a failed turn cannot leave its status visible. @@ -23832,12 +22049,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: pass logger.exception("Agent error in session %s", session_key) - # Crash-resilience for failures that happen before AIAgent enters - # run_conversation() (for example: provider/httpx client init - # failures). In that path the agent cannot persist the current - # inbound turn itself, so append the user message here once. If the - # agent already reached its early turn-start persistence, the latest - # transcript user row will match and we skip the duplicate. + # Crash-resilience for failures that happen before AIAgent enters run_conversation() + # (for example: provider/httpx client init failures). In that path the agent cannot + # persist the current inbound turn itself, so append the user message here once. If the + # agent already reached its turn-start persistence, the latest user row matches and + # we skip the duplicate. try: if 'message_text' in locals() and message_text is not None and session_entry is not None: _already_persisted = False @@ -23933,17 +22149,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _reset_notice_session_info(self, source: SessionSource) -> str: """Session-info block for the auto-reset notice, profile-scoped. - When multiplexing, resolve model/provider/context inside the profile - serving ``source`` — otherwise the banner advertises the base config's - model while the session actually runs on the profile's (#59003). - Mirrors ``_run_agent``'s gating so single-profile gateways never - enter the scope. + When multiplexing, resolve model/provider/context inside the profile serving ``source`` — + otherwise the banner advertises the base config's model while the session actually runs + on the profile's. Mirrors ``_run_agent``'s gating so single-profile gateways never enter + the scope. - Call via ``asyncio.to_thread`` from async handlers: under the scope, - resolution can do blocking work (credential refresh, context-length - HTTP probes) that must not run on the event loop. The scope is entered - inside this method, so contextvars behave correctly in the worker - thread. + Call via ``asyncio.to_thread``: under the scope, resolution can do blocking work + (credential refresh, context-length HTTP probes). The scope is entered inside this method + so contextvars behave correctly in the worker thread. """ if getattr(getattr(self, "config", None), "multiplex_profiles", False): with _profile_runtime_scope(self._resolve_profile_home_for_source(source)): @@ -23953,9 +22166,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _format_session_info(self) -> str: """Resolve current model config and return a formatted info block. - Surfaces model, provider, context length, and endpoint so gateway - users can immediately see if context detection went wrong (e.g. - local models falling to the 128K default). + Surfaces model, provider, context length, and endpoint so gateway users can immediately + see if context detection went wrong (e.g. local models falling to the 128K default). """ resolved = _resolve_gateway_model_context() model = resolved.model @@ -23991,21 +22203,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "\n".join(lines) - - - def _check_slash_access( self, source: SessionSource, canonical_cmd: str ) -> Optional[str]: - """Return a denial message if ``source`` cannot run ``canonical_cmd``, - else None. Used by both the cold and running-agent dispatch paths - in ``_handle_message`` so admin/user gating can't be bypassed by - an in-flight agent. + """Return a denial message if ``source`` cannot run ``canonical_cmd``, else None. - Backward-compat semantics live in - :func:`gateway.slash_access.policy_for_source` — when the operator - hasn't set ``allow_admin_from`` for the scope, the policy returns - ``enabled=False`` and this method always returns None. + Used by both the cold and running-agent dispatch paths in ``_handle_message`` so + admin/user gating can't be bypassed by an in-flight agent. Backward-compat: when the + operator hasn't set ``allow_admin_from`` for the scope, ``policy_for_source`` returns + ``enabled=False`` and this always returns None. """ from gateway.slash_access import policy_for_source as _policy_for_source @@ -24036,23 +22242,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return f"⛔ /{canonical_cmd} is admin-only here. {suffix}" - - - - - - def _sibling_thread_run_keys(self, source: SessionSource, own_key: str) -> list: """Find running-agent keys for OTHER participants in the same thread. - Only applies when the message originates in a thread. In per-user - thread mode (``thread_sessions_per_user=True``) each participant gets - an isolated session key of the form - ``agent:main:{platform}:{chat_type}:{chat_id}:{thread_id}:{user_id}``, - so a run started by another user is invisible to the caller's own - ``/stop``. This returns the keys of any *actually running* agents - (not the pending sentinel, not the caller's own key) whose key shares - the caller's ``{chat_id}:{thread_id}`` prefix. + Only applies when the message originates in a thread. In per-user thread mode each + participant gets an isolated key (``...:{thread_id}:{user_id}``), so another user's run is + invisible to the caller's own ``/stop``. This returns the keys of any + *actually running* agents (not the pending sentinel, not the caller's own key) whose key + shares the caller's ``{chat_id}:{thread_id}`` prefix. Returns an empty list when the source is not in a thread, or when no sibling runs exist — callers must still gate on authorization. @@ -24063,11 +22260,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return [] platform = source.platform.value chat_type = getattr(source, "chat_type", None) or "" - # Prefix that every per-user key in this thread shares, up to and - # including the thread_id segment. Matching either the exact - # shared-thread key or any key with a further (user_id) segment - # (prefix + ":") avoids cross-matching an unrelated thread whose id - # merely starts with this one. + # Prefix that every per-user key in this thread shares, up to and including the thread_id + # segment. Match the exact key or prefix + ":" (a further user_id segment) so an unrelated + # thread whose id merely starts with this one is not matched. prefix = ":".join( ["agent:main", platform, chat_type, str(chat_id), str(thread_id)] ) @@ -24081,20 +22276,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew matches.append(key) return matches - - - def _is_stale_restart_redelivery(self, event: MessageEvent) -> bool: """Return True if this /restart is a Telegram re-delivery we already handled. - The previous gateway wrote ``.restart_last_processed.json`` with the - triggering platform + update_id when it processed the /restart. If - we now see a /restart on the same platform with an update_id <= that - recorded value, it is a redelivery when this process booted from that - restart. Otherwise the marker must still be recent (< 5 minutes). - - Only applies to Telegram today (the only platform that exposes a - numeric cross-session update ordering); other platforms return False. + The previous gateway wrote ``.restart_last_processed.json`` with the triggering platform + + update_id when it processed the /restart. A /restart on the same platform with + update_id <= that value is a redelivery when this process booted from that restart; + otherwise the marker must still be recent (< 5 minutes). Telegram only (the only platform + with a numeric cross-session update ordering); other platforms return False. """ if event is None or event.source is None: return False @@ -24146,12 +22335,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if event.platform_update_id > recorded_uid: return False - # A service-managed restart can legitimately take longer than the - # marker's normal five-minute trust window while adapters, cron, and - # in-flight deliveries drain. If this process booted from the recorded - # chat restart, the first same-or-older update is still that restart's - # redelivery regardless of elapsed wall time. Consume the boot signal - # one-shot so a later genuine command is evaluated normally. + # A service-managed restart can legitimately take longer than the marker's normal five- + # minute trust window while adapters, cron, and in-flight deliveries drain. Consume the boot + # signal one-shot so a later genuine command is evaluated normally. if getattr(self, "_booted_from_restart", False): self._booted_from_restart = False return True @@ -24165,20 +22351,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False return True - - - - - - - - async def _handle_suggestions_command(self, event: MessageEvent) -> str: """Handle /suggestions in the gateway. - Delegates to the shared handler so CLI and gateway never drift. The - origin is built from the event source so an accepted suggestion's job - delivers back to this chat/thread. + Delegates to the shared handler so CLI and gateway never drift. The origin is built from + the event source so an accepted suggestion's job delivers back to this chat/thread. """ args = (event.get_command_args() or "").strip() source = event.source @@ -24206,12 +22383,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _handle_blueprint_command(self, event: MessageEvent): """Handle /blueprint in the gateway. - Delegates to the shared handler so CLI, TUI, and gateway never drift. - Returns a BlueprintCommandResult: ``text`` is shown to the user, and if - ``agent_seed`` is set the dispatch site rewrites ``event.text`` to the - seed and falls through to the agent (the ``/steer`` pattern) so the - agent gathers the slot values conversationally. Origin is built from the - event source so a directly created blueprint job delivers back to this chat. + Delegates to the shared handler so CLI, TUI, and gateway never drift. Origin is built + from the event source so a directly created blueprint job delivers back to this chat. """ args = (event.get_command_args() or "").strip() source = event.source @@ -24265,12 +22438,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _warm_goals_session_db(self, label: str) -> None: """Warm the goals SessionDB cache off-loop (best-effort). - A cold cache runs the state.db init on the loop thread behind the - bootstrap windows. That freezes the loop for the init duration. - The executor hop keeps the profile home override alive under - multiplex, so the warm cache belongs to the caller's profile. - On failure the caller falls back to the bootstrap windows, so a - dropped warm-up is a bounded stall, never a crash. + A cold cache runs the state.db init on the loop thread and freezes the loop for the init + duration. The executor hop keeps the profile home override alive under multiplex, so the + warm cache belongs to the caller's profile. On failure the caller falls back to the + bootstrap windows, so a dropped warm-up is a bounded stall, never a crash. """ try: from hermes_cli.goals import _get_session_db as _warm_goals_db @@ -24342,11 +22513,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _register_heartbeat_watch(self, quick_key: str, source: Any, session_id: str) -> None: """Track a session with an active heartbeat and start the poller. - The registry maps ``quick_key`` → ``(source, session_id)`` so the - poller can rebuild a MessageEvent and enqueue via the adapter FIFO. - In-memory by design: heartbeat STATE survives restarts in SessionDB, - but firing resumes when the user touches /heartbeat again in the new - gateway process (documented; durable schedules belong to cron). + The registry maps ``quick_key`` → ``(source, session_id)`` so the poller can rebuild a + MessageEvent and enqueue via the adapter FIFO. In-memory by design: heartbeat STATE + survives restarts in SessionDB, but firing resumes only when the user touches /heartbeat + again (durable schedules belong to cron). """ watch = getattr(self, "_heartbeat_watch", None) if watch is None: @@ -24374,10 +22544,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew watch = getattr(self, "_heartbeat_watch", None) if not watch: continue - # Warm the cache off-loop once per poll. A watch can only - # be registered through the warmed /heartbeat command, so - # this covers only the degraded path where that warm-up - # failed. + # Warm the cache off-loop once per poll. A watch can only be registered through the + # warmed /heartbeat command, so this covers only the degraded path where that warm- + # up failed. await self._warm_goals_session_db("heartbeat poll") for quick_key, (source, session_id) in list(watch.items()): try: @@ -24410,10 +22579,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew try: task = asyncio.create_task(_poll_loop()) self._heartbeat_poll_task = task - # PERMANENT once started (an infinite while-True loop, no exit - # condition) — same as a _spawn_supervised watcher. Tag it so - # _scale_to_zero_has_live_background_work() doesn't treat a - # gateway with an active heartbeat watch as busy forever. + # PERMANENT once started (an infinite while-True loop, no exit condition) — same as a + # _spawn_supervised watcher. Tag it so _scale_to_zero_has_live_background_work() doesn't + # treat a gateway with an active heartbeat watch as busy forever. task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: @@ -24422,8 +22590,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: logger.debug("Failed to start heartbeat poller", exc_info=True) - - async def _send_goal_status_notice(self, source: Any, message: str) -> None: """Send a /goal judge status line back to the originating chat/thread.""" adapter = self._adapter_for_source(source) @@ -24446,12 +22612,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _defer_goal_status_notice_after_delivery(self, source: Any, message: str) -> None: """Send a /goal status line after the main response is delivered. - The gateway message handler returns the agent response to the platform - adapter, which sends it after this method's caller has returned. For a - natural Discord/Telegram reading order, goal status belongs after that - send. Platform adapters provide a one-shot post-delivery callback for - exactly this boundary; when unavailable, fall back to direct awaited - delivery rather than silently dropping the notice. + The message handler returns the agent response to the adapter, which sends it after this + caller has returned; for natural reading order the goal status belongs after that send. + Platform adapters provide a one-shot post-delivery callback for exactly this boundary; + when unavailable, fall back to direct awaited delivery rather than silently dropping the + notice. """ adapter = self._adapter_for_source(source) if not adapter: @@ -24493,15 +22658,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew source: Any, final_response: str, ) -> None: - """Run the goal judge after a gateway turn and, if still active, - enqueue a continuation prompt for the same session. + """Run the goal judge after a gateway turn and, if still active, enqueue a continuation + prompt for the same session. - Called from ``_handle_message_with_agent`` at turn boundary, AFTER - the response has been delivered. Safe when no goal is set. - - We use the adapter's pending-message / FIFO machinery so any real - user message that arrives simultaneously is handled by the same - queue and takes priority naturally. + Called from ``_handle_message_with_agent`` at turn boundary, AFTER the response has been + delivered. We use the adapter's pending-message / FIFO machinery so any real user message + that arrives simultaneously is handled by the same queue and takes priority naturally. """ try: from hermes_cli.goals import GoalManager @@ -24515,10 +22677,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew max_turns = self._goal_max_turns_from_config() - # Warm the SessionDB cache off-loop. A cold cache runs the - # state.db init on the loop thread at the turn boundary (the - # 2026-08-14 crash-loop seam). A slow init can drop the goal - # read and silently end the goal loop. + # Warm the SessionDB cache off-loop. A cold cache runs the state.db init on the loop thread + # at the turn boundary (the 2026-08-14 crash-loop seam). A slow init can drop the goal read + # and silently end the goal loop. await self._warm_goals_session_db("goal continuation") mgr = GoalManager(session_id=sid, default_max_turns=max_turns) @@ -24531,15 +22692,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: _bg_procs = None - # evaluate_after_turn calls judge_goal() which makes a synchronous - # HTTP request to the auxiliary LLM. Running it on the event-loop - # thread would block Discord heartbeats for 10-40 s and cause - # connection flaps, so we offload it to a thread-pool executor. - # _run_in_executor_with_context (not bare run_in_executor): the - # profile secret scope and auxiliary runtime context are contextvars, - # and a default-executor hop would drop them — aux-client provider - # resolution would then read credentials unscoped and fail under - # multiplexing (same pattern as compression in slash_commands.py). + # evaluate_after_turn calls judge_goal() which makes a synchronous HTTP request to the + # auxiliary LLM. Running it on the event-loop thread would block Discord heartbeats for + # 10-40 s and cause connection flaps, so we offload it to a thread-pool executor. + # _run_in_executor_with_context (not bare run_in_executor): the profile secret scope and + # aux runtime context are contextvars; a default-executor hop drops them and aux-client + # credential resolution fails under multiplexing. decision = await self._run_in_executor_with_context( lambda: mgr.evaluate_after_turn( final_response or "", @@ -24549,12 +22707,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) msg = decision.get("message") or "" - # Defer the status line until after the adapter has delivered the - # agent's visible final response. The judge runs after the response is - # produced but before BasePlatformAdapter sends it, so sending here - # would show "✓ Goal achieved" before the answer itself. Registering - # an awaited post-delivery callback preserves delivery reliability - # without reversing the user-visible ordering. + # Defer the status line until after the adapter has delivered the agent's visible final + # response. The judge runs after the response is produced but before BasePlatformAdapter + # sends it, so sending here would show "✓ Goal achieved" before the answer itself. if msg and source is not None: await self._defer_goal_status_notice_after_delivery(source, msg) @@ -24626,9 +22781,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _final_text_for_post_turn_hooks(agent_result, event=None) -> str: """Text for /goal and /loop after a gateway turn. - Streamed turns return None from _handle_message_with_agent - (already_sent). The delivered reply is stashed on the event so - those hooks still see it. + Streamed turns return None from _handle_message_with_agent (already_sent). The delivered + reply is stashed on the event so those hooks still see it. """ text = "" if isinstance(agent_result, dict): @@ -24651,10 +22805,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Complete a /loop wakeup tick after a gateway turn. - No-op unless the session has a loop whose tick is in flight - (``awaiting_response`` — set when the wakeup was injected). Applies - the LOOP_COMPLETE marker / --until judge / caps and schedules the - next tick; the idle wakeup watcher fires it when due. + No-op unless the session has a loop whose tick is in flight (``awaiting_response`` — set + when the wakeup was injected). Applies the LOOP_COMPLETE marker / --until judge / caps + and schedules the next tick; the idle wakeup watcher fires it when due. """ try: from hermes_cli.loops import LoopManager @@ -24666,9 +22819,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not sid: return - # Warm the SessionDB cache off-loop. A cold cache at the turn - # boundary stalls the loop for the init duration and can drop - # the tick-completion write (the /goal continuation seam, one + # Warm the SessionDB cache off-loop. A cold cache at the turn boundary stalls the loop for + # the init duration and can drop the tick-completion write (the /goal continuation seam, one # sibling over). await self._warm_goals_session_db("loop completion") @@ -24688,17 +22840,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _loop_wakeup_watcher(self, interval: float = 15.0) -> None: """Fire due /loop wakeups for idle gateway sessions. - The gateway has no per-session scheduler thread, so a coarse ticker - scans persisted loops (SessionDB ``loop:*`` rows) and injects the - wakeup prompt into each due session's chat via the same synthetic- - message path used by watch notifications. Deferrals: - - - session currently running an agent turn → skip (stays due; the - adapter FIFO would race the live turn otherwise) - - active non-parked /goal on the session → skip (goal owns the - idle boundary) - - no routing metadata on the loop → skip with a one-time warning - (CLI/TUI loops carry no route and are driven by their own surfaces) + The gateway has no per-session scheduler thread, so a coarse ticker scans persisted loops + (SessionDB ``loop:*`` rows) and injects the wakeup prompt into each due session's chat + via the same synthetic-message path used by watch notifications. Deferrals: session + currently running a turn → skip (the FIFO would race the live turn); active non-parked + /goal → skip (goal owns the idle boundary); no routing metadata → skip with a one-time + warning (CLI/TUI loops carry no route). """ await asyncio.sleep(5) # let platforms finish connecting warned_no_route: set = set() @@ -24809,7 +22956,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return raw.guild.id return None - async def _handle_voice_channel_join(self, event: MessageEvent) -> str: """Join the user's current Discord voice channel.""" adapter = self._adapter_for_source(event.source) @@ -24896,10 +23042,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _handle_voice_timeout_cleanup(self, chat_id: str, *, adapter=None) -> None: """Called by the adapter when a voice channel times out. - Cleans up runner-side voice_mode state that the adapter cannot reach. - ``adapter`` is the Discord adapter that timed out (bound at join time); - under multiplexing that is a specific profile's bot, not necessarily - ``self.adapters[DISCORD]``. + Cleans up runner-side voice_mode state that the adapter cannot reach. ``adapter`` is the + Discord adapter that timed out (bound at join time); under multiplexing that is a + specific profile's bot, not necessarily ``self.adapters[DISCORD]``. """ if adapter is None: adapter = self.adapters.get(Platform.DISCORD) @@ -24911,10 +23056,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _is_duplicate_voice_transcript(self, guild_id: int, user_id: int, transcript: str) -> bool: """Suppress repeated STT outputs for the same recent utterance. - Voice capture can occasionally emit the same utterance twice a few - seconds apart, which creates a second queued agent run and overlapping - spoken replies. Dedup exact and near-exact repeats per guild/user over a - short window while allowing genuinely new turns through. + Voice capture can occasionally emit the same utterance twice a few seconds apart, which + creates a second queued agent run and overlapping spoken replies. """ from difflib import SequenceMatcher @@ -24954,11 +23097,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): """Handle transcribed voice from a user in a voice channel. - Creates a synthetic MessageEvent and processes it through the - adapter's full message pipeline (session, typing, agent, TTS reply). ``adapter`` is the Discord adapter that captured the audio (bound via - ``_bind_voice_input_callback``); under multiplexing each profile's bot - must dispatch through its own adapter, never the default profile's. + ``_bind_voice_input_callback``); under multiplexing each profile's bot must dispatch + through its own adapter, never the default profile's. """ if adapter is None: adapter = self.adapters.get(Platform.DISCORD) @@ -25099,11 +23240,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if has_agent_tts: return False - # Dedup: base adapter auto-TTS already handles voice input - # (play_tts plays in VC when connected, so runner can skip). - # When streaming already delivered the text (already_sent=True), - # the base adapter will receive None and can't run auto-TTS, - # so the runner must take over. + # Dedup: base adapter auto-TTS already handles voice input (play_tts plays in VC when + # connected, so runner can skip). When streaming already delivered the text + # (already_sent), the base adapter receives None and can't run auto-TTS, so the runner + # must take over. if is_voice_input and not already_sent: return False @@ -25124,11 +23264,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not tts_text: return - # Platform-aware output path: platforms whose native voice - # bubbles require Ogg/Opus (OPUS_VOICE_PLATFORMS — Telegram, - # Matrix, Feishu, WhatsApp, Signal) get an explicit .ogg path; - # the TTS tool's central container repair guarantees real - # Ogg/Opus bytes for every provider. Others keep MP3. + # Platform-aware output path: platforms whose native voice bubbles require Ogg/Opus + # (OPUS_VOICE_PLATFORMS — Telegram, Matrix, Feishu, WhatsApp, Signal) get an explicit + # .ogg path; the TTS tool's central container repair guarantees real Ogg/Opus bytes for + # every provider. audio_path = build_auto_tts_output_path(event.source.platform) result_json = await asyncio.to_thread( @@ -25170,13 +23309,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew reply_anchor = self._reply_anchor_for_event(event) thread_meta = self._thread_metadata_for_source(event.source, reply_anchor) if not in_voice_channel and callable(send_voice): - # Mark the auto voice reply as notify-worthy. Mirrors the - # final-text path in gateway/platforms/base.py which sets - # ``notify=True`` so platform adapters that gate push - # notifications (Telegram "important" mode) deliver the - # final voice reply as a normal notification instead of a - # silent message. Clone first so we don't mutate metadata - # shared with concurrent typing-indicator state. + # Mark the auto voice reply as notify-worthy (mirrors the final-text path in + # platforms/base.py) so adapters that gate push notifications (Telegram "important" + # mode) deliver it as a normal notification, not a silent message. Clone first so + # we don't mutate metadata shared with concurrent typing-indicator state. if thread_meta is not None: thread_meta = dict(thread_meta) thread_meta["notify"] = True @@ -25213,47 +23349,33 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Extract explicit MEDIA: tags from a response and deliver them. - Called after streaming has already sent the text to the user, so the - text itself is already delivered — this only handles file attachments - that the normal _process_message_background path would have caught. + Called after streaming has already sent the text to the user, so the text itself is + already delivered — this only handles file attachments that the normal + _process_message_background path would have caught. - Unlike the non-streaming path in ``gateway/platforms/base.py`` (which - also auto-detects bare local paths via ``extract_local_files``), this - post-stream rescan is EXPLICIT-ONLY. The visible reply has already - been streamed verbatim, so a bare path string here was either (a) - already shown to the user as text, or (b) stale tool/inspected - content that was never part of the intended visible reply. Promoting - such paths into uploads after the fact sent files the model never - asked to deliver (#20834). Only ``MEDIA:`` directives — the explicit - attachment contract — trigger post-stream uploads. + Unlike the non-streaming path in ``gateway/platforms/base.py``, this rescan is + EXPLICIT-ONLY: a bare local path in an already-streamed reply was either shown to the user + as text or is stale inspected content, and promoting it sent files the model never asked + to deliver. Only ``MEDIA:`` directives trigger post-stream uploads. """ from pathlib import Path from urllib.parse import quote as _quote try: - # Capture [[as_document]] before extract_media strips it, so the - # dispatch partition below can route image-extension files - # through send_document (preserving bytes) instead of - # send_multiple_images (Telegram sendPhoto recompresses to ~1280px). + # Capture [[as_document]] before extract_media strips it, so the dispatch partition + # below can route image-extension files through send_document (preserving bytes) instead + # of send_multiple_images (Telegram sendPhoto recompresses to ~1280px). force_document_attachments = "[[as_document]]" in response from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio media_files, cleaned = adapter.extract_media(response) media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) - # Do NOT deduplicate explicit MEDIA tags against prior turns here - # (#73771). This rescan is already EXPLICIT-ONLY (see docstring): - # a MEDIA: directive in the final streamed reply is the model - # deliberately attaching a file — including a user-requested - # resend. Stale auto-appended tags are deduped upstream in - # _collect_auto_append_media_tags with history_media_paths. - # Mirrors the same filter removal on the non-streaming path in - # gateway/platforms/base.py. - # Strip image URLs from the cleaned text for parity with the - # non-streaming chain, but do NOT run extract_local_files here: - # post-stream delivery is explicit-only (#20834). Bare local paths - # in an already-streamed reply are text the user has seen (or - # stale inspected content), not an attachment request. + # Do NOT deduplicate explicit MEDIA tags against prior turns here. This rescan is + # already EXPLICIT-ONLY (see docstring): a MEDIA: directive in the final streamed reply + # is the model deliberately attaching a file — including a user-requested resend. Stale + # auto-appended tags are deduped upstream (_collect_auto_append_media_tags). Strip image + # URLs for parity with the non-streaming chain, but do NOT run extract_local_files here. adapter.extract_images(cleaned) _thread_meta = ( @@ -25268,10 +23390,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _VIDEO_EXTS = {'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'} _IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'} - # Partition out images so they can be sent as a single batch - # (e.g. Signal's multi-attachment RPC). When [[as_document]] was - # set, image-extension files skip the photo path and route to - # send_document below — preserving original bytes. + # Partition out images so they can be sent as a single batch (e.g. Signal's multi- + # attachment RPC). When [[as_document]] was set, image-extension files skip the photo + # path and route to send_document below — preserving original bytes. image_paths: list = [] non_image_media: list = [] for media_path, is_voice in media_files: @@ -25337,14 +23458,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not text_already_delivered: text_content = _strip_response_attachments_for_direct_send(response, adapter) if text_content: - # Reconcile-by-edit first (live finding, 2026-08-16 canary): - # when the stream consumer delivered/sealed a message but its - # recorded payload didn't confirm the final (post-stream - # mutation), plain-sending here creates the duplicate — the - # sealed message already carries most of the answer. A sealed - # native stream is a regular message; chat.update on it is - # live-verified working. Fall back to plain send only when - # there is no editable message or the edit fails. + # Reconcile-by-edit first (live finding, 2026-08-16 canary): when the stream + # consumer delivered/sealed a message but its recorded payload didn't confirm the + # final (post-stream mutation), plain-sending here creates the duplicate — the + # sealed message already carries most of the answer. _reconciled = False _sc_msg_id = getattr(stream_consumer, "message_id", None) if ( @@ -25377,10 +23494,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew metadata=metadata, ) - # Failed turns still deliver their (normalized failure) text above, - # but must not upload attachments as if the turn succeeded — mirrors - # the ``not agent_result.get("failed")`` guard on the completed-turn - # delivery path. + # Failed turns still deliver their (normalized failure) text above, but must not upload + # attachments as if the turn succeeded — mirrors the ``not agent_result.get("failed")`` + # guard on the completed-turn delivery path. if not deliver_media: return @@ -25407,10 +23523,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Profile-scoping wrapper around the background agent task. - When multiplexing is active, resolve the inbound source's profile and - run the whole task inside ``_profile_runtime_scope`` so credentials - resolve from that profile's secret scope. Mirrors the pattern in - ``_run_agent``. + When multiplexing is active, resolve the inbound source's profile and run the whole task + inside ``_profile_runtime_scope`` so credentials resolve from that profile's secret + scope. Mirrors the pattern in ``_run_agent``. """ if not getattr(getattr(self, "config", None), "multiplex_profiles", False): return await self._run_background_task_inner( @@ -25431,13 +23546,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> list: """Resolve enabled toolsets for an agent run, honoring per-source overrides. - Asks the receiving adapter for a ``toolsets_for_source()`` override - (e.g. per-route webhook toolsets). When present, the override list is - validated through the SAME ``_get_platform_tools`` path as normal - platform config — by substituting it as the platform's toolset list — - so unknown names and platform-restricted toolsets are dropped rather - than trusted. When absent, falls back to standard - ``platform_toolsets.`` resolution. + Asks the receiving adapter for a ``toolsets_for_source()`` override (e.g. per-route + webhook toolsets). When present, the override list is validated through the SAME + ``_get_platform_tools`` path as normal platform config — by substituting it as the + platform's toolset list — so unknown names and platform-restricted toolsets are dropped + rather than trusted. When absent, falls back to ``platform_toolsets.``. """ from hermes_cli.tools_config import _get_platform_tools @@ -25677,12 +23790,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: pass - - - - - - async def _get_telegram_topic_capabilities(self, source: SessionSource) -> dict: """Read Telegram private-topic capability flags via Bot API getMe.""" adapter = self._adapter_for_source(source) @@ -25809,31 +23916,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _relay_auto_thread_info( self, source: SessionSource ) -> Optional[Tuple[str, str]]: - """(thread_id, initial_name) when the RELAY connector auto-threaded our - reply to this source's chat — the title-turn sibling of - _is_discord_auto_thread_lane. + """(thread_id, initial_name) when the RELAY connector auto-threaded our reply to this + source's chat — the title-turn sibling of _is_discord_auto_thread_lane. - The marker-based check above only lights up for events ARRIVING IN an - auto-created thread (turn 2+). The auto-title fires on the FIRST - exchange, whose source is the PARENT channel event — the thread did - not exist at ingest, so no markers can be present and the native lane - check never matches on the relay title turn (staging repro - 2026-07-29: initial titles fine, semantic renames never happened). + The marker-based check above only lights up for events ARRIVING IN an auto-created thread + (turn 2+); the auto-title fires on the FIRST exchange, whose source is the PARENT channel + event, so no markers exist and the native lane never matches on the relay title turn. - Preferred path: the connector stamps ``prospective_thread_id`` on the - inbound (the anchor message id, which IS the id of the thread it will - auto-create). It's deterministic and per-message, so it identifies the - EXACT thread even when several auto-threads spawn from one channel — - unlike the send-result cache below, which held a single slot per parent - chat and so only the FIRST thread in a channel ever renamed (staging - repro 2026-08-02: thread A renamed, sibling thread B stuck at raw - text). The connector's own created-name guard (prefer_connector_created) - enforces no-clobber, so no initial name is needed here. - - Fallback: the connector reports where the reply actually landed on the - send result (contract §SendResult thread_id/auto_thread_name); the - relay adapter caches it per chat and this reads it back — kept for - older connectors that don't stamp prospective_thread_id. + Preferred: the connector stamps ``prospective_thread_id`` on the inbound (the anchor + message id == the thread it will auto-create). Deterministic and per-message, so it names + the EXACT thread even when several auto-threads spawn from one channel — unlike the + per-chat send-result cache, which only ever renamed the FIRST thread. The connector's own + created-name guard (prefer_connector_created) enforces no-clobber, so no initial name is + needed here. Fallback: the send-result thread_id/auto_thread_name cached per chat by the + relay adapter, kept for older connectors. """ if source.platform != Platform.DISCORD or not source.chat_id: return None @@ -25858,11 +23954,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[Tuple[str, str]]: """``_relay_auto_thread_info``, waited out until this turn delivers. - The legacy send-result path can only answer once the reply is sent, and - the caller asks at title time — one turn early. The adapter answers on - the send either way, so the timeout is only a backstop for a turn that - never sends at all; the turn's own inactivity limit is exactly how long - that turn could still be alive. + The legacy send-result path can only answer once the reply is sent, and the caller asks + at title time — one turn early. The adapter answers on the send either way, so the + timeout is only a backstop for a turn that never sends at all; the turn's own inactivity + limit is exactly how long that turn could still be alive. """ # The connector-stamped prospective id is known at ingest, so most # sessions answer here and never wait at all. @@ -25883,9 +23978,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _sanitize_discord_thread_title(self, title: str) -> str: """Return a Discord-safe semantic thread title from a session title. - Discord thread names are capped at 100 characters measured in UTF-16 - code units (emoji count double), so truncate with the UTF-16 helpers - rather than Python code-point slices. + Discord thread names are capped at 100 characters measured in UTF-16 code units (emoji + count double), so truncate with the UTF-16 helpers rather than Python code-point slices. """ cleaned = re.sub(r"\s+", " ", str(title or "")).strip() if not cleaned: @@ -25903,21 +23997,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Best-effort semantic rename of a newly auto-created Discord thread. - ``relay_info`` is the (thread_id, initial_name) pair from the relay - connector's send-result feedback — supplied on the title turn, where - the source is the parent-channel event and carries no auto-thread - markers (see _relay_auto_thread_info). When absent, the native - marker-based lane supplies thread identity from the source itself. + ``relay_info`` is the (thread_id, initial_name) pair from the relay connector's send- + result feedback — supplied on the title turn, where the source is the parent-channel + event and carries no auto-thread markers (see _relay_auto_thread_info). """ if relay_info is None and not await asyncio.to_thread( self._is_discord_auto_thread_lane, source ): - # Relay title turn with no feedback captured at schedule time: the - # title comes off the user's opening message, so it beats the - # delivery that produces the connector's send-result feedback - # (thread_id + initial name) by the whole length of the turn. Wait - # on the adapter for that send rather than guessing how long the - # turn will take. + # Relay title turn with no feedback captured at schedule time: the title comes off the + # user's opening message, so it beats the delivery that produces the connector's send- + # result feedback (thread_id + initial name) by the whole length of the turn. if not self._is_relay_discord_channel_lane(source): return relay_info = await self._await_relay_auto_thread_info(source) @@ -25932,11 +24021,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if rename_thread is None: return target_thread_id = relay_info[0] if relay_info else str(source.thread_id) - # Relay lane (relay_info present): ask the CONNECTOR to enforce the - # no-clobber guard from its own created-name memory — the gateway - # can't reliably reproduce the thread's initial name byte-for-byte - # (normalization drift silently declined every rename before this). - # Native-marker lane keeps the legacy string guard. + # Relay lane (relay_info present): ask the CONNECTOR to enforce the no-clobber guard from + # its own created-name memory — the gateway can't reliably reproduce the thread's initial + # name byte-for-byte (normalization drift silently declined every rename before this). use_connector_guard = relay_info is not None guard_name = ( None @@ -25944,18 +24031,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else getattr(source, "auto_thread_initial_name", None) ) thread_name = self._sanitize_discord_thread_title(title) - # Relay lane only: the connector's egress guard resolves the owning - # tenant from the outbound metadata's scope_id (guild) / user_id - # (author). Those discriminator caches are keyed by the PARENT channel - # chat_id (learned at inbound), NOT the thread id. rename_thread - # defaults chat_id to the thread id when no parent is given, so the - # scope/author lookup misses and the connector declines the op - # ("target not routed to an onboarded tenant" — the live failure on - # staging 2026-08-01). Pass the parent channel id (the relay source's - # chat_id IS the parent channel; the thread came from send-result - # feedback) so the discriminators resolve. Native lane needs nothing: - # its source IS the thread and it renames via the direct Discord API, - # not the relay egress guard. + # Relay lane only: the connector's egress guard resolves the owning tenant from the outbound + # metadata's scope_id (guild) / user_id (author). Those caches are keyed by the PARENT + # channel chat_id (learned at inbound), not the thread id; rename_thread defaults chat_id + # to the thread id, so the lookup misses and the connector declines ("target not routed to + # an onboarded tenant"). Pass the parent channel id (the relay source's chat_id IS the + # parent) so the discriminators resolve. Native lane needs nothing: its source IS + # the thread and it renames via the direct Discord API, not the relay egress guard. parent_chat_id = ( str(source.chat_id) if use_connector_guard and source.chat_id else None ) @@ -26055,21 +24137,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not await asyncio.to_thread(self._is_telegram_topic_lane, source) or not source.chat_id or not source.thread_id: return - # Operator can fully disable per-topic auto-rename via - # extra.disable_topic_auto_rename. Useful when topics are managed - # by the user (ad-hoc Threaded Mode) and auto-rename would + # Operator can fully disable per-topic auto-rename via extra.disable_topic_auto_rename. + # Useful when topics are managed by the user (ad-hoc Threaded Mode) and auto-rename would # overwrite their chosen names every time the auto-title fires. if self._telegram_topic_auto_rename_disabled(source): return - # Skip rename when the topic is operator-declared via - # extra.dm_topics. Those topics have fixed names chosen by the - # operator (plus optional skill binding); auto-renaming would - # silently mutate operator config. - # - # Check the class, not the instance — getattr() on MagicMock - # auto-creates attributes, so `hasattr(adapter, "_get_dm_topic_info")` - # would return True for every test double. + # Skip rename when the topic is operator-declared via extra.dm_topics. Those topics have + # fixed names chosen by the operator (plus optional skill binding); auto-renaming would + # silently mutate operator config. Check the class, not the instance — getattr() on a + # MagicMock auto-creates attributes, so an instance hasattr() is True for every test double. adapter = self._adapter_for_source(source) if adapter is not None: get_info = getattr(type(adapter), "_get_dm_topic_info", None) @@ -26276,7 +24353,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "normal Hermes chat again. Run /topic to re-enable later." ) - async def _telegram_topic_root_status_message(self, source: SessionSource) -> str: lines = [ "Telegram multi-session topics are enabled.", @@ -26384,23 +24460,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew response += f"\n\nLast Hermes message:\n{last_assistant}" return response - - - - - - async def _execute_mcp_reload(self, event: MessageEvent) -> str: """Actually disconnect, reconnect, and notify MCP tool changes. - Split out from ``_handle_reload_mcp_command`` so the confirmation - wrapper can invoke the same path whether the user confirmed via - button, text reply, or has the confirm gate disabled. - - Under multiplex the reload runs inside the requesting profile's - runtime scope (entered here when the caller — e.g. a button-confirm - callback — did not), and only that profile's servers are torn down - and rediscovered (#95518). + Split out from ``_handle_reload_mcp_command`` so the confirmation wrapper can invoke the + same path whether the user confirmed via button, text reply, or has the confirm gate + disabled. Under multiplex the reload runs inside the requesting profile's runtime scope + (entered here when the caller — e.g. a button callback — did not), and only that + profile's servers are torn down and rediscovered. """ multiplex = bool(getattr(self.config, "multiplex_profiles", False)) if multiplex and not get_hermes_home_override(): @@ -26462,12 +24529,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else: lines.append(t("gateway.reload_mcp.tools_available", tools=len(new_tools), servers=len(connected_servers))) - # Refresh cached agents so existing sessions see new MCP tools on - # their next turn — without this, the user has to `/new` (which - # discards conversation history) to pick up tools from a server - # that was just added or reconnected. The user has already - # consented to the prompt-cache invalidation via the slash-confirm - # gate in _handle_reload_mcp_command before we reach this point. + # Refresh cached agents so existing sessions see new MCP tools on their next turn — + # without this, the user has to `/new` (which discards conversation history) to pick up + # tools from a server that was just added or reconnected. try: from tools.mcp_tool import refresh_agent_mcp_tools _cache = getattr(self, "_agent_cache", None) @@ -26490,15 +24554,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew continue if _agent is None: continue - # Preserve each cached agent's build-time toolset - # selection EXACTLY: a gateway session built with a - # restricted enabled_toolsets (e.g. ["safe"]) must - # NOT silently gain tools after a reload. This is the - # opposite of the interactive CLI/TUI /reload-mcp, - # which is a single user re-applying their own config - # edit; gateway agents are per-session and may be - # deliberately locked down. (Contract is asserted by - # test_reload_mcp_preserves_per_agent_toolset_overrides.) + # Preserve each cached agent's build-time toolset selection EXACTLY: a + # gateway session built with a restricted enabled_toolsets (e.g. + # ["safe"]) must NOT silently gain tools after a reload. Unlike the + # CLI/TUI /reload-mcp (one user re-applying their own config), gateway + # agents are per-session and may be deliberately locked down. refresh_agent_mcp_tools(_agent, quiet_mode=True) except Exception as _exc: logger.debug( @@ -26536,8 +24596,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.warning("MCP reload failed: %s", e) return t("gateway.reload_mcp.failed", error=e) - - # ------------------------------------------------------------------ # Slash-command confirmation primitive (generic) # ------------------------------------------------------------------ @@ -26565,19 +24623,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Union[str, "EphemeralReply", None]: """Gate a destructive session slash command (/new, /reset, /undo). - ``execute`` is an async callable ``execute() -> str | EphemeralReply`` - that performs the destructive action. If the - ``approvals.destructive_slash_confirm`` config gate is off, ``execute`` - runs immediately (returning its result). Otherwise this routes - through ``_request_slash_confirm`` — native yes/no buttons on - Telegram/Discord/Slack, text fallback elsewhere. - - Three-option resolution: - - - ``once`` — run ``execute`` and return its result - - ``always`` — persist ``approvals.destructive_slash_confirm: false``, - then run ``execute`` - - ``cancel`` — return a "cancelled" message; do not run ``execute`` + ``execute`` is an async callable ``execute() -> str | EphemeralReply`` that performs the + destructive action. If the ``approvals.destructive_slash_confirm`` gate is off, ``execute`` + runs immediately. Otherwise this routes through ``_request_slash_confirm`` — native + yes/no buttons on Telegram/Discord/Slack, text fallback elsewhere. Resolution: ``once`` + runs ``execute``; ``always`` persists ``destructive_slash_confirm: false`` then runs it; + ``cancel`` returns a "cancelled" message without running it. """ # Gate check. confirm_required = True @@ -26631,10 +24682,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "`approvals.destructive_slash_confirm: true` in config.yaml." ) else: - # The user did approve this run, so the action still goes - # ahead, but the preference did not stick and the prompt - # will be back next time. Say so rather than promising an - # opt-out that was never written. + # The user did approve this run, so the action still goes ahead, but the + # preference did not stick and the prompt will be back next time. Say so rather + # than promising an opt-out that was never written. note = ( "\n\n⚠️ Could not save that preference (config.yaml is not " "writable), so /clear, /new, /reset, and /undo will ask " @@ -26677,10 +24727,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[str]: """Ask the user to confirm an expensive slash command. - ``handler`` is an async callable ``handler(choice: str) -> str`` - where ``choice`` is ``"once"``, ``"always"``, or ``"cancel"``. - The handler runs on the event loop when the user responds; its - return value is sent back as a gateway message. + ``handler`` is an async callable ``handler(choice: str) -> str`` where ``choice`` is + ``"once"``, ``"always"``, or ``"cancel"``. The handler runs on the event loop when the + user responds; its return value is sent back as a gateway message. Returns a short acknowledgment string to send immediately (before the user's response). If buttons rendered successfully the ack @@ -26691,10 +24740,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew source = event.source session_key = self._session_key_for_source(source) - # Bare-runner test harnesses (object.__new__(GatewayRunner)) skip - # __init__ and don't have the counter attribute — fall back to a - # local counter so tests don't AttributeError. Real runs always - # have the instance attribute. + # Bare-runner test harnesses (object.__new__(GatewayRunner)) skip __init__ and don't have + # the counter attribute — fall back to a local counter so tests don't AttributeError. Real + # runs always have the instance attribute. counter = getattr(self, "_slash_confirm_counter", None) if counter is None: import itertools as _itertools @@ -26761,17 +24809,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew reply_to_message_id=reply_to_message_id or getattr(source, "message_id", None), ) if getattr(source, "platform", None) == Platform.SLACK: - # Per-turn egress identity (R3-5, connector PR gateway-gateway#210). - # Slack's chat.startStream requires recipient_user_id (+ - # recipient_team_id) when streaming to a channel, and the relay - # connector fills those from metadata.user_id / metadata.scope_id. - # The relay adapter's _with_scope fallback resolves BOTH from - # per-chat caches keyed only by chat_id — mutable state that a - # CONCURRENT turn overwrites: two users with overlapping turns in - # one channel would open U1's stream with U2 as the recipient. - # Stamp the authentic per-turn values from THIS turn's source here, - # where they are still turn-scoped; _with_scope only fills keys - # that are absent, so the cache degrades to what it should be — a + # Per-turn egress identity (R3-5, connector PR gateway-gateway#210). Slack's + # chat.startStream needs recipient_user_id/team_id, which the relay connector fills + # from metadata.user_id/scope_id. The relay adapter's _with_scope fallback reads both + # from per-chat caches keyed only by chat_id — mutable state a CONCURRENT turn + # overwrites (U1's stream opened with U2 as recipient). Stamp this turn's authentic + # values here; _with_scope only fills absent keys, so the cache degrades to a # restart/synthetic-send fallback. team_id = getattr(source, "scope_id", None) user_id = getattr(source, "user_id", None) @@ -26841,13 +24884,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False if chat_type == "dm": return True - # Inspect operator-declared DM topics via the adapter's lookup. Resolve - # the method on the CLASS, not the instance: getattr() on a MagicMock - # auto-creates a callable child for any attribute, so an instance-level - # lookup would report a DM topic for every test double. Only a - # dict-shaped return counts as an operator-declared topic — a bare - # MagicMock or other sentinel must not. Mirrors the guard in - # _rename_telegram_topic_for_session_title. + # Inspect operator-declared DM topics via the adapter's lookup. Resolve the method on the + # CLASS, not the instance: getattr() on a MagicMock auto-creates a callable child for any + # attribute, so an instance-level lookup would report a DM topic for every test double. + # Only a dict-shaped return counts as operator-declared — a bare MagicMock must not. if adapter is not None and chat_id: get_dm_topic_info = getattr(type(adapter), "_get_dm_topic_info", None) if callable(get_dm_topic_info): @@ -26864,20 +24904,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """Return the platform-specific reply anchor for GatewayRunner sends.""" return _reply_anchor_for_event(event) - # ------------------------------------------------------------------ # /approve & /deny — explicit dangerous-command approval # ------------------------------------------------------------------ _APPROVAL_TIMEOUT_SECONDS = 300 # 5 minutes - - # Built-in messaging platforms where the ``/update`` command is allowed. - # ACP, API server, and webhooks are programmatic interfaces that should - # not trigger system updates. Plugin-migrated platforms (discord, - # mattermost, teams, irc, line, …) are NOT listed here — they declare - # ``allow_update_command=True`` on their ``PlatformEntry`` and are - # honored via the registry fallback at ``_handle_update_command`` below. + # Built-in messaging platforms where the ``/update`` command is allowed. ACP, API server, and + # webhooks are programmatic interfaces that should not trigger system updates. Plugin-migrated + # platforms are NOT listed here — they declare ``allow_update_command=True`` on their + # ``PlatformEntry`` and are honored via the registry fallback in ``_handle_update_command``. _UPDATE_ALLOWED_PLATFORMS = frozenset({ Platform.TELEGRAM, Platform.SLACK, Platform.WHATSAPP, Platform.SIGNAL, Platform.MATRIX, @@ -26885,8 +24921,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew Platform.FEISHU, Platform.WECOM, Platform.WECOM_CALLBACK, Platform.WEIXIN, Platform.BLUEBUBBLES, Platform.QQBOT, Platform.LOCAL, }) - - def _schedule_update_notification_watch(self) -> None: """Ensure a background task is watching for update completion.""" existing_task = getattr(self, "_update_notification_task", None) @@ -26908,11 +24942,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Watch ``hermes update --gateway``, streaming output + forwarding prompts. - Polls ``.update_output.txt`` for new content and sends chunks to the - user periodically. Detects ``.update_prompt.json`` (written by the - update process when it needs user input) and forwards the prompt to - the messenger. The user's next message is intercepted by - ``_handle_message`` and written to ``.update_response``. + Polls ``.update_output.txt`` for new content and sends chunks to the user periodically. + Detects ``.update_prompt.json`` (written by the update process when it needs user input) + and forwards the prompt to the messenger. """ pending_path = _hermes_home / ".update_pending.json" claimed_path = _hermes_home / ".update_pending.claimed.json" @@ -26958,13 +24990,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not adapter or not chat_id: logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only") - # Fall back to completion-only: wait for the exit code and send the - # final notification. _send_update_notification re-resolves the - # adapter on every call, so when the target platform is still - # reconnecting it returns False and keeps the markers. Keep polling - # until it actually delivers (returns True) instead of giving up - # after the first completion check — otherwise a platform that - # reconnects a few seconds after completion never gets notified. + # Fall back to completion-only: wait for the exit code and send the final notification. + # _send_update_notification re-resolves the adapter on every call, so when the target + # platform is still reconnecting it returns False and keeps the markers. Keep polling + # until it actually delivers (True) — a platform that reconnects a few seconds after + # completion would otherwise never be notified. while (pending_path.exists() or claimed_path.exists()) and loop.time() < deadline: if exit_code_path.exists() and await self._send_update_notification(): return @@ -27073,10 +25103,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval: await _flush_buffer() - # Check for prompts — only forward if we haven't already sent - # one that's still awaiting a response. Without this guard the - # watcher would re-read the same .update_prompt.json every poll - # cycle and spam the user with duplicate prompt messages. + # Check for prompts — only forward if we haven't already sent one that's still awaiting + # a response. Without this guard the watcher would re-read the same .update_prompt.json + # every poll cycle and spam the user with duplicate prompt messages. _up_pending_state = ( self._peek_session_state(session_key) if session_key else None ) @@ -27118,11 +25147,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew f"or type your answer directly.", metadata=_non_conversational_metadata(metadata, platform=platform), ) - # Keep the prompt marker on disk until the user - # answers. If the gateway restarts mid-prompt, the - # next watcher can recover by re-forwarding it from - # disk. Duplicate sends in the same process are - # still suppressed by _update_prompt_pending. + # Keep the prompt marker on disk until the user answers. If the gateway + # restarts mid-prompt, the next watcher can recover by re-forwarding it from + # disk. self._session_state( session_key ).persistent.update_prompt_pending = True @@ -27159,10 +25186,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew Returns False when the update is still running so a caller can retry later. Returns True after a definitive send/skip decision. - - This is the legacy notification path used when the streaming watcher - cannot resolve the adapter (e.g. after a gateway restart where the - platform hasn't reconnected yet). """ pending_path = _hermes_home / ".update_pending.json" claimed_path = _hermes_home / ".update_pending.claimed.json" @@ -27211,14 +25234,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew adapter = self.adapters.get(platform) if not adapter and chat_id: - # The update finished, but the target platform has not - # reconnected yet (common right after the restart that - # `hermes update` triggers). Treating "adapter missing" as a - # definitive skip would delete the markers and silently lose the - # completion notification — the user never learns whether the - # update succeeded or timed out. Preserve the markers instead so - # a later retry (the watcher poll loop, or the next gateway - # startup) can deliver the result once the adapter is back. + # The update finished, but the target platform has not reconnected yet (common right + # after the restart that `hermes update` triggers). Treating "adapter missing" as a + # definitive skip would delete the markers and silently lose the notification; + # preserve them so a later retry (watcher poll or next startup) can deliver it. logger.info( "Update notification deferred: %s adapter not connected yet", platform_str, @@ -27436,13 +25455,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return delivered async def _send_session_db_warning_notifications(self) -> None: - """Broadcast a state.db failure warning to all home channels (#88235). + """Broadcast a state.db failure warning to all home channels. - When SessionDB init fails at gateway startup, messages may flow but - nothing is persisted — /resume, /history, and session_search all - silently break. This sends a one-time warning to each connected - platform's home channel so the user knows to investigate before - losing data. Best-effort: failures are logged, not raised. + When SessionDB init fails at gateway startup, messages may flow but nothing is persisted + — /resume, /history, and session_search all silently break. Best-effort: failures are + logged, not raised. """ error = getattr(self, "_session_db_init_error", None) if not error: @@ -27528,13 +25545,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew in a ``finally`` block. """ from gateway.session_context import set_session_vars - # Propagate the adapter's async-delivery capability so async tools - # (terminal notify_on_complete / watch_patterns, delegate_task - # background=True) know whether this channel can wake a later turn. - # Default True keeps CLI / unknown paths working; stateless adapters - # (api_server) declare supports_async_delivery=False. Use getattr so - # bare runners built via object.__new__ (tests) without self.adapters - # don't blow up — they simply default to supported. + # Propagate the adapter's async-delivery capability so async tools (terminal + # notify_on_complete / watch_patterns, delegate_task background=True) know whether this + # channel can wake a later turn. Default True keeps CLI/unknown paths working; stateless + # adapters (api_server) declare False. getattr so bare test runners without self.adapters + # simply default to supported. _adapters = getattr(self, "adapters", None) or {} _adapter = _adapters.get(context.source.platform) _async_delivery = getattr(_adapter, "supports_async_delivery", True) @@ -27648,11 +25663,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew (pre-analyze with vision_analyze and prepend the description). See agent/image_routing.py for the full decision table. - Gateway sessions can have /model overrides that live outside - config.yaml. Image preprocessing runs before AIAgent sets the - auxiliary_client runtime globals, so resolve the same per-session - runtime bundle the upcoming agent turn will use instead of consulting - only the persisted default model. + Gateway sessions can have /model overrides outside config.yaml, and image preprocessing + runs before AIAgent sets the auxiliary_client runtime globals — so resolve the same + per-session runtime bundle the upcoming turn will use, not just the persisted default. """ try: from agent.image_routing import decide_image_input_mode @@ -27711,14 +25724,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew user_text: str, image_paths: List[str], ) -> str: - """ - Auto-analyze user-attached images with the vision tool and prepend - the descriptions to the message text. + """Auto-analyze user-attached images with the vision tool and prepend the descriptions to + the message text. - Each image is analyzed with a general-purpose prompt. The resulting - description *and* the local cache path are injected so the model can: - 1. Immediately understand what the user sent (no extra tool call). - 2. Re-examine the image with vision_analyze if it needs more detail. + Each image is analyzed with a general-purpose prompt. The resulting description *and* the + local cache path are injected so the model can: 1. Immediately understand what the user + sent (no extra tool call). 2. Re-examine the image with vision_analyze if it needs detail. Args: user_text: The user's original caption / message text. @@ -27782,9 +25793,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew user_text: str, audio_paths: List[str], ) -> tuple[str, List[str]]: - """ - Auto-transcribe user voice/audio messages using the configured STT provider - and prepend the transcript to the message text. + """Auto-transcribe user voice/audio messages using the configured STT provider and prepend + the transcript to the message text. Args: user_text: The user's original caption / message text. @@ -27858,11 +25868,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew result = fallback if result["success"]: transcript = result["transcript"] - # Speech-to-text can return success=True with an empty or - # whitespace-only transcript on silence, cut-off, or - # inaudible audio. Emitting empty quotes ('""') makes the - # agent reply to nothing and can loop, so that case gets a - # clear sentinel note instead (#41603). + # Speech-to-text can return success=True with an empty or whitespace-only + # transcript on silence, cut-off, or inaudible audio. Emitting empty quotes makes + # the agent reply to nothing and can loop, so that case gets a sentinel note. if not (transcript or "").strip(): enriched_parts.append( "[The user sent a voice message but it came through " @@ -27872,23 +25880,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) continue successful_transcripts.append(transcript) - # Pass the transcript through as a plain quoted line. The - # earlier wording ("The user sent a voice message~ Here's - # what they said: ...") read as a meta-instruction and made - # the LLM volunteer commentary about voice mode rather than - # reply to the content. + # Pass the transcript through as a plain quoted line. Wrapping it in "The user + # sent a voice message..." read as a meta-instruction and made the LLM comment + # on voice mode instead of replying to the content. enriched_parts.append(f'"{transcript}"') else: error = result.get("error", "unknown error") - # All failure branches: a single, minimal, neutral marker. - # Do NOT mention "no STT provider configured", "setup - # instructions", or the "hermes-agent-setup" skill, and do - # NOT claim a direct message was sent — those phrases get - # persisted in conversation history and poison every later - # turn, so the model keeps volunteering STT-setup advice - # even after transcription starts working. The cause is - # logged for operator diagnosis but kept out of the - # LLM-visible prompt. + # All failure branches: a single, minimal, neutral marker. Do NOT mention "no STT + # provider configured", setup instructions, or claim a DM was sent — those get + # persisted in history and poison later turns (the model keeps volunteering + # STT-setup advice after transcription works). The cause is logged + # for operator diagnosis but kept out of the LLM-visible prompt. logger.info("Voice transcription failed for %s: %s", path, error) from tools.credential_files import to_agent_visible_cache_path @@ -27935,10 +25937,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> tuple[str | None, List[str]]: """Transcribe a pending audio event once and cache the result on the event. - Voice follow-ups can be inspected first by the interrupt monitor and - later consumed by the pending-drain path. Both need the same transcript, - but only one STT call and one transcript echo should happen for the - platform message. + Voice follow-ups can be inspected first by the interrupt monitor and later consumed by + the pending-drain path. Both need the same transcript, but only one STT call and one + transcript echo should happen for the platform message. """ if hasattr(event, "_gateway_pending_stt_text"): cached_text = getattr(event, "_gateway_pending_stt_text") @@ -27970,15 +25971,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Echo pending-event STT transcripts to the chat at most once. - The already-echoed transcripts are tracked as a COUNT rather than a - single boolean. ``merge_pending_message_event`` can append a second - voice note to an event whose first transcript was already echoed and - invalidates the transcription cache; the re-run transcription then - returns the earlier transcripts as a prefix of the new list, so - echoing only the unsent tail suppresses the repeat while still - surfacing the newly merged note. A count rather than a set of seen - values because two separate notes that transcribe identically are two - distinct deliveries and both must be echoed. + The already-echoed transcripts are tracked as a COUNT rather than a single boolean: + ``merge_pending_message_event`` can append a second voice note after the first transcript + was echoed and invalidate the cache; the re-run returns the earlier transcripts as a prefix + of the new list, so echoing only the unsent tail suppresses the repeat. A + count rather than a set of seen values because two separate notes that transcribe + identically are two distinct deliveries and both must be echoed. """ if ( not transcripts @@ -28011,14 +26009,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> tuple[str, List[str]]: """Transcribe a pending voice event and echo transcripts once. - Unified helper for all interrupt/monitor/backup/drain paths that need - to transcribe a pending voice event and echo the transcript to chat. - Returns ``(enriched_text, transcripts)`` so the caller can feed the - enriched text into ``agent.interrupt()`` or the pending-drain flow. - - If the event has no STT-eligible media, returns ``(text, [])`` unchanged. - The caller is responsible for the ``_build_media_placeholder`` fallback - when ``text`` is empty and the event has non-audio media. + Returns ``(enriched_text, transcripts)`` so the caller can feed the enriched text into + ``agent.interrupt()`` or the pending-drain flow. If the event has no STT-eligible media, + returns ``(text, [])`` unchanged; the caller owns the ``_build_media_placeholder`` + fallback when ``text`` is empty and the event has non-audio media. """ if not self._pending_event_audio_paths(event): return text, [] @@ -28047,9 +26041,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _build_process_event_source(self, evt: dict): """Resolve the canonical source for a synthetic background-process event. - Prefer the persisted session-store origin for the event's session key. - Falling back to the currently active foreground event is what causes - cross-topic bleed, so don't do that. + Prefer the persisted session-store origin for the event's session key. Falling back to + the currently active foreground event is what causes cross-topic bleed, so don't do that. """ from gateway.session import SessionSource @@ -28115,13 +26108,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew scope_id = str(evt.get("scope_id") or "").strip() or None if scope_id is None and chat_type not in ("dm", "thread"): - # Reconstructed (non-persisted) source for a scoped chat with no - # scope discriminator: on a relay-fronted deployment the - # connector's fail-closed tenant guard may decline the reply - # unless user_id resolves it (resolveByUser). Don't fail here — - # DMs and author-bound scoped chats still route, and native - # adapters don't need scope_id — but say so, so a post-restart - # egress decline isn't silent. + # Reconstructed (non-persisted) source for a scoped chat with no scope discriminator: on + # a relay-fronted deployment the connector's fail-closed tenant guard may decline the + # reply unless user_id resolves it (resolveByUser). Don't fail here — DMs and + # author-bound chats still route, and native adapters don't need scope_id — but warn + # so a post-restart egress decline isn't silent. logger.warning( "Synthetic event source for %s chat=%s (%s) reconstructed " "without scope_id; scoped relay egress may be declined by " @@ -28141,9 +26132,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _drain_watch_notifications(self, completion_queue) -> None: """Consume queued watch events and inject them when notifications are enabled. - The queue is ALWAYS drained (so watch events don't rot or requeue-spin) - but injection is skipped entirely when - ``display.background_process_notifications`` is ``off`` (#9290). + The queue is ALWAYS drained (so watch events don't rot or requeue-spin) but injection is + skipped entirely when ``display.background_process_notifications`` is ``off``. """ watch_events = _drain_gateway_watch_events(completion_queue) if self._load_background_notifications_mode() == "off": @@ -28163,22 +26153,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[bool]: """Inject a watch/completion notification as a synthetic message event. - Routing must come from the queued event itself, not from whatever - foreground message happened to be active when the queue was drained. - Returns ``True`` after adapter acceptance, ``False`` after a retryable - adapter failure, and ``None`` when the event has no gateway route. This - is not a transactional boundary: a process crash after adapter - acceptance can still cause durable at-least-once replay. + Routing must come from the queued event itself, not from whatever foreground message + happened to be active when the queue was drained. Returns ``True`` after adapter + acceptance, ``False`` after a retryable adapter failure, ``None`` when the event has no + gateway route. This is not a transactional boundary: a + process crash after adapter acceptance can still cause durable at-least-once replay. """ source = await asyncio.to_thread(self._build_process_event_source, evt) if not source: - # API-server-originated sessions bind a RAW session key (the - # X-Hermes-Session-Id value — see _bind_api_server_session), not a - # structured ``agent:main:...`` key, so _build_process_event_source - # cannot derive routing metadata from it and returns None above. - # Recover the raw session id and wake the real session via the API - # server's own /v1/chat/completions entry point instead of - # dropping the event. + # API-server-originated sessions bind a RAW session key (the X-Hermes-Session-Id value — + # see _bind_api_server_session), not a structured ``agent:main:...`` key, so + # _build_process_event_source cannot derive routing metadata from it and returns None + # above. raw_sid = str(evt.get("origin_session_id") or "").strip() if not raw_sid: _sk = str(evt.get("session_key") or "").strip() @@ -28193,9 +26179,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) if adapter is not None and not adapter_supports_push(adapter): if evt.get("type") == "async_delegation": - # #85957: after the parent turn's event.complete the - # CLIENT owns the next turn on this stateless surface. - # Persist the completion as a durable delivery row — + # after the parent turn's event.complete the CLIENT owns the next turn on + # this stateless surface. Persist the completion as a durable delivery row — # never self-post it as a new role=user prompt. try: logger.info( @@ -28243,13 +26228,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return None platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) - # Alias-aware resolution (relay-plane): a relay-fronted gateway - # registers ONE adapter under Platform.RELAY fronting N logical - # platforms, so a literal ``p.value == platform_name`` scan misses - # "slack" and silently drops the completion as "no gateway route" - # (staging incident 2026-08-09, second occurrence). Resolve through - # the shared transport resolver — native adapter wins; relay is - # eligible only when it advertises fronting the logical platform. + # Alias-aware resolution (relay-plane): a relay-fronted gateway registers ONE adapter under + # Platform.RELAY fronting N logical platforms, so a literal ``p.value == platform_name`` + # scan misses "slack" and silently drops the completion as "no gateway route". Resolve via + # the shared transport resolver — native adapter wins; relay is eligible only when it + # advertises fronting the logical platform. adapter = None try: _platform_enum = Platform(platform_name) @@ -28276,11 +26259,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return None from gateway.wake import adapter_supports_push as _wake_push_ok if not _wake_push_ok(adapter): - # Non-push adapter (api_server) resolved WITH routing metadata: - # its chat_id is the raw session id (see _bind_api_server_session, - # which binds chat_id = session_id). handle_message would run the - # wake under a build_session_key()-derived key that never matches - # the raw X-Hermes-Session-Id session — self-post instead. + # Non-push adapter (api_server) resolved WITH routing metadata: its chat_id is the raw + # session id (see _bind_api_server_session, which binds chat_id = session_id). + # handle_message would run the wake under a build_session_key()-derived key that never + # matches the raw X-Hermes-Session-Id session — self-post instead. from gateway.wake import deliver_wake, persist_delegation_delivery raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "") if evt.get("type") == "async_delegation": @@ -28338,12 +26320,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew source.chat_id, source.thread_id, ) - # Relay-plane egress priming (defect #4, staging 2026-08-09): a - # synthetic turn injected right after a restart reaches a relay - # adapter whose per-chat routing caches are cold (they warm only - # on inbound), so its replies egress without tenant - # discriminators and the connector's fail-closed guard declines - # them. Prime the caches from this event's session-store origin. + # Relay-plane egress priming (defect #4, staging 2026-08-09): a synthetic turn injected + # right after a restart reaches a relay adapter whose per-chat routing caches are cold + # (they warm only on inbound), so its replies egress without tenant discriminators and + # the connector's fail-closed guard declines them. _prime = getattr(adapter, "prime_routing_cache", None) if callable(_prime): _prime(synth_event) @@ -28357,11 +26337,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _completion_delivery_identity(evt: dict) -> Optional[tuple[str, str, object]]: """Return a producer-stable identity when one is available. - Delegation UUIDs identify one producer completion. Process session IDs - are normally unique too, but include the persisted spawn epoch so an - explicitly reused ID represents a distinct process incarnation. Legacy - process events without ``started_at`` are delivered without deduplication - rather than risking suppression of a real completion. + Delegation UUIDs identify one producer completion. Process session IDs are normally + unique too, but include the persisted spawn epoch so an explicitly reused ID represents a + distinct process incarnation. Legacy process events without ``started_at`` are delivered + without deduplication rather than risking suppression of a real completion. """ evt_type = str(evt.get("type") or "") if evt_type == "async_delegation": @@ -28379,19 +26358,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew Returns one of: - - ``"deliver"`` — the spawning session is live, or ended by a - compression rotation with a verified live continuation. The inner - #55578 resolver (:meth:`_resolve_async_delegation_session`) still - owns the actual route retarget; this pre-flight only proves the - completion is deliverable so the durable ack stays honest. - - ``"terminal"`` — the spawning session is gone for good (unknown, or - ended at an explicit user boundary such as /new). Delivery can never - succeed; the durable row should be terminally dropped rather than - falsely acknowledged as delivered or replayed forever as pending. - - ``"retry"`` — transient uncertainty (session DB unavailable, lookup - error, or a compression rotation caught mid-flight before its - continuation exists). The claim should be released so a later - consumer can retry; the attempt cap bounds the churn. + - ``"deliver"`` — spawning session live (or compression-rotated with a live + continuation); only proves deliverability, the resolver still retargets. + - ``"terminal"`` — parent gone for good (unknown / explicit user boundary like /new); + drop the durable row rather than falsely ack or replay forever. + - ``"retry"`` — transient uncertainty (DB unavailable, rotation mid-flight); release + the claim so a later consumer retries; the attempt cap bounds churn. """ session_db = getattr(self, "_session_db", None) if session_db is None: @@ -28410,17 +26382,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "deliver" end_reason = str(parent.get("end_reason") or "") if end_reason != "compression": - # An ended parent is only unreachable when the USER closed the - # thread of work (explicit boundary: /new -> session_reset / - # new_session, user_exit, session_switch). Idle/timeout ends are - # the norm on scale-to-zero relay deployments — the platform chat - # remains routable, and the #55578 resolver retargets the - # completion to the chat's current session. Dropping those loses - # finished work (staging incident 2026-08-09: completed - # delegation batch never delivered because the parent had - # idle-ended). The boundary set is shared with the resolver - # (_USER_BOUNDARY_END_REASONS) so this verdict and the pipeline's - # routing decision cannot drift apart. + # An ended parent is only unreachable when the USER closed the thread of work (explicit + # boundary: /new -> session_reset / new_session, user_exit, session_switch). Idle/timeout + # ends are normal on scale-to-zero relays — the chat stays routable and the resolver + # retargets, so dropping those loses finished work. Boundary set is shared with the + # resolver (_USER_BOUNDARY_END_REASONS) so the two decisions cannot drift. if end_reason in _USER_BOUNDARY_END_REASONS: return "terminal" return "deliver" @@ -28446,11 +26412,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[bool]: """Deliver once per live gateway, or return False for a retry. - ``True`` means this caller reached adapter acceptance, ``False`` means - injection failed and the claim was released for retry, and ``None`` - means either another same-lifecycle caller owns/delivered the producer - event or the event has no gateway route. No cross-process exactly-once - guarantee is claimed. + ``True``: adapter accepted; ``False``: injection failed, claim released for retry; + ``None``: another same-lifecycle caller owns/delivered it, or no gateway route. + No cross-process exactly-once guarantee is claimed. """ identity = self._completion_delivery_identity(evt) durable_claim_id = "" @@ -28516,12 +26480,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return False elif evt.get("type") == "completion": - # Background-process completions carry only session_key (chat/ - # thread routing), so after /new the notification from the OLD - # session would land in the chat's NEW session. Stamped events - # (spawn-time parent_session_id from terminal_tool) get the same - # session-boundary pre-flight as async delegations — one policy - # owner (_classify_completion_target), never a forked predicate. + # Background-process completions carry only session_key (chat/ thread routing), so after + # /new the notification from the OLD session would land in the chat's NEW session. + # Stamped events get the same pre-flight as async delegations — one policy owner + # (_classify_completion_target), never a forked predicate. # Legacy/unstamped events keep today's behavior and deliver. parent_session_id = str(evt.get("parent_session_id") or "").strip() if parent_session_id: @@ -28536,9 +26498,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) return None if verdict == "retry": - # Transient uncertainty (session DB unavailable or a - # compression rotation mid-flight): signal the watcher to - # re-poll and try again rather than dropping or + # Transient uncertainty (session DB unavailable or a compression rotation mid- + # flight): signal the watcher to re-poll and try again rather than dropping or # misrouting the result. return False if identity is not None: @@ -28622,12 +26583,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_id = str(evt.get("session_id") or "unknown") exit_code = evt.get("exit_code") reason = str(evt.get("completion_reason") or "exited") - # Completion-event output is normally passed through the terminal - # redactor at the producer seam, but that redactor is deliberately - # configurable. This synthetic turn is gateway user-facing input, - # so keep the unconditional gateway floor here as defence in depth. - # Redact before slicing: truncating first can leave a credential - # fragment that no longer matches the authoritative patterns. + # Completion-event output is normally passed through the terminal redactor at the + # producer seam, but that redactor is deliberately configurable; this is user-facing + # input so keep the unconditional gateway floor. Redact BEFORE slicing: truncating + # first can leave a credential fragment that no longer matches the patterns. output = _redact_gateway_user_facing_secrets( str(evt.get("output") or "") ).strip() @@ -28698,10 +26657,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew [evt for _text, evt, _future in entries] ) except asyncio.CancelledError: - # Shutdown may cancel us either during the fan-in window or while - # adapter delivery is blocked. Recover entries that have not yet - # detached and resolve every waiter as retryable before adapters - # are torn down. + # Shutdown may cancel us either during the fan-in window or while adapter delivery is + # blocked. Recover entries that have not yet detached and resolve every waiter as + # retryable before adapters are torn down. delivered = False if not entries: entries = self._completion_notification_batches.pop(key, []) @@ -28791,12 +26749,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _enrich_async_delegation_routing(self, evt: dict) -> None: """Fill platform/chat_id/thread_id/chat_type on an async-delegation event. - Async-delegation completion events only carry ``session_key`` (the - daemon worker has no access to the per-message routing metadata the - terminal background watcher captures at spawn time). Parse the - session_key into the routing fields ``_build_process_event_source`` - expects. Best-effort: a CLI-origin event (empty session_key) is left - as-is and simply won't route on the gateway. + Async-delegation completion events only carry ``session_key`` (the daemon worker has no + access to the per-message routing metadata the terminal background watcher captures at + spawn time). Best-effort: a CLI-origin event (empty session_key) is left as-is and simply + won't route on the gateway. """ if evt.get("platform"): return # already enriched @@ -28844,16 +26800,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[bool]: """Deliver a same-session batch of async completions as ONE turn. - A single-event group rides the existing per-event path unchanged. For - a multi-event group the primary event is delivered through - ``_deliver_completion_notification`` (which owns its durable claim, - the lifecycle dedupe, and the target preflight), carrying a - consolidated text that also contains every sibling result whose - durable row THIS runner successfully claimed up front. Only after - adapter acceptance are the sibling claims acknowledged — the durable - ledger never acks work that was not delivered, and a sibling claimed - by another consumer is excluded from the consolidated text entirely - so its content cannot be double-delivered. + Single-event groups ride the per-event path unchanged. Multi-event groups deliver the + primary via ``_deliver_completion_notification`` (owns claim, lifecycle dedupe, target + preflight) with a consolidated text of every sibling THIS runner claimed up front. + Only after adapter acceptance are the sibling claims acknowledged — the durable ledger + never acks work that was not delivered, and a sibling claimed by another consumer is + excluded from the consolidated text entirely so its content cannot be double-delivered. Returns ``True`` after adapter acceptance, ``False`` when the caller should requeue the group for retry, and ``None`` when nothing in the @@ -28947,16 +26899,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def _async_delegation_watcher(self, interval: float = 2.0) -> None: """Drain async-delegation completions and inject them as new turns. - Background subagents (``delegate_task(background=true)``) run on the - async-delegation daemon executor — they have no per-process watcher - task, so their completion events would only be seen by the post-turn - queue drain. This watcher covers the IDLE case: when a background - subagent finishes while no agent turn is running, its result still - re-enters the originating session promptly. - - Mirrors the CLI's idle ``process_loop`` drain. Stays silent when the - queue has nothing for us; ignores non-async event types (those are - handled by ``_run_process_watcher`` / the post-turn drain). + Background subagents (``delegate_task(background=true)``) run on the async-delegation + daemon executor — they have no per-process watcher task, so their completion events would + only be seen by the post-turn queue drain. This covers the IDLE case (no turn running). + Mirrors the CLI's idle ``process_loop`` drain; ignores non-async event types (handled by + ``_run_process_watcher`` / the post-turn drain). """ await asyncio.sleep(3) # let platforms finish connecting from tools.process_registry import process_registry as _pr @@ -28978,12 +26925,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew requeue.append(evt) for evt in requeue: _pr.completion_queue.put(evt) - # A same-tick drain often carries several completions for the - # SAME originating session (a fan-out of background subagents - # finishing together). Delivering each one individually floods - # the session with N synthetic turns (#70300) — group by full - # gateway route + parent session and inject one consolidated - # turn per group. Events for different sessions never coalesce. + # A same-tick drain often carries several completions for the SAME originating + # session (a fan-out of background subagents finishing together). Delivering each + # individually floods the session with N synthetic turns — group by full gateway + # route + parent session, one consolidated turn per group. Events for different + # sessions never coalesce. groups: dict[tuple[str, ...], list[dict]] = {} group_order: list[tuple[str, ...]] = [] for evt in async_events: @@ -29009,19 +26955,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await asyncio.sleep(interval) async def _run_process_watcher(self, watcher: dict) -> None: - """ - Periodically check a background process and push updates to the user. + """Periodically check a background process and push updates to the user. - Runs as an asyncio task. Stays silent when nothing changed. - Auto-removes when the process exits or is killed. - - Notification mode (from ``display.background_process_notifications``): - - ``concise`` — one-line status message on completion (default); - failures append a short output tail - - ``all`` — running-output updates + final raw-output message - - ``result`` — final raw-output completion message only - - ``error`` — final raw-output message only when exit code != 0 - - ``off`` — no messages at all + Runs as an asyncio task. Stays silent when nothing changed. Auto-removes when the process + exits or is killed. Notification mode (``display.background_process_notifications``): + concise (default, one-line; failures append output tail) / all (running updates + final + raw output) / result (final raw only) / error (final raw only if exit != 0) / off. """ from tools.process_registry import process_registry @@ -29076,10 +27015,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _raw = strip_ansi(session.output_buffer) if session.output_buffer else "" _raw = redact_terminal_output(_raw, _command) _command = _redact_gateway_user_facing_secrets(_command) - # Truncate at line boundaries so notifications never start - # mid-line (fixes #23284). Keep the last ~2000 chars but - # snap to the nearest preceding newline, then prepend a - # truncation marker when output was cut. + # Truncate at line boundaries so notifications never start mid-line (fixes + # #23284). Keep the last ~2000 chars but snap to the nearest preceding newline, + # then prepend a truncation marker when output was cut. _LIMIT = 2000 if len(_raw) > _LIMIT: _tail = _raw[-_LIMIT:] @@ -29129,16 +27067,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew break # --- Normal text-only notification --- - # Skip when the agent already consumed this completion via - # wait/log (#65379): process(wait) returned the exit code and - # output inline, so the raw "[Background process ... finished - # with exit code ...]" message would be a duplicate delivery - # of the same completion. The agent_notify branch above - # already honors _completion_consumed; without this check its - # skip FALLS THROUGH to this block and re-delivers the output - # the agent is actively summarizing. poll() is read-only and - # intentionally does not mark consumed (#10156), so a status - # check never suppresses this message. + # Skip when the agent already consumed this completion via wait/log: process(wait) + # returned the exit code and output inline, so the raw "[Background process ... + # finished with exit code ...]" message would be a duplicate delivery of the same + # completion. The agent_notify branch's skip FALLS THROUGH to here, hence this + # check. poll() is read-only and intentionally does not mark consumed. if _pr_check.is_completion_consumed(session_id): logger.debug( "Process watcher: completion for %s already consumed " @@ -29158,10 +27091,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew new_output = redact_terminal_output( new_output, getattr(session, "command", "") or "" ) - # redact_terminal_output() is unforced, so it returns raw - # text when security.redact_secrets is off. This send - # goes straight to the platform adapter, so it needs the - # same unconditional floor as the agent-notify path. + # redact_terminal_output() is unforced, so it returns raw text when + # security.redact_secrets is off. This goes straight to the platform + # adapter, so it needs the same unconditional floor as agent-notify. new_output = _redact_gateway_user_facing_secrets(new_output) if notify_mode == "concise": _cmd_disp = _redact_gateway_user_facing_secrets( @@ -29234,14 +27166,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _MAX_INTERRUPT_DEPTH = 3 # Cap recursive interrupt handling (#816) - # Config keys whose values MUST invalidate the gateway's cached agent - # when they change. The agent bakes these into its compressor / context - # handling at construction time, so a mid-running-gateway config edit - # would otherwise be silently ignored until the user triggers a - # different cache eviction (model switch, /reset, etc.). - # - # Each entry is a tuple of (section, key) read from the raw config dict. - # Add more here as new baked-at-construction config settings are added. + # Config keys whose values MUST invalidate the gateway's cached agent when they change: the + # agent bakes them in at construction, so a mid-gateway config edit would otherwise be + # silently ignored until some other eviction. (section, key) tuples from the raw config + # dict; add new baked-at-construction settings here. _CACHE_BUSTING_CONFIG_KEYS: tuple = ( ("model", "context_length"), ("model", "max_tokens"), @@ -29325,11 +27253,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew non-dict sections yield None values, which still contribute to the signature (so 'absent' vs 'present-and-null' differ). - The live tool registry generation is included too. MCP reloads and - dynamic MCP tool-list changes mutate the registry without necessarily - changing config.yaml. Cached AIAgent instances freeze their tool - schemas at construction time, so a registry generation change must - rebuild the agent before the next turn. + The live tool registry generation is included too: MCP reloads / dynamic tool-list + changes mutate the registry without touching config.yaml, and cached agents freeze + their tool schemas at construction, so a generation change must rebuild the agent. """ out: Dict[str, Any] = {} cfg = user_config if isinstance(user_config, dict) else {} @@ -29373,37 +27299,24 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> str: """Compute a stable string key from agent config values. - When this signature changes between messages, the cached AIAgent is - discarded and rebuilt. When it stays the same, the cached agent is - reused — preserving the frozen system prompt and tool schemas for - prompt cache hits. + Signature change → cached AIAgent discarded and rebuilt; unchanged → reused (frozen + system prompt + tool schemas for prompt cache hits). - ``cache_keys`` is an optional flat dict of additional config values - that should invalidate the cache when they change. Callers pass - the output of ``_extract_cache_busting_config(user_config)`` so - edits to model.context_length / compression.* in config.yaml are - picked up on the next gateway message without a manual restart. + Callers pass the output of ``_extract_cache_busting_config(user_config)`` so edits to + model.context_length / compression.* in config.yaml are picked up on the next gateway + message without a manual restart. - ``user_id`` and ``user_id_alt`` are the runtime user identities - carried by the current message's gateway source. They participate - in the cache key because the Honcho memory provider freezes them - into ``HonchoSessionManager`` at first-message init (see - ``plugins/memory/honcho/__init__.py::_do_session_init``). Without - them in the signature, a shared-thread session_key (one in which - ``build_session_key`` intentionally omits the participant ID, - e.g. ``thread_sessions_per_user=False``) would reuse the cached - AIAgent across distinct users, causing the second user's messages - to be attributed to the first user's resolved Honcho peer. This - broke #27371's per-user-peer contract in multi-user gateways. - Per-user agent rebuilds in shared threads trade prompt-cache - warmth for correct memory attribution. + ``user_id`` / ``user_id_alt`` participate because the Honcho memory provider freezes + them into ``HonchoSessionManager`` at first-message init; in shared-thread session keys + (``thread_sessions_per_user=False``) omitting them would attribute the second user's + messages to the first user's Honcho peer. Per-user rebuilds trade cache warmth for + correct memory attribution. """ import hashlib, json as _j - # Fingerprint the FULL credential string instead of using a short - # prefix. OAuth/JWT-style tokens frequently share a common prefix - # (e.g. "eyJhbGci"), which can cause false cache hits across auth - # switches if only the first few characters are considered. + # Fingerprint the FULL credential string instead of using a short prefix. OAuth/JWT-style + # tokens frequently share a common prefix (e.g. "eyJhbGci"), which can cause false cache + # hits across auth switches if only the first few characters are considered. _api_key = str(runtime.get("api_key", "") or "") _api_key_fingerprint = hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "" @@ -29438,16 +27351,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _rehydrate_session_model_override(self, session_key: str) -> None: """Lazily restore a persisted /model override after a gateway restart. - ``_session_model_overrides`` is in-memory only, so before persistence - a restart silently reverted every session to the global default model. - The non-secret parts (model/provider/base_url) are written through to - the session store when /model runs (and cleared on /new); here we read - them back on first use and re-resolve credentials via the normal - runtime provider resolution — api_key is never persisted to disk. - - No-op when an in-memory override already exists (live state wins) or - when the store has nothing persisted (e.g. the user ran /new, which - clears both the in-memory dict and the persisted field). + ``_session_model_overrides`` is in-memory only, so before persistence a restart silently + reverted every session to the global default model. Non-secret parts (model/provider/ + base_url) are written through on /model (cleared on /new); read back here on first use + and credentials re-resolved via normal runtime resolution — api_key is never persisted. + No-op when an in-memory override exists (live state wins) or nothing is persisted. """ _rehydrate_state = self._peek_session_state(session_key) if ( @@ -29474,10 +27382,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew } provider = persisted.get("provider") if provider: - # Re-resolve credentials for the persisted provider. On failure - # (e.g. credentials were removed since the switch) keep the - # credential-less override — _resolve_session_agent_runtime falls - # back to env-based resolution and applies model/provider on top. + # Re-resolve credentials for the persisted provider. On failure (e.g. credentials + # removed since the switch) keep the credential-less override — + # _resolve_session_agent_runtime falls back to env resolution and layers model/provider. try: runtime = _resolve_runtime_agent_kwargs_for_provider(provider) override["api_key"] = runtime.get("api_key") @@ -29508,11 +27415,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> tuple: """Apply /model session overrides if present, returning (model, runtime_kwargs). - The gateway /model command stores per-session overrides in - ``_session_model_overrides``. These must take precedence over - config.yaml defaults so the switched model is actually used for - subsequent messages. Fields with ``None`` values are skipped so - partial overrides don't clobber valid config defaults. + These must take precedence over config.yaml defaults so the switched model is actually + used for subsequent messages. Fields with ``None`` values are skipped so partial + overrides don't clobber valid config defaults. """ _apply_state = self._peek_session_state(session_key) override = _apply_state.conversation.model_override if _apply_state else None @@ -29532,10 +27437,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew val = override.get(key) if val is not None: runtime_kwargs[key] = val - # request_overrides reflects the switched-to provider; apply whenever - # the override recorded it (even as None) so switching to a provider - # without configured overrides clears a stale value left by the - # default provider's runtime resolution. + # request_overrides reflects the switched-to provider; apply whenever the override recorded + # it (even as None) so switching to a provider without configured overrides clears a stale + # value left by the default provider's runtime resolution. if "request_overrides" in override: override_request_overrides = override.get("request_overrides") if isinstance(override_request_overrides, dict) and override_request_overrides: @@ -29589,26 +27493,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> bool: """Pop ALL per-running-agent state entries for ``session_key``. - Replaces ad-hoc ``del self._running_agents[key]`` calls scattered - across the gateway. Those sites had drifted: some popped only - ``_running_agents``; some also ``_running_agents_ts``; only one - path also cleared ``_busy_ack_ts``. Each missed entry was a - small, persistent leak — a (str_key → float) tuple per session + Replaces ad-hoc ``del self._running_agents[key]`` calls scattered across the gateway. + Each missed entry was a small, persistent leak — a (str_key → float) tuple per session per gateway lifetime. - Use this at every site that ends a running turn, regardless of - cause (normal completion, /stop, /reset, /resume, sentinel - cleanup, stale-eviction). Per-session state that PERSISTS - across turns (``_session_model_overrides``, ``_voice_mode``, - ``_pending_approvals``, ``_update_prompt_pending``) is NOT - touched here — those have their own lifecycles. - - When ``run_generation`` is provided, only clear the slot if that - generation is still current for the session. This prevents an - older async run whose generation was bumped by /stop or /new from - clobbering a newer run's state during its own unwind. Returns - True when the slot was cleared, False when an ownership guard - blocked it. + Call at every site that ends a running turn, whatever the cause. Per-session state that + PERSISTS across turns (model overrides, voice mode, pending approvals, update prompt) is + NOT touched — those have their own lifecycles. With ``run_generation``, only clear if + that generation is still current, so a stale async unwind bumped by /stop or /new cannot + clobber a newer run. Returns True when cleared, False when the ownership guard blocked. """ if not session_key: return False @@ -29626,28 +27519,23 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug( "Failed to release active session slot", exc_info=True ) - # One structured reset instead of the old drifting pop-list - # (agent / started_ts / lease / busy_ack_ts). Turn-lease tokens - # are deliberately NOT cleared here — _release_turn_lease owns - # them (#64934). + # One structured reset instead of the old drifting pop-list (agent / started_ts / lease + # / busy_ack_ts). Turn-lease tokens are deliberately NOT cleared here — + # _release_turn_lease owns them. state.turn.clear() - # Turn boundary: a running-agent slot was just released. Persist the - # new (lower) in-flight count so the dashboard readout stays current - # between lifecycle transitions. Preserves gateway_state (see - # _persist_active_agents). + # Turn boundary: a running-agent slot was just released. Persist the new (lower) in-flight + # count so the dashboard readout stays current between lifecycle transitions. Preserves + # gateway_state (see _persist_active_agents). self._persist_active_agents() return True def _release_turn_lease(self, session_key: str, run_generation: int) -> bool: """Release the turn lease acquired by (``session_key``, ``run_generation``). - Companion to the acquisition in ``_handle_message_with_agent`` - (#64934). The token map is keyed by (routing key, run generation), so - this can only ever free the lease its own turn acquired — a stale - unwind whose generation was bumped by /stop or /new pops ITS token, - and the registry's identity check refuses it if a newer turn already - holds the lease. Idempotent and safe for bare test runners built via - ``object.__new__`` (getattr defaults). + Companion to the acquisition in ``_handle_message_with_agent``. Token map is keyed by + (routing key, run generation), so this only frees the lease its own turn acquired — a + stale unwind pops ITS token and the registry's identity check refuses it if a newer turn + holds the lease. Idempotent and safe for bare test runners built via ``object.__new__``. """ if not session_key: return False @@ -29672,14 +27560,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> bool: """Follow a mid-turn session_id rotation with the held turn lease. - Compression (session-hygiene pre-compression or the agent's own - compressor) can rotate ``session_entry.session_id`` while this turn - is in flight. The turn's flush targets the NEW id, so the - serialization boundary must follow it — otherwise an alias routing - key resolving the new id (topic tip-walk onto the fresh child) could - start a concurrent turn the lease never sees (#64934 rotation-alias - window). Call at every site that reassigns session_entry.session_id - mid-turn. Fail-open no-op when there is no held token. + Compression can rotate ``session_entry.session_id`` mid-turn. The turn's flush targets + the NEW id, so the serialization boundary must follow it — otherwise an alias routing + key resolving the new id (topic tip-walk onto the fresh child) could start a concurrent + turn the lease never sees. Call at every site that reassigns session_id mid-turn. + Fail-open no-op when no token is held. """ if not session_key or not new_session_id: return False @@ -29699,34 +27584,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _clear_conversation_scope(self, session_key: str, *, reason: str) -> None: """Clear ALL conversation-scoped per-session state for ``session_key``. - THE single conversation-boundary funnel. Call this — and nothing - else — whenever a session_key crosses a conversation boundary: - /new, /resume, auto-reset (idle/daily/suspended), expiry - finalization, and the compression-exhausted auto-reset. + THE single conversation-boundary funnel. Call this — and nothing else — whenever a + session_key crosses a conversation boundary: /new, /resume, auto-reset + (idle/daily/suspended), expiry finalization, and the compression-exhausted auto-reset. - Why a funnel: these boundaries used to each carry a hand-copied - pop-list of the per-session dicts, and the lists drifted every time - a new dict was added (#48031, #58403, #10702, #35809 were all - "boundary X forgot dict Y" bugs — e.g. /new cleared the /model - override but not the /model --once restore snapshot). Adding a new - conversation-scoped dict now means adding its attribute name to - _CONVERSATION_SCOPED_STATE below; every boundary picks it up - automatically. + Why a funnel: each boundary used to carry a hand-copied pop-list that drifted whenever a + dict was added ("boundary X forgot dict Y" bugs). New conversation-scoped dicts go in + _CONVERSATION_SCOPED_STATE and every boundary picks them up. - Scope rules: - - Conversation-scoped (cleared here): model/reasoning overrides, - one-turn restore snapshots, pending model notes, last-resolved - model cache, queued follow-up events, and the boundary security - state (approvals, /yolo, slash-confirm, update prompts). - - Turn-scoped (NOT cleared here): _running_agents/_ts, slot leases, - turn-lease tokens — owned by _release_running_agent_state and the - dispatch finally. - - Idle agent-cache eviction is NOT a conversation boundary: the - session is still alive and a resumed turn rebuilds from these - overrides. Only true boundaries call this. - - Safe on bare test runners built via ``object.__new__`` (every - access is getattr-guarded). + Scope: conversation-scoped (cleared): model/reasoning overrides, one-turn restore + snapshots, pending model notes, last-resolved model, queued follow-ups, boundary security + state. Turn-scoped (NOT cleared): _running_agents/_ts, slot leases, turn-lease tokens — + owned by _release_running_agent_state. Idle agent-cache eviction is NOT a boundary (the + session lives on and a resumed turn rebuilds from these overrides). getattr-guarded. """ if not session_key: return @@ -29735,11 +27605,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew state = self._peek_session_state(session_key) if state is not None: state.conversation.clear() - # Legacy plain-dict stores still registered in - # _CONVERSATION_SCOPED_STATE (not yet folded into SessionState), - # e.g. _pending_model_notes. SessionState-backed names resolve to - # MutableMapping views (not dict), so the isinstance(dict) guard - # skips them — already handled above. + # Legacy plain-dict stores still registered in _CONVERSATION_SCOPED_STATE (not yet folded + # into SessionState), e.g. _pending_model_notes. SessionState-backed names resolve to + # MutableMapping views (not dict), so the isinstance(dict) guard skips them — already + # handled above. for attr in _CONVERSATION_SCOPED_STATE: store = getattr(self, attr, None) if isinstance(store, dict): @@ -29796,10 +27665,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _begin_session_run_generation(self, session_key: str) -> int: """Claim a fresh run generation token for ``session_key``. - Every top-level gateway turn gets a monotonically increasing token. - If a later command like /stop or /new invalidates that token while the - old worker is still unwinding, the late result can be recognized and - dropped instead of bleeding into the fresh session. + Every top-level gateway turn gets a monotonically increasing token. If a later command + like /stop or /new invalidates that token while the old worker is still unwinding, the + late result can be recognized and dropped instead of bleeding into the fresh session. """ if not session_key: return 0 @@ -29868,13 +27736,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _process_baseline = getattr( running_agent, "_gateway_turn_process_baseline", None ) - # Bump the generation *before* scheduling the reap thread and capture - # the post-bump value: task_id is session-scoped (task_id == - # session_id), so if a replacement turn claims this session and - # spawns its own process before the reap thread actually runs, that - # claim bumps the generation again. The closure below then sees a - # stale generation and skips — the replacement turn's own baseline - # covers its own cleanup, so nothing is left permanently unreaped. + # Bump the generation *before* scheduling the reap thread and capture the post-bump value: + # task_id is session-scoped (task_id == session_id), so if a replacement turn claims this + # session and spawns its own process before the reap thread actually runs, that claim bumps + # the generation again; the closure then sees a stale generation and skips — the + # replacement's own baseline covers its cleanup, so nothing stays unreaped. _generation_at_interrupt = self._invalidate_session_run_generation( session_key, reason=invalidation_reason ) @@ -29917,15 +27783,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _iac_state.persistent.pending_command_text = None if release_running_state: self._release_running_agent_state(session_key) - # Evict the cached agent: ``_interrupt_requested`` is only - # cleared by the turn finalizer, so on a hung or still-draining - # run the flag survives the lock release and kills the session's - # NEXT message at the top of the tool loop (interrupted=True, - # api_calls=0, empty response — silently swallowed, #44212). - # Evicting mirrors the /new and /model paths: the next message - # rebuilds the agent from session history, while the old agent - # object keeps its interrupt flag so a hung drain still dies - # when it unblocks. + # Evict the cached agent: ``_interrupt_requested`` is only cleared by the turn + # finalizer, so on a hung or still-draining run the flag survives the lock release and + # kills the session's NEXT message at the top of the tool loop (interrupted=True, + # api_calls=0, empty response — silently swallowed, #44212). Evicting mirrors /new and + # /model: the next message rebuilds from history while the old agent keeps its + # interrupt flag so a hung drain still dies when it unblocks. self._evict_cached_agent(session_key) async def _refresh_agent_cache_message_count( @@ -29933,34 +27796,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> None: """Re-baseline a cached agent's stored message_count after THIS turn. - The cross-process coherence guard (#45966) compares the session's - on-disk ``message_count`` against the count snapshotted next to the - cached agent, and rebuilds the agent on a mismatch. But the snapshot - is taken at agent-BUILD time — before this turn writes its own user + - assistant (+ tool) rows — and the cache entry is never rewritten on a - reuse. So without this re-baseline, THIS process's own turn would - grow ``message_count`` and the very next turn would see a mismatch - and rebuild the agent — every turn, for every conversation — silently - destroying the per-conversation prompt caching the cache exists to - protect. - - Call this once a turn has completed and the agent has flushed its - rows to the SessionDB. It snapshots the now-current count (which - includes this process's own writes) so the guard only fires when a - DIFFERENT process changes the transcript out from under us. The - ``_sig`` is left untouched; only the count element is refreshed, and - only when the same agent is still cached (no rebuild/eviction raced - in between). Fail-safe: any DB error leaves the snapshot as-is, which - at worst costs one unnecessary rebuild on the next turn. - - When the cache entry records a ``session_id`` (4-tuple form, #54947) - that differs from the current ``session_id`` — meaning the cache - was built for a DIFFERENT conversation under the same ``session_key`` - — the snapshot is intentionally left untouched. Overwriting it with - the current session's count would corrupt the original conversation's - baseline and cause the next switch back to fire the cross-process - guard spuriously. Fail-safe: the legacy 3-tuple shape (no - ``session_id``) is still re-baselined as before. + The cross-process coherence guard compares on-disk ``message_count`` against the count + snapshotted with the cached agent and rebuilds on mismatch. But the snapshot is taken at + agent-BUILD time — before this turn writes its own user + assistant (+ tool) rows — and + the cache entry is never rewritten on a reuse, so without re-baselining our OWN writes + would force a rebuild every turn, destroying the prompt caching the cache protects. + Call this once a turn has completed and the agent has flushed its rows to the SessionDB. + Only the count element is refreshed (``_sig`` untouched), and only if the same agent is + still cached. If the entry records a different ``session_id`` (cache built for another + conversation under the same key) leave it alone — overwriting would corrupt that + conversation's baseline. Fail-safe: DB errors leave the snapshot as-is (one spare rebuild). """ if self._session_db is None or not session_id: return @@ -29985,10 +27830,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew and len(cached) > 2 and cached[0] is not _AGENT_PENDING_SENTINEL ): - # If the snapshot was taken for a different session_id - # (same session_key, different conversation), leave the - # snapshot alone — the current session_id's count belongs - # to a different DB row (#54947). + # If the snapshot was taken for a different session_id (same session_key, different + # conversation), leave the snapshot alone — the current session_id's count belongs + # to a different DB row. _snapshot_sid = cached[3] if len(cached) > 3 else None if _snapshot_sid is not None and _snapshot_sid != session_id: return @@ -30022,10 +27866,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _voice_channel_sidecar_note(self, event, source: SessionSource, session_key: str) -> Optional[str]: """Return a ``[Voice channel now: ...]`` note when VC state changed. - Compares the live Discord voice-channel context against the last - value delivered for this session and returns a note only on change - (including leaving the channel). Unchanged state returns ``None`` so - the per-turn member/speaking serialization cannot churn the prompt. + Unchanged state returns ``None`` so the per-turn member/speaking serialization cannot + churn the prompt. """ if source.platform != Platform.DISCORD: return None @@ -30054,10 +27896,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> str: """Return the session-context prompt, pinned per session. - Key hit → the pinned bytes are reused VERBATIM (immunizes the - composed system prompt against renderer nondeterminism); key miss → - re-render ``build_session_context_prompt`` and re-pin (a legitimate - cache bust: rename, topic edit, /sethome, redact_pii flip, ...). + Key hit → the pinned bytes are reused VERBATIM (immunizes the composed system prompt + against renderer nondeterminism); key miss → re-render ``build_session_context_prompt`` + and re-pin (a legitimate cache bust: rename, topic edit, /sethome, redact_pii flip, ...). """ _eph_key = self._ephemeral_change_key(context, redact_pii) _eph_pin = None @@ -30078,12 +27919,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _ephemeral_change_key(context, redact_pii: bool) -> str: """Hash the exact inputs ``build_session_context_prompt`` renders. - This key decides when the pinned per-session context-prompt bytes are - reused verbatim vs re-rendered. The maintained invariant (guarded by - the parity test in tests/gateway/test_prompt_tail_freeze.py): any - input whose change alters the rendered bytes MUST appear here — - omission means a stale pinned prompt (cosmetic staleness); inclusion - of an extra field only costs a spurious re-render. + This key decides when the pinned per-session context-prompt bytes are reused verbatim vs + re-rendered. Invariant (guarded by tests/gateway/test_prompt_tail_freeze.py): any input + whose change alters the rendered bytes MUST appear here — omission means a stale pinned + prompt; an extra field only costs a spurious re-render. """ import hashlib @@ -30107,11 +27946,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "1" if src.message_id else "0", ) - # Slack renders a capability-aware platform note gated on - # _slack_tools_loaded() — the gate state must appear in the key - # (same parity contract as the Discord gate above) so a config / - # MCP-registration flip re-renders once instead of serving a - # stale pinned note for the rest of the session. + # Slack renders a capability-aware platform note gated on _slack_tools_loaded() — the gate + # state must appear in the key (same parity contract as the Discord gate above) so a config + # / MCP-registration flip re-renders once instead of serving a stale pinned note for the + # rest of the session. slack_tools = "" if src.platform == Platform.SLACK: from gateway.session import _slack_tools_loaded @@ -30156,26 +27994,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _evict_cached_agent(self, session_key: str) -> None: """Remove a cached agent for a session (called on /new, /model, etc). - Pops the entry AND soft-releases the evicted agent's LLM client - pool so the httpx connection (sockets + held buffers) is freed - promptly rather than waiting on CPython GC — AIAgent holds - reference cycles (callbacks, tool state) that delay refcount - collection, so a manual release is required to keep gateway RSS - flat across many /new, /model, undo and reset operations (#29298, - same leak class as #25315). + Pops the entry AND soft-releases the evicted agent's LLM client pool so httpx sockets and + buffers are freed promptly — AIAgent holds reference cycles that delay refcount + collection, so without a manual release gateway RSS grows across /new, /model, reset. - The release is soft (``release_clients()``): it frees the client - pool and per-turn child subagents but PRESERVES the session's - terminal sandbox, browser daemon, and tracked bg processes (keyed - on task_id), because the session may resume with a freshly-built - agent. Call sites that want a hard teardown (true conversation - boundaries like /new) already call ``_cleanup_agent_resources`` - before evicting; ``release_clients`` is idempotent and safe to - run again after that (the client is already None). - - Cleanup runs on a daemon thread so we never block holding - ``_agent_cache_lock`` on slow socket teardown — mirrors the - cap-enforcer and idle-sweeper paths. + The release is soft (``release_clients()``): it frees the client pool and per-turn child + subagents but PRESERVES the session's terminal sandbox, browser daemon, and tracked bg + processes (keyed on task_id), because the session may resume with a freshly-built agent. + True boundaries (/new) call ``_cleanup_agent_resources`` first; ``release_clients`` is + idempotent after that. Cleanup runs on a daemon thread so ``_agent_cache_lock`` is never + held across slow socket teardown. """ # Prompt-stability state rides the agent-cache lifecycle: a fresh # agent must re-render its session-context bytes (the pin) and re-see @@ -30228,16 +28056,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _init_cached_agent_for_turn(agent: Any, interrupt_depth: int) -> None: """Reset per-turn state on a cached agent before a new turn starts. - ``_last_activity_ts``, ``_last_activity_desc``, and - ``_last_activity_provenance`` are only reset for fresh external - turns (depth 0); they are a semantic triple - description and - provenance describe the activity *at* ts, so updating one without - the others would make get_activity_summary() misleading. - For interrupt-recursive turns all three are preserved so the - inactivity watchdog can accumulate stuck-turn idle time and fire - the 30-min timeout (#15654). The depth-0 reset is still needed: - a session idle for 29 min would otherwise trip the watchdog before - the new turn makes its first API call (#9051). + ``_last_activity_ts`` / ``_desc`` / ``_provenance`` are a semantic triple (desc and + provenance describe the activity *at* ts) so they are reset together, and only for fresh + external turns (depth 0). For interrupt-recursive turns all three are preserved so the + inactivity watchdog can accumulate stuck-turn idle time and fire the 30-min timeout. The + depth-0 reset is still needed: a session idle 29 min would otherwise trip the watchdog + before the new turn's first API call. """ if interrupt_depth == 0: from agent.session_activity import ActivityProvenance @@ -30255,28 +28079,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _commit_memory_before_soft_evict(self, agent: Any, key: str) -> None: """Fire on_session_end extraction before soft-evicting a live agent. - Soft eviction (``_release_evicted_agent_soft``) deliberately keeps the - session resumable and does NOT fire ``on_session_end`` — that hook is - reserved for the true session boundary, tear-down done by - ``_session_expiry_watcher`` when the session finally expires. - - But the watcher tears down whatever agent it finds in ``_agent_cache`` - at expiry time. If cache pressure (the LRU cap) soft-evicts a - finalizable session's agent BEFORE it expires, the watcher later finds - no cached agent and ``on_session_end`` is silently skipped — memory - providers never see the transcript (#11205, LRU-cap variant). - - We hold the live, fully-scoped agent right now, so commit its - end-of-session memory extraction here using the agent's own memory - manager (correct per-user/chat scoping, no reconstruction). This uses - ``commit_memory_session`` — extraction WITHOUT provider teardown — so - the eviction stays soft and a resumed turn keeps working. - - Only fires for sessions the expiry watcher will eventually finalize - (finite reset policy). For ``mode == "none"`` sessions the watcher - never runs, so there is no missed-boundary to compensate for and we - skip the commit (the agent is simply released). Best-effort: any - failure is swallowed so eviction still proceeds. + Soft eviction (``_release_evicted_agent_soft``) deliberately keeps the session resumable + and does NOT fire ``on_session_end`` — that hook is reserved for the true session + boundary, tear-down done by ``_session_expiry_watcher`` when the session finally expires. + But the watcher tears down whatever agent it finds in ``_agent_cache``; if the LRU cap + soft-evicts first, it finds none and ``on_session_end`` is silently skipped — memory + providers never see the transcript. So commit extraction here with the live agent's own + memory manager via ``commit_memory_session`` (extraction WITHOUT teardown, eviction stays + soft). Only for finalizable sessions (finite reset policy); ``mode == "none"`` never + finalizes so nothing to compensate. Best-effort: failures swallowed, eviction proceeds. """ if agent is None or not hasattr(agent, "commit_memory_session"): return @@ -30290,10 +28101,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew entry = _store._entries.get(key) if entry is None: return - # Only compensate when the watcher would otherwise expect to find - # this agent at expiry (finite policy, not yet expired). Expired - # sessions are torn down by the watcher directly; mode="none" - # sessions are never finalized. + # Only compensate when the watcher would otherwise expect to find this agent at expiry + # (finite policy, not yet expired). Expired sessions are torn down by the watcher + # directly; mode="none" sessions are never finalized. if not _store.is_session_finalizable(entry): return if _store._is_session_expired(entry): @@ -30310,10 +28120,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _commit_then_release_soft(self, agent: Any, key: str) -> None: """Commit end-of-session memory (if warranted), then soft-release. - Runs on the daemon eviction thread so the memory-provider call and the - client teardown never block the caller's held cache lock. Order matters: - commit uses the live agent's memory manager before ``release_clients`` - drops the message buffer. + Runs on the daemon eviction thread so the memory-provider call and the client teardown + never block the caller's held cache lock. Order matters: commit uses the live agent's + memory manager before ``release_clients`` drops the message buffer. """ self._commit_memory_before_soft_evict(agent, key) self._release_evicted_agent_soft(agent) @@ -30321,12 +28130,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _release_evicted_agent_soft(self, agent: Any) -> None: """Soft cleanup for cache-evicted agents — preserves session tool state. - Called from _enforce_agent_cache_cap and _sweep_idle_cached_agents. - Distinct from _cleanup_agent_resources (full teardown) because a - cache-evicted session may resume at any time — its terminal - sandbox, browser daemon, and tracked bg processes must outlive - the Python AIAgent instance so the next agent built for the - same task_id inherits them. + Called from _enforce_agent_cache_cap and _sweep_idle_cached_agents. Distinct from + _cleanup_agent_resources (full teardown): a cache-evicted session may resume, so its + terminal sandbox, browser daemon and bg processes must outlive the AIAgent instance and + be inherited by the next agent built for the same task_id. """ if agent is None: return @@ -30339,19 +28146,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._cleanup_agent_resources(agent) except Exception: pass - # Free conversation history memory — can be tens of MB with tool - # outputs (file reads, terminal output, search results) on heavy - # 100+-tool-call sessions. release_clients() deliberately preserves - # session tool state for resume, but the message list is rebuilt from + # Free conversation history memory — can be tens of MB with tool outputs (file reads, + # terminal output, search results) on heavy 100+-tool-call sessions. release_clients() + # deliberately preserves session tool state for resume, but the message list is rebuilt from # persisted session JSON on the next turn, so dropping it here is safe. if hasattr(agent, "_session_messages"): agent._session_messages = [] - # _db_flush_scan_prefix is a shallow copy of the flushed transcript - # (run_agent.py, stamped on every successful flush) — it shares every - # message dict, so leaving it pins the multi-MB content strings the - # eviction exists to free. Pressure-evictable agents have flushed by - # definition, so this attribute is always populated on exactly the - # agents the memory valve targets. + # _db_flush_scan_prefix is a shallow copy of the flushed transcript (run_agent.py, stamped + # on every successful flush) — it shares every message dict, so leaving it pins the multi-MB + # content strings the eviction exists to free. Pressure-evictable agents have flushed by + # definition, so it is always populated on exactly the agents the valve targets. if hasattr(agent, "_db_flush_scan_prefix"): agent._db_flush_scan_prefix = None @@ -30359,8 +28163,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """Operator-configured agent-cache bounds, resolved once per process. Resolved lazily rather than in ``__init__`` so it also works for the - ``__new__``-constructed runners used by tests and by the slash-command - mixin. + ``__new__``-constructed runners used by tests and by the slash-command mixin. """ bounds = getattr(self, "_agent_cache_bounds_cache", None) if bounds is None: @@ -30370,12 +28173,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew bounds = resolve_agent_cache_bounds(_load_gateway_config()) except Exception as _e: logger.debug("Agent cache bounds config read failed: %s", _e) - # Resolve from an empty config rather than bare - # AgentCacheBounds(): the dataclass default has - # memory_high_mb=None (pressure pass OFF), but an *absent* - # config section means "auto" — a transient config read - # failure must not permanently disable the OOM valve this - # feature exists to provide. + # Resolve from an empty config rather than bare AgentCacheBounds(): the dataclass + # default has memory_high_mb=None (pressure pass OFF), but an *absent* config + # section means "auto" — a transient config read failure must not permanently + # disable the OOM valve this feature exists to provide. bounds = resolve_agent_cache_bounds({}) self._agent_cache_bounds_cache = bounds return bounds @@ -30393,22 +28194,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _sweep_agent_cache_under_pressure(self) -> int: """Shed cached transcripts once the gateway's own heap nears its budget. - The LRU cap counts entries and the idle sweep counts seconds; neither - knows that one cached agent pins a full ``_session_messages`` - transcript — tens of MB on a session with 100+ tool calls. A gateway - serving many chats therefore holds every warm transcript indefinitely: - agents that took a turn within the TTL are never idle-swept, and the - sweep additionally defers finalizable sessions until they expire. RSS - climbs until the cgroup throttles and SIGTERM can no longer flush - inside systemd's stop timeout (#80764). - - This is the missing valve. Above the configured anonymous-RSS budget - it evicts LRU agents through the same soft path the cap enforcer uses, - so the transcript is dropped and rebuilt from the persisted session on - the next turn. Three things are never touched: agents mid-turn (their - clients and sandboxes are in use), the most recently used sessions - (whose prompt cache is worth the most), and any session whose live - transcript has not finished reaching disk. + The LRU cap counts entries and the idle sweep counts seconds; neither knows one cached + agent pins a full ``_session_messages`` transcript (tens of MB on 100+ tool calls). + A gateway serving many chats therefore holds every warm transcript indefinitely: agents + that took a turn within the TTL are never idle-swept, and the sweep additionally defers + finalizable sessions until they expire. RSS climbs until the cgroup throttles and SIGTERM + can no longer flush inside the stop timeout. This is the missing valve: above the + anonymous-RSS budget it soft-evicts LRU agents (transcript rebuilt from persisted session + next turn). Never touched: agents mid-turn, the most recently used sessions (prompt cache + worth most), and sessions whose transcript has not finished reaching disk. Returns the number of entries evicted (0 when memory is fine). """ @@ -30508,17 +28302,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _release_pressure_batch(self, plan: List[tuple]) -> None: """Release a pressure-evicted batch, then return the heap to the OS. - Sequential on one daemon thread rather than a thread per agent: the - batch is already capped, and the point of the pass is to reclaim - memory, not to race N teardowns. The trailing ``malloc_trim`` is what - turns "Python dropped the transcript" into "RSS actually fell" — - without it glibc keeps the freed arenas and the cgroup never notices. - - The plan is drained (``pop`` + ``del``) rather than iterated so that - no local reference pins the evicted agents when ``gc.collect`` + - ``malloc_trim`` run — otherwise the trim frees almost nothing in this - pass, the next tick re-reads a still-high RSS, and the valve - over-evicts an extra batch of warm prompt caches every cycle. + Sequential on one daemon thread rather than a thread per agent: the batch is already + capped, and the point of the pass is to reclaim memory, not to race N teardowns. The + trailing ``malloc_trim`` is what turns "Python dropped it" into "RSS actually fell" — + glibc otherwise keeps the freed arenas. The plan is drained (``pop`` + ``del``), not + iterated, so no local reference pins evicted agents during ``gc.collect`` + trim; + otherwise the trim frees little and the valve over-evicts warm caches every cycle. """ while plan: key, agent = plan.pop(0) # FIFO — evict LRU-first order preserved @@ -30537,17 +28326,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _enforce_agent_cache_cap(self) -> None: """Evict oldest cached agents when cache exceeds the LRU cap. - Must be called with _agent_cache_lock held. Resource cleanup - (memory provider shutdown, tool resource close) is scheduled - on a daemon thread so the caller doesn't block on slow teardown - while holding the cache lock. - - Agents currently in _running_agents are SKIPPED — their clients, - terminal sandboxes, background processes, and child subagents - are all in active use by the running turn. Evicting them would - tear down those resources mid-turn and crash the request. If - every candidate in the LRU order is active, we simply leave the - cache over the cap; it will be re-checked on the next insert. + Must be called with _agent_cache_lock held. Resource cleanup (memory provider shutdown, + tool resource close) is scheduled on a daemon thread so the caller doesn't block on slow + teardown while holding the cache lock. Agents in _running_agents are SKIPPED — their + clients, sandboxes, bg processes and child subagents are in use; evicting would crash + the turn. If every LRU candidate is active the cache stays over cap until the next insert. """ _cache = getattr(self, "_agent_cache", None) if _cache is None: @@ -30566,15 +28349,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if a is not None and a is not _AGENT_PENDING_SENTINEL } - # Walk LRU → MRU and evict excess-LRU entries that aren't mid-turn. - # We only consider entries in the first (size - cap) LRU positions - # as eviction candidates. If one of those slots is held by an - # active agent, we SKIP it without compensating by evicting a - # newer entry — that would penalise a freshly-inserted session - # (which has no cache history to retain) while protecting an - # already-cached long-running one. The cache may therefore stay - # temporarily over cap; it will re-check on the next insert, - # after active turns have finished. + # Walk LRU → MRU and evict excess-LRU entries that aren't mid-turn. We only consider entries + # in the first (size - cap) LRU positions as eviction candidates. An active slot is SKIPPED + # without evicting a newer entry instead — that would penalise a fresh session (no cache + # history) to protect a long-running one. Cache may stay over cap until the next insert. cap = self._agent_cache_cap() excess = max(0, len(_cache) - cap) evict_plan: List[tuple] = [] # [(key, agent), ...] @@ -30604,12 +28382,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew key, len(_cache), ) if agent is not None: - # Commit end-of-session memory extraction, then soft-release, - # both on the daemon thread so the (possibly network-bound) - # provider call never blocks the held cache lock. The commit - # only fires for finalizable-not-yet-expired sessions whose - # agent would otherwise vanish before the expiry watcher can - # fire on_session_end (#11205, LRU-cap variant). + # Commit end-of-session memory extraction, then soft-release, both on the daemon + # thread so the (possibly network-bound) provider call never blocks the held cache + # lock. threading.Thread( target=self._commit_then_release_soft, args=(agent, key), @@ -30620,12 +28395,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _sweep_idle_cached_agents(self) -> int: """Evict cached agents whose AIAgent has been idle past the idle TTL. - Safe to call from the session expiry watcher without holding the - cache lock — acquires it internally. Returns the number of entries - evicted. Resource cleanup is scheduled on daemon threads. - - Agents currently in _running_agents are SKIPPED for the same reason - as _enforce_agent_cache_cap: tearing down an active turn's clients + Safe to call from the expiry watcher without the cache lock — acquires it internally; + cleanup is scheduled on daemon threads. + Returns the number of entries evicted. Agents currently in _running_agents are SKIPPED + for the same reason as _enforce_agent_cache_cap: tearing down an active turn's clients mid-flight would crash the request. """ _cache = getattr(self, "_agent_cache", None) @@ -30734,16 +28507,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew *, on_missing_cursor: str, ) -> "tuple[Any, Optional[Callable[[], None]]]": - """Build the shared ``StreamConsumerConfig`` and the optional - Telegram pause-typing closure used by both agent-run paths. + """Build the shared ``StreamConsumerConfig`` and the optional Telegram pause-typing closure + used by both agent-run paths. ``on_missing_cursor`` controls how platforms whose adapter sets - ``SUPPORTS_MESSAGE_EDITING = False`` are handled — both semantics - are preserved verbatim from the pre-refactor call sites: - - - ``"fallback"`` (proxy path): stream anyway with an empty cursor. - - ``"raise"`` (in-process agent path): raise ``RuntimeError`` so - the caller's ``except`` skips streaming entirely. + ``SUPPORTS_MESSAGE_EDITING = False`` are handled — both semantics are preserved verbatim + from the pre-refactor call sites: ``"fallback"`` (proxy path) streams with an empty + cursor; ``"raise"`` (in-process path) raises ``RuntimeError`` so the caller's ``except`` + skips streaming entirely. Returns ``(consumer_cfg, pause_typing_before_finalize)``. """ @@ -30756,18 +28527,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _chat_id=source.chat_id, ) -> None: _adapter.pause_typing_for_chat(_chat_id) - # Platforms that don't support editing sent messages - # (e.g. QQ, WeChat) should skip streaming entirely — - # without edit support, the consumer sends a partial - # first message that can never be updated, resulting in - # duplicate messages (partial + final). - # (The proxy path instead opts into a cursorless fallback - # via on_missing_cursor="fallback".) + # Platforms that don't support editing sent messages (e.g. QQ, WeChat) should skip streaming + # entirely — without edit support, the consumer sends a partial first message that can never + # be updated, resulting in duplicate messages (partial + final). _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True) - # Adapters that can't edit messages but provide a native-streaming - # transport (e.g. WeCom's msgtype: "stream" via send_stream_frame) - # get past the gate — the consumer's native branch delivers the full - # turn through that transport. + # Adapters that can't edit messages but provide a native-streaming transport (e.g. WeCom's + # msgtype: "stream" via send_stream_frame) get past the gate — the consumer's native branch + # delivers the full turn through that transport. _adapter_supports_native_stream = bool(getattr( adapter, "SUPPORTS_NATIVE_STREAMING", False, )) @@ -30785,10 +28551,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if source.platform == Platform.MATRIX: _effective_cursor = "" _buffer_only = True - # Fresh-final applies to Telegram only — other - # platforms either edit in place cheaply (Discord, - # Slack) or don't have the timestamp-on-edit / - # edit-timestamp-stays-stale problem. + # Fresh-final applies to Telegram only — other platforms either edit in place cheaply + # (Discord, Slack) or don't have the timestamp-on-edit / edit-timestamp-stays-stale problem. # (Ported from openclaw/openclaw#72038.) _fresh_final_secs = ( float(getattr(scfg, "fresh_final_after_seconds", 0.0) or 0.0) @@ -30817,17 +28581,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew run_generation: Optional[int] = None, event_message_id: Optional[str] = None, ) -> Dict[str, Any]: - """Forward the message to a remote Hermes API server instead of - running a local AIAgent. + """Forward the message to a remote Hermes API server instead of running a local AIAgent. - When ``GATEWAY_PROXY_URL`` (or ``gateway.proxy_url`` in config.yaml) - is set, the gateway becomes a thin relay: it handles platform I/O - (encryption, threading, media) and delegates all agent work to the - remote server via ``POST /v1/chat/completions`` with SSE streaming. - - This lets a Docker container handle Matrix E2EE while the actual - agent runs on the host with full access to local files, memory, - skills, and a unified session store. + This lets a Docker container handle Matrix E2EE while the actual agent runs on the host + with full access to local files, memory, skills, and a unified session store. """ try: from aiohttp import ClientSession as _AioClientSession, ClientTimeout @@ -30867,16 +28624,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return self._is_session_run_current(session_key, run_generation) # Build messages in OpenAI chat format -------------------------- - # - # The remote api_server can maintain session continuity via - # X-Hermes-Session-Id, so it loads its own history. We only - # need to send the current user message. If the remote has - # no history for this session yet, include what we have locally - # so the first exchange has context. - # - # We always include the current message. For history, send a - # compact version (text-only user/assistant turns) — the remote - # handles tool replay and system prompts. + # The remote api_server keeps session continuity via X-Hermes-Session-Id and loads its + # own history, so we only send the current message; if the remote has no history yet, + # include a compact text-only local history for context (remote handles tool replay). api_messages: List[Dict[str, str]] = [] if context_prompt: @@ -31112,12 +28862,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Dict[str, Any]: """Profile-scoping wrapper around the agent run. - When multiplexing is active, resolve the inbound source's profile and - run the whole turn inside ``_profile_runtime_scope`` so config/skills/ - memory resolve to that profile's home AND credentials resolve from that - profile's secret scope (never the process-global ``os.environ``). When - multiplexing is off this is a transparent pass-through — zero behavior - change for single-profile gateways. + When multiplexing is active, resolve the source's profile and run the whole turn inside + ``_profile_runtime_scope`` so config/skills/memory resolve to that profile's home AND + credentials come from its secret scope (never process-global ``os.environ``). + When multiplexing is off this is a transparent pass-through — zero behavior change for + single-profile gateways. """ if not getattr(getattr(self, "config", None), "multiplex_profiles", False): return await self._run_agent_inner( @@ -31156,12 +28905,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew the most specific matching route wins (guild < channel < thread). See :mod:`gateway.profile_routing` for matching rules. - Gated on ``gateway.multiplex_profiles``: routing stamps - ``source.profile``, which selects the session-key namespace and batch - keys — but the profile-scoped agent run only activates under - multiplexing. Without this gate, a configured route with multiplexing - off would namespace batch/session keys by profile while the agent - still runs in ``agent:main``, splitting the two out of agreement. + Gated on ``gateway.multiplex_profiles``: routing stamps ``source.profile`` (session-key + namespace + batch keys) but the profile-scoped run only activates under multiplexing; + without the gate, keys would be namespaced by profile while the agent still ran in + ``agent:main``. """ config = getattr(self, "config", None) if not getattr(config, "multiplex_profiles", False): @@ -31214,12 +28961,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _resolve_profile_home_for_source(self, source: SessionSource) -> "Path": """Resolve which profile's HERMES_HOME should serve this inbound source. - Resolution order: - 1. ``source.profile`` — set by /p// URL prefix, per-credential - adapter ownership, OR profile_routes matching at ``build_source`` time. - 2. ``_profile_name_for_source`` — re-run routing here as a defensive - fallback for sources that bypass ``build_source``. - 3. The active profile (the multiplexer's own home). + Resolution order: 1. ``source.profile`` — set by /p// URL prefix, per-credential + adapter ownership, OR profile_routes matching at ``build_source`` time. + 2. ``_profile_name_for_source`` — re-run routing as a defensive fallback for sources + that bypass ``build_source``. 3. The active profile (the multiplexer's own home). """ from gateway.profile_routing import ProfileRouteRejected from hermes_cli.profiles import ( @@ -31228,7 +28973,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew profile_exists, ) from hermes_constants import get_hermes_home - + # Track whether a profile was explicitly requested (vs. falling back to default) explicit_profile = None try: @@ -31241,7 +28986,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew explicit_profile = name # Routing explicitly set this profile if not name: name = get_active_profile_name() or "default" - + profile_dir = get_profile_dir(name) # Warn if an explicit profile doesn't exist on disk if explicit_profile and not profile_exists(name): @@ -31289,17 +29034,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None, ) -> Dict[str, Any]: - """ - Run the agent with the given message and context. - + """Run the agent with the given message and context. + Returns the full result dict from run_conversation, including: - "final_response": str (the text to send back) - "messages": list (full conversation including tool calls) - "api_calls": int - "completed": bool - - This is run in a thread pool to not block the event loop. - Supports interruption via new messages. """ # ---- Proxy mode: delegate to remote API server ---- if self._get_proxy_url(): @@ -31321,7 +29062,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if run_generation is None or not session_key: return True return self._is_session_run_current(session_key, run_generation) - + user_config = _load_gateway_config() platform_key = _platform_config_key(source.platform) @@ -31428,12 +29169,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # so each progress line would be sent as a separate message. from gateway.config import Platform tool_progress_enabled = progress_mode not in {"off", "log"} and source.platform != Platform.WEBHOOK - # Live working-state status for text-rendering typing indicators - # (Slack's assistant status line). Independent of tool_progress — - # Slack defaults tool_progress off (permanent lines spam channels) - # but the status line is ephemeral, so live status stays useful - # there. Rendering rides the existing _keep_typing refresh: the - # callback only stores a phrase on the adapter, costing zero extra + # Live working-state status for text-rendering typing indicators (Slack's assistant status + # line). Independent of tool_progress — Slack defaults tool_progress off (permanent lines + # spam channels) but the status line is ephemeral, so live status stays useful there. + # Rides the existing _keep_typing refresh (callback only stores a phrase) — zero extra # platform API calls. _live_status_mode = resolve_display_setting( user_config, platform_key, "live_status", "full" @@ -31459,21 +29198,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew source.platform != Platform.WEBHOOK and interim_assistant_messages_mode != "off" ) - # thinking_progress is independent — if enabled, we need the progress - # queue even when tool_progress is off (thinking relay uses same infra). - # Mattermost requires a per-platform opt-in: global scratch-text display - # is too easy to leak into busy public threads. + # thinking_progress is independent — if enabled, we need the progress queue even when + # tool_progress is off (thinking relay uses same infra). Mattermost requires a per-platform + # opt-in: global scratch-text display is too easy to leak into busy public threads. _thinking_mode = _display_surface_mode( "thinking_progress", default=False, require_platform_override_for={Platform.MATTERMOST}, ) _thinking_enabled = _thinking_mode != "off" - # Slack-native task cards (#29483): when the Slack adapter's opt-in - # is set, tool progress renders as native plan/task cards via - # chat.startStream — the progress queue is needed even though Slack - # keeps ordinary text tool_progress off by default (requiring both - # flags would silently leave the native feature inactive). + # Slack-native task cards: when the Slack adapter's opt-in is set, tool progress renders as + # native plan/task cards via chat.startStream — the progress queue is needed even though + # Slack keeps ordinary text tool_progress off by default (requiring both flags would + # silently leave the native feature inactive). _progress_adapter_for_native = self._adapter_for_source(source) _native_slack_task_cards = False if ( @@ -31491,7 +29228,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew tool_progress_enabled or _thinking_enabled or _native_slack_task_cards ) - # Queue for progress messages (thread-safe) progress_queue = queue.Queue() if needs_progress_queue else None last_tool = [None] # Mutable container for tracking in closure @@ -31503,12 +29239,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew last_was_terminal_block = [False] # ── Discord voice "verbal ack before tool calls" ──────────────── - # When the bot is in a voice channel with the continuous mixer - # installed (discord.voice_fx.enabled), speak a short phrase ("let me - # look into that") over the ambient idle bed on the FIRST tool call of - # the turn. Fires from tool_start_callback (independent of the - # tool-progress text gate), at most once per turn. No-op on every - # other platform / when not in a voice channel. + # When the bot is in a voice channel with the continuous mixer installed + # (discord.voice_fx.enabled), speak a short phrase ("let me look into that") over the + # ambient idle bed on the FIRST tool call of the turn. Fires from tool_start_callback + # (independent of the tool-progress text gate), at most once per turn; no-op elsewhere. _voice_ack_fired = [False] _voice_ack_guild: List[Optional[int]] = [None] if source.platform == Platform.DISCORD: @@ -31526,12 +29260,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # voice_ack_callback extracted to TurnRunner.voice_ack_callback # (published onto turn_ctx after the runner is constructed below). - # Auto-cleanup of temporary progress bubbles (Telegram + any adapter - # that implements ``delete_message``). When enabled via - # ``display.platforms..cleanup_progress: true``, message IDs - # from the tool-progress / "⏳ Working — N min" / status-callback bubbles - # are collected here and deleted after the final response lands. - # Failed runs skip cleanup so the bubbles remain as breadcrumbs. + # Auto-cleanup of temporary progress bubbles (Telegram + any adapter that implements + # ``delete_message``). Failed runs skip cleanup so the bubbles remain as breadcrumbs. _cleanup_progress = bool( resolve_display_setting(user_config, platform_key, "cleanup_progress") ) @@ -31608,7 +29338,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew turn_ctx.native_tool_complete_callback = ( turn_runner.native_tool_complete_callback ) - + # Background task to send progress messages # Accumulates tool lines into a single message that gets edited. # @@ -31665,15 +29395,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew source.platform, source.thread_id, event_message_id, reply_in_thread=_progress_reply_in_thread, ) - # Relay Discord auto-thread lane: a channel-initiating message has no - # thread_id at ingest (the thread is born on the connector's FIRST - # send). The connector stamps prospective_thread_id (the anchor message - # id, == the id of the thread it will create) and auto-threads any - # outbound carrying that anchor as reply_to. Without it, the progress / - # tool-status bubble is sent flat (no thread, no anchor) and lands in - # the PARENT channel while the final reply threads — the search-status - # updates leaked outside the thread (staging repro 2026-08-02). Carry - # the anchor on the progress send so it routes into the SAME auto-thread. + # Relay Discord auto-thread lane: a channel-initiating message has no thread_id at ingest + # (the thread is born on the connector's FIRST send). The connector stamps + # prospective_thread_id (anchor message id == the thread it will create) and auto-threads + # outbound carrying it as reply_to; without it progress bubbles land flat in the PARENT + # channel. Carry the anchor on the progress send so it routes into the SAME auto-thread. _relay_prospective_thread_id = ( str(getattr(source, "prospective_thread_id", None)) if source.platform == Platform.DISCORD @@ -31729,10 +29455,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew async def write_tool_log(): """Drain log_queue and append tool-call lines to tool_calls.log. - Only active when ``display.tool_progress`` is ``log``. Uses a - RotatingFileHandler (5MB × 3 backups) so the audit log can't grow - unbounded, and the shared RedactingFormatter so secrets never land - on disk. + Only active when ``display.tool_progress`` is ``log``. Uses a RotatingFileHandler + (5MB × 3 backups) so the audit log can't grow unbounded, and the shared + RedactingFormatter so secrets never land on disk. """ if log_queue is None: return @@ -31787,23 +29512,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew turn_ctx._progress_metadata = _progress_metadata turn_ctx._progress_reply_to = _progress_reply_to send_progress_messages = turn_runner.send_progress_messages - + # We need to share the agent instance for interrupt support agent_holder = [None] # Mutable container for the agent instance turn_ctx.agent_holder = agent_holder result_holder = [None] # Mutable container for the result tools_holder = [None] # Mutable container for the tool definitions stream_consumer_holder = [None] # Mutable container for stream consumer - # #60671 — streaming PCM audio consumer. Created on the gateway - # event-loop thread (NOT inside run_sync's executor worker) so the - # outer finalisation / interrupt paths can reference it without a - # cross-scope NameError. + # streaming PCM audio consumer. Created on the gateway event-loop thread (NOT inside + # run_sync's executor worker) so the outer finalisation / interrupt paths can reference it + # without a cross-scope NameError. streaming_tts_consumer_holder: list = [None] turn_ctx.result_holder = result_holder turn_ctx.tools_holder = tools_holder turn_ctx.stream_consumer_holder = stream_consumer_holder turn_ctx.streaming_tts_consumer_holder = streaming_tts_consumer_holder - + # Bridge sync step_callback → async hooks.emit for agent:step events _loop_for_step = asyncio.get_running_loop() _hooks_ref = self.hooks @@ -31823,10 +29547,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _status_adapter = self._adapter_for_source(source) _status_chat_id = source.chat_id if source.platform == Platform.FEISHU and source.thread_id and event_message_id: - # Feishu topics only keep messages inside the topic when they are - # sent via the reply API with reply_in_thread=true. Status/interim, - # approval, and stream-consumer paths usually only receive metadata, - # so carry the triggering message id as a Feishu-specific fallback. + # Feishu topics only keep messages inside the topic when they are sent via the reply API + # with reply_in_thread=true. Status/approval/stream paths usually only get metadata, so + # carry the triggering message id as a Feishu-specific fallback. _status_thread_metadata: Optional[Dict[str, Any]] = { "thread_id": _progress_thread_id, "reply_to_message_id": event_message_id, @@ -31860,13 +29583,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew turn_ctx._status_callback_sync = turn_runner._status_callback_sync # ---- Streaming TTS consumer setup (#60671) ---- - # Created on the gateway event-loop thread (here, in _run_agent_inner), - # NOT inside run_sync's executor worker. This avoids a cross-scope - # NameError: the outer interrupt / finalisation paths reference the - # consumer via ``streaming_tts_consumer_holder[0]``. - # - # Gates: voice input, auto-TTS enabled for this chat, adapter - # supports streaming, and a usable streaming TTS provider configured. + # Created on the gateway event-loop thread (here, in _run_agent_inner), NOT inside + # run_sync's executor worker. This avoids a cross-scope NameError: the outer interrupt / + # finalisation paths reference the consumer via ``streaming_tts_consumer_holder[0]``. _stts_adapter = self._adapter_for_source(source) _is_voice_input = ( message_type is not None @@ -31901,13 +29620,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # executor call below is unchanged). Its closed-over locals travel # on turn_ctx; `nonlocal message` rebinds became ctx.message writes. run_sync = turn_runner.run_sync - - # Start progress message sender if enabled. Gate on needs_progress_queue - # (tool_progress OR thinking_progress), not tool_progress alone: the - # sender drains BOTH tool-progress lines and _thinking scratch bubbles. - # With the old tool_progress-only gate, a thinking_progress:true / - # tool_progress:off user had the callback queue _thinking messages that - # no task ever drained — so they silently never appeared. + + # Start progress message sender if enabled. Gate on needs_progress_queue (tool_progress OR + # thinking_progress), not tool_progress alone: the sender drains BOTH tool-progress lines + # and _thinking scratch bubbles — with a tool_progress-only gate, thinking_progress:true / + # tool_progress:off queued _thinking messages nothing ever drained. progress_task = None if needs_progress_queue: progress_task = asyncio.create_task(send_progress_messages()) @@ -31930,7 +29647,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await asyncio.sleep(0.05) stream_task = asyncio.create_task(_start_stream_consumer()) - + # Track this agent as running for this session (for interrupt support) # We do this in a callback after the agent is created async def track_agent(): @@ -31939,10 +29656,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await asyncio.sleep(0.05) if not session_key: return - # Only promote the sentinel to the real agent if this run is still - # current. If /stop or /new bumped the generation while we were - # spinning up, leave the newer run's slot alone — we'll be - # discarded by the stale-result check in _handle_message_with_agent. + # Only promote the sentinel to the real agent if this run is still current. If /stop or + # /new bumped the generation while we were spinning up, leave the newer run's slot alone + # — we'll be discarded by the stale-result check in _handle_message_with_agent. if run_generation is not None and not self._is_session_run_current( session_key, run_generation ): @@ -31955,15 +29671,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._session_state(session_key).turn.agent = agent_holder[0] if self._draining: self._update_runtime_status("draining") - + tracking_task = asyncio.create_task(track_agent()) - - # Monitor for interrupts from the adapter (new messages arriving). - # This is the PRIMARY interrupt path for regular text messages — - # Level 1 (base.py) catches them before _handle_message() is reached, - # so the Level 2 running_agent.interrupt() path never fires. - # The inactivity poll loop below has a BACKUP check in case this - # task dies (no error handling = silent death = lost interrupts). + + # Monitor for interrupts from the adapter (new messages arriving). This is the PRIMARY + # interrupt path for regular text messages — Level 1 (base.py) catches them before + # _handle_message() is reached, so the Level 2 running_agent.interrupt() path never fires. + # The inactivity poll loop has a BACKUP check in case this task dies silently. _interrupt_detected = asyncio.Event() # shared with backup check async def monitor_for_interrupt(): @@ -31978,31 +29692,23 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _adapter = self._adapter_for_source(source) if not _adapter: continue - # Check if adapter has a pending interrupt for this session. - # Must use session_key (build_session_key output) — NOT - # source.chat_id — because the adapter stores interrupt events - # under the full session key. + # Check if adapter has a pending interrupt for this session. Must use + # session_key (build_session_key output) — NOT source.chat_id — because the + # adapter stores interrupt events under the full session key. if hasattr(_adapter, 'has_pending_interrupt') and _adapter.has_pending_interrupt(session_key): agent = agent_holder[0] if agent: - # Peek at the pending message text WITHOUT consuming it. - # The message must remain in _pending_messages so the - # post-run dequeue at _dequeue_pending_event() can - # retrieve the full MessageEvent (with media metadata). - # If we pop here, a race exists: the agent may finish - # before checking _interrupt_requested, and the message - # is lost — neither the interrupt path nor the dequeue - # path finds it. + # Peek at the pending message text WITHOUT consuming it: the message must + # stay in _pending_messages for the post-run _dequeue_pending_event() + # (full MessageEvent with media). Popping here races: the agent may finish + # before checking _interrupt_requested and the message is lost to both paths. _peek_event = _adapter._pending_messages.get(session_key) pending_text = None if _peek_event is not None: pending_text = _peek_event.text or "" - # Transcribe audio media BEFORE signaling the - # agent, so voice messages interrupt with the - # real transcript instead of an empty string - # (or file-path placeholder). Matches the UX - # of fresh voice messages including the - # optional 🎙️ echo back to the user. + # Transcribe audio media BEFORE signaling the agent, so voice + # messages interrupt with the real transcript instead of an empty + # string (or file-path placeholder). _media_urls = getattr(_peek_event, "media_urls", None) or [] if self._pending_event_audio_paths(_peek_event): pending_text, _ = await self._transcribe_and_echo_pending_voice( @@ -32027,14 +29733,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew raise except Exception as _mon_err: logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err) - + interrupt_monitor = asyncio.create_task(monitor_for_interrupt()) - # Periodic "still working" notifications for long-running tasks. - # Fires every N seconds so the user knows the agent hasn't died. - # Config: agent.gateway_notify_interval in config.yaml, or - # HERMES_AGENT_NOTIFY_INTERVAL env var. Default 180s (3 min). - # 0 = disable notifications. + # Periodic "still working" notifications for long-running tasks. Fires every N seconds so + # the user knows the agent hasn't died. Config: agent.gateway_notify_interval in + # config.yaml, or HERMES_AGENT_NOTIFY_INTERVAL env var. Default 180s (3 min). _NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) _NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None _long_running_mode = _display_surface_mode( @@ -32052,20 +29756,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _notify_adapter = self._adapter_for_source(source) if not _notify_adapter: return - # Track the heartbeat message id so we can edit-in-place on - # platforms that support it (Telegram, Discord, Slack, etc.) - # instead of spamming a new "Still working" bubble every - # interval. Falls back to send-new when edit fails or isn't - # supported by the adapter. + # Track the heartbeat message id so we can edit-in-place on platforms that support it + # (Telegram, Discord, Slack, etc.) instead of spamming a new "Still working" bubble + # every interval. _heartbeat_msg_id: Optional[str] = None while True: await asyncio.sleep(_NOTIFY_INTERVAL) - # Stop heartbeating once this run no longer owns the session - # slot or the executor has finished — otherwise a stale - # "running: delegate_task" bubble can outlive the run that - # spawned it (#12029). _executor_task is a closure var bound - # just after this task is scheduled; tolerate the brief window - # before then (the first wake is _NOTIFY_INTERVAL away anyway). + # Stop heartbeating once this run no longer owns the session slot or the executor + # has finished — otherwise a stale "running: delegate_task" bubble can outlive the + # run that spawned it. _executor_task is a closure var bound just after this task + # is scheduled; tolerate the brief window before then. try: _exec_ref = _executor_task except NameError: @@ -32075,10 +29775,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ): break _elapsed_mins = int((time.time() - _notify_start) // 60) - # Include agent activity context if available. Default - # heartbeat is terse: elapsed + current tool. Verbose - # iteration counter is gated on busy_ack_detail so users - # who want it can opt in per platform. + # Include agent activity context if available. Default heartbeat is terse: elapsed + + # current tool. Verbose iteration counter is gated on busy_ack_detail so users who + # want it can opt in per platform. _agent_ref = agent_holder[0] _status_detail = "" _want_iteration_detail = bool( @@ -32148,16 +29847,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if consumer is None: return False if getattr(consumer, "final_response_sent", False): - # A successful finalize call is not proof the *content* was - # final: the edit may have carried only the last preview - # snapshot while the tail generated between that snapshot and - # stream completion never reached any API call (#71643). - # Reconcile the recorded turn-final payload against the - # completed response; only a demonstrable mismatch (False) - # overrides the flag — including payload-less multi-message - # split delivery (#78541). None (no record on a non-split - # legacy path) keeps the legacy trust so ambiguous-timeout - # dedup is not regressed. + # A successful finalize call is not proof the *content* was final: the edit may have + # carried only the last preview snapshot while the tail generated between that + # snapshot and stream completion never reached any API call. Reconcile the recorded + # turn-final payload: only a demonstrable mismatch (False, incl. payload-less split + # delivery) overrides the flag; None keeps legacy trust so timeout dedup isn't regressed. matcher = getattr(consumer, "delivered_final_matches", None) if callable(matcher): try: @@ -32176,27 +29870,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False try: - # Run in thread pool to not block. Use an *inactivity*-based - # timeout instead of a wall-clock limit: the agent can run for - # hours if it's actively calling tools / receiving stream tokens, - # but a hung API call or stuck tool with no activity for the - # configured duration is caught and killed. (#4815) - # - # Config: agent.gateway_timeout in config.yaml, or - # HERMES_AGENT_TIMEOUT env var (env var takes precedence). - # Default 1800s (30 min inactivity). 0 = unlimited. + # Run in thread pool to not block. *Inactivity*-based timeout, not wall-clock: the agent + # may run for hours while actively calling tools / streaming, but a hung API call or + # stuck tool with no activity is killed. Config agent.gateway_timeout or + # HERMES_AGENT_TIMEOUT (env wins); default 1800s; 0 = unlimited. _agent_timeout_raw = _float_env("HERMES_AGENT_TIMEOUT", 1800) _agent_timeout = _agent_timeout_raw if _agent_timeout_raw > 0 else None _agent_warning_raw = _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900) _agent_warning = _agent_warning_raw if _agent_warning_raw > 0 else None _warning_fired = False - # A background=true process intentionally survives a successful - # turn, so capture existing IDs and reap only children created by - # THIS turn if it times out. The daemon watchdog is independent of - # asyncio: cgroup memory reclaim may starve the event loop that runs - # the normal timeout poll, but it need not also postpone cleanup - # until the loop recovers (#76115). + # A background=true process intentionally survives a successful turn, so capture + # existing IDs and reap only children created by THIS turn if it times out. The daemon + # watchdog is independent of asyncio: cgroup memory reclaim can starve the loop that + # runs the normal timeout poll, and cleanup must not wait for the loop to recover. from tools.process_registry import process_registry _turn_task_id = session_id or "" @@ -32206,11 +29893,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _turn_worker_done = threading.Event() _turn_timeout_fired = threading.Event() _turn_cleanup_lock = threading.Lock() - # task_id above is session-scoped, not turn-scoped (#76115 - # review): gate the eventual reap on this exact claim still - # being current, so a replacement turn that starts on the same - # session before the watchdog fires doesn't get its own fresh - # process killed by this turn's stale baseline. + # task_id above is session-scoped, not turn-scoped: gate the eventual reap on this exact + # claim still being current, so a replacement turn that starts on the same session + # before the watchdog fires doesn't get its own fresh process killed by this turn's + # stale baseline. _turn_run_generation = run_generation _turn_is_current = ( (lambda: self._is_session_run_current(session_key, _turn_run_generation)) @@ -32223,17 +29909,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return run_sync() finally: _turn_worker_done.set() - # `.turn.agent` on the session state is only reset to - # _AGENT_PENDING_SENTINEL when the *next* turn is - # claimed (see _session_state(...).turn.agent = ... at - # claim time), so a stale reference to this exact agent - # instance stays reachable from - # _interrupt_and_clear_session() until then. Clearing - # the ownership markers here — the instant this turn's - # own worker finishes — closes that window: an - # explicit /stop landing on the already-finished turn - # no longer reaps background work the turn deliberately - # left running (#76115). + # `.turn.agent` on the session state is only reset to _AGENT_PENDING_SENTINEL + # when the *next* turn is claimed (see _session_state(...).turn.agent = ... at + # claim time), so a stale reference to this exact agent instance stays reachable + # from _interrupt_and_clear_session() until then. Clearing ownership markers the + # instant our worker finishes closes that window: a /stop landing on the finished + # turn no longer reaps background work it deliberately left running. _finished_agent = agent_holder[0] if agent_holder else None if _finished_agent is not None: _finished_agent._gateway_turn_process_task_id = "" @@ -32320,13 +30001,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew {_executor_task}, timeout=_POLL_INTERVAL ) if done: - # Prefer the real result when the worker finished, - # even if the watchdog fired in the same window: the - # completed run already persisted its reply to session - # history, so surfacing the "agent inactive" diagnostic - # here would contradict the stored transcript. This - # mirrors _abandon_timed_out_gateway_turn's own - # worker_done-wins tiebreak (under cleanup_lock). + # Prefer the real result when the worker finished, even if the watchdog + # fired in the same window: the completed run already persisted its reply to + # session history, so surfacing the "agent inactive" diagnostic here would + # contradict the stored transcript. response = _executor_task.result() break if _turn_timeout_fired.is_set(): @@ -32475,27 +30153,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "failed": True, } - # Track fallback model state: if the agent switched to a - # fallback model during this run, persist it so /model shows - # the actually-active model instead of the config default. - # Skip eviction when the run failed — evicting a failed agent - # forces MCP reinit on the next message for no benefit (the - # same error will recur). This was the root cause of #7130: - # a bad model ID triggered fallback → eviction → recreation → - # MCP reinit → same 400 → loop, burning 91% CPU for hours. + # Track fallback model state: if the agent switched to a fallback model during this run, + # persist it so /model shows the actually-active model instead of the config default. + # Skip eviction when the run failed — evicting a failed agent forces MCP reinit on the + # next message for no benefit (same error recurs): bad model → fallback → evict → + # recreate → reinit → same 400 loop burned CPU for hours. _agent = agent_holder[0] _result_for_fb = result_holder[0] _run_failed = _result_for_fb.get("failed") if _result_for_fb else False if _agent is not None and hasattr(_agent, 'model') and not _run_failed: _cfg_model = _resolve_gateway_model() - # Normalize _cfg_model the same way AIAgent.__init__ does, so a - # vendor-prefixed config value (e.g. "deepseek/deepseek-v4-pro") - # matches the agent's stripped model ("deepseek-v4-pro") on - # native providers. Without this, _agent.model != _cfg_model is - # always true for vendor-prefixed config and the cached agent is - # evicted on every successful turn — destroying prompt caching. - # Aggregators (openrouter, etc.) keep the vendor/model slug, so - # they're left untouched. + # Normalize _cfg_model the same way AIAgent.__init__ does, so a vendor-prefixed + # config value (e.g. "deepseek/deepseek-v4-pro") matches the agent's stripped model on + # native providers — otherwise _agent.model != _cfg_model is always true and the + # cached agent is evicted every turn, destroying prompt caching. Aggregators + # (openrouter, etc.) keep the vendor/model slug, so they're left untouched. try: from hermes_cli.model_normalize import ( _AGGREGATOR_PROVIDERS, @@ -32515,15 +30187,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew result = result_holder[0] adapter = self._adapter_for_source(source) - # Finalize the streaming-TTS consumer (#60671). - # - # finish() is called from the outer event-loop thread (not the - # executor worker) so early returns from run_sync are also - # finalised. wait_complete() drains queued audio; on timeout - # the consumer is aborted unconditionally — if audio was - # audible, suppression is preserved so the gateway does not - # replay from the beginning; if no audio was audible, the - # whole-file fallback path is permitted. + # Finalize the streaming-TTS consumer. finish() runs on the outer event-loop thread so + # early returns from run_sync are also finalised. wait_complete() drains queued audio; + # on timeout abort unconditionally — if audio was audible keep suppression (no replay + # from the start); if not, the whole-file fallback is permitted. _stts = streaming_tts_consumer_holder[0] if _stts is not None: _stts.finish() @@ -32541,19 +30208,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None) if callable(_mark_turn): _mark_turn(session_key, run_generation) - + # Get pending message from adapter. # Use session_key (not source.chat_id) to match adapter's storage keys. pending_event = None pending = None if result and adapter and session_key: pending_event = _dequeue_pending_event(adapter, session_key) - # /queue overflow: after consuming the adapter's "next-up" - # slot, promote the next queued event into it so the - # recursive run's drain will see it. This keeps the slot - # occupied for the full FIFO chain, which (a) preserves - # order, and (b) causes any mid-chain /queue to correctly - # route to overflow rather than jumping the queue. + # /queue overflow: after consuming the adapter's "next-up" slot, promote the next + # queued event into it so the recursive run's drain will see it. Keeping the slot + # occupied for the whole FIFO chain preserves order and makes a mid-chain /queue + # route to overflow instead of jumping the queue. pending_event = self._promote_queued_event(session_key, adapter, pending_event) if result.get("interrupted") and not pending_event and result.get("interrupt_message"): interrupt_message = result.get("interrupt_message") @@ -32566,12 +30231,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew else: pending = interrupt_message elif pending_event: - # Transcribe audio media on the dequeued event BEFORE it is - # handed back as the next user turn, so queued/interrupting - # voice messages drain with the real transcript instead of - # a file-path placeholder. When configured, echo each - # transcript back to the user in the same 🎙️ format as - # fresh voice messages. + # Transcribe audio media on the dequeued event BEFORE it is handed back as the + # next user turn, so queued/interrupting voice messages drain with the real + # transcript instead of a file-path placeholder. _pending_text = pending_event.text or "" _media_urls = getattr(pending_event, "media_urls", None) or [] if self._pending_event_audio_paths(pending_event): @@ -32590,21 +30252,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if pending: logger.debug("Processing queued message after agent completion: '%s...'", pending[:40]) - # Leftover /steer: if a steer arrived after the last tool batch - # (e.g. during the final API call), the agent couldn't inject it - # and returned it in result["pending_steer"]. Deliver it as the - # next user turn so it isn't silently dropped. + # Leftover /steer: if a steer arrived after the last tool batch (e.g. during the final + # API call), the agent couldn't inject it and returned it in result["pending_steer"]. + # Deliver it as the next user turn so it isn't silently dropped. if result and not pending and not pending_event: _leftover_steer = result.get("pending_steer") if _leftover_steer: pending = _leftover_steer logger.debug("Delivering leftover /steer as next turn: '%s...'", pending[:40]) - # Safety net: if the pending text is a slash command (e.g. "/stop", - # "/new"), discard it — commands should never be passed to the agent - # as user input. The primary fix is in base.py (commands bypass the - # active-session guard), but this catches edge cases where command - # text leaks through the interrupt_message fallback. + # Safety net: if the pending text is a slash command (e.g. "/stop", "/new"), discard it + # — commands should never be passed to the agent as user input. if pending and pending.strip().startswith("/"): _pending_parts = pending.strip().split(None, 1) _pending_cmd_word = _pending_parts[0][1:].lower() if _pending_parts else "" @@ -32672,11 +30330,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew pass except Exception as e: logger.debug("Stream consumer wait before queued message failed: %s", e) - # The queued branch needs raw ``result`` for interruption, - # history, and recursion state, but delivery must use the - # finalized task result. The latter contains empty/failure - # normalization and any final response processing applied by - # _run_agent_task; sending the raw copy bypasses those steps. + # The queued branch needs raw ``result`` for interruption, history, and + # recursion state, but delivery must use the finalized task result — it carries + # empty/failure normalization and final-response processing from _run_agent_task. _delivery_result = response if isinstance(response, dict) else (result or {}) _previewed = bool(_delivery_result.get("response_previewed")) first_response = _delivery_result.get("final_response", "") @@ -32724,9 +30380,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) except Exception as e: logger.warning("Failed to send first response before queued message: %s", e) - # Release deferred bg-review notifications now that the - # first response has been delivered. Pop from the - # adapter's callback dict (prevents double-fire in + # Release deferred bg-review notifications now that the first response has been + # delivered. Pop from the adapter's callback dict (prevents double-fire in # base.py's finally block) and call it. if getattr(type(adapter), "pop_post_delivery_callback", None) is not None: _bg_cb = adapter.pop_post_delivery_callback( @@ -32771,11 +30426,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_key or "?", ) return result - # Resolve the follow-up's session key BEFORE preparing the - # inbound text: _prepare_inbound_message_text buffers native - # image paths under the key it is given, and the recursive - # _run_agent below consumes them under next_session_key. - # The write and consume keys must match or the images drop. + # Resolve the follow-up's session key BEFORE preparing the inbound text: + # _prepare_inbound_message_text buffers native image paths under the key it is + # given, and the recursive _run_agent below consumes them under + # next_session_key. Write and consume keys must match or the images drop. try: next_session_key = self._session_key_for_source(next_source) except Exception: @@ -32822,20 +30476,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: pass - # Re-baseline the cached agent's message_count snapshot before - # recursing into the in-band queued (/queue) follow-up turn. - # The first turn has completed and flushed its own user + - # assistant rows to the SessionDB, so the cross-process - # coherence guard (#45966) — which this recursive _run_agent - # call re-enters — would otherwise see the grown on-disk count - # against the stale build-time snapshot and rebuild the agent - # on THIS process's OWN writes, destroying the prompt-cache - # prefix #46237 was merged to preserve. The existing - # re-baseline in _handle_message_with_agent only runs after the - # whole _run_agent chain unwinds — too late for the in-band - # follow-up. Use the same (session_key, session_id) the - # recursive call runs under so the snapshot matches exactly - # what the follow-up's guard will consult. Fail-safe in helper. + # Re-baseline the cached agent's message_count snapshot before recursing into the + # in-band queued (/queue) follow-up turn: the first turn flushed its own rows, so the + # coherence guard the recursive call re-enters would rebuild on OUR OWN writes and + # destroy the prompt-cache prefix. The re-baseline in _handle_message_with_agent runs + # only after the whole chain unwinds — too late. Fail-safe in helper. await self._refresh_agent_cache_message_count(session_key, session_id) followup_result = await self._run_agent( @@ -32863,12 +30508,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Wait for stream consumer to finish its final edit if stream_task: - # If the agent never created a stream consumer (e.g. non- - # streaming code path, or a test stub returning synchronously) - # there is nothing to flush — cancel immediately instead of - # waiting out the 5s timeout on a task that's just polling for - # a consumer that will never arrive. This was a 5-second - # cost per non-streaming test run. + # If the agent never created a stream consumer (e.g. non- streaming code path, or a + # test stub returning synchronously) there is nothing to flush — cancel immediately + # instead of waiting out the 5s timeout on a task that's just polling for a consumer + # that will never arrive. _has_stream_consumer = ( stream_consumer_holder and stream_consumer_holder[0] is not None @@ -32888,7 +30531,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await stream_task except asyncio.CancelledError: pass - + # Unconditional abort + bounded wait for the streaming-TTS # consumer (#60671 hardening). Covers cancellation / exception # paths where the normal finalisation block was skipped. @@ -32903,17 +30546,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Clean up tracking tracking_task.cancel() if session_key: - # Only release the slot if this run's generation still owns - # it. A /stop or /new that bumped the generation while we - # were unwinding has already installed its own state; this - # guard prevents an old run from clobbering it on the way - # out. + # Only release the slot if this run's generation still owns it. A /stop or /new that + # bumped the generation while we were unwinding has already installed its own state; + # this guard prevents an old run from clobbering it on the way out. self._release_running_agent_state( session_key, run_generation=run_generation ) if self._draining: self._update_runtime_status("draining") - + # Wait for cancelled tasks for task in [progress_task, log_task, interrupt_monitor, tracking_task, _notify_task]: if task: @@ -32922,53 +30563,35 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except asyncio.CancelledError: pass except Exception: - # A background task that died of a non-cancellation - # error (transport drop in a progress/card publish) - # must not abort the cleanup path — everything after - # this loop (final-delivery bookkeeping) still runs - # (review B7). + # A background task that died of a non-cancellation error (transport drop in + # a progress/card publish) must not abort the cleanup path — everything + # after this loop (final-delivery bookkeeping) still runs (review B7). logger.debug( "background turn task failed during cleanup", exc_info=True, ) - # If streaming already delivered the response, mark it so the - # caller's send() is skipped (avoiding duplicate messages). - # BUT: never suppress delivery when the agent failed — the error - # message is new content the user hasn't seen, and it must reach - # them even if streaming had sent earlier partial output. - # - # Also never suppress when the final response is "(empty)" — this - # means the model failed to produce content after tool calls (common - # with mimo-v2-pro, GLM-5, etc.). The stream consumer may have - # sent intermediate text ("Let me search for that…") alongside the - # tool call, setting already_sent=True, but that text is NOT the - # final answer. Suppressing delivery here leaves the user staring - # at silence. (#10xxx — "agent stops after web search") + # If streaming already delivered the response, mark it so the caller's send() is skipped + # (avoiding duplicate messages). BUT never suppress when the agent failed — the error is + # new content the user hasn't seen. Also never suppress on "(empty)": the model produced + # nothing after tool calls, and interim text ("Let me search…") set already_sent but is + # NOT the final answer; suppressing would leave the user staring at silence. _sc = stream_consumer_holder[0] if isinstance(response, dict) and not response.get("failed"): _final = response.get("final_response") or "" _is_empty_sentinel = not _final or _final == "(empty)" - # response_previewed means the interim_assistant_callback already - # saw the final text, but only suppress the normal send if that - # exact final text was delivered. Unrelated commentary/progress - # must not be mistaken for the final response (#14238). + # response_previewed means the interim_assistant_callback already saw the final text, + # but only suppress the normal send if that exact final text was delivered. Unrelated + # commentary/progress must not be mistaken for the final response. _previewed = bool(response.get("response_previewed")) _content_delivered = bool( _sc and getattr(_sc, "final_content_delivered", False) ) - # #71643: a *successful* finalize edit can still carry only the - # last preview snapshot — deltas generated between that edit and - # stream completion never reach any API call, and both suppression - # flags are set from the call's success rather than its content. - # Reconcile the consumer's recorded turn-final payload against the - # completed response: on a demonstrable mismatch (False) neither - # final_response_sent nor final_content_delivered may suppress the - # normal final send. False also covers payload-less multi-message - # split delivery (#78541). None (no record on a non-split legacy - # path) keeps legacy trust; the failed-finalize family - # (#51828 / #33793) is unaffected because those paths leave the - # flags False or record the complete fallback payload. + # A *successful* finalize edit can still carry only the last preview snapshot — deltas + # generated between that edit and stream completion never reach any API call, and both + # suppression flags are set from the call's success rather than its content. Reconcile + # the recorded turn-final payload: on mismatch (False, incl. payload-less split delivery) + # neither flag may suppress the final send; None (no record) keeps legacy trust. _stale_finalized = False if _content_delivered and not _is_empty_sentinel: _matcher = getattr(_sc, "delivered_final_matches", None) @@ -33004,18 +30627,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) response["already_sent"] = True elif not _is_empty_sentinel and not _transformed and _stale_finalized and _sc is not None: - # Stale finalize (#71643): the streamed message holds only the - # last preview snapshot. Prefer editing it up to the complete - # response (same shape as the transformed branch below) so the - # user gets one corrected message; on edit failure fall through - # with already_sent unset so the normal final send delivers the - # complete text. - # - # Not valid for a multi-message split delivery: there - # ``message_id`` is only the LAST chunk, so editing it with the - # complete response would repeat every sealed head chunk's text - # inside the tail message. Fall through to the normal final send - # instead (#78541). + # Stale finalize: the streamed message holds only the last preview snapshot. Prefer + # editing it up to the complete response (one corrected message); on edit failure + # fall through with already_sent unset so the normal send delivers. Not valid for + # multi-message split delivery: message_id is only the LAST chunk, so editing it + # would repeat every sealed head chunk — fall through to the normal send instead. _sc_msg_id = _sc.message_id _sc_adapter = getattr(_sc, "adapter", None) if getattr(_sc, "_turn_split_delivery", False): @@ -33076,14 +30692,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_key or "?", _edit_err, ) elif _sc is not None and not _is_empty_sentinel: - # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this - # turn but suppression did NOT fire, so the gateway's normal - # final-send is about to run. On WeCom this is the exact window - # that produced "回复了两条" — a final-frame ack still in flight - # (final_content_delivered not yet set) while this send races - # ahead. Log the decision inputs so a recurrence can be pinned to - # "signal never set" vs "ack-pending race". - # See docs/rca-wecom-stream-final-ack-timeout-duplicate.md. + # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this turn but suppression + # did NOT fire, so the gateway's normal final-send is about to run. Log the decision + # inputs so a recurrence can be pinned to "signal never set" vs "ack-pending race". logger.warning( "Normal final-send NOT suppressed despite active stream " "consumer for session %s: streamed=%s previewed=%s " @@ -33097,11 +30708,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew len(_final), ) - # Schedule deletion of tracked temporary progress bubbles after the - # final response lands. Failed runs skip this so bubbles remain as - # breadcrumbs for the user to see what work happened. Only fires on - # adapters that support ``delete_message`` (see init above); failures - # are swallowed — deletion is best-effort. + # Schedule deletion of tracked temporary progress bubbles after the final response lands. + # Failed runs skip this so bubbles remain as breadcrumbs for the user to see what work + # happened. Only on adapters with ``delete_message``; failures swallowed (best-effort). if ( _cleanup_progress and _cleanup_adapter is not None @@ -33156,40 +30765,17 @@ def _run_planned_stop_watcher( ) -> None: """Poll for the planned-stop marker and trigger graceful shutdown. - On Windows, ``asyncio.add_signal_handler`` raises NotImplementedError - for SIGTERM/SIGINT, so the standard signal-driven shutdown path - never runs when ``hermes gateway stop`` signals the gateway. The - consequence is that the drain loop is skipped — in-flight agent - sessions are killed mid-turn and ``resume_pending`` is never set, - so the next gateway boot has no idea those sessions need to be - auto-resumed (issue #33778, v0.13.0 session-resume feature broken - on native Windows). + On Windows, ``asyncio.add_signal_handler`` raises NotImplementedError for SIGTERM/SIGINT, so + the standard signal-driven shutdown path never runs when ``hermes gateway stop`` signals the + gateway — the drain loop is skipped, sessions die mid-turn and ``resume_pending`` is never + set, so the next boot cannot auto-resume them. - This watcher runs on every platform (cheap, defensive) and bridges - the gap on Windows by translating a filesystem marker into the - same shutdown-handler invocation a real SIGTERM would have produced - on POSIX. The CLI's ``hermes_cli.gateway_windows.stop()`` writes - the marker via ``write_planned_stop_marker(pid)`` and then waits - for the gateway PID to exit; this watcher is what makes that - exit happen cleanly. - - On POSIX this is a no-op safety net — the signal handler always - races us to consuming the marker file because it fires synchronously - from the kernel's signal delivery. - - Args: - stop_event: cleared by start_gateway() during normal shutdown - to tell the watcher to exit. - runner: the GatewayRunner instance; we check ``_running`` and - ``_draining`` to avoid triggering shutdown if the gateway - is already in one of those states. - loop: the asyncio event loop the shutdown handler must run on. - shutdown_handler: same callable that's wired to SIGTERM — - tolerates a ``None`` signal argument (planned stop case) - and consumes the marker via - ``consume_planned_stop_marker_for_self()``. - poll_interval: seconds between marker checks. 0.5s gives a - responsive shutdown without burning CPU. + This watcher runs on every platform (cheap, defensive) and translates the filesystem marker + written by ``hermes_cli.gateway_windows.stop()`` (``write_planned_stop_marker(pid)``) into + the same shutdown-handler invocation a real SIGTERM would produce. On POSIX it is a no-op + safety net: the synchronous signal handler always consumes the marker first. + ``runner._running`` / ``_draining`` are checked to avoid re-triggering shutdown; the handler + tolerates ``signal=None`` and consumes the marker via ``consume_planned_stop_marker_for_self()``. """ from gateway.status import ( _get_planned_stop_marker_path, @@ -33203,29 +30789,18 @@ def _run_planned_stop_watcher( and not getattr(runner, "_draining", False) and getattr(runner, "_running", False) ): - # A marker existing is NOT sufficient — it may have been - # written for a PREVIOUS gateway instance (different PID) - # and left behind because that process exited before the - # CLI's stop() could clean it up. Firing the handler on a - # stale/foreign marker drives the gateway into shutdown, - # then consume_planned_stop_marker_for_self() correctly - # reports a PID mismatch — but by then we're already - # stopping, so it's logged as an unexpected "UNKNOWN" exit - # and the watchdog crash-loops the gateway (issue #34597, - # a regression from PR #33798 which added this watcher - # without the PID check). - # - # Only fire when the marker actually targets us. The probe - # is non-destructive on a match (the handler does the - # authoritative consume on the loop thread) and self-heals - # by unlinking stale/malformed markers so they cannot wedge - # a freshly booted gateway. + # A marker existing is NOT sufficient — it may have been written for a PREVIOUS + # gateway instance (different PID) and left behind because that process exited + # before the CLI's stop() could clean it up. Firing on a foreign marker drives us + # into shutdown, then the PID-mismatch is logged as an "UNKNOWN" exit and the + # watchdog crash-loops. Only fire when the marker targets us; the probe is + # non-destructive on match and unlinks stale/malformed markers so they can't wedge + # a fresh gateway. if not planned_stop_marker_targets_self(): stop_event.wait(poll_interval) continue - # Drive the same path as a real signal handler. - # Pass signal=None — the handler tolerates that and consumes - # the marker via consume_planned_stop_marker_for_self, + # Drive the same path as a real signal handler. Pass signal=None — the handler + # tolerates that and consumes the marker via consume_planned_stop_marker_for_self, # which also validates target_pid + start_time match us. loop.call_soon_threadsafe(shutdown_handler, None) # Done — the handler will set _draining; we exit on next tick. @@ -33238,16 +30813,12 @@ def _run_planned_stop_watcher( def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop=None, interval: int = 60, cron_provider=None): """Background thread for gateway-only periodic chores (NOT cron). - Split out of the historical ``_start_cron_ticker`` so the cron *trigger* - can live behind the ``CronScheduler`` provider (built-in or external) while - these gateway-specific chores keep running independently of which provider - fires cron. An external scale-to-zero provider has no 60s loop at all, but - this housekeeping still wants its hourly cadence — so it owns its own loop. - - Refreshes the channel directory every 5 minutes and prunes the - image/audio/video/document/screenshot caches + expired ``hermes debug - share`` pastes once per hour, and polls the curator hourly (its inner - gate enforces the real weekly cadence). + Split out of the historical ``_start_cron_ticker`` so the cron *trigger* can live behind the + ``CronScheduler`` provider (built-in or external) while these gateway-specific chores keep + running independently of which provider fires cron. An external scale-to-zero provider has + no 60s loop at all, so housekeeping owns its own loop. Refreshes the channel directory every + 5 min; prunes media caches + expired share pastes hourly; polls the curator hourly (its + inner gate enforces the weekly cadence). """ from gateway.platforms.base import ( cleanup_audio_cache, @@ -33294,10 +30865,9 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop try: from gateway.channel_directory import build_channel_directory if loop is not None: - # build_channel_directory is async (Slack web calls), and - # this runs in a background thread. Schedule onto the - # gateway event loop and wait briefly for completion so - # refresh failures are still logged via the except. + # build_channel_directory is async (Slack web calls), and this runs in a + # background thread. Schedule onto the gateway event loop and wait briefly for + # completion so refresh failures are still logged via the except. fut = safe_schedule_threadsafe( build_channel_directory(adapters), loop, logger=logger, @@ -33328,12 +30898,10 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop except Exception as e: logger.debug("Paste sweep error: %s", e) - # Misfire catch-up (external cron providers only): fire jobs whose - # scheduled time passed with no external fire delivered — the - # backstop for a dead loopback fire hop (gateway restart window, - # api_server not bound, scheduler retries exhausted). The helper - # no-ops for the built-in ticker and enforces the - # cron.misfire_grace_minutes window; the store CAS claim de-dupes + # Misfire catch-up (external cron providers only): fire jobs whose scheduled time passed + # with no external fire delivered — the backstop for a dead loopback fire hop (gateway + # restart window, api_server not bound, scheduler retries exhausted). The helper no-ops for + # the built-in ticker, enforces cron.misfire_grace_minutes, and the store CAS claim de-dupes # against a late external retry arriving concurrently. if cron_provider is not None and tick_count % MISFIRE_SWEEP_EVERY == 0: try: @@ -33349,10 +30917,9 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop except Exception as e: logger.debug("Misfire catch-up sweep error: %s", e) - # Curator — piggy-back on the housekeeping loop so long-running - # gateways get weekly skill maintenance without needing restarts. - # maybe_run_curator() is internally gated by config.interval_hours - # (7 days by default), so CURATOR_EVERY is just the poll rate — the + # Curator — piggy-back on the housekeeping loop so long-running gateways get weekly skill + # maintenance without needing restarts. maybe_run_curator() is internally gated by + # config.interval_hours (7 days by default), so CURATOR_EVERY is just the poll rate — the # real work only fires once per config interval. if tick_count % CURATOR_EVERY == 0: try: @@ -33404,14 +30971,11 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop except Exception as e: logger.debug("Auto-archive tick error: %s", e) - # Deferred stale-FTS rebuild retry (#100108). A SessionDB that opened - # while another process held state.db / the rebuild lock fails closed - # and leaves search on the LIKE fallback; a short-lived CLI clears - # that on its next open, but the gateway opens once and stays up for - # days. Retry here, on the existing tick, against the shared - # instances this process already holds: non-blocking admission, no - # new thread, rate-limited inside SessionDB. No-op when nothing is - # stale (one attribute read per instance). + # Deferred stale-FTS rebuild retry. A SessionDB that opened while another process held + # state.db / the rebuild lock fails closed and leaves search on the LIKE fallback; a short- + # lived CLI clears that on its next open, but the gateway opens once and stays up for days. + # Retry against the shared instances this process holds: non-blocking admission, no new + # thread, rate-limited inside SessionDB; no-op when nothing is stale. if tick_count % FTS_STALE_RETRY_EVERY == 0: try: from hermes_state_registry import live_shared_session_dbs @@ -33436,10 +31000,9 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop trim_memory(reason="messaging gateway housekeeping") except Exception as exc: - # debug, not warning: sibling housekeeping branches all log - # failures at debug, and a persistent failure (e.g. broken - # import after a partial update) would otherwise warn every - # 60s forever. + # debug, not warning: sibling housekeeping branches all log failures at debug, and a + # persistent failure (e.g. broken import after a partial update) would otherwise + # warn every 60s forever. logger.debug( "gateway housekeeping memory trim failed: %s: %s", type(exc).__name__, @@ -33454,11 +31017,9 @@ def _start_cron_ticker(stop_event: threading.Event, adapters=None, loop=None, in """DEPRECATED shim — preserved for backward compatibility. The cron trigger now lives behind the ``CronScheduler`` provider - (``cron.scheduler_provider``); the gateway resolves a provider and runs its - ``start()`` directly (see ``start_gateway``). This shim runs ONLY the - built-in in-process tick loop, exactly as before, for any external caller - or test that still references this symbol (e.g. hermes_cli/debug.py). It no - longer runs gateway housekeeping — that moved to + (``cron.scheduler_provider``); the gateway resolves a provider and runs its ``start()`` + directly (see ``start_gateway``). This shim runs ONLY the built-in in-process tick loop for + external callers/tests still referencing this symbol; housekeeping moved to ``_start_gateway_housekeeping``. """ from cron.scheduler_provider import InProcessCronScheduler @@ -33478,22 +31039,16 @@ def _stop_cron_provider(provider) -> None: logger.debug("Cron provider stop() error: %s", exc) -# Upper bound for cooperatively draining the cron ticker on shutdown. The cron -# thread delivers via ``safe_schedule_threadsafe`` and blocks on -# ``future.result(timeout=60)`` (see cron/scheduler.py::_deliver_result), so a -# single in-flight delivery unblocks within ~60s. The extra margin covers the -# hop back through run_one_job's bookkeeping. +# Upper bound for cooperatively draining the cron ticker on shutdown. The cron thread delivers via +# ``safe_schedule_threadsafe`` and blocks on ``future.result(timeout=60)`` (see +# cron/scheduler.py::_deliver_result), so a single in-flight delivery unblocks within ~60s. _CRON_SHUTDOWN_DRAIN_TIMEOUT = 65.0 -# Upper bound for cooperatively draining the housekeeping ticker on shutdown. -# Housekeeping periodically refreshes the channel directory via -# ``safe_schedule_threadsafe(build_channel_directory(...), loop)`` and blocks on -# ``fut.result(timeout=30)`` (see ``_start_gateway_housekeeping``) — the same -# loop-scheduled-future pattern as cron. So the cooperative bound must cover -# that 30s future (plus margin) rather than the old 5s join, otherwise a -# channel-directory refresh in flight at shutdown gets abandoned mid-resolve. -# Unlike a dropped cron delivery this is not user-facing (it self-heals on the -# next tick), but bounding it correctly keeps the drain honest. +# Upper bound for cooperatively draining the housekeeping ticker on shutdown. Housekeeping refreshes +# the channel directory via ``safe_schedule_threadsafe(...)`` and blocks on ``fut.result(timeout=30)`` +# (same loop-scheduled-future pattern as cron). So the cooperative bound must cover that 30s future +# (plus margin) rather than the old 5s join, otherwise a channel-directory refresh in flight at +# shutdown gets abandoned mid-resolve (not user-facing — self-heals next tick — but keeps the drain honest). _HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT = 35.0 @@ -33502,15 +31057,12 @@ async def _await_thread_exit( ) -> bool: """Wait for a daemon thread to exit WITHOUT blocking the event loop. - A synchronous ``thread.join()`` here would freeze the event loop — fatal - for the cron ticker, whose in-flight delivery is a coroutine scheduled onto - *this* loop via ``safe_schedule_threadsafe``. Blocking the loop deadlocks - that delivery (the loop can never run it), so ``join(timeout=5)`` always - times out and the message is silently dropped on restart (#58818). - - Polling ``is_alive()`` with ``await asyncio.sleep`` keeps the loop running - so the pending delivery completes, then the ticker sees ``stop_event`` and - exits. Returns True if the thread exited within ``timeout``. + A synchronous ``thread.join()`` here would freeze the event loop — fatal for the cron ticker, + whose in-flight delivery is a coroutine scheduled onto *this* loop via + ``safe_schedule_threadsafe``: the loop could never run it, so ``join(timeout=5)`` always timed + out and the message was silently dropped on restart. Polling ``is_alive()`` with + ``await asyncio.sleep`` keeps the loop running so the delivery completes and the ticker sees + ``stop_event``. Returns True if the thread exited within ``timeout``. """ if thread is None: return True @@ -33521,22 +31073,18 @@ async def _await_thread_exit( async def _shutdown_mcp_servers_nonblocking(timeout: float = 5.0) -> bool: - """Close MCP servers off-loop with a bounded wait (#82874). + """Close MCP servers off-loop with a bounded wait. - ``shutdown_mcp_servers()`` is synchronous and can block for its full - internal 15s future wait when the MCP loop and its stdio children are - torn down concurrently (every process in the tree gets SIGTERM at once - under a container/supervisor stop). Calling it directly from the gateway - event-loop thread freezes the loop for that whole window, so supervisors - with a shorter kill grace (s6-overlay defaults to 3s) SIGKILL the gateway - before ``lifecycle_ledger.mark_exited()`` runs and every subsequent boot - reports a phantom unclean death. + ``shutdown_mcp_servers()`` is synchronous and can block for its full 15s internal wait when + the MCP loop and stdio children are torn down concurrently (supervisor SIGTERMs the whole + tree). Calling it on the loop thread freezes the loop, so supervisors with a short kill grace + (s6 default 3s) SIGKILL us before ``lifecycle_ledger.mark_exited()`` runs and every later + boot reports a phantom unclean death. - Run it on a daemon thread instead and poll with ``_await_thread_exit`` so - the loop keeps servicing teardown. If it does not finish within - ``timeout`` we proceed with shutdown; the daemon thread is left to finish - (or die with the process) in the background. Returns True when the MCP - shutdown completed within the budget. + Run it on a daemon thread instead and poll with ``_await_thread_exit`` so the loop keeps + servicing teardown. If it does not finish within ``timeout`` we proceed with shutdown; the + daemon thread is left to finish (or die with the process) in the background. Returns True + when it completed within budget. """ def _do() -> None: @@ -33577,31 +31125,23 @@ def _gateway_stderr_formatter() -> logging.Formatter: return RedactingFormatter("%(asctime)s %(levelname)s %(name)s: %(message)s") - # ownership guard inserted below (PR #93084) def _replace_target_belongs_to_other_profile(existing_pid: int) -> bool: """Return True when ``--replace`` must refuse to signal ``existing_pid``. - The PID file is HERMES_HOME-scoped, but a poisoned/stale record can point - at another profile's LIVE gateway; signaling it starts the cross-profile - SIGTERM restart loop this guard exists to prevent (#89315). This is a - destructive-action authority check, so ownership is decided by the - persisted identity record ALONE — exact ``_same_hermes_home`` equality — - and only while that record stays bound to the live target by exact PID + - start-time identity: + The PID file is HERMES_HOME-scoped, but a poisoned/stale record can point at another + profile's LIVE gateway; signaling it starts a cross-profile SIGTERM restart loop. + This is a destructive-action authority check, so ownership is decided by the persisted + identity record ALONE — exact ``_same_hermes_home`` equality — and only while that record + stays bound to the live target by exact PID + start-time identity: - * The authorizing record is whichever source produced the PID for this - destructive decision (PID file, gateway lock record, or runtime-status - fallback). A readable live argv carries no HERMES_HOME (it travels in - the environment), so it can never prove home ownership; it is used only - as an additional CONSISTENCY check — token-exact profile flags that - clearly contradict our home refuse the signal even when the record - agrees. - * Missing, legacy, conflicting, stale-bound, or unprovable identity → - refuse (fail closed). + * The authorizing record is whichever source produced the PID (PID file, lock record, or + runtime-status fallback). Live argv carries no HERMES_HOME so it can never PROVE ownership; + it is only a consistency check — profile flags that clearly contradict our home refuse. + * Missing, legacy, conflicting, stale-bound, or unprovable identity → refuse (fail closed). - Same-home targets keep replacing normally; every refusal path here only - narrows what the legacy start_time check alone used to allow. + Same-home targets replace normally; every refusal here only narrows what the legacy + start_time check alone used to allow. """ try: from gateway.status import ( @@ -33702,10 +31242,9 @@ def _replace_target_belongs_to_other_profile(existing_pid: int) -> bool: def _looks_like_profile_conflict_from_cmdline(command: str, our_home) -> bool: """Token-exact contradiction check between a target argv and our home. - Authority lives in the pid record; this only catches argv that EXPLICITLY - advertises a different profile than ours. Substring matching is not - identity: ``--profile timothy`` must NOT read as profile ``tim``. Returns - False whenever the argv does not clearly contradict our home. + Authority lives in the pid record; this only catches argv that EXPLICITLY advertises another + profile. Substring matching is not identity: ``--profile timothy`` must NOT read as profile + ``tim``. Returns False whenever the argv does not clearly contradict our home. """ from gateway.status import _profile_name_for_home @@ -33761,15 +31300,13 @@ def _looks_like_profile_conflict_from_cmdline(command: str, our_home) -> bool: return False - async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool: - """ - Start the gateway and run until interrupted. - - This is the main entry point for running the gateway. - Returns True if the gateway ran successfully, False if it failed to start. - A False return causes a non-zero exit code so systemd can auto-restart. - + """Start the gateway and run until interrupted. + + This is the main entry point for running the gateway. Returns True if the gateway ran + successfully, False if it failed to start. A False return causes a non-zero exit code so + systemd can auto-restart. + Args: config: Optional gateway configuration override. replace: If True, kill any existing gateway instance before starting. @@ -33785,18 +31322,16 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = apply_nofile_soft_limit() - # Snapshot the checkout revision now, while sys.modules still matches disk, - # so a later `git pull` under this long-lived process can be detected (and - # risky work like model switching refused) instead of crashing on a stale - # in-memory module. + # Snapshot the checkout revision now, while sys.modules still matches disk, so a later `git + # pull` under this long-lived process can be detected (and risky work like model switching + # refused) instead of crashing on a stale in-memory module. from gateway.code_skew import record_boot_fingerprint record_boot_fingerprint() # ── Duplicate-instance guard ────────────────────────────────────── - # Prevent two gateways from running under the same HERMES_HOME. - # The PID file is scoped to HERMES_HOME, so future multi-profile - # setups (each profile using a distinct HERMES_HOME) will naturally - # allow concurrent instances without tripping this guard. + # Prevent two gateways from running under the same HERMES_HOME. The PID file is scoped to + # HERMES_HOME, so future multi-profile setups (each profile using a distinct HERMES_HOME) will + # naturally allow concurrent instances without tripping this guard. from gateway.status import ( acquire_gateway_runtime_lock, get_running_pid, @@ -33808,10 +31343,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = existing_pid = get_running_pid() if existing_pid is not None and existing_pid != os.getpid(): if replace: - # Cross-profile ownership gate (#89315): never signal a live - # process we cannot prove belongs to this HERMES_HOME. A poisoned - # PID record steering --replace at another profile's gateway is - # exactly the restart-loop shape this flow must not allow. + # Cross-profile ownership gate: never signal a live process we cannot prove belongs to + # this HERMES_HOME. A poisoned PID record steering --replace at another profile's + # gateway is exactly the restart-loop shape this flow must not allow. if _replace_target_belongs_to_other_profile(existing_pid): from gateway.status import _get_process_hermes_home @@ -33828,22 +31362,19 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = "Replacing existing gateway instance (PID %d) with --replace.", existing_pid, ) - # Record a takeover marker so the target's shutdown handler - # recognises its SIGTERM as a planned takeover and exits 0 - # (rather than exit 1, which would trigger systemd's - # Restart=on-failure and start a flap loop against us). - # Best-effort — proceed even if the write fails. + # Record a takeover marker so the target's shutdown handler recognises its SIGTERM as a + # planned takeover and exits 0 (rather than exit 1, which would trigger systemd's + # Restart=on-failure and start a flap loop against us). Best-effort — proceed on failure. try: from gateway.status import write_takeover_marker write_takeover_marker(existing_pid) except Exception as e: logger.debug("Could not write takeover marker: %s", e) - # Snapshot the old gateway's child processes BEFORE signalling it: - # once it exits, orphans are reparented and can no longer be found - # by a parent walk. On POSIX, adapter subprocesses that outlive - # the gateway keep holding scoped token locks and block the - # replacement (Windows terminate_pid(force=True) already - # tree-kills via taskkill /T). Best-effort — [] on any failure. + # Snapshot the old gateway's child processes BEFORE signalling it: once it exits, + # orphans are reparented and can no longer be found by a parent walk. On POSIX, adapter + # subprocesses outliving the gateway keep holding scoped token locks and block the + # replacement (Windows terminate_pid(force=True) already tree-kills). Best-effort — [] + # on any failure. try: from gateway.status import _snapshot_gateway_children _old_gateway_children = _snapshot_gateway_children(existing_pid) @@ -33920,10 +31451,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = except Exception: pass return False - # Old gateway confirmed dead — reap any orphaned child processes - # it left behind (POSIX; mirrors Windows taskkill /T tree-kill). - # Orphaned adapter subprocesses would otherwise keep holding - # scoped token locks against us. Best-effort, never raises. + # Old gateway confirmed dead — reap any orphaned child processes it left behind (POSIX; + # mirrors Windows taskkill /T tree-kill). Orphaned adapter subprocesses would otherwise + # keep holding scoped token locks against us. Best-effort, never raises. try: from gateway.status import reap_gateway_children reap_gateway_children( @@ -33990,10 +31520,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = from hermes_logging import setup_logging, _safe_stderr setup_logging(hermes_home=_hermes_home, mode="gateway") - # Startup security posture audit — warn-on-load, never blocks. Surfaces - # root / weak-SSH / ephemeral-container / unauthenticated-listener posture - # so operators get the "you're exposed" signal the June 2026 MCP-config - # persistence campaign victims never had. + # Startup security posture audit — warn-on-load, never blocks. Surfaces root / weak-SSH / + # ephemeral-container / unauthenticated-listener posture so operators get the "you're exposed" + # signal the June 2026 MCP-config persistence campaign victims never had. try: from hermes_cli.security_audit_startup import log_startup_security_warnings @@ -34032,23 +31561,19 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # policy. GatewayRunner scopes this bit to cold adapter connects and clears # it before the background reconnect watcher starts. runner._platform_lock_takeover_on_start = bool(replace) - - # Track whether an unexpected signal initiated the shutdown. When an - # unexpected SIGTERM kills the gateway, we exit non-zero so service - # managers can revive the process. Planned stop paths write a marker - # before signalling us so they can exit cleanly instead. + + # Track whether an unexpected signal initiated the shutdown. When an unexpected SIGTERM kills + # the gateway, we exit non-zero so service managers can revive the process. Planned stop paths + # write a marker before signalling us so they can exit cleanly instead. _signal_initiated_shutdown = False # Set up signal handlers def shutdown_signal_handler(received_signal=None): nonlocal _signal_initiated_shutdown - # Planned --replace takeover check: when a sibling gateway is - # taking over via --replace, it wrote a marker naming this PID - # before sending SIGTERM. If present, treat the signal as a - # planned shutdown and exit 0 so systemd's Restart=on-failure - # doesn't revive us (which would flap-fight the replacer when - # both services are enabled, e.g. hermes.service + hermes- - # gateway.service from pre-rename installs). + # Planned --replace takeover check: when a sibling gateway is taking over via --replace, it + # wrote a marker naming this PID before sending SIGTERM. Treat as planned and exit 0 so + # systemd's Restart=on-failure doesn't revive us and flap-fight the replacer (e.g. when + # both hermes.service and hermes-gateway.service are enabled). planned_takeover = False try: from gateway.status import consume_takeover_marker_for_self @@ -34056,10 +31581,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = except Exception as e: logger.debug("Takeover marker check failed: %s", e) - # Planned stop check: service managers and `hermes gateway stop` - # also send SIGTERM, which is indistinguishable from an unexpected - # external kill unless the CLI marks it first. SIGINT comes from an - # interactive Ctrl+C and is likewise an intentional foreground stop. + # Planned stop check: service managers and `hermes gateway stop` also send SIGTERM, which is + # indistinguishable from an unexpected external kill unless the CLI marks it first. SIGINT + # comes from an interactive Ctrl+C and is likewise an intentional foreground stop. planned_stop = False if received_signal == signal.SIGINT: planned_stop = True @@ -34070,12 +31594,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = except Exception as e: logger.debug("Planned stop marker check failed: %s", e) - # Fast (<10ms) snapshot of who's asking us to shut down — runs - # synchronously inside the asyncio signal handler, so we keep it - # purely stdlib + /proc reads, no subprocesses. See PR #15826 - # (May 2026): the previous implementation called `ps aux` here - # synchronously, blocking the event loop for up to 3s while - # adapter teardown couldn't begin. + # Fast (<10ms) snapshot of who's asking us to shut down — runs synchronously inside the + # asyncio signal handler, so we keep it purely stdlib + /proc reads, no subprocesses (a + # synchronous `ps aux` here once blocked the loop ~3s and delayed adapter teardown). try: from gateway.shutdown_forensics import ( format_context_for_log, @@ -34099,12 +31620,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = ) else: _signal_initiated_shutdown = True - # Mirror onto the runner so _stop_impl can suppress the - # gateway_state=stopped persist for unexpected signals - # (container/s6 SIGTERM on restart, OOM, bare kill) — see - # issue #42675. Operator-initiated stops set a planned-stop - # marker first, land in the `planned_stop` branch above, and - # leave this flag False so they DO persist "stopped". + # Mirror onto the runner so _stop_impl can suppress the gateway_state=stopped persist + # for unexpected signals (container/s6 SIGTERM on restart, OOM, bare kill). Operator + # stops set a planned-stop marker, take the `planned_stop` branch above and leave this + # False so they DO persist "stopped". runner._signal_initiated_shutdown = True logger.info( "Received %s — initiating shutdown", @@ -34137,21 +31656,14 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = def restart_signal_handler(): runner.request_restart(detached=False, via_service=True) - + loop = asyncio.get_running_loop() - # Install a loop-level exception handler that swallows transient - # network errors from background tasks. Issues #31066 / #31110: - # an unhandled ``telegram.error.TimedOut`` (or peer NetworkError / - # httpx connection error) in any awaited coroutine would propagate - # to the loop and kill the gateway process, taking down every - # profile attached to the same runner. systemd then restarts the - # service after ~5s but the active conversation turn is lost. - # - # The fix is intentionally narrow: only well-known transient - # network errors are swallowed (and logged with full traceback so - # the originating call site is still discoverable). Anything else - # is forwarded to the default handler so real bugs still surface. + # Install a loop-level exception handler that swallows transient network errors from background + # tasks: an unhandled telegram TimedOut / NetworkError / httpx connection error in any awaited + # coroutine would otherwise kill the whole gateway (every profile) and lose the active turn. + # Deliberately narrow: only well-known transient network errors are swallowed (logged with + # traceback); everything else goes to the default handler so real bugs still surface. loop.set_exception_handler(_gateway_loop_exception_handler) if threading.current_thread() is threading.main_thread(): @@ -34168,18 +31680,12 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = else: logger.info("Skipping signal handlers (not running in main thread).") - # Windows fallback: asyncio.add_signal_handler raises NotImplementedError - # on Windows, so `hermes gateway stop`'s SIGTERM (which Python maps to - # TerminateProcess on Windows) never invokes shutdown_signal_handler. - # That means the drain loop never runs, mark_resume_pending never fires, - # and sessions are silently lost across restarts (issue #33778). - # - # The fix is a marker-polling thread: `hermes gateway stop` writes the - # planned-stop marker BEFORE killing, and this thread notices it and - # drives the same shutdown path the signal handler would have. Runs - # on every platform (cheap, defensive) so non-signal-bearing - # environments (Windows native, sandboxed CI runners that mask - # SIGTERM) still get a clean drain. + # Windows fallback: asyncio.add_signal_handler raises NotImplementedError on Windows, so `hermes + # gateway stop`'s SIGTERM (which Python maps to TerminateProcess on Windows) never invokes + # shutdown_signal_handler — no drain loop, no mark_resume_pending, sessions lost on restart. + # A marker-polling thread notices the planned-stop marker `hermes gateway stop` writes BEFORE + # killing and drives the same shutdown path. Runs on every platform (cheap, defensive) so + # non-signal-bearing environments (Windows native, CI runners masking SIGTERM) drain cleanly. _planned_stop_watcher_stop = threading.Event() _planned_stop_watcher_thread = threading.Thread( target=_run_planned_stop_watcher, @@ -34220,13 +31726,11 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = atexit.register(remove_pid_file) atexit.register(release_gateway_runtime_lock) - # Control socket (#92091 step 1) — the gateway-owned identify/status - # surface. Started immediately after the PID-file claim: winning that - # O_EXCL race is the moment this process becomes the authoritative - # gateway for its HERMES_HOME, so from here on "does a socket answer?" - # is a truthful liveness/identity query for updater and fleet consumers. - # Strictly non-fatal: a bind failure only means consumers fall back to - # the process-scan/state-file layer, exactly as before this feature. + # Control socket (#92091 step 1) — the gateway-owned identify/status surface. Started right + # after the PID-file claim: winning that O_EXCL race is when this process becomes the + # authoritative gateway for its HERMES_HOME, so "does a socket answer?" is a truthful + # liveness/identity query from here on. Strictly non-fatal: a bind failure only means + # consumers fall back to the process-scan/state-file layer, exactly as before this feature. _control_server = None try: from gateway.control_socket import GatewayControlServer @@ -34300,12 +31804,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = _ensure_windows_gateway_venv_imports() - # MCP tool discovery — run in an executor so the asyncio event loop - # stays responsive even when a configured MCP server is slow or - # unreachable. discover_mcp_tools() uses a blocking 120s wait - # internally; calling it from the loop thread would freeze platform - # heartbeats (Discord shard, Telegram polling) until it returned. - # See #16856. + # MCP tool discovery — run in an executor so the asyncio event loop stays responsive even when a + # configured MCP server is slow or unreachable. discover_mcp_tools() uses a blocking 120s wait + # internally; calling it from the loop thread would freeze platform heartbeats (Discord shard, + # Telegram polling) until it returned. try: await _discover_gateway_mcp_tools(runner.config) except Exception as e: @@ -34334,13 +31836,11 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = _shutdown_gateway_health_export(runner) if runner.exit_reason: logger.error("Gateway exiting cleanly: %s", runner.exit_reason) - # A clean exit that carries an explicit exit code (e.g. a fatal - # config error stamped with GATEWAY_FATAL_CONFIG_EXIT_CODE) must - # propagate that code to the process so the s6 finish script can - # translate it (78 → 125) and stop the supervisor restart loop. - # Without this, the early `return True` below makes main() exit 0, - # the finish script's `[ "$1" = "78" ]` check never matches, and - # s6 crash-loops the gateway anyway (#51228). + # A clean exit that carries an explicit exit code (e.g. a fatal config error stamped with + # GATEWAY_FATAL_CONFIG_EXIT_CODE) must propagate that code to the process so the s6 finish + # script can translate it (78 → 125) and stop the supervisor restart loop. Otherwise the + # early `return True` below exits 0, the finish script's 78 check never matches, and s6 + # crash-loops the gateway anyway. if runner.exit_code is not None: raise SystemExit(runner.exit_code) return True @@ -34363,11 +31863,8 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = finally: _shutdown_gateway_health_export(runner) - # Start the background cron scheduler via the resolved provider so - # scheduled jobs fire automatically. The built-in provider is the - # historical in-process 60s ticker; an external provider (e.g. chronos) - # may arm a schedule and return. Pass the event loop so cron delivery can - # use live adapters (E2EE support). + # Start the background cron scheduler via the resolved provider so scheduled jobs fire + # automatically. Pass the event loop so cron delivery can use live adapters (E2EE support). from cron.scheduler_provider import ( InProcessCronScheduler, resolve_cron_scheduler, @@ -34435,15 +31932,11 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = ) cron_thread.start() - # Preflight tell for the hosted fire path: an external cron provider - # (Chronos) delivers scheduled fires over HTTP to THIS process's - # api_server adapter on loopback. If that adapter never came up (most - # commonly API_SERVER_KEY missing from this process's environment — - # e.g. a gateway relaunched outside its supervisor without the profile - # env), every scheduled fire will fail with ConnectError at the - # dashboard forwarder while manual runs keep working, which users - # reliably misread as a job bug. Say it loudly ONCE at startup, when - # it is fixable, instead of letting the first miss say it at 2am. + # Preflight tell for the hosted fire path: an external cron provider (Chronos) delivers + # scheduled fires over HTTP to THIS process's api_server adapter on loopback. If that adapter + # never came up (typically API_SERVER_KEY missing, e.g. relaunched outside the supervisor's + # env), every fire fails with ConnectError while manual runs work — misread as a job bug. Say + # it loudly ONCE at startup, when it is fixable, instead of letting the first miss say it at 2am. if not isinstance(cron_provider, InProcessCronScheduler): try: _has_api_server = Platform.API_SERVER in (runner.adapters or {}) @@ -34487,11 +31980,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # Wait for shutdown await runner.wait_for_shutdown() - # Stop the control socket first: once shutdown begins this process is no - # longer a truthful "the gateway is serving here" answer, and a successor - # (--replace / supervisor respawn) must be able to bind. Early-exit paths - # above don't reach this; their process exit runs the atexit - # cleanup_files hook, and a successor clears any stale socket on bind. + # Stop the control socket first: once shutdown begins this process is no longer a truthful "the + # gateway is serving here" answer, and a successor (--replace / supervisor respawn) must be able + # to bind. Early-exit paths above don't reach this; their atexit cleanup_files hook runs, and a + # successor clears any stale socket on bind. if _control_server is not None: try: await _control_server.stop() @@ -34509,16 +32001,11 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = if runner.exit_reason: logger.error("Gateway exiting with failure: %s", runner.exit_reason) return False - - # Stop cron scheduler + housekeeping cleanly. - # - # These MUST be awaited cooperatively, not join()ed. A cron delivery in - # flight when the gateway restarts is a coroutine scheduled onto THIS event - # loop (safe_schedule_threadsafe); the ticker thread is blocked on its - # future.result(). A synchronous cron_thread.join() would block the loop, - # so that delivery could never run — it timed out and the message was - # silently dropped (#58818). Awaiting keeps the loop alive so the in-flight - # delivery finishes before we tear down. + + # Stop cron scheduler + housekeeping cleanly. These MUST be awaited cooperatively, not + # join()ed: an in-flight cron delivery is a coroutine scheduled onto THIS loop while the + # ticker thread blocks on future.result(); a synchronous join would block the loop so the + # delivery could never run and the message was silently dropped. cron_stop.set() _stop_cron_provider(cron_provider) if not await _await_thread_exit(cron_thread, timeout=_CRON_SHUTDOWN_DRAIN_TIMEOUT): @@ -34573,12 +32060,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = def _guard_corrupt_user_config() -> None: """Fail closed when the active profile's config.yaml cannot be parsed. - The gateway is a fully non-interactive surface: nobody is present to - repair a corrupt ``config.yaml``, and silently continuing on built-in - defaults lets provider auto-detection adopt credentials from ``.env`` - that the config never named (issue #81952). Same policy and escape - hatch (``HERMES_IGNORE_USER_CONFIG=1``) as the non-interactive CLI - guard in ``hermes_cli/main.py``. + The gateway is a fully non-interactive surface: nobody is present to repair a corrupt + ``config.yaml``, and silently continuing on built-in defaults lets provider auto-detection + adopt credentials from ``.env`` that the config never named. Same policy and escape hatch + (``HERMES_IGNORE_USER_CONFIG=1``) as the non-interactive CLI guard in ``hermes_cli/main.py``. """ from hermes_cli.config import ( InvalidUserConfigError, @@ -34598,12 +32083,10 @@ def main(): # startup (watchdog, DB opens, provider resolution). See _guard docstring. _guard_corrupt_user_config() - # Advertise the agent harness to child processes (AI_AGENT is the - # cross-agent standard; HERMES_AGENT the Hermes-specific marker — see - # _advertise_agent_env in hermes_cli/main.py, kept inline here to avoid - # importing that module's startup side effects). The value must equal our - # public agent-harness registry id (``hermes-agent``) — standard-var - # matching is exact. setdefault so an outer harness is never clobbered. + # Advertise the agent harness to child processes (AI_AGENT is the cross-agent standard; + # HERMES_AGENT the Hermes-specific marker — see _advertise_agent_env in hermes_cli/main.py, kept + # inline here to avoid importing that module's startup side effects). Value must equal our + # registry id ``hermes-agent`` (matching is exact); setdefault never clobbers an outer harness. os.environ.setdefault("AI_AGENT", "hermes-agent") os.environ.setdefault("HERMES_AGENT", "true") @@ -34621,13 +32104,10 @@ def main(): except Exception: pass - # Startup-liveness watchdog (OOF-298): armed before config load, DB - # opens, and the rest of pre-loop startup so a deadlock in that window - # still gets the process respawned by the service supervisor instead of - # wedging as a live-PID zombie. (Import-time coverage for the standard - # ``hermes gateway run`` path is provided even earlier, by the argv - # fast-path in hermes_cli.main.) Disarmed by GatewayRunner once the - # event loop is confirmed live. + # Startup-liveness watchdog (OOF-298): armed before config load, DB opens, and the rest of pre- + # loop startup so a deadlock in that window still gets the process respawned by the service + # supervisor instead of wedging as a live-PID zombie. (The argv fast-path in hermes_cli.main + # covers import time even earlier.) Disarmed by GatewayRunner once the loop is confirmed live. try: from gateway.startup_watchdog import arm_startup_watchdog arm_startup_watchdog() @@ -34643,34 +32123,27 @@ def main(): pass import argparse - + parser = argparse.ArgumentParser(description="Hermes Gateway - Multi-platform messaging") parser.add_argument("--config", "-c", help="Path to gateway config file") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") - + args = parser.parse_args() - + config = None if args.config: import yaml with open(args.config, encoding="utf-8") as f: data = yaml.safe_load(f) or {} config = GatewayConfig.from_dict(data) - - # start_gateway() performs the full graceful teardown (adapters - # disconnected, sessions saved + flushed, SQLite closed, cron/MCP stopped, - # PID file + runtime lock released) before it returns OR raises SystemExit - # with an explicit code. Force-exit afterwards so a wedged non-daemon worker - # thread (e.g. a ThreadPoolExecutor tool/LLM call blocked with no timeout) - # cannot block interpreter finalization (Py_FinalizeEx joins all non-daemon - # threads, incl. concurrent.futures' _python_exit) and strand the gateway - # half-shut down with the supervisor unable to restart it (#53107). - # - # SystemExit is caught explicitly: start_gateway raises it on the - # clean-fatal-config (#51228), planned-restart, and service-restart paths, - # all of which complete teardown first. Routing those codes through the - # same os._exit backstop means EVERY exit path is wedge-proof, not just the - # boolean-return ones. + + # start_gateway() performs the full graceful teardown (adapters disconnected, sessions saved + + # flushed, SQLite closed, cron/MCP stopped, PID file + runtime lock released) before it returns + # OR raises SystemExit with an explicit code. Force-exit afterwards so a wedged non-daemon + # worker thread (e.g. a ThreadPoolExecutor call with no timeout) cannot block interpreter + # finalization (Py_FinalizeEx joins all non-daemon threads) and strand the gateway half-shut. + # SystemExit is caught explicitly (clean-fatal-config, planned-restart, service-restart paths + # all complete teardown first) so EVERY exit path goes through the os._exit backstop. try: success = asyncio.run(start_gateway(config)) exit_code = 0 if success else 1 @@ -34688,34 +32161,20 @@ def main(): def _exit_after_graceful_shutdown(exit_code: int) -> None: """Flush stdio, release the PID file + runtime lock, then hard-exit. - Graceful teardown is already complete by the time this runs, so there is - nothing left that needs a clean interpreter shutdown. We deliberately use - ``os._exit`` (not ``sys.exit``): ``sys.exit`` raises ``SystemExit``, which - triggers ``Py_FinalizeEx`` → ``wait_for_thread_shutdown`` and joins every - non-daemon thread — exactly the hang (#53107) a wedged tool-worker causes. + Graceful teardown is already complete by the time this runs, so there is nothing left that + needs a clean interpreter shutdown. Deliberately ``os._exit`` (not ``sys.exit``): SystemExit + triggers ``Py_FinalizeEx`` → joins every non-daemon thread — exactly the hang a wedged + tool-worker causes. - ``os._exit`` bypasses ``atexit`` handlers, so we cannot rely on the - ``atexit``-registered ``remove_pid_file`` / ``release_gateway_runtime_lock`` - (registered in ``start_gateway``) to run. The full-shutdown path releases - both explicitly in ``_stop_impl``, but the EARLY exit paths — - clean-fatal-config (#51228) and startup-aborted-before-running — raise - ``SystemExit`` right after ``runner.start()`` without going through - ``_stop_impl``, so on those paths ``atexit`` was the only thing releasing - them. Now that those paths are routed through this backstop (#53107), - release both here explicitly. Both calls are idempotent — - ``remove_pid_file`` only unlinks a PID file that belongs to this process, - and ``release_gateway_runtime_lock`` no-ops when the lock is already - released — so this is a no-op on the normal shutdown path and the actual - cleanup on the early-exit paths. + ``os._exit`` bypasses ``atexit``, so the atexit-registered ``remove_pid_file`` / + ``release_gateway_runtime_lock`` never run. The full-shutdown path releases both in + ``_stop_impl``, but the EARLY exit paths (clean-fatal-config, startup-aborted) relied on + atexit; now that those paths are routed through this backstop, release both here explicitly + (both idempotent — no-op on the normal path). - Logging IS drained here: the rotating file handlers are driven by an - async ``QueueListener`` on a dedicated thread (see - ``hermes_logging._register_queued_handler``), so records emitted right - before shutdown may still be sitting in the in-memory queue. ``os._exit`` - below bypasses ``atexit``, so the ``atexit``-registered listener drain - never runs on this path — we drain explicitly (bounded, via - ``drain_log_queue``) or lose the last log lines (including the shutdown - reason on the early-exit paths). Stdio is flushed too. + Logging IS drained here: file handlers run behind an async ``QueueListener`` thread, so + records emitted just before shutdown may sit in the queue and the atexit listener drain + never runs — drain explicitly (bounded) or lose the last lines. Stdio is flushed too. """ for stream in (sys.stdout, sys.stderr): try: @@ -34732,22 +32191,19 @@ def _exit_after_graceful_shutdown(exit_code: int) -> None: release_gateway_runtime_lock() except Exception: pass - # Mark this life cleanly exited in the lifecycle sentinel (NS-608). This - # is the single funnel every graceful exit passes through, so the next - # boot's unclean-death detector only fires for genuine SIGKILL/OOM/VM - # deaths. Ownership-guarded internally: a --replace old life won't + # Mark this life cleanly exited in the lifecycle sentinel (NS-608). This is the single funnel + # every graceful exit passes through, so the next boot's unclean-death detector only fires for + # genuine SIGKILL/OOM/VM deaths. Ownership-guarded internally: a --replace old life won't # clobber the replacement's freshly claimed "running" sentinel. try: from gateway.lifecycle_ledger import mark_exited mark_exited(exit_code, reason="graceful_shutdown") except Exception: pass - # Drain the async log queue: os._exit bypasses atexit, so the listener's - # atexit drain won't fire. Use drain_log_queue() (bounded, no restart), NOT - # flush_log_queue(): if the listener is wedged on the rotation lock — the - # exact failure this async-logging change survives — an unbounded stop() - # join would re-freeze the shutdown. drain_log_queue() no-ops when logging - # never initialized a queue (very early aborts), so this is always safe. + # Drain the async log queue: os._exit bypasses atexit, so the listener's atexit drain won't + # fire. Use drain_log_queue() (bounded, no restart), NOT flush_log_queue(): if the listener is + # wedged on the rotation lock an unbounded stop() join would re-freeze the shutdown. No-ops + # when logging never initialized a queue (very early aborts). try: from hermes_logging import drain_log_queue drain_log_queue(timeout=1.0) diff --git a/gateway/runtime_footer.py b/gateway/runtime_footer.py index 2526bc3e10..d2ba7991f7 100644 --- a/gateway/runtime_footer.py +++ b/gateway/runtime_footer.py @@ -1,35 +1,21 @@ -"""Gateway runtime-metadata footer. - -Renders a compact footer showing runtime state (model, context %, cwd) and -appends it to the FINAL message of an agent turn when enabled. Off by default +"""Gateway runtime-metadata footer (model · context % · cwd), off by default to keep replies minimal. -Config (``~/.hermes/config.yaml``):: +Config (``display.runtime_footer``, per-platform override under +``display.platforms..runtime_footer``; toggled by ``/footer on|off``):: display: runtime_footer: - enabled: true # off by default + enabled: true fields: [model, context_pct, cwd] # order shown; drop any to hide -Available fields: - model — bare model id, vendor prefix dropped (``gpt-5.4``) - context_pct — last-call context occupancy as a percent (``5%``) - latency — wall-clock duration of the turn (``22s``, ``1m05s``) - cwd — home-relative working dir (``~``) +Fields: ``model`` (vendor prefix dropped), ``context_pct`` (last-call +occupancy), ``latency`` (turn wall-clock, opt-in — NOT in the default set so an +unset ``fields`` renders exactly as before), ``cwd`` (home-relative). -``latency`` is opt-in: it is NOT in the default field set, so a footer whose -``fields`` are unset renders exactly as before. - -Per-platform overrides live under ``display.platforms..runtime_footer``. -Users can toggle the global setting with ``/footer on|off`` from both the CLI -and any gateway platform. - -The footer is appended to the final response text in ``gateway/run.py`` right -before returning the response to the adapter send path — so it only lands on -the final message a user sees, not on tool-progress updates or streaming -partials. When streaming is on and the final text has already been delivered -piecemeal, the footer is sent as a separate trailing message via -``send_trailing_footer()``. +``gateway/run.py`` appends the footer to the final response only (never to +tool-progress or streaming partials); when streaming already delivered the text, +it goes out as a trailing message via ``send_trailing_footer()``. """ from __future__ import annotations @@ -56,44 +42,40 @@ def _home_relative_cwd(cwd: str) -> str: def _model_short(model: Optional[str]) -> str: - """Drop ``vendor/`` prefix for readability (``openai/gpt-5.4`` → ``gpt-5.4``).""" - if not model: - return "" - return model.rsplit("/", 1)[-1] + """Drop ``vendor/`` prefix (``openai/gpt-5.4`` → ``gpt-5.4``).""" + return model.rsplit("/", 1)[-1] if model else "" + + +def _env_cwd() -> str: + try: + from tools.terminal_scope import terminal_env + except ImportError: + return os.environ.get("TERMINAL_CWD", "") + return terminal_env("TERMINAL_CWD", "") + + +def _apply_footer_section(resolved: dict[str, Any], section: Any) -> None: + if not isinstance(section, dict): + return + if "enabled" in section: + resolved["enabled"] = bool(section.get("enabled")) + if isinstance(section.get("fields"), list) and section["fields"]: + resolved["fields"] = [str(f) for f in section["fields"]] def resolve_footer_config( user_config: dict[str, Any] | None, platform_key: str | None = None, ) -> dict[str, Any]: - """Resolve effective runtime-footer config for *platform_key*. - - Merge order (later wins): - 1. Built-in defaults (enabled=False) - 2. ``display.runtime_footer`` - 3. ``display.platforms..runtime_footer`` - """ + """Resolve effective footer config: defaults (enabled=False) < + ``display.runtime_footer`` < ``display.platforms..runtime_footer``.""" resolved = {"enabled": False, "fields": list(_DEFAULT_FIELDS)} cfg = (user_config or {}).get("display") or {} - - global_cfg = cfg.get("runtime_footer") - if isinstance(global_cfg, dict): - if "enabled" in global_cfg: - resolved["enabled"] = bool(global_cfg.get("enabled")) - if isinstance(global_cfg.get("fields"), list) and global_cfg["fields"]: - resolved["fields"] = [str(f) for f in global_cfg["fields"]] - + _apply_footer_section(resolved, cfg.get("runtime_footer")) if platform_key: - platforms = cfg.get("platforms") or {} - plat_cfg = platforms.get(platform_key) + plat_cfg = (cfg.get("platforms") or {}).get(platform_key) if isinstance(plat_cfg, dict): - plat_footer = plat_cfg.get("runtime_footer") - if isinstance(plat_footer, dict): - if "enabled" in plat_footer: - resolved["enabled"] = bool(plat_footer.get("enabled")) - if isinstance(plat_footer.get("fields"), list) and plat_footer["fields"]: - resolved["fields"] = [str(f) for f in plat_footer["fields"]] - + _apply_footer_section(resolved, plat_cfg.get("runtime_footer")) return resolved @@ -108,6 +90,12 @@ def _format_latency(seconds: float) -> str: return f"{m}m{sec:02d}s" +def _context_pct(context_tokens: int, context_length: Optional[int]) -> str: + if context_length and context_length > 0 and context_tokens >= 0: + return f"{max(0, min(100, round((context_tokens / context_length) * 100)))}%" + return "" + + def format_runtime_footer( *, model: Optional[str], @@ -117,40 +105,19 @@ def format_runtime_footer( turn_seconds: Optional[float] = None, fields: Iterable[str] = _DEFAULT_FIELDS, ) -> str: - """Render the footer line, or return "" if no fields have data. + """Render the footer line, or "" if no fields have data. - Fields are skipped silently when their underlying data is missing — a - partially-populated footer is better than a line with ``?%`` or empty slots. + Fields whose data is missing (and unknown field names) are skipped silently — + a partial footer beats ``?%`` or empty slots. """ - parts: list[str] = [] - for field in fields: - if field == "model": - m = _model_short(model) - if m: - parts.append(m) - elif field == "context_pct": - if context_length and context_length > 0 and context_tokens >= 0: - pct = max(0, min(100, round((context_tokens / context_length) * 100))) - parts.append(f"{pct}%") - elif field == "latency": - # Wall-clock turn duration. Skipped when the caller supplied no - # timing (call sites that don't measure) or the value is negative. - if turn_seconds is not None and turn_seconds >= 0: - parts.append(_format_latency(turn_seconds)) - elif field == "cwd": - try: - from tools.terminal_scope import terminal_env as _tenv - except ImportError: - env_cwd = os.environ.get("TERMINAL_CWD", "") - else: - env_cwd = _tenv("TERMINAL_CWD", "") - rel = _home_relative_cwd(cwd or env_cwd) - if rel: - parts.append(rel) - # Unknown field names are silently ignored. - - if not parts: - return "" + renderers = { + "model": lambda: _model_short(model), + "context_pct": lambda: _context_pct(context_tokens, context_length), + # Skipped when the caller did not measure (None) or the value is negative. + "latency": lambda: _format_latency(turn_seconds) if turn_seconds is not None and turn_seconds >= 0 else "", + "cwd": lambda: _home_relative_cwd(cwd or _env_cwd()), + } + parts = [value for field in fields if (render := renderers.get(field)) and (value := render())] return _SEP.join(parts) @@ -164,15 +131,12 @@ def build_footer_line( cwd: Optional[str] = None, turn_seconds: Optional[float] = None, ) -> str: - """Top-level entry point used by gateway/run.py. + """Entry point for gateway/run.py: footer text, or "" when disabled / no data. + Callers append it to the final response themselves, preserving a single + blank line of separation. - Returns the footer text (empty string when disabled or no data). Callers - append this to the final response themselves, preserving a single blank - line of separation. - - ``turn_seconds`` is the wall-clock duration of the agent run, measured by - the caller with ``time.monotonic()``. Callers that don't measure it leave - it ``None`` and the ``latency`` field is skipped. + ``turn_seconds`` is the caller-measured (``time.monotonic()``) run duration; + ``None`` skips the ``latency`` field. """ cfg = resolve_footer_config(user_config, platform_key) if not cfg.get("enabled"): diff --git a/gateway/scale_to_zero.py b/gateway/scale_to_zero.py index e1a64159db..875e868b4d 100644 --- a/gateway/scale_to_zero.py +++ b/gateway/scale_to_zero.py @@ -1,39 +1,29 @@ -"""Scale-to-zero idle detection + dormant-quiesce for the gateway (Phase 0). +"""Scale-to-zero idle detection + dormant-quiesce for the gateway. -This is the gateway-side BEHAVIOUR layer that consumes the relay scale-to-zero -PRIMITIVES (gateway-gateway Phase 5: the buffered-flip, the durable per-instance -buffer, the wakeUrl poke, the reconnect supervisor). It owns the *decision* to go -idle, drives the relay transport's ``go_dormant()`` (D12), and then SUSPENDS the -machine itself through the local Fly Machines API socket. Wake stays platform-side: -autostart-on-wakeUrl (decisions.md Q3=C′). +Gateway-side behaviour layer over the relay scale-to-zero primitives: it owns +the *decision* to go idle, drives the relay transport's ``go_dormant()``, then +SUSPENDS the machine through the local Fly Machines API socket. Wake stays +platform-side (autostart-on-wakeUrl). -Why the gateway self-suspends instead of relying on ``autostop:"suspend"``: Fly -Proxy judges idle exclusively on INBOUND proxied connections — it cannot see an -in-flight agent turn (outbound-only LLM traffic) and there is no way for the app -to signal "not ready to suspend". Mid-2026 the proxy also stopped treating open -OUTBOUND sockets as activity, so the relay WebSocket no longer masks the race: -Fly would suspend a machine mid-job, and could suspend BEFORE ``go_dormant()`` -flipped the relay destination (the buffered-event black hole). Owning the suspend -call closes both: it only ever fires after the idle predicate (no running agents, -no live background work, inbound-quiet) holds AND the dormant quiesce completed. +Why the gateway self-suspends instead of relying on Fly ``autostop:"suspend"``: +Fly Proxy judges idle only on INBOUND proxied connections — it cannot see an +in-flight agent turn (outbound-only LLM traffic) and no longer counts open +outbound sockets, so it would suspend mid-job or BEFORE ``go_dormant()`` flipped +the relay destination (buffered-event black hole). Owning the call means it only +fires after the idle predicate holds AND the dormant quiesce completed. -Design constraints (decisions.md): - - Per-instance enable is gated SOLELY by the NAS "Labs" toggle, carried to the - gateway as the ``HERMES_SCALE_TO_ZERO`` env stamp (D11/Q8=A). NOT a user - config key; ``scale_to_zero.idle_timeout_minutes`` IS config.yaml (D2). - - Arm only when messaging is relay-only or absent (D1/F6) AND a wakeUrl is - registered (§3.4(1)) AND the flag is set. - - Idle = no in-flight agent turn AND no inbound for N min AND no live - background work (D2/D3/F7). - - The quiesce uses ``go_dormant()`` (socket closed + supervisor preserved), - NEVER the stop/restart drain or ``disconnect()`` (F12/F14). The process stays - alive; Fly freezes+resumes it. - - ``mark_resume_pending`` is deliberately NOT called here (D13 — suspend - preserves RAM; revive only if we move to autostop:"stop" or see kills). +Design constraints: + - Per-instance enable is gated SOLELY by the NAS "Labs" toggle, carried as + the ``HERMES_SCALE_TO_ZERO`` env stamp — not a config key; + ``scale_to_zero.idle_timeout_minutes`` IS config.yaml. + - Arm only when messaging is relay-only or absent AND a wakeUrl is registered + AND the flag is set. + - Idle = no in-flight work AND no inbound for N min AND no live background work. + - Quiesce uses ``go_dormant()`` (socket closed, supervisor preserved), NEVER + the stop/restart drain or ``disconnect()``. The process stays alive. + - ``mark_resume_pending`` is deliberately NOT called: suspend preserves RAM. -The pure helpers (``parse_idle_timeout_seconds``, ``scale_to_zero_enabled``, -``messaging_is_relay_only_or_absent``, ``is_idle``, ``should_arm``) take plain -inputs so they unit-test without a live gateway. +The pure helpers take plain inputs so they unit-test without a live gateway. """ from __future__ import annotations @@ -42,56 +32,59 @@ import json import logging import os import socket +import time +from pathlib import Path from typing import Any, Iterable, Optional logger = logging.getLogger(__name__) -# Env flag stamped by NAS when the scaleToZero Labs toggle is on (D11/Q8=A), -# mirroring how the `relay` feature stamps GATEWAY_RELAY_URL. Truthy values only. +# Env flag stamped by NAS when the scaleToZero Labs toggle is on. Truthy values only. SCALE_TO_ZERO_ENV = "HERMES_SCALE_TO_ZERO" -# Fly-injected machine identity (present on every Fly machine). Used by the -# self-suspend call; both must be present for self_suspend_available(). +# Fly-injected machine identity; both must be present for self_suspend_available(). FLY_APP_NAME_ENV = "FLY_APP_NAME" FLY_MACHINE_ID_ENV = "FLY_MACHINE_ID" -# The local flaps (Fly Machines API) unix socket, available inside every Fly -# machine. A POST to /v1/apps/{app}/machines/{id}/suspend snapshots RAM and -# suspends THIS machine — the Fly-endorsed replacement for proxy autostop when -# the app must own the idle decision (https://fly.io/docs/reference/suspend-resume/). +# Local flaps (Fly Machines API) unix socket. POST .../suspend snapshots RAM and +# suspends THIS machine (https://fly.io/docs/reference/suspend-resume/). FLY_API_SOCKET = "/.fly/api" - -# config.yaml default (D2). Behavioural setting -> config, not env. -# 2 minutes: with the gateway owning the suspend (idle predicate covers agent -# turns, cron, API runs, and background work; the relay drains + flips before -# the freeze), a short window is safe — real work always blocks the suspend and -# resume-from-suspend is sub-second, so the only cost of waking "too eagerly" -# after a quiet spell is a Fly-proxied poke away. Longer windows just bill idle -# RAM. Raise per-instance via gateway.scale_to_zero.idle_timeout_minutes. +# config.yaml default (behavioural setting -> config, not env). Short is safe +# because real work always blocks the suspend and resume is sub-second; longer +# windows just bill idle RAM. Raise per-instance via +# gateway.scale_to_zero.idle_timeout_minutes. DEFAULT_IDLE_TIMEOUT_MINUTES = 2 _TRUTHY = {"1", "true", "yes", "on"} +# Dashboard-client liveness marker. The dashboard process (tui_gateway/ws.py, a +# DIFFERENT process on hosted instances) touches this on every /api/ws connect +# and inbound frame (clients ping every 15s). The gateway folds the mtime into +# its inbound clock so an open client holds the box awake like a chat message +# does (and gets the same idle_timeout grace after it disconnects) — otherwise +# the box suspends under the client, whose reconnect re-pokes +# the wake URL and the instance flaps every ~60s. Deliberately NO staleness +# cutoff: the mtime is a real inbound timestamp and is_idle already decides +# whether it is recent enough. +DASHBOARD_CLIENT_HEARTBEAT_REL = os.path.join("state", "dashboard_clients.heartbeat") + + +def _env_str(env: Optional[dict], key: str) -> str: + return str((os.environ if env is None else env).get(key, "")).strip() + def scale_to_zero_enabled(environ: Optional[dict] = None) -> bool: - """Whether the per-instance Labs toggle is on (the HERMES_SCALE_TO_ZERO stamp). - - D11/Q8=A: this env flag is the SOLE per-instance enable signal reaching the - gateway. Absent/blank/falsey -> disabled (fail-safe default off). - """ - env = environ if environ is not None else os.environ - return str(env.get(SCALE_TO_ZERO_ENV, "")).strip().lower() in _TRUTHY + """Whether the Labs toggle stamp is set. Absent/blank/falsey -> disabled.""" + return _env_str(environ, SCALE_TO_ZERO_ENV).lower() in _TRUTHY def parse_idle_timeout_seconds( cfg_value: Any, default_minutes: int = DEFAULT_IDLE_TIMEOUT_MINUTES ) -> float: - """Coerce ``scale_to_zero.idle_timeout_minutes`` (config.yaml, D2) to seconds. + """Coerce ``scale_to_zero.idle_timeout_minutes`` to seconds. - Degrades to the default on any non-numeric / non-positive value (never raises, - never returns <= 0 — a zero/negative timeout would make the gateway go dormant - instantly, which is never the intent). + Any non-numeric / non-positive value degrades to the default (never <= 0: + that would make the gateway go dormant instantly). """ try: minutes = float(cfg_value) @@ -103,22 +96,18 @@ def parse_idle_timeout_seconds( def messaging_is_relay_only_or_absent(platforms: Iterable[Any]) -> bool: - """True iff the only connected messaging platform is RELAY, or there is none - (a Chronos-only / no-platform agent) — the F6/D1 structural precondition. + """True iff the only connected platform is RELAY, or there is none. - A directly-connected platform (Discord/Telegram/Slack/...) holds a live - socket and cannot scale to zero, so its presence disarms the feature. We - compare by the platform's ``.value``/name to avoid importing the enum here - (keeps this module import-light and unit-testable). + A directly-connected platform holds a live socket and cannot scale to zero. + Compared by ``.value``/name so this module stays enum-import-free. """ names = {_platform_name(p) for p in platforms} names.discard("relay") - return len(names) == 0 + return not names def _platform_name(platform: Any) -> str: - value = getattr(platform, "value", platform) - return str(value).strip().lower() + return str(getattr(platform, "value", platform)).strip().lower() def should_arm( @@ -127,13 +116,10 @@ def should_arm( relay_only_or_absent: bool, wake_url: Optional[str], ) -> bool: - """Whether to start the idle watcher at all (D1/D11/§3.4(1)). - - ALL must hold: the Labs flag is on, messaging is relay-only/absent, and a - wakeUrl is registered (a suspended instance with no reachable wake target is - a black hole — §3.4(1)). Any unmet -> the watcher never starts (no idle - timer, no dormancy), so a non-opted instance behaves exactly as today. - """ + """Start the idle watcher only if ALL hold: flag on, relay-only/absent messaging, + wakeUrl registered (a suspended instance with no wake target is a black hole). + Any unmet -> the watcher never starts (no idle timer, no dormancy), so a + non-opted instance behaves exactly as before.""" return bool(enabled) and bool(relay_only_or_absent) and bool(wake_url) @@ -144,48 +130,22 @@ def is_idle( idle_timeout_seconds: float, has_live_background_work: bool, ) -> bool: - """The idle predicate (D2/D3/F7). Pure — composes the three conjuncts. + """The pure idle predicate: no counted active work, no inbound within the + timeout window, and no live background work (backgrounded delegate_task / + kanban / bg terminal) — suspending mid-flight would lose it. - Idle iff: no counted active work (in-flight agent turns + cron jobs + - API-server runs — the caller aggregates every foreground work source), - no inbound within the timeout window, and no live background work - (backgrounded delegate_task / kanban / bg terminal). Any active work - keeps the gateway awake — suspending mid-flight would lose it. - - ``active_work_count`` deliberately names the BROAD aggregate, not just - agents: a caller passing only ``len(_running_agents)`` reopens the - mid-cron-job suspend hole. Callers that cannot read a work source must - fail AWAKE (pass a positive sentinel), never fail to 0. + ``active_work_count`` is the BROAD aggregate (agent turns + cron + API runs), + not just agents — passing only ``len(_running_agents)`` reopens the + mid-cron-job suspend hole. Callers that cannot read a work source must fail + AWAKE (pass a positive sentinel), never fail to 0. """ - if active_work_count > 0: - return False - if has_live_background_work: + if active_work_count > 0 or has_live_background_work: return False return seconds_since_last_inbound >= idle_timeout_seconds -# Dashboard-client liveness marker. The dashboard process (tui_gateway/ws.py, -# a DIFFERENT process from the gateway on hosted instances) touches this file -# on every /api/ws connect and inbound frame — the desktop app, web dashboard -# and TUI all send `gateway.ping` every 15s (apps/shared/src/json-rpc-gateway.ts, -# ui-tui/src/gatewayClient.ts). The gateway folds the mtime into its inbound -# clock, so an open client holds the box awake exactly like a chat message does -# and gets the same idle_timeout grace after it disconnects. Without this the -# box suspends under the open client, the client's reconnect loop re-pokes the -# Fly-proxied hostname, autostart resumes it, and the instance flaps every ~60s -# (13 of 72 active opted-in prod instances, 2026-09-02). -# -# There is deliberately NO staleness cutoff here: the mtime is a timestamp of -# real inbound, and is_idle already decides whether it is recent enough. A -# lingering marker cannot pin the box — once it is older than idle_timeout it -# no longer counts, exactly like an old _last_inbound_at. -DASHBOARD_CLIENT_HEARTBEAT_REL = os.path.join("state", "dashboard_clients.heartbeat") - - def dashboard_client_heartbeat_path(hermes_home: Optional[os.PathLike | str] = None): """Path of the dashboard-client liveness marker under HERMES_HOME.""" - from pathlib import Path - if hermes_home is None: from hermes_constants import get_hermes_home @@ -214,18 +174,15 @@ def dashboard_client_last_seen( ) -> Optional[float]: """Epoch seconds a dashboard client last sent a WS frame, or None if never. - Missing marker -> None (the steady state on a box nobody has the dashboard - open on — NOT fail-awake, or every instance would never sleep). An - unreadable marker -> ``now`` (fail-awake: an unreadable source counts as - activity, same rule as the work counters in ``is_idle``). + Missing marker -> None (steady state when nobody has the dashboard open — + NOT fail-awake, or no instance would ever sleep). Unreadable marker -> + ``now`` (fail-awake, same rule as the work counters in ``is_idle``). """ - import time - current = time.time() if now is None else now p = dashboard_client_heartbeat_path() if path is None else path try: - # Clamp to now: a wall-clock step-back (NTP) can leave the mtime in the - # future, which would push idle out by the step size for no reason. + # Clamp to now: an NTP step-back can leave the mtime in the future, + # which would push idle out by the step size for no reason. return min(os.stat(p).st_mtime, current) except FileNotFoundError: return None @@ -234,17 +191,14 @@ def dashboard_client_last_seen( def self_suspend_available(environ: Optional[dict] = None) -> bool: - """Whether this process can suspend its own machine via the flaps socket. + """True iff Fly machine identity is present AND the local Machines API socket exists. - True iff the Fly-injected machine identity is present AND the local Machines - API socket exists. Off-Fly (local dev, Azure ACA, tests) this is False and - the watcher skips the quiesce entirely: the platform owns the freeze, so - the gateway stays connected until it lands. + Off-Fly (local dev, other clouds, tests) the watcher skips the quiesce: + the platform owns the freeze, so the gateway stays connected until it lands. """ - env = environ if environ is not None else os.environ return bool( - str(env.get(FLY_APP_NAME_ENV, "")).strip() - and str(env.get(FLY_MACHINE_ID_ENV, "")).strip() + _env_str(environ, FLY_APP_NAME_ENV) + and _env_str(environ, FLY_MACHINE_ID_ENV) and os.path.exists(FLY_API_SOCKET) ) @@ -257,24 +211,14 @@ def suspend_self( ) -> bool: """POST /v1/apps/{app}/machines/{id}/suspend on the local flaps socket. - Fly's in-machine Machines API needs no token — the socket itself is the - credential. Equivalent to: - curl --unix-socket /.fly/api -X POST \\ - http://flaps/v1/apps/$FLY_APP_NAME/machines/$FLY_MACHINE_ID/suspend - - Returns True when flaps accepted the request (2xx). The caller should treat - this as fire-and-forget: on success the kernel freezes this process shortly - after, so there may be nothing meaningful to run afterwards. Never raises — - a failed suspend just leaves the machine running (fail-awake, never - fail-frozen), which costs money but loses no work. - - stdlib-only on purpose: a plain unix-socket HTTP/1.1 request, no httpx/ - requests dependency in the hot path and no async plumbing to freeze - mid-await. + No token needed — the socket is the credential. Returns True when flaps + accepted (2xx); on success the kernel freezes this process shortly after, + so treat as fire-and-forget. Never raises: a failed suspend just leaves the + machine running (fail-awake, never fail-frozen). stdlib-only on purpose — + a plain unix-socket HTTP/1.1 request, no async plumbing to freeze mid-await. """ - env = environ if environ is not None else os.environ - app = str(env.get(FLY_APP_NAME_ENV, "")).strip() - machine_id = str(env.get(FLY_MACHINE_ID_ENV, "")).strip() + app = _env_str(environ, FLY_APP_NAME_ENV) + machine_id = _env_str(environ, FLY_MACHINE_ID_ENV) if not app or not machine_id: logger.warning("scale-to-zero: suspend_self called without Fly machine identity") return False diff --git a/gateway/session_context.py b/gateway/session_context.py index 9a6a8c4226..a09fabdf00 100644 --- a/gateway/session_context.py +++ b/gateway/session_context.py @@ -1,198 +1,152 @@ -""" -Session-scoped context variables for the Hermes gateway. +"""Session-scoped context variables for the Hermes gateway. -Replaces the previous ``os.environ``-based session state -(``HERMES_SESSION_PLATFORM``, ``HERMES_SESSION_CHAT_ID``, etc.) with -Python's ``contextvars.ContextVar``. +Replaces the old ``os.environ``-based session state (``HERMES_SESSION_*``) +with ``contextvars.ContextVar``. The gateway processes messages concurrently +via asyncio; ``os.environ`` is process-global, so message B silently +overwrote message A's thread id before A's agent finished and notifications +routed to the wrong thread. ContextVar values are task-local (inherited by +``run_in_executor`` threads), so concurrent messages never interfere. -**Why this matters** - -The gateway processes messages concurrently via ``asyncio``. When two -messages arrive at the same time the old code did: - - os.environ["HERMES_SESSION_THREAD_ID"] = str(context.source.thread_id) - -Because ``os.environ`` is *process-global*, Message A's value was -silently overwritten by Message B before Message A's agent finished -running. Background-task notifications and tool calls therefore routed -to the wrong thread. - -``contextvars.ContextVar`` values are *task-local*: each ``asyncio`` -task (and any ``run_in_executor`` thread it spawns) gets its own copy, -so concurrent messages never interfere. - -**Backward compatibility** - -The public helper ``get_session_env(name, default="")`` mirrors the old -``os.getenv("HERMES_SESSION_*", ...)`` calls. Existing tool code only -needs to replace the import + call site: - - # before - import os - platform = os.getenv("HERMES_SESSION_PLATFORM", "") - - # after - from gateway.session_context import get_session_env - platform = get_session_env("HERMES_SESSION_PLATFORM", "") +``get_session_env(name, default="")`` is a drop-in for +``os.getenv("HERMES_SESSION_*", default)`` at existing tool call sites. """ from contextlib import contextmanager from contextvars import ContextVar from typing import Any, Iterator -# Sentinel to distinguish "never set in this context" from "explicitly set to empty". -# When a contextvar holds _UNSET, we fall back to os.environ (CLI/cron compat). -# When it holds "" (after clear_session_vars resets it), we return "" — no fallback. +# Distinguishes "never set in this context" (fall back to os.environ for +# CLI/cron compat) from "explicitly set to empty" by clear_session_vars (no fallback). _UNSET: Any = object() -# Process-level flag: has any code in this process bound a session via -# set_session_vars()? Concurrent multi-session hosts (the messaging gateway, the -# ACP adapter, the API server, the TUI, cron) all do; a pure single-process -# CLI/one-shot that never engages the session-context system does not. -# -# The subprocess-env bridge (tools/environments/local.py) reads this to choose -# its leak policy: when engaged, the ContextVars are authoritative and an _UNSET -# var means "no session bound in THIS task" — so a process-global os.environ -# mirror (written last-writer-wins by whatever concurrent session ran most -# recently) must NOT be inherited into a child process. When never engaged, the -# os.environ fallback is preserved (no concurrency to leak across). Monotonic -# latch — once any host binds a session, the process stays engaged for life. +# Process-level monotonic latch: has any code in this process bound a session via +# set_session_vars()? Concurrent multi-session hosts (gateway, ACP, API server, +# TUI, cron) do; a pure single-process CLI/one-shot does not. The subprocess-env +# bridge (tools/environments/local.py) reads this to pick its leak policy: when +# engaged, the ContextVars are authoritative and an _UNSET var means "no session +# bound in THIS task", so the last-writer-wins os.environ mirror must NOT be +# inherited by a child process. When never engaged, the os.environ fallback is +# preserved (no concurrency to leak across). _session_context_engaged: bool = False def session_context_engaged() -> bool: - """True if any session has been bound via set_session_vars in this process. - - See the ``_session_context_engaged`` comment for the leak-policy rationale. - """ + """True if any session has been bound via set_session_vars in this process.""" return _session_context_engaged -# --------------------------------------------------------------------------- -# Per-task session variables -# --------------------------------------------------------------------------- -_SESSION_PLATFORM: ContextVar = ContextVar("HERMES_SESSION_PLATFORM", default=_UNSET) -_SESSION_SOURCE: ContextVar = ContextVar("HERMES_SESSION_SOURCE", default=_UNSET) -_SESSION_CHAT_ID: ContextVar = ContextVar("HERMES_SESSION_CHAT_ID", default=_UNSET) -_SESSION_CHAT_TYPE: ContextVar = ContextVar("HERMES_SESSION_CHAT_TYPE", default=_UNSET) -_SESSION_CHAT_NAME: ContextVar = ContextVar("HERMES_SESSION_CHAT_NAME", default=_UNSET) -_SESSION_THREAD_ID: ContextVar = ContextVar("HERMES_SESSION_THREAD_ID", default=_UNSET) -_SESSION_USER_ID: ContextVar = ContextVar("HERMES_SESSION_USER_ID", default=_UNSET) -_SESSION_USER_ID_ALT: ContextVar = ContextVar("HERMES_SESSION_USER_ID_ALT", default=_UNSET) -_SESSION_USER_NAME: ContextVar = ContextVar("HERMES_SESSION_USER_NAME", default=_UNSET) -# Platform-neutral scope discriminator (Discord guild / Slack workspace / -# Matrix server) of the originating chat. Captured at session-bind time so -# async producers (delegate_task background=True, terminal watchers) can -# persist a completion's full routing origin — on a relay-fronted deployment -# the connector's fail-closed egress guard needs scope_id (or a user binding) -# to resolve the tenant for a scoped reply after a restart. -_SESSION_SCOPE_ID: ContextVar = ContextVar("HERMES_SESSION_SCOPE_ID", default=_UNSET) -_SESSION_KEY: ContextVar = ContextVar("HERMES_SESSION_KEY", default=_UNSET) -_SESSION_ID: ContextVar = ContextVar("HERMES_SESSION_ID", default=_UNSET) -# In-process UI session/window id for multi-session desktop/TUI hosts. This is -# intentionally separate from HERMES_SESSION_ID: the latter is the durable -# conversation/session-db id, while the UI id is the live frontend tab/window -# that commissioned a detached completion. Background completions use it as a -# precise return address so a stale/rotated durable session key cannot be -# consumed by whichever desktop poller wakes first. -_SESSION_UI_SESSION_ID: ContextVar = ContextVar("HERMES_UI_SESSION_ID", default=_UNSET) -# ID of the message that triggered the current turn. Used as a reply anchor -# so background-process notifications stay inside the originating Telegram -# private-chat topic (those lanes route only with thread id + reply anchor). -_SESSION_MESSAGE_ID: ContextVar = ContextVar("HERMES_SESSION_MESSAGE_ID", default=_UNSET) +def _var(name: str) -> ContextVar: + return ContextVar(name, default=_UNSET) -_SESSION_PROFILE: ContextVar = ContextVar("HERMES_SESSION_PROFILE", default=_UNSET) -_BROWSER_CONTROL_PRINCIPAL: ContextVar = ContextVar( - "HERMES_BROWSER_CONTROL_PRINCIPAL", default=_UNSET -) -_BROWSER_CONTROL_TRANSPORT_FAMILY: ContextVar = ContextVar( - "HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY", default=_UNSET + +# --- Per-task session variables -------------------------------------------- +_SESSION_PLATFORM = _var("HERMES_SESSION_PLATFORM") +_SESSION_SOURCE = _var("HERMES_SESSION_SOURCE") +_SESSION_CHAT_ID = _var("HERMES_SESSION_CHAT_ID") +_SESSION_CHAT_TYPE = _var("HERMES_SESSION_CHAT_TYPE") +_SESSION_CHAT_NAME = _var("HERMES_SESSION_CHAT_NAME") +_SESSION_THREAD_ID = _var("HERMES_SESSION_THREAD_ID") +_SESSION_USER_ID = _var("HERMES_SESSION_USER_ID") +_SESSION_USER_ID_ALT = _var("HERMES_SESSION_USER_ID_ALT") +_SESSION_USER_NAME = _var("HERMES_SESSION_USER_NAME") +# Platform-neutral scope discriminator (Discord guild / Slack workspace / Matrix +# server). Captured at bind time so async producers (delegate_task +# background=True, terminal watchers) can persist a completion's full routing +# origin: a relay connector's fail-closed egress guard needs scope_id (or a user +# binding) to resolve the tenant for a scoped reply after a restart. +_SESSION_SCOPE_ID = _var("HERMES_SESSION_SCOPE_ID") +_SESSION_KEY = _var("HERMES_SESSION_KEY") +_SESSION_ID = _var("HERMES_SESSION_ID") +# In-process UI tab/window id for multi-session desktop/TUI hosts — deliberately +# separate from the durable HERMES_SESSION_ID. Background completions use it as +# a precise return address so a stale/rotated durable key cannot be consumed by +# whichever desktop poller wakes first. +_SESSION_UI_SESSION_ID = _var("HERMES_UI_SESSION_ID") +# Triggering message id: reply anchor so background notifications stay inside +# the originating Telegram private-chat topic (routes only with thread id + anchor). +_SESSION_MESSAGE_ID = _var("HERMES_SESSION_MESSAGE_ID") +_SESSION_PROFILE = _var("HERMES_SESSION_PROFILE") +_BROWSER_CONTROL_PRINCIPAL = _var("HERMES_BROWSER_CONTROL_PRINCIPAL") +_BROWSER_CONTROL_TRANSPORT_FAMILY = _var("HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY") +# Per-session cron marker, tri-state: _UNSET keeps the legacy env fallback for +# CLI/tests; "1" marks cron; "" explicitly marks non-cron and masks leaked env. +_CRON_SESSION = _var("HERMES_CRON_SESSION") + +# Whether this session's channel can route an ASYNC completion back to the agent +# AFTER the turn ends (wake a fresh turn). True for long-lived CLI sessions and +# real gateway platforms (persistent outbound channel + watcher/drain loops); +# False for finite runtimes that may exit before a detached completion returns +# (stateless API-server requests, dispatcher-spawned Kanban workers). Tools that +# promise async delivery (terminal notify_on_complete / watch_patterns, +# delegate_task background=True) read ``async_delivery_supported()`` and refuse a +# promise the channel can't keep. Default _UNSET => supported, so the CLI (never +# sets a platform) and contextvar-unaware paths keep working; stateless adapters +# opt OUT via ``supports_async_delivery = False`` on the adapter class, which the +# gateway propagates here at session-bind time. +_SESSION_ASYNC_DELIVERY = _var("HERMES_SESSION_ASYNC_DELIVERY") + +# Cron auto-delivery vars — set per-job in run_job() so concurrent jobs don't +# clobber each other's delivery targets. +_CRON_AUTO_DELIVER_PLATFORM = _var("HERMES_CRON_AUTO_DELIVER_PLATFORM") +_CRON_AUTO_DELIVER_CHAT_ID = _var("HERMES_CRON_AUTO_DELIVER_CHAT_ID") +_CRON_AUTO_DELIVER_THREAD_ID = _var("HERMES_CRON_AUTO_DELIVER_THREAD_ID") + +# Vars bound by set_session_vars / cleared to "" by clear_session_vars, in order. +_SESSION_VARS = ( + _SESSION_PLATFORM, + _SESSION_SOURCE, + _SESSION_CHAT_ID, + _SESSION_CHAT_TYPE, + _SESSION_CHAT_NAME, + _SESSION_THREAD_ID, + _SESSION_USER_ID, + _SESSION_USER_ID_ALT, + _SESSION_USER_NAME, + _SESSION_SCOPE_ID, + _SESSION_KEY, + _SESSION_ID, + _SESSION_UI_SESSION_ID, + _SESSION_MESSAGE_ID, + _SESSION_PROFILE, + _BROWSER_CONTROL_PRINCIPAL, + _BROWSER_CONTROL_TRANSPORT_FAMILY, + _CRON_SESSION, ) -# Per-session cron marker. Unlike the process-global legacy env var, this is -# scoped to one cron job / inbound session. _UNSET preserves the legacy env -# fallback for CLI/tests; "1" marks cron; "" explicitly marks non-cron and -# masks any leaked process env value. -_CRON_SESSION: ContextVar = ContextVar("HERMES_CRON_SESSION", default=_UNSET) - -# Whether the current session's delivery channel can route an ASYNC completion -# back to the agent AFTER the current turn ends (i.e. wake a fresh turn). -# -# True — long-lived CLI sessions (in-process completion_queue drain) and the -# real gateway platforms (Telegram/Discord/Slack/...), which hold a -# persistent outbound channel and run the watcher/drain loops. -# False — finite runtimes that can end before a detached completion returns: -# stateless API-server requests and dispatcher-spawned Kanban workers. -# -# Tools that promise async delivery (terminal notify_on_complete / -# watch_patterns, delegate_task background=True) read this via -# ``async_delivery_supported()`` and refuse to hand out a promise the channel -# can't keep — turning a silent no-op into an explicit contract. -# -# Default _UNSET => treated as supported, so CLI (which never sets a platform) -# and any contextvar-unaware path keep working. Stateless adapters opt OUT by -# setting ``supports_async_delivery = False`` on the adapter class; the gateway -# propagates that into this contextvar at session-bind time. -_SESSION_ASYNC_DELIVERY: ContextVar = ContextVar("HERMES_SESSION_ASYNC_DELIVERY", default=_UNSET) - -# Cron auto-delivery vars — set per-job in run_job() so concurrent jobs -# don't clobber each other's delivery targets. -_CRON_AUTO_DELIVER_PLATFORM: ContextVar = ContextVar("HERMES_CRON_AUTO_DELIVER_PLATFORM", default=_UNSET) -_CRON_AUTO_DELIVER_CHAT_ID: ContextVar = ContextVar("HERMES_CRON_AUTO_DELIVER_CHAT_ID", default=_UNSET) -_CRON_AUTO_DELIVER_THREAD_ID: ContextVar = ContextVar("HERMES_CRON_AUTO_DELIVER_THREAD_ID", default=_UNSET) - +# Legacy env-var name -> ContextVar, for get_session_env. _SESSION_ASYNC_DELIVERY +# is deliberately absent: it is a bool capability read via async_delivery_supported. _VAR_MAP = { - "HERMES_SESSION_PLATFORM": _SESSION_PLATFORM, - "HERMES_SESSION_SOURCE": _SESSION_SOURCE, - "HERMES_SESSION_CHAT_ID": _SESSION_CHAT_ID, - "HERMES_SESSION_CHAT_TYPE": _SESSION_CHAT_TYPE, - "HERMES_SESSION_CHAT_NAME": _SESSION_CHAT_NAME, - "HERMES_SESSION_THREAD_ID": _SESSION_THREAD_ID, - "HERMES_SESSION_USER_ID": _SESSION_USER_ID, - "HERMES_SESSION_USER_ID_ALT": _SESSION_USER_ID_ALT, - "HERMES_SESSION_USER_NAME": _SESSION_USER_NAME, - "HERMES_SESSION_SCOPE_ID": _SESSION_SCOPE_ID, - "HERMES_SESSION_KEY": _SESSION_KEY, - "HERMES_SESSION_ID": _SESSION_ID, - "HERMES_UI_SESSION_ID": _SESSION_UI_SESSION_ID, - "HERMES_SESSION_MESSAGE_ID": _SESSION_MESSAGE_ID, - "HERMES_SESSION_PROFILE": _SESSION_PROFILE, - "HERMES_BROWSER_CONTROL_PRINCIPAL": _BROWSER_CONTROL_PRINCIPAL, - "HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY": _BROWSER_CONTROL_TRANSPORT_FAMILY, - "HERMES_CRON_SESSION": _CRON_SESSION, - "HERMES_CRON_AUTO_DELIVER_PLATFORM": _CRON_AUTO_DELIVER_PLATFORM, - "HERMES_CRON_AUTO_DELIVER_CHAT_ID": _CRON_AUTO_DELIVER_CHAT_ID, - "HERMES_CRON_AUTO_DELIVER_THREAD_ID": _CRON_AUTO_DELIVER_THREAD_ID, + var.name: var + for var in (*_SESSION_VARS, _CRON_AUTO_DELIVER_PLATFORM, _CRON_AUTO_DELIVER_CHAT_ID, _CRON_AUTO_DELIVER_THREAD_ID) } +def _clear_session_cwd() -> None: + try: + from agent.runtime_cwd import clear_session_cwd + + clear_session_cwd() + except Exception: + pass + + def set_current_session_id(session_id: str) -> None: """Synchronize ``HERMES_SESSION_ID`` across ContextVar and ``os.environ``. - Long-lived single-process entrypoints like the CLI can rotate sessions via - ``/new``, ``/resume``, ``/branch``, or compression splits without - reconstructing the entire agent. Tools still consult - ``get_session_env("HERMES_SESSION_ID")`` with an ``os.environ`` fallback, - so both storage paths must move together when the active session changes. + Long-lived single-process entrypoints (CLI) rotate sessions via /new, + /resume, /branch, or compression splits without rebuilding the agent; tools + read ``get_session_env("HERMES_SESSION_ID")`` with an os.environ fallback, + so both stores must move together. - Delegated subagent children are the exception: they are constructed inside - the parent process within ``delegated_child_context()``, and their - ``AIAgent.__init__`` calls this same helper. Writing a child's internal - session id to ``os.environ`` (process-global) would clobber the parent's - ``HERMES_SESSION_ID`` for the rest of the process — leaking the child id - into parent tools and subprocesses spawned after the child was built. The - ContextVar write below is task-local and safe for concurrent children; only - the process-global ``os.environ`` mirror is suppressed for delegated - children. Root agents (CLI, gateway, cron) keep both paths. + Delegated subagent children are the exception: they are built in the parent + process inside ``delegated_child_context()`` and their ``AIAgent.__init__`` + calls this helper. Writing the child's id to process-global ``os.environ`` + would clobber the parent's id for the rest of the process, so only the + task-local ContextVar write happens for them. Root agents keep both paths. """ import os _SESSION_ID.set(session_id) - - # Skip the process-global os.environ write for delegated children. The - # child's own tools and subprocesses still resolve their id through the - # ContextVar (task-local), while the parent's process-wide env keeps the - # parent's session identity. See HermesPRDelegationSessionContext task. try: from agent.delegation_context import is_delegated_child_context @@ -200,7 +154,6 @@ def set_current_session_id(session_id: str) -> None: return except Exception: pass - os.environ["HERMES_SESSION_ID"] = session_id @@ -208,9 +161,9 @@ def set_current_session_id(session_id: str) -> None: def scoped_current_session_id(session_id: str | None = None) -> Iterator[None]: """Bind a task-local session id and restore the prior value on exit. - With ``session_id=None`` this acts as a save/restore boundary around code - that may call :func:`set_current_session_id` itself (notably delegated - ``AIAgent`` construction). It intentionally never mutates ``os.environ``. + ``session_id=None`` makes this a pure save/restore boundary around code that + may call :func:`set_current_session_id` itself (delegated ``AIAgent`` + construction). Never mutates ``os.environ``. """ previous = _SESSION_ID.get() if session_id is not None: @@ -245,49 +198,26 @@ def set_session_vars( ) -> list: """Set all session context variables and return reset tokens. - Call ``clear_session_vars(tokens)`` in a ``finally`` block when the handler - exits. Note ``clear_session_vars`` resets every var to ``""`` (to suppress - the ``os.environ`` fallback) rather than restoring prior values — these - helpers are not nestable/stack-safe, and the returned tokens are accepted - only for API compatibility. + Call ``clear_session_vars(tokens)`` in a ``finally`` when the handler exits. + These helpers are not nestable: clearing resets every var to ``""`` rather + than restoring prior values, and the tokens are accepted only for API compat. - ``cwd`` pins the logical working directory for this context. - - ``async_delivery`` declares whether this session's channel can route a - background completion back to the agent after the turn ends (see - ``_SESSION_ASYNC_DELIVERY`` / ``async_delivery_supported``). Stateless - request/response adapters (the API server) pass ``False``. - - ``cron_session`` is tri-state: ``_UNSET`` preserves legacy - ``os.environ["HERMES_CRON_SESSION"]`` fallback, ``"1"`` marks a cron job, - and ``""`` explicitly marks a non-cron session while masking leaked env. + ``cwd`` pins the logical working directory. ``async_delivery`` declares + whether the channel can route a background completion back after the turn + (stateless adapters such as the API server pass ``False``). ``cron_session`` + is tri-state; see ``_CRON_SESSION``. """ - # Mark the session-context machinery engaged for this process. The - # subprocess-env bridge uses this to switch from "os.environ fallback" to - # "ContextVar-authoritative, strip on _UNSET" — see session_context_engaged. + # Latch the process as engaged — see _session_context_engaged. global _session_context_engaged _session_context_engaged = True - tokens = [ - _SESSION_PLATFORM.set(platform), - _SESSION_SOURCE.set(source), - _SESSION_CHAT_ID.set(chat_id), - _SESSION_CHAT_TYPE.set(chat_type), - _SESSION_CHAT_NAME.set(chat_name), - _SESSION_THREAD_ID.set(thread_id), - _SESSION_USER_ID.set(user_id), - _SESSION_USER_ID_ALT.set(user_id_alt), - _SESSION_USER_NAME.set(user_name), - _SESSION_SCOPE_ID.set(scope_id), - _SESSION_KEY.set(session_key), - _SESSION_ID.set(session_id), - _SESSION_UI_SESSION_ID.set(ui_session_id), - _SESSION_MESSAGE_ID.set(message_id), - _SESSION_PROFILE.set(profile), - _BROWSER_CONTROL_PRINCIPAL.set(browser_control_principal), - _BROWSER_CONTROL_TRANSPORT_FAMILY.set(browser_control_transport_family), - _CRON_SESSION.set(cron_session), - _SESSION_ASYNC_DELIVERY.set(bool(async_delivery)), - ] + values = ( + platform, source, chat_id, chat_type, chat_name, thread_id, user_id, + user_id_alt, user_name, scope_id, session_key, session_id, ui_session_id, + message_id, profile, browser_control_principal, + browser_control_transport_family, cron_session, + ) + tokens = [var.set(value) for var, value in zip(_SESSION_VARS, values)] + tokens.append(_SESSION_ASYNC_DELIVERY.set(bool(async_delivery))) try: from agent.runtime_cwd import set_session_cwd @@ -300,133 +230,68 @@ def set_session_vars( def clear_session_vars(tokens: list) -> None: """Mark session context variables as explicitly cleared. - Sets all variables to ``""`` so that ``get_session_env`` returns an empty - string instead of falling back to (potentially stale) ``os.environ`` - values. The *tokens* argument is accepted for API compatibility with - callers that saved the return value of ``set_session_vars``, but the - actual clearing uses ``var.set("")`` rather than ``var.reset(token)`` - to ensure the "explicitly cleared" state is distinguishable from - "never set" (which holds the ``_UNSET`` sentinel). + Sets every var to ``""`` (not ``var.reset(token)``) so ``get_session_env`` + returns empty instead of falling back to stale ``os.environ`` values while + staying distinguishable from "never set" (``_UNSET``). Async-delivery is + reset to ``_UNSET`` rather than a falsy value: a cleared context must fall + back to default-supported, not look like an opted-out stateless adapter. """ - for var in ( - _SESSION_PLATFORM, - _SESSION_SOURCE, - _SESSION_CHAT_ID, - _SESSION_CHAT_TYPE, - _SESSION_CHAT_NAME, - _SESSION_THREAD_ID, - _SESSION_USER_ID, - _SESSION_USER_ID_ALT, - _SESSION_USER_NAME, - _SESSION_SCOPE_ID, - _SESSION_KEY, - _SESSION_ID, - _SESSION_UI_SESSION_ID, - _SESSION_MESSAGE_ID, - _SESSION_PROFILE, - _BROWSER_CONTROL_PRINCIPAL, - _BROWSER_CONTROL_TRANSPORT_FAMILY, - _CRON_SESSION, - ): + for var in _SESSION_VARS: var.set("") - # Reset async-delivery capability to the "never set" sentinel rather than a - # falsy value: a cleared context should fall back to the default-supported - # behavior (CLI / unaware paths), not be mistaken for an opted-out - # stateless adapter. _SESSION_ASYNC_DELIVERY.set(_UNSET) - try: - from agent.runtime_cwd import clear_session_cwd - - clear_session_cwd() - except Exception: - pass + _clear_session_cwd() def reset_session_vars() -> None: """Reset every session context variable to ``_UNSET`` for THIS context. - Distinct from :func:`clear_session_vars`, which sets the vars to ``""`` - ("explicitly cleared" — suppresses the os.environ fallback and is used when - a handler *finishes*). This helper restores the ``_UNSET`` sentinel - ("never bound in this context"), which is what a freshly-spawned task should - look like *before* it binds its own session. + Unlike :func:`clear_session_vars` (``""`` = "explicitly cleared", used when a + handler *finishes*), this restores "never bound here" — what a freshly + spawned task should look like *before* binding its own session. - 🔴 Why this exists — the cross-session ContextVar inheritance leak. - Each gateway message is processed in its own ``asyncio`` task, created via - ``create_task`` (which snapshots the *current* context with - ``copy_context``). When message B's task is spawned from a context where a - concurrent message A had already called :func:`set_session_vars`, B inherits - A's **set** ContextVars. Until B calls its own ``set_session_vars`` there is - a window where any subprocess B spawns (e.g. a tool shelling out) reads - *A's* ``HERMES_SESSION_*`` identity via the subprocess-env bridge. The - bridge's ``_UNSET``-strip guard cannot help: the vars are not ``_UNSET``, - they are set-to-A. Calling ``reset_session_vars`` at the top of the - per-message handler drops the inherited identity so the window strips safe - (no session) instead of leaking the foreign one; the handler then binds its - own via ``set_session_vars`` a few steps later. See + Why: ``create_task`` snapshots the current context, so message B's task can + inherit message A's already-**set** vars. Until B binds its own session, + any subprocess it spawns reads A's identity through the subprocess-env + bridge — whose _UNSET-strip guard cannot help because the vars are set-to-A. + Calling this at the top of the per-message handler makes that window strip + safe (no session) instead of leaking the foreign one. See tests/tools/test_local_env_session_leak.py and tests/gateway/test_session_context_inheritance.py. - Note ``_SESSION_ASYNC_DELIVERY`` lives outside ``_VAR_MAP`` (it is a bool - capability flag read via :func:`async_delivery_supported`, not a string - ``HERMES_SESSION_*`` env var read via :func:`get_session_env`), so it is - reset explicitly below. Without it, a task spawned from a context where a - sibling adapter bound ``async_delivery=False`` (the stateless API server) - inherits that ``False`` through the pre-bind window, and - ``async_delivery_supported`` wrongly reports the new turn's channel as - unable to route a background completion until ``set_session_vars`` runs. + ``_SESSION_ASYNC_DELIVERY`` is reset explicitly (it lives outside + ``_VAR_MAP``): otherwise a task spawned from a context where a sibling + adapter bound ``async_delivery=False`` inherits that ``False`` through the + pre-bind window and misreports the new channel as unable to deliver. """ for var in _VAR_MAP.values(): var.set(_UNSET) - # Reset the async-delivery capability to "never bound here" (_UNSET) for the - # same inheritance-leak reason as the mapped vars above — see clear_session_vars, - # which resets this var on the handler-exit path for the symmetric concern. _SESSION_ASYNC_DELIVERY.set(_UNSET) - try: - from agent.runtime_cwd import clear_session_cwd - - clear_session_cwd() - except Exception: - pass + _clear_session_cwd() def get_session_env(name: str, default: str = "") -> str: """Read a session context variable by its legacy ``HERMES_SESSION_*`` name. - Drop-in replacement for ``os.getenv("HERMES_SESSION_*", default)``. - - Resolution order: - 1. Context variable (set by the gateway for concurrency-safe access). - If the variable was explicitly set (even to ``""``) via - ``set_session_vars`` or ``clear_session_vars``, that value is - returned — **no fallback to os.environ**. - 2. ``os.environ`` (only when the context variable was never set in - this context — i.e. CLI, cron scheduler, and test processes that - don't use ``set_session_vars`` at all). - 3. *default* + Drop-in for ``os.getenv(name, default)``. Resolution: the ContextVar if it + was ever set in this context (even to ``""`` — no fallback); else + ``os.environ`` (CLI, cron scheduler, tests that never bind); else *default*. """ import os var = _VAR_MAP.get(name) - if var is not None: - value = var.get() - if value is not _UNSET: - return value - # Fall back to os.environ for CLI, cron, and test compatibility + if var is not None and (value := var.get()) is not _UNSET: + return value return os.getenv(name, default) -# Surfaces that are not a human chat channel. The gateway binds a platform -# value (``telegram``) to HERMES_SESSION_PLATFORM, while the CLI, TUI, and -# desktop bind HERMES_SESSION_SOURCE (``cli``, ``tui``, ``desktop``) and leave -# the platform empty — so both have to be consulted. ``local``, ``api_server``, -# ``webhook``, and ``msgraph_webhook`` are real Platform values that reach -# HERMES_SESSION_PLATFORM but have no attachment channel behind them. -# Default-deny: an unrecognized identity counts as messaging so a newly added -# chat platform is never treated as a private surface before this set is -# updated. Mirrors LOCAL_SESSION_SOURCE_IDS in -# apps/desktop/src/lib/session-source.ts; keep roughly in sync when adding a -# local or programmatic surface. +# Surfaces that are not a human chat channel. The gateway binds a platform value +# (``telegram``) to HERMES_SESSION_PLATFORM while the CLI/TUI/desktop bind +# HERMES_SESSION_SOURCE and leave platform empty, so both are consulted. +# ``local``, ``api_server``, ``webhook``, ``msgraph_webhook`` are real Platform +# values with no attachment channel behind them. Default-deny: an unrecognized +# identity counts as messaging so a new chat platform is never treated as a +# private surface before this set is updated. Mirrors LOCAL_SESSION_SOURCE_IDS +# in apps/desktop/src/lib/session-source.ts; keep roughly in sync. NON_MESSAGING_SESSION_SURFACES = frozenset( { "", @@ -448,47 +313,33 @@ NON_MESSAGING_SESSION_SURFACES = frozenset( def session_is_messaging_surface() -> bool: """Whether this turn is delivered over a human messaging channel. - Callers use this to decide anything that differs between "the user is - reading a chat message" and "the user is at a machine they own": whether - to emit a delivery tag, whether a file has to land somewhere the gateway - is allowed to send from, whether narration would read as chat noise. - - Resolves ``HERMES_PLATFORM``, then the session platform, then the session - source, and reports messaging when any of them names a surface outside + Decides "user is reading a chat message" vs "user is at a machine they own": + delivery tags, whether a file must land somewhere the gateway can send from, + whether narration reads as chat noise. Checks ``HERMES_PLATFORM``, then the + session platform, then the session source against :data:`NON_MESSAGING_SESSION_SURFACES`. """ import os platform = os.getenv("HERMES_PLATFORM") or get_session_env("HERMES_SESSION_PLATFORM", "") source = get_session_env("HERMES_SESSION_SOURCE", "") - for identity in (platform, source): - identity = str(identity or "").strip().lower() - if identity and identity not in NON_MESSAGING_SESSION_SURFACES: - return True - return False + return any( + (ident := str(identity or "").strip().lower()) and ident not in NON_MESSAGING_SESSION_SURFACES + for identity in (platform, source) + ) def declare_stateless_channel() -> None: """Declare that this session cannot receive an async background completion. - Binds only the delivery capability, leaving every other session var unset. - Use this instead of ``set_session_vars(async_delivery=False)`` on a pure - single-process runner: ``set_session_vars`` also latches - ``_session_context_engaged`` (see above), which switches the subprocess - env bridge from "os.environ fallback" to "ContextVar-authoritative, strip on - _UNSET" in ``tools/environments/local.py``. A one-shot CLI that never engages - the session-context system must not flip that latch as a side effect of - declaring a capability. - - Callers that already build a full session context (cron's ``run_job``) get - the same state by passing ``async_delivery=False`` to ``set_session_vars``. - - A session that cannot take a late completion makes ``delegate_task`` fall - through to its existing inline/synchronous path, so subagent results are - returned within the turn instead of being dispatched to a channel that will - never deliver them. - - See NousResearch/hermes-agent#53027 and #63142. + Binds only the delivery capability. Use this instead of + ``set_session_vars(async_delivery=False)`` on a pure single-process runner: + ``set_session_vars`` also latches ``_session_context_engaged``, which flips + the subprocess env bridge to ContextVar-authoritative — a one-shot CLI must + not flip that latch as a side effect of declaring a capability. Callers that + build a full context (cron's ``run_job``) pass ``async_delivery=False``. + ``delegate_task`` then falls through to its inline path so results return + within the turn instead of going to a channel that never delivers. """ _SESSION_ASYNC_DELIVERY.set(False) @@ -496,30 +347,17 @@ def declare_stateless_channel() -> None: def async_delivery_supported() -> bool: """Whether the current session can deliver a background completion later. - Returns ``False`` for finite runtimes that can end before a detached result - is delivered: sessions explicitly bound by a stateless channel — an adapter - that cannot route a notification back after the turn ends (the API server), - or a one-shot runner that exits after its final response (``hermes -z``, - cron — see :func:`declare_stateless_channel`) — and dispatcher-spawned - Kanban workers (identified by ``HERMES_KANBAN_TASK``), which are one-shot - ``chat -q`` subprocesses. The real gateway platforms, the interactive CLI, - and any other path that never bound the contextvar return ``True``. - - Tools that promise async delivery (``terminal`` notify_on_complete / - watch_patterns, ``delegate_task`` background=True) consult this before - registering a watcher / dispatching a detached child, so they can refuse a - promise the channel can't keep instead of silently no-op'ing. + False for finite runtimes: sessions bound by a stateless channel (API + server, ``hermes -z``, cron — see :func:`declare_stateless_channel`) and + dispatcher-spawned Kanban workers (``HERMES_KANBAN_TASK``), which are + one-shot ``chat -q`` subprocesses whose parent disappears after the quiet + turn, so a later completion has no durable consumer. Gateway platforms, + the interactive CLI, and any path that never bound the var return True. """ import os - # A Kanban worker is a one-shot subprocess. Its parent session and process - # disappear after the quiet turn returns, so a completion queued later has - # no durable consumer even though an ordinary CLI session can drain that - # queue. Force tools onto their existing synchronous/polling fallbacks. + # Kanban worker: force tools onto their synchronous/polling fallbacks. if os.environ.get("HERMES_KANBAN_TASK"): return False - value = _SESSION_ASYNC_DELIVERY.get() - if value is _UNSET: - return True - return bool(value) + return True if value is _UNSET else bool(value) diff --git a/gateway/session_db_recovery.py b/gateway/session_db_recovery.py index 96e67a06a9..0a7be4ace3 100644 --- a/gateway/session_db_recovery.py +++ b/gateway/session_db_recovery.py @@ -22,7 +22,7 @@ class _Unavailable: class _HealthSource: - pass + """Weak-keyable identity for one cache's health entries.""" _health_lock = threading.Lock() @@ -32,29 +32,26 @@ _health_states: weakref.WeakKeyDictionary[_HealthSource, dict[Path, str]] = ( def _publish_health(source: _HealthSource, path: Path, state: str) -> None: - """Publish one privacy-safe aggregate for every live gateway DB cache.""" + """Publish one privacy-safe aggregate (no paths, no errors) across all live caches.""" with _health_lock: - states = _health_states.setdefault(source, {}) - states[path] = state - all_states = [value for item in _health_states.values() for value in item.values()] - if "retrying" in all_states: - aggregate = "retrying" - elif "unavailable" in all_states: - aggregate = "unavailable" - else: - aggregate = "ok" - + _health_states.setdefault(source, {})[path] = state + all_states = {value for item in _health_states.values() for value in item.values()} + aggregate = next((s for s in ("retrying", "unavailable") if s in all_states), "ok") try: from gateway.status import write_runtime_status write_runtime_status(session_store={"status": aggregate}) except Exception: - # Runtime health is diagnostic only; persistence must not depend on it. - pass + pass # Runtime health is diagnostic only; persistence must not depend on it. class RecoverableHandleCache: - """Cache handles by path while allowing failed opens to heal in-process.""" + """Cache handles by path while allowing failed opens to heal in-process. + + Opens run OUTSIDE ``lock`` (single-flight per path via ``in_flight``); a + ``close_all`` bumps ``_generation`` so any open that completes afterwards is + treated as stale and rejected rather than resurrecting a drained cache. + """ def __init__( self, @@ -69,14 +66,16 @@ class RecoverableHandleCache: self.lock = lock if lock is not None else threading.Lock() self._clock = clock self._initial_retry_delay = max(0.0, float(initial_retry_delay)) - self._max_retry_delay = max( - self._initial_retry_delay, float(max_retry_delay) - ) + self._max_retry_delay = max(self._initial_retry_delay, float(max_retry_delay)) self._unavailable: dict[Path, _Unavailable] = {} self._health_source = _HealthSource() self._generation = 0 self._close_rejected: Callable[[Any], None] | None = None + def _is_stale(self, path: Path, unavailable: _Unavailable, generation: int) -> bool: + """Caller holds ``lock``: True when close_all ran or the slot was replaced mid-open.""" + return generation != self._generation or self._unavailable.get(path) is not unavailable + def get( self, path: Path, @@ -86,15 +85,18 @@ class RecoverableHandleCache: on_recovered: Callable[[], None] | None = None, non_cacheable: Callable[[Exception], bool] | None = None, ) -> Any: - """Return a cached handle or make one bounded, single-flight open attempt.""" + """Return a cached handle or make one bounded, single-flight open attempt. + + Returns None while a retry is in flight or backing off (callers fall back). + ``non_cacheable`` exceptions (e.g. a live-system guard) are re-raised + without recording a failure so the next call retries immediately. + """ path = Path(path) with self.lock: if path in self.handles: return self.handles[path] - unavailable = self._unavailable.setdefault(path, _Unavailable()) - now = self._clock() - if unavailable.in_flight or now < unavailable.next_retry_at: + if unavailable.in_flight or self._clock() < unavailable.next_retry_at: return None unavailable.in_flight = True was_unavailable = unavailable.failures > 0 @@ -106,46 +108,33 @@ class RecoverableHandleCache: try: handle = opener() except Exception as exc: - if non_cacheable is not None and non_cacheable(exc): - with self.lock: - if ( - generation == self._generation - and self._unavailable.get(path) is unavailable - ): - self._unavailable.pop(path, None) - raise + uncacheable = non_cacheable is not None and non_cacheable(exc) with self.lock: - current = self._unavailable.get(path) - stale = generation != self._generation or current is not unavailable + stale = self._is_stale(path, unavailable, generation) + if uncacheable: + if not stale: + self._unavailable.pop(path, None) + raise if not stale: unavailable.failures += 1 delay = min( - self._initial_retry_delay - * (2 ** min(unavailable.failures - 1, 30)), + self._initial_retry_delay * (2 ** min(unavailable.failures - 1, 30)), self._max_retry_delay, ) unavailable.next_retry_at = self._clock() + delay unavailable.in_flight = False - if stale: - if raise_on_error: - raise - return None - _publish_health(self._health_source, path, "unavailable") + if not stale: + _publish_health(self._health_source, path, "unavailable") if raise_on_error: raise return None with self.lock: - stale = ( - generation != self._generation - or self._unavailable.get(path) is not unavailable - ) - if stale: - close_rejected = self._close_rejected - else: + stale = self._is_stale(path, unavailable, generation) + if not stale: self.handles[path] = handle self._unavailable.pop(path, None) - close_rejected = None + close_rejected = self._close_rejected if stale else None if stale: if close_rejected is not None: try: @@ -177,13 +166,3 @@ class RecoverableHandleCache: if states is not None: for path in paths: states.pop(path, None) - - def status_for(self, path: Path) -> str: - """Return a sanitized state for tests and internal diagnostics.""" - with self.lock: - if Path(path) in self.handles: - return "ok" - unavailable = self._unavailable.get(Path(path)) - if unavailable is None: - return "unknown" - return "retrying" if unavailable.in_flight else "unavailable" diff --git a/gateway/session_stall.py b/gateway/session_stall.py index 76dafcbb9b..118981af6f 100644 --- a/gateway/session_stall.py +++ b/gateway/session_stall.py @@ -1,21 +1,15 @@ -"""Gateway session stall notification policy (#72016 item 2). +"""Gateway session stall notification policy. -Consumes the shared activity observation contract from -``agent.session_activity`` / ``AIAgent.get_activity_summary()`` -(#72039) as the **single progress source**. This module owns only the -notify-once policy for "pending inbound + stale progress"; it does not -invent a parallel progress clock from turn-start or inbound event -timestamps. +Consumes the shared activity observation contract from ``agent.session_activity`` +/ ``AIAgent.get_activity_summary()`` as the **single progress source**. This +module owns only the notify-once policy for "pending inbound + stale progress"; +it never derives a parallel progress clock from turn-start or inbound timestamps. -Boundaries (keep separate): -- ``gateway/shutdown_watchdog.py`` — process / event-loop liveness -- ``gateway/delivery_ledger.py`` — outbound delivery obligations -- Pending inbound here is a stall *policy gate* (queued follow-up exists), - not an outbound obligation and not a progress timestamp. - -Notification / timeout / kill / retry policy stay in their own components; -the shared contract remains observation-only (timestamp + bounded -description + provenance). +Boundaries (keep separate): ``gateway/shutdown_watchdog.py`` is process / +event-loop liveness; ``gateway/delivery_ledger.py`` is outbound delivery +obligations. Pending inbound here is a stall *policy gate* (a queued follow-up +exists), not an obligation and not a progress timestamp. Timeout / kill / retry +policy stay in their own components. """ from __future__ import annotations @@ -32,15 +26,13 @@ def should_emit_session_stall_notification( already_notified: bool, ) -> bool: """Return True when a stall warning should be sent for this session.""" - if timeout_seconds <= 0: - return False - if not has_pending_inbound: - return False - if already_notified: - return False - if idle_seconds is None: - return False - return idle_seconds >= timeout_seconds + return ( + timeout_seconds > 0 + and has_pending_inbound + and not already_notified + and idle_seconds is not None + and idle_seconds >= timeout_seconds + ) def should_clear_session_stall_notification( @@ -50,23 +42,27 @@ def should_clear_session_stall_notification( has_pending_inbound: bool, ) -> bool: """Return True when a prior stall notice may be cleared (episode ended).""" - if not has_pending_inbound: - return True - if timeout_seconds <= 0: + if not has_pending_inbound or timeout_seconds <= 0: return True # Unknown progress: hold the latch. Do not treat observation gaps as recovery. - if idle_seconds is None: - return False - return idle_seconds < timeout_seconds + return idle_seconds is not None and idle_seconds < timeout_seconds def format_session_stall_notification(idle_seconds: float) -> str: - """User-facing stall warning (ASCII minutes; matches issue #72016 copy).""" + """User-facing stall warning (ASCII minutes).""" mins = max(1, int(idle_seconds // 60)) - return ( - f"⚠️ Agent session appears stalled (last activity {mins} min ago). " - f"Try /new to reset." - ) + return f"⚠️ Agent session appears stalled (last activity {mins} min ago). Try /new to reset." + + +def _finite_float(value: Any) -> Optional[float]: + """``value`` as a finite float, or None (bools are rejected as non-numeric).""" + if value is None or isinstance(value, bool): + return None + try: + result = float(value) + except (TypeError, ValueError): + return None + return result if math.isfinite(result) else None def resolve_session_idle_seconds_from_activity( @@ -74,52 +70,26 @@ def resolve_session_idle_seconds_from_activity( *, now: Optional[float] = None, ) -> Optional[float]: - """Idle seconds from a shared activity snapshot only (#72039 contract). + """Idle seconds from a shared activity snapshot only. - Prefers ``seconds_since_activity`` when present and finite; otherwise - derives from ``last_activity_at`` / ``last_activity_ts``. Returns - ``None`` when there is no usable progress timestamp — callers must - not fall back to turn-start or pending-inbound clocks. + Prefers ``seconds_since_activity`` when present and finite; otherwise derives + from ``last_activity_at`` / ``last_activity_ts``. Returns None when there is + no usable progress timestamp — callers must not fall back to turn-start or + pending-inbound clocks. """ if not activity: return None - elapsed = activity.get("seconds_since_activity") - if isinstance(elapsed, bool): - elapsed = None - if elapsed is not None: - try: - idle = float(elapsed) - except (TypeError, ValueError): - idle = None - else: - if math.isfinite(idle): - if idle < 0: - return 0.0 - return idle - # Non-finite: fall through to last_activity_at / last_activity_ts + idle = _finite_float(activity.get("seconds_since_activity")) + if idle is not None: + return max(0.0, idle) ts = activity.get("last_activity_at") - if ts is None: - ts = activity.get("last_activity_ts") - if ts is None: + when = _finite_float(activity.get("last_activity_ts") if ts is None else ts) + if when is None: return None - if isinstance(ts, bool): - return None - try: - when = float(ts) - except (TypeError, ValueError): - return None - if not math.isfinite(when): - return None - if now is None: import time as _time - clock = float(_time.time()) - else: - clock = float(now) - idle = clock - when - if idle < 0: - return 0.0 - return idle + now = _time.time() + return max(0.0, float(now) - when) diff --git a/gateway/session_state.py b/gateway/session_state.py index d10f8eb031..66ad914616 100644 --- a/gateway/session_state.py +++ b/gateway/session_state.py @@ -1,85 +1,66 @@ """Per-session gateway state consolidated into one container. GatewayRunner historically carried ~19 separate ``Dict[str, ...]`` attributes -keyed by session_key, each with its own ad-hoc lifecycle. Three failure -classes grew out of that shape: +keyed by session_key, each with an ad-hoc lifecycle. That shape bred three +bug classes, all structurally closed here: -1. Boundary drift — every conversation boundary carried a hand-copied - pop-list that went stale when a new dict was added (#48031, #58403, - #10702, #35809). Mitigated by the ``_CONVERSATION_SCOPED_STATE`` registry, - now structurally fixed: the fields live in one ``ConversationState`` - dataclass with a single ``clear()``. -2. Turn-release drift — ad-hoc ``del self._running_agents[key]`` sites that - popped different subsets of the turn dicts. Mitigated by - ``_release_running_agent_state``, now ``TurnState.clear()``. -3. Wholesale-reset races — lazy-init paths like - ``self._session_reasoning_overrides = {}`` replaced the ENTIRE dict, - discarding concurrent sessions' entries when raced. Structurally - impossible now: state is per-session, resets touch one field of one - ``SessionState``. +1. Boundary drift — hand-copied pop-lists at conversation boundaries went + stale when a dict was added. Now one ``ConversationState.clear()``. +2. Turn-release drift — ad-hoc ``del self._running_agents[key]`` sites popped + different subsets of the turn dicts. Now ``TurnState.clear()``. +3. Wholesale-reset races — lazy-init ``self._x = {}`` replaced the ENTIRE dict, + discarding concurrent sessions' entries. Resets now touch one field of + one ``SessionState``. -Scopes (placement follows where each dict is CLEARED today): - -- ``SessionState.turn`` — reset at end of every running turn. -- ``SessionState.conversation`` — reset at conversation boundaries - (/new, /resume, auto-reset, expiry, compression-exhausted reset). -- ``SessionState.persistent`` — own lifecycles (approval resolution, update - prompt answer, native-image consumption); ``run_generation`` is monotonic - and NEVER reset (#28686). +Scopes follow where each dict was CLEARED: ``turn`` resets at the end of every +running turn; ``conversation`` at conversation boundaries (/new, /resume, +auto-reset, expiry, compression-exhausted reset); ``persistent`` fields have +their own lifecycles and ``run_generation`` is monotonic and NEVER reset. Entries in ``GatewayRunner._sessions`` are never evicted (matching the old -dicts, most of which also leaked empty/stale entries for dead sessions — -see the migration table in the consolidating commit). Follow-up work may -add eviction of fully-default SessionStates. +dicts, which also leaked empty/stale entries for dead sessions). Eviction of +fully-default SessionStates is possible follow-up work. """ from __future__ import annotations from collections.abc import MutableMapping from dataclasses import dataclass, field -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterator, List, NamedTuple, Optional, Tuple # Presence-sensitive sentinel: /fast stores "priority" or None (explicit # normal), so key PRESENCE — not value truthiness — decides whether the # override applies. ``_UNSET_TIER`` means "no override recorded". _UNSET_TIER = object() - -# Public alias for callers outside this module. -SERVICE_TIER_UNSET = _UNSET_TIER +SERVICE_TIER_UNSET = _UNSET_TIER # public alias @dataclass class TurnState: """State scoped to one running gateway turn. - Cleared by ``GatewayRunner._release_running_agent_state`` (via - ``clear()``) at every site that ends a running turn. ``lease_token`` / - ``lease_generation`` are deliberately NOT cleared here — they are owned - by ``_release_turn_lease`` (#64934), which must release the registry - lease exactly once per acquiring turn. + ``clear()`` runs at every site that ends a running turn. ``lease_token`` / + ``lease_generation`` are deliberately NOT cleared by it — they are owned by + ``_release_turn_lease``, which must release the registry lease exactly once + per acquiring turn. """ # Running AIAgent instance (or _AGENT_PENDING_SENTINEL); None = idle. agent: Any = None - # Turn start timestamp (0.0 = not running). - started_ts: float = 0.0 - # Cross-process active-session slot lease (None = none held). - lease: Any = None - # Last busy-ack timestamp (debounce; 0.0 = never acked). - busy_ack_ts: float = 0.0 - # Held turn-lease token + the run generation that acquired it. The old - # ``_turn_lease_tokens`` dict was keyed by (session_key, generation) so a - # stale unwind could never free a newer turn's lease; the pair encoding - # preserves that: release/rebind only match when generation is current. + started_ts: float = 0.0 # 0.0 = not running + lease: Any = None # cross-process active-session slot lease + busy_ack_ts: float = 0.0 # debounce; 0.0 = never acked + # Held turn-lease token + the run generation that acquired it. The pair + # replaces the old (session_key, generation)-keyed dict so a stale unwind + # can never free a newer turn's lease: release/rebind only match when the + # generation is current. lease_token: Any = None lease_generation: Optional[int] = None def clear(self) -> None: """Reset the per-turn slot (agent / start ts / lease / busy-ack). - Mirrors the exact clear set of the old _release_running_agent_state: - _running_agents, _running_agents_ts, _active_session_leases (popped - by the caller so it can call ``lease.release()``), _busy_ack_ts. + The caller pops ``lease`` first so it can call ``lease.release()``. """ self.agent = None self.started_ts = 0.0 @@ -93,78 +74,45 @@ class ConversationState: # /model per-session override (model/provider/api_key/base_url/api_mode). model_override: Optional[Dict[str, Any]] = None - # /model --once restore snapshot. - one_turn_restore: Optional[Dict[str, Any]] = None - # /reasoning per-session override. - reasoning_override: Optional[Dict[str, Any]] = None + one_turn_restore: Optional[Dict[str, Any]] = None # /model --once snapshot + reasoning_override: Optional[Dict[str, Any]] = None # /reasoning override # /fast per-session override: "priority" or None; _UNSET_TIER = absent. service_tier_override: Any = _UNSET_TIER - # Last successfully-resolved non-empty model (#35314 recovery). - last_resolved_model: str = "" - # /queue overflow FIFO (adapter slot holds the head). - queued_events: List[Any] = field(default_factory=list) - # Per-turn must-deliver sidecar notes (one-shot). - sidecar_notes: List[str] = field(default_factory=list) - # Pinned session-context bytes: (change_key, text). - ephemeral_pin: Optional[Tuple[Any, ...]] = None - # Last voice-channel context delivered (None = never delivered). - vc_last: Optional[str] = None + last_resolved_model: str = "" # last successfully-resolved non-empty model + queued_events: List[Any] = field(default_factory=list) # /queue overflow FIFO (adapter slot holds the head) + sidecar_notes: List[str] = field(default_factory=list) # one-shot must-deliver notes + ephemeral_pin: Optional[Tuple[Any, ...]] = None # pinned session-context bytes: (change_key, text) + vc_last: Optional[str] = None # last voice-channel context delivered def clear(self) -> None: - """Reset every conversation-scoped field to its default. - - The structural successor of the ``_CONVERSATION_SCOPED_STATE`` - pop-loop: adding a field here means every boundary clears it - automatically. - """ - self.model_override = None - self.one_turn_restore = None - self.reasoning_override = None - self.service_tier_override = _UNSET_TIER - self.last_resolved_model = "" - self.queued_events = [] - self.sidecar_notes = [] - self.ephemeral_pin = None - self.vc_last = None + """Reset every field to its default — adding a field here means every + conversation boundary clears it automatically.""" + self.__dict__.update(ConversationState().__dict__) @dataclass class PersistentState: - """State with its own lifecycle — NOT cleared by turn or boundary resets - wholesale (approvals/update prompts ARE cleared by the boundary - *security* funnel, but individually, matching the old behavior).""" + """State with its own lifecycle — NOT cleared wholesale by turn or boundary + resets (approvals/update prompts ARE cleared by the boundary *security* + funnel, but individually).""" - # Pending exec approval ({"command": ..., "pattern_key": ...}). - approvals: Optional[Dict[str, Any]] = None - # /update prompt awaiting a user response. - update_prompt_pending: bool = False - # Image paths staged for native (inline) attachment; consumed one-shot. - native_image_paths: List[str] = field(default_factory=list) - # Legacy runner-level pending message text (write-mostly; flushed to - # disk on shutdown — see #72680). NOTE: distinct from the adapter-level - # ``_pending_messages`` (Dict[str, MessageEvent]) in gateway/base.py, - # which is a different store that happens to share the old name. + approvals: Optional[Dict[str, Any]] = None # {"command": ..., "pattern_key": ...} + update_prompt_pending: bool = False # /update prompt awaiting a reply + native_image_paths: List[str] = field(default_factory=list) # consumed one-shot + # Legacy runner-level pending message text (write-mostly; flushed to disk on + # shutdown). Distinct from the adapter-level ``_pending_messages`` + # (Dict[str, MessageEvent]) in gateway/base.py, which shares the old name. pending_command_text: Optional[str] = None - # Monotonic run-generation counter (#28686). NEVER reset: clearing it - # would break stale-run detection. + # Monotonic run-generation counter. NEVER reset: stale-run detection depends on it. run_generation: int = 0 - # Consecutive session-hygiene compression failures for this session - # (#79624). The in-agent compressor escalates repeat timeouts via - # ContextCompressor._consecutive_timeout_failures, but hygiene builds a - # FRESH AIAgent per run and bind_session_state() zeroes that counter, so - # the in-agent ladder is structurally unreachable from the gateway. - # Tracking the streak here — outside the per-run agent — lets hygiene - # escalate its cooldown instead of retrying on a flat interval forever. - # Reset on a successful compression, not by turn/boundary resets. - # - # PROCESS-LOCAL, deliberately: `PersistentState` means "survives turn and - # boundary resets", NOT "survives a restart" — this field has no disk flush - # (unlike `pending_command_text` above, #72680), so a gateway restart drops - # escalation back to rung 1 while the DB-backed deadline itself survives - # (#74136). Keying on `session_key` rather than `session_id` is what buys - # correctness across compaction ROTATION (the sid changes, the chat does - # not). gateway.run mirrors this value to the DB keyed by session_key so - # the same semantics also survive gateway restarts. + # Consecutive session-hygiene compression failures. The in-agent compressor's + # own timeout ladder is unreachable from the gateway (hygiene builds a FRESH + # AIAgent per run and bind_session_state() zeroes that counter), so the streak + # lives here and lets hygiene escalate its cooldown. Reset on a successful + # compression only. PROCESS-LOCAL by design: no disk flush, so a restart drops + # escalation to rung 1 while the DB-backed deadline survives; gateway.run + # mirrors it to the DB keyed by session_key (not session_id) so it also holds + # across compaction ROTATION, where the sid changes but the chat does not. hygiene_failure_streak: int = 0 @@ -181,106 +129,44 @@ class SessionState: # Legacy dict-view adapters. # # Dozens of tests construct bare runners (object.__new__) and read/write the -# old dict attributes directly (``runner._running_agents = {}``, -# ``assert key in runner._pending_approvals``...). These views keep that -# surface working: each is a LIVE MutableMapping over one SessionState field -# across all sessions. Production code in gateway/run.py accesses state via -# ``self._session_state(key)..`` — the views exist for tests -# and for the thin remaining mixin/adapter call sites documented in the -# consolidating commit. +# old dict attributes directly (``runner._running_agents = {}``, ``assert key +# in runner._pending_approvals``). Each view is a LIVE MutableMapping over one +# SessionState field across all sessions. Production code in gateway/run.py +# uses ``self._session_state(key)..``. # --------------------------------------------------------------------------- -class _FieldSpec: - """Describes one legacy dict: scope attr, field name, absence test.""" +class _FieldSpec(NamedTuple): + """One legacy dict: scope attr, field name, default factory, presence test.""" - __slots__ = ("scope", "name", "default", "is_present") - - def __init__( - self, - scope: str, - name: str, - default: Callable[[], Any], - is_present: Callable[[Any], bool], - ) -> None: - self.scope = scope - self.name = name - self.default = default - self.is_present = is_present + scope: str + name: str + default: Callable[[], Any] + is_present: Callable[[Any], bool] -def _present_not_none(v: Any) -> bool: - return v is not None +def _spec(scope: str, name: str, default: Any) -> _FieldSpec: + """``default`` is either a type (presence = truthiness) or a sentinel value + such as ``None`` / ``_UNSET_TIER`` (presence = ``is not`` sentinel).""" + if isinstance(default, type): + return _FieldSpec(scope, name, default, bool) + return _FieldSpec(scope, name, lambda: default, lambda v: v is not default) -def _present_nonzero(v: Any) -> bool: - return bool(v) +class _RunnerView(MutableMapping): + """Shared plumbing: live view over ``runner._sessions``, dict-comparable.""" + __slots__ = ("_runner",) -def _present_not_unset(v: Any) -> bool: - return v is not _UNSET_TIER - - -class SessionFieldView(MutableMapping): - """Live dict-like view of one SessionState field across sessions.""" - - __slots__ = ("_runner", "_spec") - - def __init__(self, runner: Any, spec: _FieldSpec) -> None: + def __init__(self, runner: Any) -> None: self._runner = runner - self._spec = spec - # -- internals -------------------------------------------------------- def _sessions(self) -> Dict[str, SessionState]: return self._runner.__dict__.get("_sessions") or {} - def _value(self, state: SessionState) -> Any: - return getattr(getattr(state, self._spec.scope), self._spec.name) - - # -- mapping protocol --------------------------------------------------- - def __getitem__(self, key: str) -> Any: - state = self._sessions().get(key) - if state is None: - raise KeyError(key) - value = self._value(state) - if not self._spec.is_present(value): - raise KeyError(key) - return value - - def __setitem__(self, key: str, value: Any) -> None: - state = self._runner._session_state(key) - setattr(getattr(state, self._spec.scope), self._spec.name, value) - - def __delitem__(self, key: str) -> None: - state = self._sessions().get(key) - if state is None or not self._spec.is_present(self._value(state)): - raise KeyError(key) - setattr( - getattr(state, self._spec.scope), - self._spec.name, - self._spec.default(), - ) - - def __iter__(self) -> Iterator[str]: - for key, state in list(self._sessions().items()): - if self._spec.is_present(self._value(state)): - yield key - def __len__(self) -> int: return sum(1 for _ in self) - def __contains__(self, key: object) -> bool: - state = self._sessions().get(key) # type: ignore[arg-type] - return state is not None and self._spec.is_present(self._value(state)) - - def clear(self) -> None: # avoid MutableMapping's popitem loop - for state in list(self._sessions().values()): - setattr( - getattr(state, self._spec.scope), - self._spec.name, - self._spec.default(), - ) - # Mapping doesn't provide __eq__; tests compare against plain dicts. def __eq__(self, other: object) -> bool: if isinstance(other, (dict, MutableMapping)): @@ -291,185 +177,149 @@ class SessionFieldView(MutableMapping): result = self.__eq__(other) return NotImplemented if result is NotImplemented else not result + +class SessionFieldView(_RunnerView): + """Live dict-like view of one SessionState field across sessions.""" + + __slots__ = ("_spec",) + + def __init__(self, runner: Any, spec: _FieldSpec) -> None: + super().__init__(runner) + self._spec = spec + + def _value(self, state: SessionState) -> Any: + return getattr(getattr(state, self._spec.scope), self._spec.name) + + def _set(self, state: SessionState, value: Any) -> None: + setattr(getattr(state, self._spec.scope), self._spec.name, value) + + def __getitem__(self, key: str) -> Any: + state = self._sessions().get(key) + if state is None or not self._spec.is_present(value := self._value(state)): + raise KeyError(key) + return value + + def __setitem__(self, key: str, value: Any) -> None: + self._set(self._runner._session_state(key), value) + + def __delitem__(self, key: str) -> None: + state = self._sessions().get(key) + if state is None or not self._spec.is_present(self._value(state)): + raise KeyError(key) + self._set(state, self._spec.default()) + + def __iter__(self) -> Iterator[str]: + for key, state in list(self._sessions().items()): + if self._spec.is_present(self._value(state)): + yield key + + def __contains__(self, key: object) -> bool: + state = self._sessions().get(key) # type: ignore[arg-type] + return state is not None and self._spec.is_present(self._value(state)) + + def clear(self) -> None: # avoid MutableMapping's popitem loop + for state in list(self._sessions().values()): + self._set(state, self._spec.default()) + def __repr__(self) -> str: # pragma: no cover - debug aid return f"SessionFieldView({self._spec.scope}.{self._spec.name}, {dict(self.items())!r})" -class TurnLeaseTokenView(MutableMapping): +class TurnLeaseTokenView(_RunnerView): """Legacy view of ``_turn_lease_tokens``: keyed by (session_key, generation). - The pair now lives on ``TurnState.lease_token`` / ``lease_generation``; - at most one held token exists per session key (the lease registry - serializes acquisition per session), so the single slot is equivalent to - the old tuple-keyed dict. + The pair lives on ``TurnState.lease_token`` / ``lease_generation``; the lease + registry serializes acquisition per session, so at most one held token + exists per session key and the single slot equals the old tuple-keyed dict. """ - __slots__ = ("_runner",) + __slots__ = () - def __init__(self, runner: Any) -> None: - self._runner = runner - - def _sessions(self) -> Dict[str, SessionState]: - return self._runner.__dict__.get("_sessions") or {} - - @staticmethod - def _split(key: Any) -> Tuple[str, int]: + def _held(self, key: Any) -> Tuple[Any, SessionState]: + """Return (session_key, state) for a currently-held (key, gen) or raise KeyError.""" if not isinstance(key, tuple) or len(key) != 2: raise KeyError(key) - return key[0], key[1] + state = self._sessions().get(key[0]) + if state is None or state.turn.lease_token is None or state.turn.lease_generation != key[1]: + raise KeyError(key) + return key[0], state def __getitem__(self, key: Any) -> Any: - session_key, generation = self._split(key) - state = self._sessions().get(session_key) - if ( - state is None - or state.turn.lease_token is None - or state.turn.lease_generation != generation - ): - raise KeyError(key) - return state.turn.lease_token + return self._held(key)[1].turn.lease_token def __setitem__(self, key: Any, value: Any) -> None: - session_key, generation = self._split(key) - state = self._runner._session_state(session_key) - state.turn.lease_token = value - state.turn.lease_generation = generation + if not isinstance(key, tuple) or len(key) != 2: + raise KeyError(key) + turn = self._runner._session_state(key[0]).turn + turn.lease_token, turn.lease_generation = value, key[1] def __delitem__(self, key: Any) -> None: - session_key, generation = self._split(key) - state = self._sessions().get(session_key) - if ( - state is None - or state.turn.lease_token is None - or state.turn.lease_generation != generation - ): - raise KeyError(key) - state.turn.lease_token = None - state.turn.lease_generation = None + turn = self._held(key)[1].turn + turn.lease_token = turn.lease_generation = None def __iter__(self) -> Iterator[Tuple[str, Any]]: for key, state in list(self._sessions().items()): if state.turn.lease_token is not None: yield (key, state.turn.lease_generation) - def __len__(self) -> int: - return sum(1 for _ in self) - - def __eq__(self, other: object) -> bool: - if isinstance(other, (dict, MutableMapping)): - return dict(self.items()) == dict(other) - return NotImplemented - - def __ne__(self, other: object) -> bool: - result = self.__eq__(other) - return NotImplemented if result is NotImplemented else not result + def clear(self) -> None: # avoid MutableMapping's popitem loop + for key in list(self): + del self[key] -# One spec per legacy dict attribute (see the consolidating commit's -# migration table for declared-line / clear-path provenance). +# One spec per legacy dict attribute. LEGACY_FIELD_SPECS: Dict[str, _FieldSpec] = { - "_running_agents": _FieldSpec("turn", "agent", lambda: None, _present_not_none), - "_running_agents_ts": _FieldSpec("turn", "started_ts", float, _present_nonzero), - "_active_session_leases": _FieldSpec("turn", "lease", lambda: None, _present_not_none), - "_busy_ack_ts": _FieldSpec("turn", "busy_ack_ts", float, _present_nonzero), - "_session_model_overrides": _FieldSpec( - "conversation", "model_override", lambda: None, _present_not_none - ), - "_pending_one_turn_model_restores": _FieldSpec( - "conversation", "one_turn_restore", lambda: None, _present_not_none - ), - "_session_reasoning_overrides": _FieldSpec( - "conversation", "reasoning_override", lambda: None, _present_not_none - ), - "_session_service_tier_overrides": _FieldSpec( - "conversation", "service_tier_override", lambda: _UNSET_TIER, _present_not_unset - ), - "_last_resolved_model": _FieldSpec( - "conversation", "last_resolved_model", str, _present_nonzero - ), - "_queued_events": _FieldSpec("conversation", "queued_events", list, _present_nonzero), - "_pending_turn_sidecar_notes": _FieldSpec( - "conversation", "sidecar_notes", list, _present_nonzero - ), - "_session_ephemeral_pin": _FieldSpec( - "conversation", "ephemeral_pin", lambda: None, _present_not_none - ), - "_session_vc_last": _FieldSpec( - "conversation", "vc_last", lambda: None, _present_not_none - ), - "_pending_approvals": _FieldSpec( - "persistent", "approvals", lambda: None, _present_not_none - ), - "_update_prompt_pending": _FieldSpec( - "persistent", "update_prompt_pending", bool, _present_nonzero - ), - "_pending_native_image_paths_by_session": _FieldSpec( - "persistent", "native_image_paths", list, _present_nonzero - ), - "_pending_messages": _FieldSpec( - "persistent", "pending_command_text", lambda: None, _present_not_none - ), - "_session_run_generation": _FieldSpec( - "persistent", "run_generation", int, _present_nonzero - ), + "_running_agents": _spec("turn", "agent", None), + "_running_agents_ts": _spec("turn", "started_ts", float), + "_active_session_leases": _spec("turn", "lease", None), + "_busy_ack_ts": _spec("turn", "busy_ack_ts", float), + "_session_model_overrides": _spec("conversation", "model_override", None), + "_pending_one_turn_model_restores": _spec("conversation", "one_turn_restore", None), + "_session_reasoning_overrides": _spec("conversation", "reasoning_override", None), + "_session_service_tier_overrides": _spec("conversation", "service_tier_override", _UNSET_TIER), + "_last_resolved_model": _spec("conversation", "last_resolved_model", str), + "_queued_events": _spec("conversation", "queued_events", list), + "_pending_turn_sidecar_notes": _spec("conversation", "sidecar_notes", list), + "_session_ephemeral_pin": _spec("conversation", "ephemeral_pin", None), + "_session_vc_last": _spec("conversation", "vc_last", None), + "_pending_approvals": _spec("persistent", "approvals", None), + "_update_prompt_pending": _spec("persistent", "update_prompt_pending", bool), + "_pending_native_image_paths_by_session": _spec("persistent", "native_image_paths", list), + "_pending_messages": _spec("persistent", "pending_command_text", None), + "_session_run_generation": _spec("persistent", "run_generation", int), } -def legacy_dict_property(attr_name: str) -> property: - """Build a legacy dict-shaped @property for one migrated attribute. +def _legacy_property(make_view: Callable[[Any], MutableMapping], doc: str) -> property: + """Dict-shaped @property over a live view. - Getter returns a live :class:`SessionFieldView`; setter accepts a plain - dict (the ubiquitous test pattern ``runner._X = {...}``), resetting the - field on every known session and then applying the given entries. + Getter returns the view; setter accepts a plain dict (the ubiquitous test + pattern ``runner._X = {...}``), resetting the field on every known session + and then applying the given entries; ``del runner._X`` (older tests + simulating a runner without the attribute) means "no entries". """ - spec = LEGACY_FIELD_SPECS[attr_name] - def fget(self: Any) -> SessionFieldView: - return SessionFieldView(self, spec) - - def fset(self: Any, mapping: Optional[Dict[str, Any]]) -> None: - view = SessionFieldView(self, spec) + def fset(self: Any, mapping: Optional[Dict[Any, Any]]) -> None: + view = make_view(self) view.clear() for key, value in (mapping or {}).items(): view[key] = value - def fdel(self: Any) -> None: - # ``del runner._busy_ack_ts`` in older tests simulated a runner - # without the attribute; the closest equivalent is "no entries". - SessionFieldView(self, spec).clear() + return property(make_view, fset, lambda self: make_view(self).clear(), doc=doc) - return property( - fget, - fset, - fdel, - doc=( - f"Legacy dict view over SessionState.{spec.scope}.{spec.name} " - "(kept for tests that access the pre-SessionState attribute)." - ), + +def legacy_dict_property(attr_name: str) -> property: + """Legacy dict-shaped @property for one migrated attribute.""" + spec = LEGACY_FIELD_SPECS[attr_name] + return _legacy_property( + lambda self: SessionFieldView(self, spec), + f"Legacy dict view over SessionState.{spec.scope}.{spec.name} " + "(kept for tests that access the pre-SessionState attribute).", ) def legacy_lease_token_property() -> property: """Legacy (session_key, generation)-keyed view of held turn-lease tokens.""" - - def fget(self: Any) -> TurnLeaseTokenView: - return TurnLeaseTokenView(self) - - def fset(self: Any, mapping: Optional[Dict[Tuple[str, int], Any]]) -> None: - view = TurnLeaseTokenView(self) - for key in list(view): - del view[key] - for key, value in (mapping or {}).items(): - view[key] = value - - def fdel(self: Any) -> None: - view = TurnLeaseTokenView(self) - for key in list(view): - del view[key] - - return property( - fget, - fset, - fdel, - doc="Legacy (session_key, generation)-keyed turn-lease token view.", + return _legacy_property( + TurnLeaseTokenView, "Legacy (session_key, generation)-keyed turn-lease token view." ) diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 099471f356..5304ee1eb8 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -49,10 +49,9 @@ from utils import ( logger = logging.getLogger("gateway.run") -# Upper bound on the off-loop agent-resource cleanup during a /new or /reset -# (see _handle_reset_command). A stuck teardown must not block the event loop; -# past this the reset proceeds and the cleanup is left to finish (or leak) in -# its worker thread. (#35994) +# Upper bound on the off-loop agent-resource cleanup during a /new or /reset (see +# _handle_reset_command). A stuck teardown must not block the event loop; past this the reset +# proceeds and the cleanup is left to finish (or leak) in its worker thread. _RESET_CLEANUP_TIMEOUT_S = 30.0 @@ -72,15 +71,12 @@ def _int_value(value: Any) -> int: def _model_switch_skew_guard() -> Optional[str]: """Refuse a model switch when the gateway is running stale code. - A long-lived gateway holds its modules in memory from boot. If the checkout - changed underneath it (e.g. a manual ``git pull``), switching models can hit - a first-time lazy import on a new code path and crash on a stale cached - dependency — the cryptic ``cannot import name 'env_float' from 'utils'``. - Detect the drift and tell the user to restart instead. - - Intentionally scoped to model switching — the known, highest-risk trigger. - Any first-time lazy import on a stale process is technically exposed; we - don't guard every import site, only this one. + A long-lived gateway holds its modules in memory from boot. If the checkout changed + underneath it (e.g. a manual ``git pull``), switching models can hit a first-time lazy import + on a new code path and crash on a stale cached dependency — the cryptic + ``cannot import name 'env_float' from 'utils'``. Detect the drift and tell the user to + restart instead. Deliberately scoped to model switching (the highest-risk trigger); other + lazy-import sites are not guarded. """ from gateway.code_skew import detect_code_skew @@ -101,15 +97,11 @@ def _model_switch_skew_guard() -> Optional[str]: def _home_thread_from_source(source) -> Optional[str]: """The thread id /sethome should persist on the home target, or None. - Slack thread-per-message session keying stamps a top-level message's own - id as ``source.thread_id`` (a session KEY, not a durable location). - Persisting it would pin the HOME target itself to the ephemeral thread - spawned around the /sethome message — every bare-platform delivery - (``deliver="slack"``) would then land in that thread forever. Same - recognition as cron origin capture: a Slack thread id equal to the - message's own id is synthetic. A /sethome run inside a genuine thread - (thread id = the parent's id, not this message's own) keeps that thread - as the home target. + Slack thread-per-message session keying stamps a top-level message's own id as + ``source.thread_id`` (a session KEY, not a durable location). Persisting it would pin HOME to + the ephemeral thread around the /sethome message, so every bare ``deliver="slack"`` would land + there forever. Same recognition as cron origin capture: a Slack thread id equal to the + message's own id is synthetic; a genuine thread (id = parent's) is kept as the home target. """ thread_id = getattr(source, "thread_id", None) if not thread_id: @@ -131,12 +123,9 @@ class GatewaySlashCommandsMixin: def _typed_command_prefix_for(self, platform) -> str: """Return the prefix users can always type to reach Hermes commands. - Reads the adapter's ``typed_command_prefix`` capability flag - (default "/"). Slack and Matrix return "!" because typed "/" - commands are blocked in Slack threads / reserved by Matrix clients; - their adapters rewrite "!command" to "/command" on receive. - Instruction text built for those platforms must show the prefix - that actually works when typed. + Reads the adapter's ``typed_command_prefix`` capability flag (default "/"). Slack and + Matrix return "!" because typed "/" commands are blocked in Slack threads / reserved by + Matrix clients; their adapters rewrite "!command" to "/command" on receive. """ adapter = self.adapters.get(platform) if getattr(self, "adapters", None) else None return getattr(adapter, "typed_command_prefix", "/") if adapter is not None else "/" @@ -144,33 +133,25 @@ class GatewaySlashCommandsMixin: async def _handle_reset_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /new or /reset command.""" source = event.source - + # Get existing session key session_key = self._session_key_for_source(source) self._invalidate_session_run_generation(session_key, reason="session_reset") - # Evict the running-agent slot now that the generation is bumped. The - # in-flight run's own guarded release (run_generation=old) will return - # False and leave its dead agent behind; clearing here keeps the slot - # from becoming a zombie that silently drops all later messages (#28686). - # Idempotent, so the run's finally calling it again is harmless. + # Evict the running-agent slot now that the generation is bumped: the in-flight run's own + # guarded release (old generation) returns False and would leave a zombie slot that silently + # drops all later messages. Idempotent, so the run's finally calling it again is harmless. self._release_running_agent_state(session_key) # Snapshot the old entry so on_session_finalize can report the # expiring session id before reset_session() rotates it. old_entry = self.session_store._entries.get(session_key) - # Close tool resources on the old agent (terminal sandboxes, browser - # daemons, background processes) before evicting from cache. - # Guard with getattr because test fixtures may skip __init__. - # - # _cleanup_agent_resources is synchronous and can block for a long time - # (agent.close() does subprocess teardown; shutdown_memory_provider() - # may do network IO). This handler runs ON the event loop when a - # Telegram/Discord/Slack confirm-button click resolves the slash-confirm - # (see _request_slash_confirm), so an inline call wedges the whole loop - # and the bot goes silent until restart (#35994). Offload it to a worker - # thread (via the contextvar-preserving executor helper) with a bounded - # timeout so the loop is never blocked. + # Close tool resources on the old agent (terminal sandboxes, browser daemons, background + # processes) before evicting from cache. Guard with getattr because test fixtures may skip + # __init__. _cleanup_agent_resources is synchronous and can block (subprocess teardown, + # memory-provider network IO) and this handler runs ON the event loop when a confirm-button + # click resolves the slash-confirm — an inline call wedges the loop and the bot goes silent. + # Offload to a worker thread with a bounded timeout. _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None: with _cache_lock: @@ -202,19 +183,15 @@ class GatewaySlashCommandsMixin: ) self._evict_cached_agent(session_key) - # Conversation boundary: clear ALL conversation-scoped per-session - # state (model/reasoning overrides, one-turn restores, model notes, - # last-resolved cache, /queue overflow) + security state in one - # funnel call. See _CONVERSATION_SCOPED_STATE in gateway/run.py. + # Conversation boundary: clear ALL conversation-scoped per-session state (model/reasoning + # overrides, one-turn restores, model notes, last-resolved cache, /queue overflow) + + # security state in one funnel call. See _CONVERSATION_SCOPED_STATE in gateway/run.py. self._clear_conversation_scope(session_key, reason="session_reset") - # The old conversation's in-flight async delegations end WITH it - # (#55578): after the reset rotates the session id, their completions - # would have no live owner — a dangling subagent can only burn tokens - # and park an orphaned payload on the shared queue. Interrupt by the - # expiring durable session id (delegations dispatched from gateway - # chats are pinned to it via parent_session_id) and by the routing - # key as a fallback for older records. + # The old conversation's in-flight async delegations end WITH it: after the reset rotates + # the session id, their completions would have no live owner — a dangling subagent can only + # burn tokens and park an orphaned payload on the shared queue. Interrupt by the expiring + # durable session id (parent_session_id) and by routing key as a fallback for older records. try: from tools.async_delegation import interrupt_for_session @@ -246,9 +223,8 @@ class GatewaySlashCommandsMixin: _old_sid = old_entry.session_id if old_entry else None - # Fire plugin on_session_finalize hook (session boundary). - # Off-loop + bounded: finalize hooks can block arbitrarily - # (observability trace exports) and this handler runs on the + # Fire plugin on_session_finalize hook (session boundary). Off-loop + bounded: finalize + # hooks can block arbitrarily (observability trace exports) and this handler runs on the # gateway event loop (see GatewayRunner._finalize_session_off_loop). try: await self._finalize_session_off_loop( @@ -315,11 +291,9 @@ class GatewaySlashCommandsMixin: _title_note = t("gateway.reset.title_empty_untitled") header = header + _title_note - # When /new runs inside a Telegram DM topic lane, rewrite the - # (chat_id, thread_id) → session_id binding so the next message - # uses the freshly-created session. Without this, the binding - # still points at the old session and the binding-lookup at the - # top of _handle_message_with_agent would switch right back. + # When /new runs inside a Telegram DM topic lane, rewrite the (chat_id, thread_id) → + # session_id binding so the next message uses the freshly-created session. Otherwise the + # binding-lookup at the top of _handle_message_with_agent switches right back to the old one. if await asyncio.to_thread(self._is_telegram_topic_lane, source) and new_entry is not None: try: await asyncio.to_thread(self._record_telegram_topic_binding, source, new_entry) @@ -355,17 +329,12 @@ class GatewaySlashCommandsMixin: async def _handle_profile_command(self, event: MessageEvent) -> str: """Handle /profile — show the profile serving this source and its home. - On a multiplexed gateway the process-level active profile is always - the multiplexer's own (usually ``default``), so reporting it would - answer "default" in every chat regardless of which profile actually - serves the room/channel (``source.profile`` — stamped by the - ``/p//`` URL prefix, a per-credential adapter, or a room→ - profile map). When ``multiplex_profiles`` is on, report the stamped - profile and, like the scoped /reset banner (#59003), resolve the - displayed home under that profile's runtime scope. When multiplexing - is off (the default) the stamp is ignored — mirroring the gating in - ``_run_agent`` and ``_reset_notice_session_info`` — and the command - reports the active profile and default home, byte-identical to before. + On a multiplexed gateway the process-level active profile is always the multiplexer's own, + so reporting it would answer "default" in every chat regardless of which profile serves the + room (``source.profile``). When ``multiplex_profiles`` is on, report the stamped profile + and, like the scoped /reset banner, resolve the displayed home under that profile's runtime + scope. When off (default) the stamp is ignored — mirroring ``_run_agent`` — and output is + byte-identical to before. """ from hermes_constants import display_hermes_home from hermes_cli.slash_exec import CommandContext, execute_command @@ -408,10 +377,9 @@ class GatewaySlashCommandsMixin: async def _handle_whoami_command(self, event: MessageEvent) -> str: """Handle /whoami — show the user's slash command access on this scope. - Always works (it's in the always-allowed floor of slash_access). - Reports: platform, scope (DM vs group), the user's tier - (admin / user / unrestricted), and the slash commands they can - actually run on this scope. + Always works (it's in the always-allowed floor of slash_access). Reports: platform, scope + (DM vs group), the user's tier (admin / user / unrestricted), and the slash commands they + can actually run on this scope. """ from gateway.slash_access import policy_for_source as _policy_for_source @@ -459,16 +427,9 @@ class GatewaySlashCommandsMixin: async def _handle_kanban_command(self, event: MessageEvent) -> str: """Handle /kanban — delegate to the shared kanban CLI. - Run the potentially-blocking DB work in a thread pool so the - gateway event loop stays responsive. Read operations (list, - show, context, tail) are permitted while an agent is running; - mutations are allowed too because the board is profile-agnostic - and does not touch the running agent's state. - - For ``/kanban create`` invocations we also auto-subscribe the - originating gateway source (platform + chat + thread) to the new - task's terminal events, so the user hears back when the worker - completes / blocks / auto-blocks / crashes without having to poll. + Run the potentially-blocking DB work in a thread pool so the gateway event loop stays + responsive. Reads and mutations are both allowed while an agent runs: the board is + profile-agnostic and never touches the running agent's state. """ import asyncio import re @@ -508,10 +469,9 @@ class GatewaySlashCommandsMixin: except Exception as exc: # pragma: no cover - defensive return t("gateway.kanban.error_prefix", error=exc) - # Auto-subscribe on create. Parse the task id from the CLI's standard - # success line ("Created t_abcd (ready, assignee=...)"). If the user - # passed --json we don't subscribe; they're clearly scripting and - # can call /kanban notify-subscribe explicitly. + # Auto-subscribe on create. Parse the task id from the CLI's standard success line ("Created + # t_abcd (ready, assignee=...)"). If the user passed --json we don't subscribe; they're + # clearly scripting and can call /kanban notify-subscribe explicitly. if is_create and output: m = re.search(r"Created\s+(t_[0-9a-f]+)\b", output) if m: @@ -604,12 +564,9 @@ class GatewaySlashCommandsMixin: title = None session_row: dict[str, Any] = {} - # Pull token totals from the SQLite session DB rather than the - # in-memory SessionStore. The agent's per-turn token deltas are - # persisted into sessions_db (run_agent.py), not into SessionEntry, - # so session_entry.total_tokens is always 0. SessionDB is the - # single source of truth; reading it here keeps /status accurate - # without duplicating token writes into two stores. + # Pull token totals from the SQLite session DB rather than the in-memory SessionStore. The + # agent's per-turn token deltas are persisted into sessions_db (run_agent.py), not into + # SessionEntry, so session_entry.total_tokens is always 0. db_total_tokens = 0 persisted_route: dict[str, Any] = {} if self._session_db: @@ -639,11 +596,9 @@ class GatewaySlashCommandsMixin: except Exception: persisted_route = {} - # Resolve model/context for cockpit-style status. Prefer the live or - # cached agent because it carries the actual runtime route and context - # compressor. Fall back to persisted SessionDB metadata plus the - # SessionStore's last_prompt_tokens so /status remains useful between - # turns without making billing/account calls. + # Resolve model/context for cockpit-style status. Prefer the live or cached agent because it + # carries the actual runtime route and context compressor; fall back to SessionDB metadata + + # last_prompt_tokens so /status stays useful between turns without billing/account calls. status_agent = agent if is_running else None if status_agent is None: cache_lock = getattr(self, "_agent_cache_lock", None) @@ -780,18 +735,11 @@ class GatewaySlashCommandsMixin: async def _handle_context_command(self, event: MessageEvent) -> str: """Handle /context — the dedicated context-window view. - /status shows a one-line ``used / total`` summary; this command is the - deep view: a usage gauge, auto-compression threshold and headroom, - compression count and last savings, and cumulative throughput — the last - clearly labelled as throughput, NOT context size. - - Resolves from the running agent (mid-turn), then the cached agent - (between turns), then the SessionStore/SessionDB metadata for a gauge - even when no agent is resident. Falls back to a transcript estimate only - as a last resort. - - ``/context all`` appends the expanded per-skill / per-toolset cost - listings (requires a resident agent). + /status shows a one-line ``used / total`` summary; this command is the deep view: a usage + gauge, auto-compression threshold and headroom, compression count and last savings, and + cumulative throughput — the last clearly labelled as throughput, NOT context size. + Resolution order: running agent, cached agent, SessionStore/SessionDB metadata, and a + transcript estimate only as last resort. ``/context all`` adds per-skill/toolset listings. """ from gateway.run import _AGENT_PENDING_SENTINEL @@ -948,10 +896,9 @@ class GatewaySlashCommandsMixin: lines.append("") lines.append(t("gateway.context.detail_after_first")) - # Per-category estimated breakdown (+ optional expanded listings). - # Same chars/4 engine the desktop popover and /usage use; plain - # text (no glyph grid — monospace isn't guaranteed on messaging - # platforms). Fail-open: rendering errors never break /context. + # Per-category estimated breakdown (+ optional expanded listings). Same chars/4 engine + # the desktop popover and /usage use; plain text (no glyph grid — monospace isn't + # guaranteed on messaging platforms). Fail-open: rendering errors never break /context. if has_agent: breakdown = await asyncio.to_thread( self._context_breakdown_block, agent, source, expanded @@ -1023,15 +970,11 @@ class GatewaySlashCommandsMixin: def _same_origin_chat(self, current: SessionSource, origin: Optional[SessionSource]) -> bool: """Platform-agnostic counterpart to ``_same_matrix_room``. - True when *origin* shares *current*'s platform and chat, and the same - participant whenever the session key for this source is per-user. Group - and thread sessions that ``build_session_key`` isolates per participant - (the default ``group_sessions_per_user=True``) must also be scoped by - participant here — otherwise a co-member could resume another member's - live per-user group session (IDOR). Only an explicitly shared - group/thread (``group_sessions_per_user=False`` / - ``thread_sessions_per_user``) lets co-members share, mirroring the key - contract via ``is_shared_multi_user_session``. + Group and thread sessions that ``build_session_key`` isolates per participant (the + default ``group_sessions_per_user=True``) must also be scoped by participant here — + otherwise a co-member could resume another member's live per-user group session (IDOR). + Only an explicitly shared group/thread lets co-members share, mirroring the key contract + via ``is_shared_multi_user_session``. """ if origin is None or current is None: return False @@ -1087,15 +1030,13 @@ class GatewaySlashCommandsMixin: return False def _resume_caller_is_admin(self, source: SessionSource) -> bool: - """Whether *source* is an EXPLICITLY-configured admin allowed to make a - cross-origin /resume or /sessions listing. + """Whether *source* is an EXPLICITLY-configured admin allowed to make a cross-origin /resume + or /sessions listing. - Deliberately stricter than ``SlashAccessPolicy.is_admin()``: that returns - True for every allowed caller when slash gating is DISABLED (so commands - stay runnable by default), but cross-ORIGIN DATA ACCESS must require a - real, configured admin. Otherwise the default (no admin list) config - would treat every gateway caller as cross-origin-capable and re-open the - enumeration IDOR. + Deliberately stricter than ``SlashAccessPolicy.is_admin()``: that returns True for every + allowed caller when slash gating is DISABLED (so commands stay runnable by default), but + cross-ORIGIN DATA ACCESS must require a real, configured admin — otherwise the default + (no admin list) config would make every caller cross-origin-capable and re-open the IDOR. """ try: from gateway.slash_access import policy_for_source @@ -1110,13 +1051,11 @@ class GatewaySlashCommandsMixin: ) -> bool: """Whether *source* may resume the persisted session *target_id*. - Generalizes the Matrix-only room guard to every adapter so a caller - cannot bind their gateway session to another user's/room's persisted - session id (IDOR). Uses the live origin when the target is active; - otherwise falls back to the DB row's source + user_id (the sessions - table has no chat_id). An identity-bearing caller is allowed only when - the row PROVES the same owner; a row that lacks enough ownership data - fails closed. An explicit admin ``--all`` override bypasses scoping. + Generalizes the Matrix-only room guard to every adapter so a caller cannot bind their + gateway session to another user's/room's persisted session id (IDOR). Uses the live origin + when the target is active; otherwise the DB row's source + user_id. A row must PROVE the + same owner; insufficient ownership data fails closed. An explicit admin ``--all`` override + bypasses scoping. """ if allow_override and self._resume_caller_is_admin(source): return True @@ -1140,51 +1079,28 @@ class GatewaySlashCommandsMixin: return False # different platform / source caller_uid = str(getattr(source, "user_id", "") or "") row_uid = str(row.get("user_id") or "") - # Chat/thread origin recorded at session creation (see - # SessionDB._insert_session_row). The sessions table historically stored - # only source + user_id, so a same-user row could belong to a DIFFERENT - # chat; comparing the persisted origin closes that gap. Legacy rows - # created before origin capture have NULL here and therefore fail closed - # (they cannot prove the caller's chat) — resume them via a live session - # or an admin override. + # Chat/thread origin recorded at session creation (see SessionDB._insert_session_row). The + # sessions table historically stored only source + user_id, so a same-user row could belong + # to a DIFFERENT chat; comparing the persisted origin closes that gap. Legacy rows have NULL + # here and fail closed — resume them via a live session or an admin override. caller_chat = str(getattr(source, "chat_id", "") or "") row_chat = str(row.get("chat_id") or "") caller_thread = str(getattr(source, "thread_id", "") or "") row_thread = str(row.get("thread_id") or "") chat_type = (getattr(source, "chat_type", "") or "").lower() caller_is_dm = chat_type in {"dm", "direct", "private", ""} - # build_session_key keys the participant on ``user_id_alt or user_id`` - # (Signal/Feishu carry the canonical participant in user_id_alt), but the - # sessions table only ever stored user_id — it has no user_id_alt column. - # So when the caller carries a user_id_alt, the row CANNOT prove the - # canonical participant that the live session key is built from: two - # members sharing one user_id but different user_id_alt map to DIFFERENT - # session keys, yet the persisted row's user_id would match both. The - # live-origin guard (_same_origin_chat) compares user_id_alt correctly; - # the persisted fallback cannot, so any per-user comparison that would - # otherwise rely on row_uid == caller_uid must fail closed here to stay - # in lock-step with the key boundary (CWE-639). Shared group/thread - # sessions are unaffected (they don't scope by participant at all), and - # an admin --all override still bypasses this above. + # build_session_key keys the participant on ``user_id_alt or user_id`` (Signal/Feishu carry + # the canonical participant in user_id_alt), but the sessions table only ever stored user_id + # — it has no user_id_alt column. So a row cannot prove the canonical participant for an + # alt-keyed caller: any per-user comparison relying on row_uid == caller_uid must fail + # closed to stay in lock-step with the key boundary (CWE-639). Shared sessions unaffected. caller_keys_on_alt = bool(str(getattr(source, "user_id_alt", "") or "")) if caller_uid: - # Identity-bearing caller: allow only when the row PROVES the same - # owner AND the same platform/origin AND the same chat/thread. A row - # with no/blank user_id cannot be proven to belong to this caller; a - # row with no/blank source cannot be proven to share the caller's - # platform (the row_src check above only rejects a *mismatching* - # non-blank source, so a blank/legacy source would otherwise slip - # through on user_id equality alone); and a row whose origin chat - # (or thread) differs from the caller's belongs to a different - # conversation. Any gap fails closed — an identified user must not - # bind to an unowned, other-owned, other-chat, or unproven-origin - # persisted session by id/title. (Legacy NULL-owner/blank-source/ - # NULL-chat rows are intentionally not resumable this way; use a - # live session or an explicit admin override.) - # Common origin proof for any identity-bearing caller: a non-blank - # source that matches the caller's platform, and the same thread. A - # blank/legacy source can't prove the platform; a different thread is - # a different session (build_session_key appends thread_id). + # Identity-bearing caller: allow only when the row PROVES the same owner AND the same + # platform/origin AND the same chat/thread. A blank/legacy source can't prove the + # platform (the row_src check above only rejects a *mismatching* non-blank source); a + # different thread is a different session (build_session_key appends thread_id). Any + # gap fails closed — legacy NULL-owner/blank-source/NULL-chat rows are not resumable here. origin_ok = ( bool(row_src) and bool(caller_src) and str(row_src) == str(caller_src) @@ -1193,28 +1109,20 @@ class GatewaySlashCommandsMixin: if not origin_ok: return False if caller_is_dm: - # DMs are keyed on user_id; require the same owner. chat_id is - # legitimately absent on both sides for a no-chat_id DM (scoped - # by user_id), but a mismatching chat_id (when present) is still - # rejected. - # - # A no-chat_id DM is keyed PURELY on the participant - # (``user_id_alt or user_id``). If the caller keys on user_id_alt - # the persisted row (user_id only) cannot prove that participant, - # so fail closed. When chat_id is present on both sides it is the - # DM key and equal chat_id is sufficient, so the alt gap doesn't - # apply there. + # DMs are keyed on user_id; require the same owner. chat_id is legitimately absent + # on both sides for a no-chat_id DM (scoped by user_id), but a mismatching chat_id + # (when present) is still rejected. A no-chat_id DM is keyed PURELY on the + # participant, so an alt-keyed caller fails closed there; with chat_id on both sides + # equal chat_id is the DM key and suffices. if caller_keys_on_alt and not (bool(row_chat) and bool(caller_chat)): return False return ( bool(row_uid) and row_uid == caller_uid and row_chat == caller_chat ) - # Non-DM (group/channel/forum/thread): build_session_key includes - # chat_id, so a row (or caller) with NO chat provenance cannot prove - # same-chat. Require both sides non-blank and equal — a legacy - # NULL-chat row (or a caller missing its chat_id) fails closed even - # when both normalize to "". (CWE-639) + # Non-DM (group/channel/forum/thread): build_session_key includes chat_id, so a row (or + # caller) with NO chat provenance cannot prove same-chat. Require both non-blank and + # equal — a legacy NULL-chat row fails closed even when both normalize to "". (CWE-639) if not (bool(row_chat) and bool(caller_chat) and row_chat == caller_chat): return False # Within the same non-DM chat/thread, mirror build_session_key's @@ -1231,21 +1139,16 @@ class GatewaySlashCommandsMixin: ) if shared: return True - # Per-user non-DM: the session key includes the participant - # (``user_id_alt or user_id``). If the caller keys on user_id_alt, - # the persisted row (user_id only) cannot prove the canonical - # participant, so fail closed rather than matching on user_id alone. + # Per-user non-DM: the session key includes the participant (``user_id_alt or + # user_id``). If the caller keys on user_id_alt, the persisted row (user_id only) cannot + # prove the canonical participant, so fail closed rather than matching on user_id alone. if caller_keys_on_alt: return False return bool(row_uid) and row_uid == caller_uid - # No caller identity: the persisted row carries only source + user_id - # (the sessions table has no chat_id), so a same-platform row can belong - # to a DIFFERENT chat or user. Same-platform alone is therefore NOT - # ownership proof — an identity-less caller must not bind to, or - # enumerate, a persisted session by id/title. Fail closed. A legitimate - # same-chat resume of an ACTIVE session still works through the - # live-origin branch above (which compares chat_id), and an operator can - # use the admin --all override. (CWE-639: IDOR on session routing.) + # No caller identity: the persisted row carries only source + user_id (the sessions table + # has no chat_id), so a same-platform row can belong to a DIFFERENT chat or user. Same + # platform alone is NOT ownership proof — fail closed. Same-chat resume of an ACTIVE session + # still works via the live-origin branch above. (CWE-639: IDOR on session routing.) return False async def _resume_row_visible( @@ -1253,18 +1156,14 @@ class GatewaySlashCommandsMixin: ) -> bool: """Whether a titled-session listing *row* belongs to the caller's origin. - Prevents cross-origin enumeration of session ids/previews via the - numbered /resume list. Preserves the existing Matrix room-scoping - semantics; scopes every other platform to the caller's own sessions - unless an admin passes ``--all``. + Prevents cross-origin enumeration of session ids/previews via the numbered /resume list. + Preserves the existing Matrix room-scoping semantics; scopes every other platform to the + caller's own sessions unless an admin passes ``--all``. """ sid = str(row.get("id") or "") if source.platform == Platform.MATRIX: - # Cross-room enumeration is cross-ORIGIN data access: gate the - # ``--all`` short-circuit behind a real configured admin, exactly - # like the non-Matrix branch below. A non-admin Matrix ``--all`` - # falls back to same-room scoping rather than exposing every Matrix - # titled session. + # Cross-room enumeration is cross-ORIGIN data access: gate the ``--all`` short-circuit + # behind a real configured admin, exactly like the non-Matrix branch below. if allow_all and self._resume_caller_is_admin(source): return True return self._same_matrix_room(source, self._gateway_session_origin_for_id(sid)) @@ -1426,13 +1325,10 @@ class GatewaySlashCommandsMixin: async def _handle_stop_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /stop command - interrupt a running agent. - When an agent is truly hung (blocked thread that never checks - _interrupt_requested), the early intercept in _handle_message() - handles /stop before this method is reached. This handler fires - only through normal command dispatch (no running agent) or as a - fallback. Force-clean the session lock in all cases for safety. - - The session is preserved so the user can continue the conversation. + When an agent is truly hung (blocked thread that never checks _interrupt_requested), the + early intercept in _handle_message() handles /stop before this method is reached; this + handler fires only via normal dispatch (no running agent) or as a fallback, and force-cleans + the session lock in all cases. The session is preserved so the user can continue. """ from gateway.run import _AGENT_PENDING_SENTINEL, _INTERRUPT_REASON_STOP source = event.source @@ -1461,12 +1357,10 @@ class GatewaySlashCommandsMixin: ) return EphemeralReply(t("gateway.stop.stopped")) - # No run under the caller's own session key. In a per-user thread - # (thread_sessions_per_user=True) each participant is isolated even - # inside one shared thread, so a run another user started lives under - # a different key. Authorized users should still be able to /stop it - # (#bernard-thread-stop). Fall back to interrupting any running - # agent(s) that share this thread, gated on authorization. + # No run under the caller's own session key. In a per-user thread + # (thread_sessions_per_user=True) each participant is isolated even inside one shared + # thread, so a run another user started lives under a different key. Authorized users must + # still be able to /stop it: fall back to sibling runs in this thread, gated on authorization. sibling_keys = self._sibling_thread_run_keys(source, session_key) if sibling_keys and self._is_user_authorized(source): for sibling_key in sibling_keys: @@ -1484,11 +1378,9 @@ class GatewaySlashCommandsMixin: ) return EphemeralReply(t("gateway.stop.stopped")) - # No running agent anywhere for this scope. A platform status - # indicator can still be stuck — e.g. Slack's persistent - # assistant.threads.setStatus survives a gateway restart or a turn - # that died without a final send (#32295). Best-effort clear so - # /stop always dismisses a phantom "is thinking...". + # No running agent anywhere for this scope. A platform status indicator can still be stuck — + # e.g. Slack's persistent assistant.threads.setStatus survives a gateway restart or a turn + # that died without a final send. adapter = getattr(self, "adapters", {}).get(source.platform) if adapter and hasattr(adapter, "_stop_typing_with_metadata"): try: @@ -1507,12 +1399,12 @@ class GatewaySlashCommandsMixin: return t("gateway.stop.no_active") async def _handle_platform_command(self, event: MessageEvent) -> str: - """Handle ``/platform list|pause|resume [name]`` — surface and - manually control failed/paused gateway adapters. + """Handle ``/platform list|pause|resume [name]`` — surface and manually control + failed/paused gateway adapters. Examples: - ``/platform list`` — show connected + failed/paused platforms - ``/platform pause whatsapp`` — stop the reconnect watcher hammering whatsapp + ``/platform list`` — show connected + failed/paused platforms + ``/platform pause whatsapp`` — stop the reconnect watcher hammering whatsapp ``/platform resume whatsapp`` — re-queue a paused platform for retry """ text = (getattr(event, "content", "") or "").strip() @@ -1602,17 +1494,11 @@ class GatewaySlashCommandsMixin: async def _handle_restart_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: """Handle /restart command - drain active work, then restart the gateway.""" from gateway.run import _hermes_home - # Defensive idempotency check: if the previous gateway process - # recorded this same /restart (same platform + update_id) and the new - # process is seeing it *again*, this is a re-delivery caused by PTB's - # graceful-shutdown `get_updates` ACK failing on the way out ("Error - # while calling `get_updates` one more time to mark all fetched - # updates. Suppressing error to ensure graceful shutdown. When - # polling for updates is restarted, updates may be received twice." - # in gateway.log). Ignoring the stale redelivery prevents a - # self-perpetuating restart loop where every fresh gateway - # re-processes the same /restart command and immediately restarts - # again. + # Defensive idempotency check: if the previous gateway process recorded this same /restart + # (same platform + update_id) and the new process is seeing it *again*, this is a re- + # delivery caused by PTB's graceful-shutdown `get_updates` ACK failing on the way out + # ("updates may be received twice" in gateway.log). Ignoring the stale redelivery prevents + # a self-perpetuating loop where every fresh gateway re-runs /restart and restarts again. if self._is_stale_restart_redelivery(event): logger.info( "Ignoring redelivered /restart (platform=%s, update_id=%s) — " @@ -1665,11 +1551,9 @@ class GatewaySlashCommandsMixin: except Exception as e: logger.debug("Failed to write restart notify file: %s", e) - # Record the triggering platform + update_id in a dedicated dedup - # marker. Unlike .restart_notify.json (which gets unlinked once the - # new gateway sends the "gateway restarted" notification), this - # marker persists so the new gateway can still detect a delayed - # /restart redelivery from Telegram. Overwritten on every /restart. + # Record the triggering platform + update_id in a dedicated dedup marker. Unlike + # .restart_notify.json (unlinked once the new gateway sends its notification) this persists + # so a delayed Telegram redelivery is still detectable. Overwritten on every /restart. try: dedup_data = { "platform": event.source.platform.value if event.source.platform else None, @@ -1687,15 +1571,11 @@ class GatewaySlashCommandsMixin: logger.debug("Failed to write restart dedup marker: %s", e) active_agents = self._running_agent_count() - # When running under a service manager (systemd/launchd) or inside a - # Docker/Podman container, use the service restart path: exit with - # code 75 so the service manager / container restart policy restarts - # us. The detached subprocess approach (setsid + bash) doesn't work - # under systemd (KillMode=mixed kills the cgroup) or Docker (tini - # exits when the gateway dies, taking the detached helper with it). - # Native supervisor markers cover direct systemd/launchd starts. The - # explicit marker covers wrappers such as ``sudo env -i`` that strip - # those markers before execing the foreground gateway. + # When running under a service manager (systemd/launchd) or inside a Docker/Podman + # container, use the service restart path: exit with code 75 so the service manager / + # container restart policy restarts us. The detached setsid+bash approach fails there + # (systemd KillMode=mixed kills the cgroup; tini exits with the gateway). Native supervisor + # markers cover direct starts; the explicit marker covers wrappers like ``sudo env -i``. from gateway.restart import ( is_container_restart_context, is_gateway_supervisor_process, @@ -1749,17 +1629,7 @@ class GatewaySlashCommandsMixin: ) async def _handle_model_command(self, event: MessageEvent) -> Optional[str]: - """Handle /model command — switch model. - - Supports: - /model — interactive picker (Telegram/Discord) or text list - /model — switch model (this session only) - /model --once — switch for the next turn only - /model --session — switch for this session only (explicit) - /model --global — switch and persist to config.yaml - /model --provider — switch provider + model - /model --provider — switch to provider, auto-detect model - """ + """Handle /model command — switch model.""" from gateway.run import _hermes_home, _load_gateway_config from hermes_cli.model_switch import ( switch_model as _switch_model, parse_model_switch_args, @@ -1833,11 +1703,9 @@ class GatewaySlashCommandsMixin: except Exception: pass - # Check for session override. Normalize the source the same way a normal - # message turn does - # (Telegram DM topic recovery) before deriving the override key, so - # the override is stored under the key the next message turn reads - # (#30479). + # Check for session override. Normalize the source the same way a normal message turn does + # (Telegram DM topic recovery) before deriving the override key, so the override is stored + # under the key the next message turn reads. source = await asyncio.to_thread(self._normalize_source_for_session_key, source) session_key = self._session_key_for_source(source) override = self._session_model_overrides.get(session_key, {}) @@ -1896,10 +1764,9 @@ class GatewaySlashCommandsMixin: skew_error = _model_switch_skew_guard() if skew_error: return skew_error - # Offload the switch off the event loop — switch_model() - # can fall through to a synchronous models.dev HTTP fetch - # (requests.get, 15s timeout) on a cold/expired cache, - # which freezes the gateway otherwise. See #20525, #41289. + # Offload the switch off the event loop — switch_model() can fall through to + # a synchronous models.dev HTTP fetch (requests.get, 15s timeout) on a + # cold/expired cache, which freezes the gateway otherwise. result = await asyncio.to_thread( _switch_model, raw_input=model_id, @@ -1955,15 +1822,11 @@ class GatewaySlashCommandsMixin: ), ) except Exception as exc: - # The in-place swap rolled the agent back to the - # OLD working model/client and re-raised. Abort - # the rest of the commit: do NOT persist the - # failed model to the DB, do NOT set a session - # override pointing at the broken model, and do - # NOT evict the working cached agent. Otherwise - # the next message rebuilds a dead agent from the - # broken override and the conversation is lost - # (#50163). A failed switch must be a no-op. + # The in-place swap rolled the agent back to the OLD working + # model/client and re-raised. Abort the commit: do NOT persist the + # failed model, set a session override, or evict the working cached + # agent — otherwise the next message rebuilds a dead agent from the + # broken override and the conversation is lost. A failed switch is a no-op. logger.warning( "Picker model switch failed for cached agent: %s", exc ) @@ -1992,10 +1855,9 @@ class GatewaySlashCommandsMixin: "Failed to persist model switch to DB: %s", exc ) - # Store model note + session override. Use display - # form (strips opaque Palantir prefix) for the user- - # visible note; session-override map still gets the - # full opaque ID, which is what the wire needs. + # Store model note + session override. Use display form (strips opaque + # Palantir prefix) for the user- visible note; session-override map still + # gets the full opaque ID, which is what the wire needs. from hermes_cli.model_switch import format_model_for_display _display_cur = format_model_for_display(_cur_model) _display_new = format_model_for_display(result.new_model) @@ -2070,13 +1932,10 @@ class GatewaySlashCommandsMixin: _persist_model_cfg.pop("context_length", None) _persist_model_cfg["default"] = result.new_model _persist_model_cfg["provider"] = result.target_provider - # Named providers always resolve base_url/api_mode fresh, - # so any leftover is cleared unconditionally below. Custom - # providers have no registry entry to re-derive from, so - # they need an explicit set-or-clear here — the previous - # lone `if result.base_url:` left a stale base_url behind - # when switching to a custom provider whose resolver - # returned an empty base_url (#25107). + # Named providers always resolve base_url/api_mode fresh, so any + # leftover is cleared unconditionally below. Custom providers have no + # registry entry to re-derive from, so they need an explicit + # set-or-clear here (a lone `if base_url:` left stale values behind). _is_custom_target = str(result.target_provider or "").strip().lower() == "custom" if result.base_url: _persist_model_cfg["base_url"] = result.base_url @@ -2210,10 +2069,9 @@ class GatewaySlashCommandsMixin: skew_error = _model_switch_skew_guard() if skew_error: return skew_error - # Offload the switch off the event loop — switch_model() can fall - # through to a synchronous models.dev HTTP fetch (requests.get, 15s - # timeout) on a cold/expired cache, which freezes the gateway - # otherwise. See #20525, #41289. + # Offload the switch off the event loop — switch_model() can fall through to a synchronous + # models.dev HTTP fetch (requests.get, 15s timeout) on a cold/expired cache, which freezes + # the gateway otherwise. result = await asyncio.to_thread( _switch_model, raw_input=model_input, @@ -2271,12 +2129,10 @@ class GatewaySlashCommandsMixin: capabilities=getattr(result, "runtime_capabilities", None), ) except Exception as exc: - # In-place swap rolled the agent back to the OLD working - # model/client and re-raised. Abort the commit: skip DB - # persist, session override, cache eviction, and config - # write so a failed switch is a no-op rather than a dead - # conversation (#50163). Without this early return the - # next message rebuilds a broken agent from the override. + # In-place swap rolled the agent back to the OLD working model/client and re- + # raised. Abort the commit (skip DB persist, session override, cache eviction, + # config write) so a failed switch is a no-op. Without this early return the next + # message rebuilds a broken agent from the override. logger.warning("In-place model switch failed for cached agent: %s", exc) return t( "gateway.model.error_prefix", @@ -2306,10 +2162,9 @@ class GatewaySlashCommandsMixin: "Failed to persist model switch to DB: %s", exc ) - # Store a note to prepend to the next user message so the model - # knows about the switch (avoids system messages mid-history). - # Display form strips opaque Palantir RID prefixes; the override - # map below keeps the full ID for the wire. + # Store a note to prepend to the next user message so the model knows about the switch + # (avoids system messages mid-history). Display form strips opaque Palantir RID + # prefixes; the override map below keeps the full ID for the wire. from hermes_cli.model_switch import format_model_for_display if not hasattr(self, "_pending_model_notes"): self._pending_model_notes = {} @@ -2339,18 +2194,11 @@ class GatewaySlashCommandsMixin: elif hasattr(self, "_pending_one_turn_model_restores"): self._pending_one_turn_model_restores.pop(session_key, None) - # Write-through the non-secret parts (model/provider/base_url) to - # the session store so the override survives a gateway restart. - # api_key/api_mode are never persisted — they are re-resolved via - # runtime provider resolution on rehydration. - # - # /model --once is intentionally EXCLUDED from the write-through: - # a one-turn override must never survive a restart. The persisted - # value stays at the pre-once state (the prior session override, - # or nothing), which is exactly what the finally-restore reverts - # the in-memory dict to. (#29923 review defect: the original - # implementation wrote through, so a crash before the restore - # rehydrated the once-model permanently.) + # Write-through the non-secret parts (model/provider/base_url) to the session store so + # the override survives a gateway restart. api_key/api_mode are never persisted — they + # are re-resolved via runtime provider resolution on rehydration. /model --once is + # deliberately EXCLUDED: a one-turn override must never survive a restart, so the + # persisted value stays at the pre-once state the finally-restore reverts to. if not one_turn: try: await self.async_session_store.set_model_override( @@ -2373,12 +2221,10 @@ class GatewaySlashCommandsMixin: # defaults must not be persisted back to the user's file). from hermes_cli.config import read_user_config_raw cfg = read_user_config_raw(config_path) - # Coerce scalar/None ``model:`` into a dict before mutation — - # otherwise ``cfg.setdefault("model", {})`` returns the existing - # scalar and the next assignment raises - # ``TypeError: 'str' object does not support item assignment``. - # Reproduces when ``config.yaml`` has ``model: `` (flat - # string) instead of the proper nested ``model: {default: ...}``. + # Coerce scalar/None ``model:`` into a dict before mutation — otherwise + # ``cfg.setdefault("model", {})`` returns the existing scalar and the next + # assignment raises ``TypeError: 'str' object does not support item + # assignment``. raw_model = cfg.get("model") if isinstance(raw_model, dict): model_cfg = raw_model @@ -2487,13 +2333,10 @@ class GatewaySlashCommandsMixin: return "\n".join(lines) - # Selection-guard confirmation gate (typed /model path). - # The pickers (Telegram/Discord inline keyboards, TUI, dashboard) - # already confirm via their own UI affordances; this covers the - # direct text command, which previously bypassed the guard. - # Runs the unified registry (cost + data-policy + future guards). - # Pricing lookups may hit models.dev or a /models endpoint on a - # cache miss, so run it off the event loop. + # Selection-guard confirmation gate (typed /model path). The pickers already confirm + # via their own UI; this covers the direct text command, which previously bypassed the + # guard. Runs the unified registry (cost + data-policy + future guards). Pricing lookups may + # hit models.dev or a /models endpoint on a cache miss, so run it off the event loop. _cost_warning = None try: from hermes_cli.model_selection_guards import combined_selection_warning @@ -2538,15 +2381,9 @@ class GatewaySlashCommandsMixin: async def _handle_codex_runtime_command(self, event: MessageEvent) -> str: """Handle /codex-runtime command in the gateway. - Same surface as the CLI handler in cli.py: - /codex-runtime — show current state - /codex-runtime auto — Hermes default runtime - /codex-runtime codex_app_server — codex subprocess runtime - /codex-runtime on / off — synonyms - - On change, the cached agent for this session is evicted so the next - message creates a fresh AIAgent with the new api_mode wired in - (avoids prompt-cache invalidation mid-session).""" + On change the cached agent is evicted so the next message builds a fresh AIAgent with the + new api_mode (avoids prompt-cache invalidation mid-session). + """ from hermes_cli import codex_runtime_switch as crs raw_args = event.get_command_args().strip() if event else "" @@ -2625,11 +2462,10 @@ class GatewaySlashCommandsMixin: available = "`none`, " + ", ".join(f"`{n}`" for n in personalities) return t("gateway.personality.unknown", name=args.lower(), available=available) - # Persist the selection only — hermes_cli.personality never writes - # agent.system_prompt (user-owned manual overlay). persist_personality - # writes get_hermes_home()/config.yaml, i.e. the routed profile under - # multiplex; the next turn re-resolves the prompt from that file - # (_get_system_prompt_for_channel), so no process-global state to update. + # Persist the selection only — hermes_cli.personality never writes agent.system_prompt + # (user-owned manual overlay). persist_personality writes get_hermes_home()/config.yaml, + # i.e. the routed profile under multiplex; the next turn re-resolves the prompt from that + # file (_get_system_prompt_for_channel), so no process-global state to update. if not persist_personality(name): return t("gateway.personality.save_failed", error="config write failed") @@ -2642,12 +2478,10 @@ class GatewaySlashCommandsMixin: source = event.source session_entry = await self.async_session_store.get_or_create_session(source) history = await self.async_session_store.load_transcript(session_entry.session_id) - - # Find the last *real* user message. Timeline bookkeeping rows carry - # role=user + display_kind (model_switch / async_delegation_complete / - # auto_continue / hidden); clients never count them as user turns. - # Without this filter /retry rewrote the transcript around a marker - # and re-sent opaque bookkeeping text (same class as the TUI ordinal). + + # Find the last *real* user message. Timeline bookkeeping rows carry role=user + + # display_kind (model_switch / async_delegation_complete / auto_continue / hidden); clients + # never count them as user turns. last_user_idx = None # The canonical projection excludes bookkeeping and pure handoffs while # still recognizing a real ask embedded in a compaction carrier. @@ -2680,11 +2514,9 @@ class GatewaySlashCommandsMixin: return f"Cannot retry that message safely: {exc}" if handoff is not None: - # A composite carrier is one physical row containing both the - # retained summary and the live ask. Let the carrier-aware rewind - # archive that row/tail and insert its pure scaffold atomically. - # Plain turns keep the existing rewrite path below; #84078 owns - # its separate archive_dropped/prefix-CAS semantics. + # A composite carrier is one physical row containing both the retained summary and the + # live ask. Let the carrier-aware rewind archive that row/tail and insert its pure + # scaffold atomically. try: rewind_result = await self.async_session_store.rewind_session( session_entry.session_id, @@ -2700,10 +2532,9 @@ class GatewaySlashCommandsMixin: # therefore never be removed while this handler resends stale text. last_user_msg = rewind_result["target_text"] else: - # After in-place compaction the pre-compaction transcript lives on - # as active=0/compacted=1 rows under this session id. active_only - # preserves that archive; a separate existence probe could fail - # open or race with the write. + # After in-place compaction the pre-compaction transcript lives on as + # active=0/compacted=1 rows under this session id. active_only preserves that archive; a + # separate existence probe could fail open or race with the write. if not await self.async_session_store.rewrite_transcript( session_entry.session_id, truncated, @@ -2722,20 +2553,16 @@ class GatewaySlashCommandsMixin: raw_message=event.raw_message, channel_prompt=event.channel_prompt, ) - + # Let the normal message handler process it return await self._handle_message(retry_event) async def _handle_goal_command(self, event: "MessageEvent") -> str: """Handle /goal for gateway platforms. - Subcommands: ``/goal`` / ``/goal status`` / ``/goal pause`` / - ``/goal resume`` / ``/goal clear``. Any other text becomes the - new goal. - - Setting a new goal queues the goal text as the next turn so the - agent starts working on it immediately — the post-turn - continuation hook then takes over from there. + Subcommands: ``/goal`` / ``/goal status`` / ``/goal pause`` / ``/goal resume`` / ``/goal + clear``. Setting a new goal queues the goal text as the next turn so the agent starts + working on it immediately — the post-turn continuation hook then takes over from there. """ args = (event.get_command_args() or "").strip() lower = args.lower() @@ -2768,12 +2595,9 @@ class GatewaySlashCommandsMixin: state = mgr.resume() if state is None: return t("gateway.goal.no_resume") - # Resume must restart work, not just flip persisted state - # (#75362): enqueue the canonical continuation through the - # adapter FIFO — the same path the post-turn judge uses — so - # the next turn fires as soon as this reply is delivered. A - # real user message already queued still preempts naturally, - # and pause/clear's stale-continuation cleanup recognizes it. + # Resume must restart work, not just flip persisted state: enqueue the canonical + # continuation through the adapter FIFO — the same path the post-turn judge uses — so + # the next turn fires as soon as this reply is delivered. prompt = mgr.next_continuation_prompt() try: adapter = self.adapters.get(event.source.platform) if event.source else None @@ -2886,10 +2710,9 @@ class GatewaySlashCommandsMixin: import asyncio from hermes_cli.goals import draft_contract - # _run_in_executor_with_context, not a bare hop: drafting a - # contract calls the auxiliary LLM, whose provider/credential - # resolution reads the profile secret scope — a contextvar that - # a default-executor hop drops, leaving it unscoped. + # _run_in_executor_with_context, not a bare hop: drafting a contract calls the + # auxiliary LLM, whose provider/credential resolution reads the profile secret scope + # — a contextvar that a default-executor hop drops, leaving it unscoped. draft_contract_obj = await self._run_in_executor_with_context( draft_contract, objective ) @@ -3014,10 +2837,9 @@ class GatewaySlashCommandsMixin: async def _handle_refine_command(self, event: "MessageEvent") -> str: """Handle /refine — run the memory/skill review fork on demand. - Uses the session's cached AIAgent (idle agents live in - ``_agent_cache``). The review runs in a daemon thread against a - snapshot of the conversation; the live session and prompt cache are - untouched. Requires the session to have at least one completed turn. + Uses the session's cached AIAgent (idle agents live in ``_agent_cache``). The review runs + in a daemon thread against a snapshot of the conversation; the live session and prompt + cache are untouched. Requires the session to have at least one completed turn. """ args = (event.get_command_args() or "").strip() quick_key = self._session_key_for_source(event.source) if event.source else None @@ -3058,15 +2880,9 @@ class GatewaySlashCommandsMixin: async def _handle_review_command(self, event: "MessageEvent") -> str: """Handle /review — spawn an independent reviewer subagent. - Snapshots the last 10 chat messages from the session's cached agent, - wraps them (plus any argument text) in a reviewer briefing, and - dispatches a full-privilege background subagent on the async - delegation rail. The completed review re-enters this session as a - normal async-delegation completion turn. - - The approval session-key contextvar is only bound during agent - turns, so it is bound explicitly here — without it the completion - event would carry no gateway route and never re-enter this chat. + The approval session-key contextvar is only bound during agent turns, so it is bound + explicitly here — without it the completion event would carry no gateway route and never + re-enter this chat. """ args = (event.get_command_args() or "").strip() quick_key = self._session_key_for_source(event.source) if event.source else None @@ -3117,9 +2933,8 @@ class GatewaySlashCommandsMixin: async def _handle_subgoal_command(self, event: "MessageEvent") -> str: """Handle /subgoal for gateway platforms (mirror of CLI handler). - Subgoals are extra criteria appended to the active goal mid-loop. - They modify state read at the next turn boundary, so this is safe - to invoke while the agent is running. + Subgoals are extra criteria appended to the active goal mid-loop. They modify state read + at the next turn boundary, so this is safe to invoke while the agent is running. """ args = (event.get_command_args() or "").strip() mgr, _session_entry = await self._get_goal_manager_for_event(event) @@ -3193,10 +3008,9 @@ class GatewaySlashCommandsMixin: async def _handle_loop_command(self, event: "MessageEvent") -> str: """Handle /loop for gateway platforms — recurring in-session wakeups. - Mirrors the CLI handler via the shared ``dispatch_loop_command``. - New loops capture the event's routing (platform/chat/thread) so the - gateway's idle loop-wakeup watcher can inject ticks back into this - chat even after a restart. + Mirrors the CLI handler via the shared ``dispatch_loop_command``. New loops capture the + event's routing (platform/chat/thread) so the gateway's idle loop-wakeup watcher can + inject ticks back into this chat even after a restart. """ try: from hermes_cli.loops import dispatch_loop_command, goal_blocks_loop_tick @@ -3240,15 +3054,12 @@ class GatewaySlashCommandsMixin: return output async def _handle_undo_command(self, event: MessageEvent) -> str: - """Handle /undo [N] — back up N user turns (default 1), soft-deleting - the truncated rows on disk and echoing the backed-up message text so - the user can copy/edit and resend. + """Handle /undo [N] — back up N user turns (default 1), soft-deleting the truncated rows on + disk and echoing the backed-up message text so the user can copy/edit and resend. - Mirrors the CLI/TUI /undo: rewound rows stay in state.db (active=0) - for audit and are hidden from re-prompts and search. The cached agent - is evicted so the next message rebuilds context from the truncated - (active-only) transcript — the gateway's equivalent of the CLI's - in-place history surgery + memory-cache invalidation. + The cached agent is evicted so the next message rebuilds context from the truncated + (active-only) transcript — the gateway's equivalent of the CLI's in-place history surgery + + memory-cache invalidation. """ source = event.source @@ -3429,10 +3240,9 @@ class GatewaySlashCommandsMixin: if adapter: self._set_adapter_auto_tts_disabled(adapter, chat_id, disabled=True) toggle_line = t("gateway.voice.disabled_short") - # Bare /voice still toggles, but append an explainer so users - # discover the on/off/tts/status subcommands (and, on Discord, - # live voice-channel join/leave). The toggle result is shown - # first via the {toggle} placeholder. + # Bare /voice still toggles, but append an explainer so users discover the + # on/off/tts/status subcommands (and, on Discord, live voice-channel join/leave). The + # toggle result is shown first via the {toggle} placeholder. supports_voice_channels = adapter is not None and hasattr( adapter, "join_voice_channel" ) @@ -3529,15 +3339,9 @@ class GatewaySlashCommandsMixin: async def _handle_diff_command(self, event: MessageEvent) -> str: """Handle /diff — show git changes in the working directory. - ``/diff`` (default) shows unstaged + untracked changes, ``/diff - staged`` the staged ones, ``/diff all`` everything since HEAD, and - ``/diff session`` the cumulative checkpoint-baseline diff of what - Hermes itself changed. ``--stat`` limits output to the summary. - - The diff body is truncated hard here (messaging surfaces are not a - pager); platform senders additionally split/clamp long messages to - per-platform limits, the same way tool-progress output is truncated - in three layers before delivery. + The diff body is truncated hard here (messaging surfaces are not a pager); platform + senders additionally split/clamp long messages to per-platform limits, the same way tool- + progress output is truncated in three layers before delivery. """ args = event.get_command_args().strip() @@ -3678,14 +3482,10 @@ class GatewaySlashCommandsMixin: async def _handle_btw_command(self, event: MessageEvent) -> str: """Handle /btw — answer a side question about this conversation. - Snapshots the session transcript and answers the question with a - one-shot auxiliary LLM call (main model by default) — the live - session's history is never touched, so role alternation and the - prompt cache stay intact and the current turn keeps running. The - answer is delivered to the chat when ready. - - Deliberately different from /bg, which spawns a fresh contextless - agent session for independent work. + Snapshots the session transcript and answers the question with a one-shot auxiliary LLM + call (main model by default) — the live session's history is never touched, so role + alternation and the prompt cache stay intact and the current turn keeps running. + Deliberately different from /bg, which spawns a fresh contextless agent session. """ question = event.get_command_args().strip() if not question: @@ -3714,13 +3514,11 @@ class GatewaySlashCommandsMixin: "api_mode": runtime_kwargs.get("api_mode"), } history_snapshot = list(history) - # Prefer the cache-parity fork when this chat has a live cached - # AIAgent: the fork replays the snapshot against the warm provider - # prefix cache (same mechanism as the background self-improvement - # review), giving the side answer FULL conversation context at - # cache-read prices. If no cached agent exists (evicted / first - # message), the provider cache is cold anyway — the one-shot digest - # fallback inside answer_side_question handles it. + # Prefer the cache-parity fork when this chat has a live cached AIAgent: the fork replays + # the snapshot against the warm provider prefix cache (same mechanism as the background + # self-improvement review), giving the side answer FULL conversation context at cache-read + # prices. With no cached agent the provider cache is cold anyway — answer_side_question's + # one-shot digest fallback handles it. parent_agent = None try: session_key = self._session_key_for_source(source) @@ -3802,9 +3600,8 @@ class GatewaySlashCommandsMixin: ) -> str: """Apply a /reasoning argument (typed or picked) and return the reply. - Single application path shared by the typed `/reasoning ` branch - and the interactive choice picker, so both surfaces stay in lockstep - with the canonical parser. + Single application path shared by the typed `/reasoning ` branch and the interactive + choice picker, so both surfaces stay in lockstep with the canonical parser. """ from hermes_constants import parse_reasoning_effort @@ -3917,16 +3714,7 @@ class GatewaySlashCommandsMixin: return False async def _handle_reasoning_command(self, event: MessageEvent) -> Optional[str]: - """Handle /reasoning command — manage reasoning effort and display toggle. - - Usage: - /reasoning Show current effort level and display state - /reasoning Set reasoning effort for this session only - /reasoning --global Persist reasoning effort to config.yaml - /reasoning reset Clear this session's reasoning override - /reasoning show|on Show model reasoning in responses - /reasoning hide|off Hide model reasoning from responses - """ + """Handle /reasoning command — manage reasoning effort and display toggle.""" from gateway.run import _platform_config_key raw_args = event.get_command_args().strip() @@ -4012,10 +3800,8 @@ class GatewaySlashCommandsMixin: async def _handle_memory_command(self, event: MessageEvent) -> str: """Handle /memory — review pending memory writes + toggle the approval gate. - Memory entries are small enough to review inline in a chat bubble, so - the full pending/approve/reject/approval flow works on every platform. - Gate changes persist to config.yaml and evict the cached agent so the - new setting takes effect on the next message. + Memory entries are small enough to review inline in a chat bubble, so the full + pending/approve/reject/approval flow works on every platform. """ from gateway.run import _gateway_config_home from hermes_cli.write_approval_commands import handle_pending_subcommand @@ -4053,18 +3839,11 @@ class GatewaySlashCommandsMixin: async def _handle_skills_command(self, event: MessageEvent) -> str: """Handle /skills on the gateway — pending skill-write review only. - The full skills hub (search/browse/install) stays CLI-only; this - handler covers the write-approval review surface (pending / approve / - reject / diff / approval) so a skill staged from a gateway session can - be reviewed from that same session. Gated by ``skills.write_approval`` - via the CommandDef's ``gateway_config_gate``; also answers when staged - writes still exist after the gate was turned off (so they are never - stranded). - - ``diff`` output is truncated for chat bubbles — the full diff lives in - the pending JSON file under ``~/.hermes/pending/skills/``. (Note this is - the write-approval ``diff ``; the CLI also has an unrelated - ``hermes skills diff `` that diffs a bundled skill vs stock.) + The full skills hub (search/browse/install) stays CLI-only; this handler covers the + write-approval review surface (pending / approve / reject / diff / approval) so a skill + staged from a gateway session can be reviewed from that same session. Gated by + ``skills.write_approval``, but still answers when staged writes exist after the gate was + turned off (so they are never stranded). ``diff`` output is truncated for chat bubbles. """ from gateway.run import _gateway_config_home from hermes_cli.write_approval_commands import handle_pending_subcommand @@ -4247,11 +4026,10 @@ class GatewaySlashCommandsMixin: async def _handle_verbose_command(self, event: MessageEvent) -> str: """Handle /verbose command — cycle tool progress display mode. - Gated by ``display.tool_progress_command`` in config.yaml (default off). - When enabled, cycles the tool progress mode through off → new → all → - verbose → off for the *current platform*. The setting is saved to - ``display.platforms..tool_progress`` so each channel can - have its own verbosity level independently. + Gated by ``display.tool_progress_command`` in config.yaml (default off). Cycles the mode + off → new → all → verbose → off for the *current platform*. The setting is saved to + ``display.platforms..tool_progress`` so each channel can have its own + verbosity level independently. """ from gateway.run import _gateway_config_home, _load_gateway_config, _platform_config_key @@ -4309,15 +4087,7 @@ class GatewaySlashCommandsMixin: return f"{descriptions[new_mode]}\n" + t("gateway.verbose.save_failed", error=e) async def _handle_busy_command(self, event: MessageEvent) -> Union[str, EphemeralReply]: - """Handle /busy — control what happens when messaging while Hermes is working. - - Usage: - /busy Show current busy input mode - /busy status Show current busy input mode - /busy queue Queue messages for the next turn - /busy steer Inject messages mid-run without interrupting - /busy interrupt Interrupt the current run (default) - """ + """Handle /busy — control what happens when messaging while Hermes is working.""" arg = event.get_command_args().strip().lower() if not arg or arg == "status": mode = self._effective_busy_input_mode(event.source) @@ -4351,11 +4121,10 @@ class GatewaySlashCommandsMixin: ) else: self._busy_input_mode = arg - # busy_input_mode is the source of truth for the text mode - # too (run.py:_load_busy_text_mode) — re-derive it so the - # adapter refresh below doesn't read a stale value and keep - # interrupting after e.g. /busy queue (config IS saved; only - # the live session lagged until restart). + # busy_input_mode is the source of truth for the text mode too + # (run.py:_load_busy_text_mode) — re-derive it so the adapter refresh below doesn't + # read a stale value and keep interrupting after e.g. /busy queue (config IS saved; + # only the live session lagged until restart). self._busy_text_mode = self._load_busy_text_mode() adapter = self._adapter_for_source(event.source) @@ -4378,19 +4147,7 @@ class GatewaySlashCommandsMixin: ) async def _handle_footer_command(self, event: MessageEvent) -> str: - """Handle /footer command — toggle the runtime-metadata footer. - - Usage: - /footer → toggle on/off - /footer on → enable globally - /footer off → disable globally - /footer status → show current state + fields - - The footer is saved to ``display.runtime_footer.enabled`` (global). - Per-platform overrides under ``display.platforms..runtime_footer`` - are respected but not modified here — edit config.yaml directly for - per-platform control. - """ + """Handle /footer command — toggle the runtime-metadata footer.""" from gateway.run import _gateway_config_home, _load_gateway_config, _platform_config_key, _resolve_gateway_model from gateway.runtime_footer import resolve_footer_config @@ -4466,15 +4223,11 @@ class GatewaySlashCommandsMixin: async def _handle_compress_command(self, event: MessageEvent) -> str: """Profile-scoping wrapper around manual /compress. - Multiplexed gateways resolve credentials through the fail-closed - per-profile secret scope (``agent.secret_scope``, Workstream A). The - agent turn installs it via ``_run_agent``'s wrapper, but slash-command - dispatch does not — so manual /compress reached the compressor's - provider resolution unscoped and died with ``UnscopedSecretError`` - (``get_secret('OPENROUTER_BASE_URL') called with no profile secret - scope active``). Install the source profile's scope around the whole - handler, mirroring ``_run_agent``. Single-profile gateways skip this - — zero behavior change. + Multiplexed gateways resolve credentials through the fail-closed per-profile secret scope + (``agent.secret_scope``, Workstream A). Agent turns install it via ``_run_agent``'s + wrapper but slash dispatch does not, so an unscoped /compress died with + ``UnscopedSecretError``. Install the source profile's scope around the whole handler, + mirroring ``_run_agent``. Single-profile gateways skip this — zero behavior change. """ if not getattr(getattr(self, "config", None), "multiplex_profiles", False): return await self._handle_compress_command_inner(event) @@ -4488,16 +4241,13 @@ class GatewaySlashCommandsMixin: async def _compress_codex_app_server_session( self, session_key: str, session_id: str ) -> str: - """Manual /compress for codex_app_server sessions (#73503). + """Manual /compress for codex_app_server sessions. - Compacts the LIVE cached agent's app-server thread via - ``thread/compact/start`` (through ``_compress_context``'s codex route - with ``force=True``, which bypasses the automatic-mode gate in every - ``compression.codex_app_server_auto`` mode — a manual /compress is an - explicit user decision) and keeps that agent cached, so the compacted - thread is what the next turn continues from. Never builds a temporary - compression agent and never rewrites the transcript mirror: neither - can shrink the server-side thread that is the model's real context. + Compacts the LIVE cached agent's app-server thread via ``thread/compact/start`` with + ``force=True`` (bypasses the ``codex_app_server_auto`` mode gate — a manual /compress is an + explicit user decision) and keeps that agent cached so the next turn continues from it. + Never builds a temporary compression agent and never rewrites the transcript mirror: + neither can shrink the server-side thread that is the model's real context. """ agent = None lock = getattr(self, "_agent_cache_lock", None) @@ -4549,15 +4299,9 @@ class GatewaySlashCommandsMixin: async def _handle_compress_command_inner(self, event: MessageEvent) -> str: """Handle /compress command -- manually compress conversation context. - Accepts an optional focus topic: ``/compress `` guides the - summariser to preserve information related to *focus* while being - more aggressive about discarding everything else. - - Also accepts the boundary-aware form ``/compress here [N]``: - summarize everything except the most recent ``N`` exchanges - (default 2), kept verbatim. Inspired by Claude Code's Rewind - "Summarize up to here" action (v2.1.139, May 2026, - https://code.claude.com/docs/en/whats-new/2026-w20). + Accepts an optional focus topic: ``/compress `` guides the summariser to preserve + information related to *focus* while being more aggressive about discarding everything + else. """ source = event.source session_entry = await self.async_session_store.get_or_create_session(source) @@ -4616,13 +4360,10 @@ class GatewaySlashCommandsMixin: from agent.model_metadata import estimate_request_tokens_rough session_key = self._session_key_for_source(source) - # Preserve the same platform + stable gateway session identity that a - # normal gateway turn passes (gateway/run.py main turn), so external - # context engines bind this temporary compression agent to the - # original platform conversation instead of falling back to an - # unbound/default "cli" host source — see #50422. _platform_config_key - # maps LOCAL->"cli" exactly like the live turn, avoiding a new - # "local" vs "cli" mismatch. + # Preserve the same platform + stable gateway session identity that a normal gateway + # turn passes (gateway/run.py main turn), so external context engines bind this + # temporary compression agent to the original platform conversation instead of falling + # back to an unbound/default "cli" host source — see #50422. from gateway.run import ( _GATEWAY_HYGIENE_PLATFORM, _platform_config_key, @@ -4636,39 +4377,30 @@ class GatewaySlashCommandsMixin: session_key=session_key, ) if str(runtime_kwargs.get("api_mode") or "").lower() == "codex_app_server": - # codex app-server runtime (#73503): the model's working - # context is the app-server's server-side thread, owned by the - # LIVE cached agent (agent/codex_runtime.py — one - # CodexAppServerSession per AIAgent, spawned lazily on first - # turn). A temporary compression agent has no thread, so the - # codex route in _compress_context_via_codex_app_server bailed - # at its "no active codex thread" guard, the transcript came - # back unchanged, and the finally-clause eviction below then - # destroyed the only real context. Compact the live agent's - # thread via thread/compact/start instead — and KEEP the agent - # cached so the compacted thread survives to the next turn. - # No local transcript fallback in any mode: rewriting the - # mirror cannot shrink the thread. + # codex app-server runtime: the model's working context is the app-server's server- + # side thread, owned by the LIVE cached agent (agent/codex_runtime.py — one + # CodexAppServerSession per AIAgent, spawned lazily on first turn). A temporary + # compression agent has no thread (codex route bails, and the finally-eviction below + # would destroy the only real context), so compact the live thread and KEEP the agent + # cached. No transcript fallback: rewriting the mirror cannot shrink the thread. return await self._compress_codex_app_server_session( session_key, session_entry.session_id ) if not runtime_kwargs.get("api_key"): return t("gateway.compress.no_provider") - # Pass the FULL transcript (tool results included) — same - # rationale as the session-hygiene auto-compress in - # gateway/run.py (#3854): filtering to user/assistant-only - # starves the compressor's tool-result pruning and can trip the - # protect-first/last early-return on short filtered histories. + # Pass the FULL transcript (tool results included) — same rationale as the session- + # hygiene auto-compress in gateway/run.py: filtering to user/assistant-only starves the + # compressor's tool-result pruning and can trip the protect-first/last early-return on + # short filtered histories. msgs = [ m for m in history if m.get("role") in {"user", "assistant", "tool"} ] - # Boundary-aware split: only the head is summarized; the most - # recent `keep_last` exchanges are preserved verbatim. The - # split snaps the tail to a user-turn start so the rejoined - # transcript keeps role alternation valid. + # Boundary-aware split: only the head is summarized; the most recent `keep_last` + # exchanges are preserved verbatim. The split snaps the tail to a user-turn start so the + # rejoined transcript keeps role alternation valid. tail: list = [] head = msgs if partial: @@ -4678,28 +4410,20 @@ class GatewaySlashCommandsMixin: partial = False head = msgs - # Bind the temporary compression agent to the originating source's - # platform + stable gateway session key. These are *authoritative* - # identity invariants (derived from `source`), so assign them into - # runtime_kwargs directly rather than via setdefault: a value already - # present there from the resolver would be a placeholder/stale - # identity and must not win. Assigning (vs passing a second explicit - # kwarg) also keeps each key single-valued, avoiding a "got multiple - # values for keyword argument" TypeError. platform is only set when - # known: for a source without platform metadata we leave it unset so - # AIAgent's default (platform=None -> source "cli") applies, exactly - # the prior behavior. _resolve_session_agent_runtime does not set - # either key today, so in practice this just adds them. + # Bind the temporary compression agent to the originating source's platform + stable + # gateway session key. These are authoritative identity invariants, so assign directly + # (not setdefault — a resolver-supplied value would be a stale placeholder and must not + # win; assigning also avoids a duplicate-kwarg TypeError). platform is only set when + # known so AIAgent's default (None -> "cli") still applies. _resolve_session_agent_runtime + # does not set either key today, so in practice this just adds them. if platform_key is not None: runtime_kwargs["platform"] = platform_key runtime_kwargs["gateway_session_key"] = session_key - # The manual compression helper runs outside the live session's - # fully initialized prompt environment (it loads the memory - # provider only when compression.checkpoint_required demands it), - # and _compress_context may persist its cached system prompt. - # Restore the exact live-session prompt so provider blocks are - # retained. + # The manual compression helper runs outside the live session's fully initialized prompt + # environment (it loads the memory provider only when compression.checkpoint_required + # demands it), and _compress_context may persist its cached system prompt. Restore the + # exact live-session prompt so provider blocks are retained. session_row = None get_session = getattr(self._session_db, "get_session", None) if callable(get_session): @@ -4715,11 +4439,9 @@ class GatewaySlashCommandsMixin: exc_info=True, ) - # This agent performs a lossy rewrite. When the operator enabled - # compression.checkpoint_required, the memory provider must be - # loaded so _compress_context() can create the required - # pre-compression checkpoint; otherwise keep the historical fast - # path (no provider init, no best-effort hook) for this helper. + # This agent performs a lossy rewrite. When compression.checkpoint_required is on, the + # memory provider must be loaded so _compress_context() can write the pre-compression + # checkpoint; otherwise keep the historical fast path (no provider init). from hermes_cli.config import load_config as _load_cfg from utils import is_truthy_value as _is_truthy @@ -4740,10 +4462,9 @@ class GatewaySlashCommandsMixin: session_db=getattr(self._session_db, "_db", self._session_db), ) _seed_hygiene_system_prompt(tmp_agent, session_row) - # Keep the real source platform during construction so external - # context engines bind correctly. If compression has to rebuild the - # prompt, stamp that provider-less fallback as stale for the next - # real gateway turn. + # Keep the real source platform during construction so external context engines bind + # correctly. If compression has to rebuild the prompt, stamp that provider-less fallback + # as stale for the next real gateway turn. tmp_agent.platform = _GATEWAY_HYGIENE_PLATFORM try: tmp_agent._print_fn = lambda *a, **kw: None @@ -4752,10 +4473,9 @@ class GatewaySlashCommandsMixin: # must remain open for the next user turn. tmp_agent._end_session_on_close = False - # Estimate with system prompt + tool schemas included so the - # figure reflects real request pressure, not a transcript-only - # underestimate (#6217). Must be computed after tmp_agent is - # built so _cached_system_prompt/tools are populated. + # Estimate with system prompt + tool schemas included so the figure reflects real + # request pressure, not a transcript-only underestimate. Must be computed after + # tmp_agent is built so _cached_system_prompt/tools are populated. _sys_prompt = getattr(tmp_agent, "_cached_system_prompt", "") or "" _tools = getattr(tmp_agent, "tools", None) or None approx_tokens = estimate_request_tokens_rough( @@ -4766,12 +4486,10 @@ class GatewaySlashCommandsMixin: if not compressor.has_content_to_compress(head): return t("gateway.compress.nothing_to_do") - # _run_in_executor_with_context (not a bare run_in_executor): - # the profile secret scope installed by the wrapper is a - # contextvar, and the default-executor hop would drop it — - # the compressor's aux-client provider resolution would then - # read credentials unscoped and fail closed under - # multiplexing. + # _run_in_executor_with_context (not a bare run_in_executor): the profile secret + # scope installed by the wrapper is a contextvar, and the default-executor hop would + # drop it — the compressor's aux-client provider resolution would then read + # credentials unscoped and fail closed under multiplexing. compressed, _ = await self._run_in_executor_with_context( lambda: tmp_agent._compress_context( head, @@ -4783,14 +4501,9 @@ class GatewaySlashCommandsMixin: ) ) - # If _compress_context returned unchanged because a - # concurrent compression lock is held, tell the user - # clearly instead of showing the misleading - # "No changes from compression" no-op text. The wording - # distinguishes a confirmed holder from an unconfirmed - # acquisition failure (describe_compression_lock_skip). - # The deferred context-engine notification is discarded by - # the finally block below (finalize committed=False). + # If _compress_context returned unchanged because a concurrent compression lock is + # held, tell the user clearly instead of showing the misleading "No changes from + # compression" no-op text. _lock_skipped = getattr(tmp_agent, "_compression_skipped_due_to_lock", None) if _lock_skipped is True or isinstance(_lock_skipped, str): from agent.manual_compression_feedback import ( @@ -4801,43 +4514,25 @@ class GatewaySlashCommandsMixin: if partial and tail: compressed = rejoin_compressed_head_and_tail(compressed, tail) - # _compress_context either rotated (legacy: ended the old - # session, created a continuation id — write compressed messages - # into the NEW session so the original stays searchable) or - # compacted in place (compression.in_place / #38763: same id, + # _compress_context either rotated (legacy: ended the old session, created a + # continuation id — write compressed messages into the NEW session so the original + # stays searchable) or compacted in place (compression.in_place / #38763: same id, # transcript replaced with the compacted set). new_session_id = tmp_agent.session_id rotated = new_session_id != session_entry.session_id _in_place = bool(getattr(tmp_agent, "_last_compaction_in_place", False)) - # Persist the compressed transcript BEFORE repointing the live - # session onto the new session_id. Order matters: if we - # repointed first and the canonical DB write then failed (lock - # contention under concurrent writes, ENOSPC, a disk/IO error), - # the session entry would already reference a brand-new, empty - # session_id while the handler still reported success — the - # user's active conversation would silently vanish from view. - # Writing first, and treating a write failure as fatal, keeps - # the old history reachable (on rotation the entry still points - # at it; in place the original transcript is untouched) and lets - # the outer handler surface a "compress failed" banner instead. + # Persist the compressed transcript BEFORE repointing the live session onto the new + # session_id. Order matters: repoint first + failed DB write (lock contention, ENOSPC, + # IO error) would leave the entry on an empty session while reporting success — the + # conversation silently vanishes. Write first and treat failure as fatal so old + # history stays reachable and the outer handler can surface a "compress failed". # - # Only rewrite the transcript when rotation produced a NEW - # session id. In-place compaction does NOT need a rewrite: - # archive_and_compact() has already soft-archived the previous - # active rows and inserted the compacted messages as the new - # active set inside _compress_context(). Calling - # rewrite_transcript() after in-place compaction would invoke - # replace_messages(active_only=False) which DELETEs ALL rows — - # including the archived turns that archive_and_compact() - # deliberately preserved (silent data loss, #61145). - # - # The third case: _compress_context could NOT rotate AND was - # not in-place (e.g. legacy mode but _session_db unavailable / - # the DB split raised) — there session_id is unchanged for a - # FAILURE reason, and rewrite_transcript() would DELETE the - # original messages and replace them with only the compressed - # summary (permanent data loss #44794, #39704). + # Only rewrite when rotation produced a NEW session id. In-place compaction already + # archived + inserted rows inside _compress_context(); rewrite_transcript() would call + # replace_messages(active_only=False) and DELETE the archived turns (silent data loss). + # Unchanged id without in-place means _compress_context FAILED to rotate — a rewrite + # there would replace the original messages with only the summary (permanent loss). if rotated: if not await self.async_session_store.rewrite_transcript( new_session_id, compressed @@ -4881,13 +4576,10 @@ class GatewaySlashCommandsMixin: new_tokens, compression_state=compressor, ) - # Detect summary-generation failure so we can surface a - # visible warning to the user even on the manual /compress - # path (otherwise the failure is silently logged). - # _last_compress_aborted means the aux LLM returned no - # usable summary and the compressor preserved messages - # unchanged (no drop, no placeholder). force=True was - # passed above so any active cooldown is bypassed. + # Detect summary-generation failure so we can surface a visible warning to the user + # even on the manual /compress path (otherwise the failure is silently logged). + # _last_compress_aborted = aux LLM returned no usable summary and messages were kept + # unchanged. force=True above bypasses any active cooldown. _summary_aborted = bool(getattr(compressor, "_last_compress_aborted", False)) _summary_err = getattr(compressor, "_last_summary_error", None) # Force-redact provider exception text at this UI boundary @@ -4909,10 +4601,7 @@ class GatewaySlashCommandsMixin: # from current files (SOUL.md, memory, etc.). self._evict_cached_agent(session_key) # Off-loop + bounded: temporary-agent teardown can block on - # subprocess/network/SQLite work. Running it inline freezes the - # gateway loop and stalls platform polling / heartbeat, the same - # wedge class fixed for /new (#35994) and hygiene/shutdown - # (#53175). + # subprocess/network/SQLite work. await self._cleanup_agent_resources_off_loop( tmp_agent, context="manual compression" ) @@ -4951,10 +4640,9 @@ class GatewaySlashCommandsMixin: from hermes_state import format_session_db_unavailable return format_session_db_unavailable(prefix=t("gateway.shared.session_db_unavailable_prefix")) - # Authorization: /topic activates multi-session mode and mutates - # SQLite side tables. Unauthorized senders (not in allowlist) must - # not be able to do that. Gateway routes already authorize the - # message before reaching here, but defense in depth. + # Authorization: /topic activates multi-session mode and mutates SQLite side tables. + # Unauthorized senders (not in allowlist) must not be able to do that. Gateway routes + # already authorize the message before reaching here, but defense in depth. auth_fn = getattr(self, "_is_user_authorized", None) if callable(auth_fn): try: @@ -5034,10 +4722,7 @@ class GatewaySlashCommandsMixin: return await self._telegram_topic_root_status_message(source) async def _handle_save_command(self, event: MessageEvent) -> str: - """Handle /save — export the current session and send it as a document. - - Usage: ``/save [json|md|html] [filename] [redact]`` - """ + """Handle /save — export the current session and send it as a document.""" from hermes_cli.session_export import ( SAVE_USAGE, default_save_filename, @@ -5087,10 +4772,9 @@ class GatewaySlashCommandsMixin: temp_dir = tempfile.mkdtemp(prefix="hermes_save_") temp_path = os.path.join(temp_dir, filename) try: - # Off-loop: rendering a long session and writing it to disk are - # CPU/disk-bound and scale with transcript size (multi-MB for - # long sessions). Inline they stall every other chat on the - # gateway event loop (Pattern A). One thread hop covers both. + # Off-loop: rendering a long session and writing it to disk are CPU/disk-bound and scale + # with transcript size (multi-MB for long sessions). Inline they stall every other chat + # on the gateway event loop (Pattern A). One thread hop covers both. def _render_and_write() -> None: rendered = render_session_for_save(export_data, fmt) with open(temp_path, "w", encoding="utf-8") as f: @@ -5161,11 +4845,9 @@ class GatewaySlashCommandsMixin: # Set the title try: if await self._session_db.set_session_title(session_id, sanitized): - # Propagate the user-chosen title to the visible Telegram - # forum topic name too. Auto-generated titles already rename - # the topic; without this, /title only updated the DB title - # and the topic kept its auto-assigned name. No-ops off - # Telegram topic lanes and when auto-rename is disabled. + # Propagate the user-chosen title to the visible Telegram forum topic name too. + # Auto-generated titles already rename the topic; without this, /title only + # updated the DB title and the topic kept its auto-assigned name. schedule_rename = getattr( self, "_schedule_telegram_topic_title_rename", None ) @@ -5315,10 +4997,9 @@ class GatewaySlashCommandsMixin: elif not await self._resume_target_allowed( source, target_id, allow_override=(allow_all or allow_cross_room) ): - # IDOR guard: a session id/title is a routing handle, not authority. - # Bind /resume to the caller's own platform/user/chat on every - # non-Matrix adapter so one user can't attach to another's - # persisted transcript. + # IDOR guard: a session id/title is a routing handle, not authority. Bind /resume to the + # caller's own platform/user/chat on every non-Matrix adapter so one user can't attach + # to another's persisted transcript. return t("gateway.resume.blocked_not_owner", name=name) # Check if already on that session @@ -5334,18 +5015,14 @@ class GatewaySlashCommandsMixin: if not new_entry: return t("gateway.resume.switch_failed") - # Conversation boundary: clear ALL conversation-scoped per-session - # state (model/reasoning overrides #10702, one-turn restores, model - # notes, last-resolved cache #58403, /queue overflow) + security - # state in one funnel call. See _CONVERSATION_SCOPED_STATE in - # gateway/run.py. + # Conversation boundary: clear ALL conversation-scoped per-session state (model/reasoning + # overrides #10702, one-turn restores, model notes, last-resolved cache #58403, /queue + # overflow) + security state in one funnel call. self._clear_conversation_scope(session_key, reason="resume") - # Evict any cached agent for this session so the next message - # rebuilds with the correct session_id end-to-end — mirrors - # /branch and /reset. Without this, the cached AIAgent (and its - # memory provider, which cached `_session_id` during initialize()) - # keeps writing into the wrong session's record. See #6672. + # Evict any cached agent for this session so the next message rebuilds with the correct + # session_id end-to-end — mirrors /branch and /reset. Otherwise the cached AIAgent (and its + # memory provider, which cached _session_id at initialize()) keeps writing to the wrong session. self._evict_cached_agent(session_key) # Get the title for confirmation @@ -5401,11 +5078,9 @@ class GatewaySlashCommandsMixin: ) session_key = self._session_key_for_source(source) - # A cross-origin listing (`/sessions all`) is honored only for an - # admin, mirroring the `/resume --all` override. `all` is just a parsed - # user argument, so without this gate any caller could run - # `/sessions all` and enumerate other origins' session ids / titles / - # previews / sources — the enumeration half of the /resume IDOR. + # A cross-origin listing (`/sessions all`) is honored only for an admin, mirroring the + # `/resume --all` override. `all` is just a parsed user argument; ungated, any caller could + # enumerate other origins' session ids/titles/previews — the enumeration half of the IDOR. cross_origin = include_all and self._resume_caller_is_admin(source) # Don't silently no-op a requested widening: a non-admin `/sessions all` # used to render the same scoped list with zero feedback, which reads @@ -5454,9 +5129,8 @@ class GatewaySlashCommandsMixin: async def _handle_branch_command(self, event: MessageEvent) -> str: """Handle /branch [name] — fork the current session into a new independent copy. - Copies conversation history to a new session so the user can explore - a different approach without losing the original. - Inspired by Claude Code's /branch command. + Copies conversation history to a new session so the user can explore a different approach + without losing the original. Inspired by Claude Code's /branch command. """ import uuid as _uuid @@ -5492,10 +5166,9 @@ class GatewaySlashCommandsMixin: parent_session_id = current_entry.session_id - # Serialize the parent's full origin (same shape as the reset path's - # db_create_kwargs in gateway/session.py, #82633) so the branch row - # carries complete identity from birth. Prefer the live entry's origin - # (it may hold richer metadata than the triggering event's source). + # Serialize the parent's full origin (same shape as the reset path's db_create_kwargs in + # gateway/session.py, #82633) so the branch row carries complete identity from birth. Prefer + # the live entry's origin (it may hold richer metadata than the triggering event's source). _branch_origin = current_entry.origin or source _branch_origin_json = None if _branch_origin is not None: @@ -5506,11 +5179,9 @@ class GatewaySlashCommandsMixin: except Exception: _branch_origin_json = None - # Create the new session with parent link. - # Persist a stable ``_branched_from`` marker in model_config so - # list_sessions_rich() keeps the branch visible in /resume and - # /sessions even after the parent is reopened and re-ended with a - # different end_reason (e.g. tui_shutdown overwriting 'branched'). + # Create the new session with parent link. Persist a stable ``_branched_from`` marker in + # model_config so list_sessions_rich() keeps the branch visible in /resume and /sessions + # even after the parent is reopened and re-ended with a different end_reason. try: await self._session_db.create_session( session_id=new_session_id, @@ -5548,11 +5219,9 @@ class GatewaySlashCommandsMixin: logger.error("Failed to create branch session: %s", e) return t("gateway.branch.create_failed", error=e) - # Copy conversation history to the new session in bounded-chunk - # transactions (see #23254): one txn per row was the removed - # write-amplification pattern, and a history can be hundreds of rows. - # Best-effort like the old loop — a failed copy still yields a - # usable (partial) branch. + # Copy conversation history to the new session in bounded-chunk transactions: one txn per + # row was the removed write-amplification pattern, and a history can be hundreds of rows. + # Best-effort like the old loop — a failed copy still yields a usable (partial) branch. try: await self._session_db.append_messages_batch( new_session_id, @@ -5604,12 +5273,9 @@ class GatewaySlashCommandsMixin: async def _handle_topup_command(self, event: MessageEvent) -> str: """Handle /topup -- show the Nous balance and hand off to the portal. - Renders the balance block + identity line + a tappable portal URL that - opens the billing page. Remote spending is managed on the portal: this - messaging command does NOT charge, confirm, or track payment here — - everything happens in the browser and the next /topup shows the new balance. The - tappable URL is the affordance and works on every platform (button-capable - or plain text like SMS/email). Fetched off the event loop; fail-open. + Remote spending is managed on the portal: this messaging command does NOT charge, + confirm, or track payment here — everything happens in the browser and the next /topup + shows the new balance. Fetched off the event loop; fail-open. """ from agent.account_usage import build_credits_view @@ -5638,10 +5304,8 @@ class GatewaySlashCommandsMixin: def _context_breakdown_block(self, agent, source, expanded: bool) -> list[str]: """Render the /context per-category block (plain text, no grid). - Estimated (chars/4) — same engine as the desktop popover and /usage. - ``expanded`` appends the per-skill / per-toolset listings from the - prompt-size attribution mechanism. Runs in a thread (sync store reads); - returns [] and never raises so /context stays robust. + Estimated (chars/4) — same engine as the desktop popover and /usage. Runs in a thread + (sync store reads); returns [] and never raises so /context stays robust. """ try: from agent.context_breakdown import ( @@ -5744,10 +5408,9 @@ class GatewaySlashCommandsMixin: if cached: agent = cached[0] - # Resolve provider/base_url/api_key for the account-usage fetch. - # Prefer the live agent; fall back to persisted billing data on the - # SessionDB row so `/usage` still returns account info between turns - # when no agent is resident. + # Resolve provider/base_url/api_key for the account-usage fetch. Prefer the live agent; fall + # back to persisted billing data on the SessionDB row so `/usage` still returns account info + # between turns when no agent is resident. provider = getattr(agent, "provider", None) if agent and agent is not _AGENT_PENDING_SENTINEL else None base_url = getattr(agent, "base_url", None) if agent and agent is not _AGENT_PENDING_SENTINEL else None api_key = getattr(agent, "api_key", None) if agent and agent is not _AGENT_PENDING_SENTINEL else None @@ -5802,13 +5465,11 @@ class GatewaySlashCommandsMixin: account_lines = render_account_usage_lines(account_snapshot, markdown=True) # ── Nous credits magnitudes + monthly-grant % gauge ───────────── - # Shared with the CLI / TUI /usage block via nous_credits_lines(): a single - # auth-gate + portal-fetch + render path (which also honors the dev fixture). - # Run off the event loop. The helper gates on "a Nous account is logged in" - # — NOT the inference provider and NOT nested under `if provider:` — so a - # Nous-credentialled user running inference elsewhere (or with none resident) - # still sees their balance. NO recovery trigger: messaging binds no notice - # consumer, so /usage only displays. Fail-open: never break /usage. + # Shared with the CLI / TUI /usage block via nous_credits_lines(): a single auth-gate + + # portal-fetch + render path (which also honors the dev fixture). Run off the event loop. + # Gates on "a Nous account is logged in" — NOT the inference provider, NOT nested under + # `if provider:` — so a Nous-credentialled user inferring elsewhere still sees a balance. + # No recovery trigger (messaging binds no notice consumer). Fail-open: never break /usage. try: from agent.account_usage import nous_credits_lines @@ -5846,10 +5507,9 @@ class GatewaySlashCommandsMixin: if ctx.compression_count: lines.append(t("gateway.usage.label_compressions", count=ctx.compression_count)) - # Per-category context breakdown (estimated — chars/4 heuristic). - # Same engine the desktop popover uses (PR #54907). The system - # prompt / tools / skills / memory slices read off the live agent; - # the conversation slice is estimated from the session transcript. + # Per-category context breakdown (estimated — chars/4 heuristic). Same engine the + # desktop popover uses. The system prompt / tools / skills / memory slices read off the + # live agent; the conversation slice is estimated from the session transcript. breakdown_lines = await asyncio.to_thread( self._context_breakdown_lines, agent, source ) @@ -5941,12 +5601,10 @@ class GatewaySlashCommandsMixin: from hermes_state import release_or_close release_or_close(db) - # _run_in_executor_with_context, not a bare hop: ``SessionDB()`` - # with no explicit path resolves ``get_hermes_home()`` at call - # time, and that override is a contextvar installed by - # ``_profile_runtime_scope``. A default-executor hop starts the - # worker with an EMPTY context, so /insights read the DEFAULT - # profile's state.db and reported another profile's conversations. + # _run_in_executor_with_context, not a bare hop: ``SessionDB()`` with no explicit path + # resolves ``get_hermes_home()`` at call time, and that override is a contextvar + # installed by ``_profile_runtime_scope``. A default-executor hop starts with an EMPTY + # context, so /insights would read the DEFAULT profile's state.db. return await self._run_in_executor_with_context(_run_insights) except Exception as e: logger.error("Insights command error: %s", e, exc_info=True) @@ -5955,18 +5613,11 @@ class GatewaySlashCommandsMixin: async def _handle_reload_mcp_command(self, event: MessageEvent) -> Optional[str]: """Handle /reload-mcp — reconnect MCP servers and rebuild the cached agent. - Reloading MCP tools invalidates the provider prompt cache for the - active session (tool schemas are baked into the system prompt). The - next message re-sends full input tokens, which is expensive on - long-context or high-reasoning models. - - To surface that cost, the command routes through the slash-confirm - primitive: users get an Approve Once / Always Approve / Cancel - prompt before the reload actually runs. "Always Approve" persists - ``approvals.mcp_reload_confirm: false`` so the prompt is silenced - for subsequent reloads in any session. - - Users can also skip the confirm by flipping the config key directly. + Reloading MCP tools invalidates the provider prompt cache (tool schemas are baked into the + system prompt): the next message re-sends full input tokens, which is expensive on + long-context or high-reasoning models. To surface that cost the command routes through the + slash-confirm primitive. "Always Approve" persists ``approvals.mcp_reload_confirm: false`` + so the prompt is silenced for subsequent reloads in any session. """ source = event.source session_key = self._session_key_for_source(source) @@ -5982,10 +5633,9 @@ class GatewaySlashCommandsMixin: if not confirm_required: return await self._execute_mcp_reload(event) - # Route through slash-confirm. The primitive sends the prompt and - # stores the resume handler; the button/text response triggers - # ``_resolve_slash_confirm`` which invokes the handler with the - # chosen outcome. + # Route through slash-confirm. The primitive sends the prompt and stores the resume handler; + # the button/text response triggers ``_resolve_slash_confirm`` which invokes the handler + # with the chosen outcome. async def _on_confirm(choice: str) -> Optional[str]: if choice == "cancel": return t("gateway.reload_mcp.cancelled") @@ -6018,17 +5668,12 @@ class GatewaySlashCommandsMixin: async def _handle_reload_skills_command(self, event: MessageEvent) -> str: """Handle /reload-skills — rescan skills dir, queue a note for next turn. - Skills don't need to be in the system prompt for the model to use - them (they're invoked via ``/skill-name``, ``skills_list``, or - ``skill_view`` at runtime), so this does NOT clear the prompt cache - — prefix caching stays intact. - - If any skills were added or removed, a one-shot note is queued on - ``self._pending_skills_reload_notes[session_key]``. The gateway - prepends it to the NEXT user message in this session (see the - consumer at ~L11025 in ``_run_agent_turn``), then clears it. Nothing - is written to the session transcript out-of-band, so message - alternation is preserved. + Skills don't need to be in the system prompt for the model to use them (they're invoked + via ``/skill-name``, ``skills_list``, or ``skill_view`` at runtime), so this does NOT + clear the prompt cache — prefix caching stays intact. Added/removed skills are reported via + a one-shot note in ``_pending_skills_reload_notes[session_key]`` that the gateway prepends + to the NEXT user message (then clears) — nothing is written to the transcript out-of-band, + so message alternation is preserved. """ try: from agent.skill_commands import reload_skills @@ -6085,10 +5730,9 @@ class GatewaySlashCommandsMixin: lines.append(_fmt_line(item)) lines.append(t("gateway.reload_skills.total", count=total)) - # Queue the one-shot note for the next user turn in this session. - # Format matches how the system prompt renders pre-existing - # skills (`` - name: description``) so the model reads the - # diff in the same shape as its original skill catalog. + # Queue the one-shot note for the next user turn in this session. Format matches how the + # system prompt renders pre-existing skills (`` - name: description``) so the model + # reads the diff in the same shape as its original skill catalog. sections = ["[USER INITIATED SKILLS RELOAD:"] if added: sections.append("") @@ -6155,22 +5799,9 @@ class GatewaySlashCommandsMixin: async def _handle_approve_command(self, event: MessageEvent) -> Optional[str]: """Handle /approve command — unblock waiting agent thread(s). - The agent thread(s) are blocked inside tools/approval.py waiting for - the user to respond. This handler signals the event so the agent - resumes and the terminal_tool executes the command inline — the same - flow as the CLI's synchronous input() approval. - - Supports multiple concurrent approvals (parallel subagents, - execute_code). ``/approve`` resolves the oldest pending command; - ``/approve all`` resolves every pending command at once. - - Usage: - /approve — approve oldest pending command once - /approve all — approve ALL pending commands at once - /approve session — approve oldest + remember for session - /approve all session — approve all + remember for session - /approve always — approve oldest + remember permanently - /approve all always — approve all + remember permanently + The agent thread(s) are blocked inside tools/approval.py waiting for the user to respond. + This handler signals the event so the agent resumes and the terminal_tool executes the + command inline — the same flow as the CLI's synchronous input() approval. """ source = event.source session_key = self._session_key_for_source(source) @@ -6209,11 +5840,9 @@ class GatewaySlashCommandsMixin: logger.info("User approved %d dangerous command(s) via /approve (%s)", count, choice) plural = "plural" if count > 1 else "singular" confirmation_text = t(f"gateway.approve.{choice}_{plural}", count=count) - # Native-streaming adapters (WeCom msgtype:"stream") need the - # confirmation sent directly with control-lane metadata so it lands - # via a reliable proactive send instead of the (already-finalized) - # reply stream. Every other platform keeps the normal contract: - # else: return the text and let the gateway deliver it. + # Native-streaming adapters (WeCom msgtype:"stream") need the confirmation sent directly + # with control-lane metadata so it lands via a reliable proactive send instead of the + # (already-finalized) reply stream. Everyone else returns the text for normal delivery. # (`is not True` — mock adapters auto-create truthy attributes.) if getattr(_adapter, "SUPPORTS_NATIVE_STREAMING", False) is not True: return confirmation_text @@ -6241,13 +5870,9 @@ class GatewaySlashCommandsMixin: async def _handle_deny_command(self, event: MessageEvent) -> str: """Handle /deny command — reject pending dangerous command(s). - Signals blocked agent thread(s) with a 'deny' result so they receive - a definitive BLOCKED message, same as the CLI deny flow. - - ``/deny`` denies the oldest; ``/deny all`` denies everything. - ``/deny `` (or ``/deny all ``) attaches a one-line - reason that is relayed back to the agent so it can adapt instead of - only hearing "denied". Ported from qwibitai/nanoclaw#2832. + Signals blocked agent thread(s) with a 'deny' result so they receive a definitive BLOCKED + message, same as the CLI deny flow. ``/deny`` denies the oldest; ``/deny all`` denies + everything. Ported from qwibitai/nanoclaw#2832. """ source = event.source session_key = self._session_key_for_source(source) @@ -6302,10 +5927,9 @@ class GatewaySlashCommandsMixin: else: confirmation_text = t("gateway.deny.denied_singular") - # Same native-streaming carve-out as /approve above: only WeCom-style - # native-stream adapters take the direct control-lane send; everyone - # else returns the text for normal gateway delivery. - # (`is not True` — mock adapters auto-create truthy attributes.) + # Same native-streaming carve-out as /approve above: only WeCom-style native-stream adapters + # take the direct control-lane send; everyone else returns the text for normal gateway + # delivery. (`is not True` — mock adapters auto-create truthy attributes.) if getattr(_adapter, "SUPPORTS_NATIVE_STREAMING", False) is not True: return confirmation_text if _adapter: @@ -6369,19 +5993,17 @@ class GatewaySlashCommandsMixin: lines.append(t("gateway.debug.share_hint")) return "\n".join(lines) - # _run_in_executor_with_context, not a bare hop: this collects the - # profile's logs/config off ``get_hermes_home()`` and uploads them to a - # public paste. Losing the contextvar override would publish the DEFAULT - # profile's diagnostics from another profile's chat. + # _run_in_executor_with_context, not a bare hop: this collects the profile's logs/config off + # ``get_hermes_home()`` and uploads them to a public paste. Losing the contextvar override + # would publish the DEFAULT profile's diagnostics from another profile's chat. return await self._run_in_executor_with_context(_collect_and_upload) async def _handle_update_command(self, event: MessageEvent) -> str: """Handle /update command — update Hermes Agent to the latest version. - Spawns ``hermes update`` in a detached session (via ``setsid``) so it - survives the gateway restart that ``hermes update`` may trigger. Marker - files are written so either the current gateway process or the next one - can notify the user when the update finishes. + Spawns ``hermes update`` in a detached session (via ``setsid``) so it survives the + gateway restart that ``hermes update`` may trigger. Marker files are written so either + the current gateway process or the next one can notify the user when the update finishes. """ from gateway.run import _hermes_home, _resolve_hermes_bin import json @@ -6466,11 +6088,9 @@ class GatewaySlashCommandsMixin: import textwrap from hermes_cli._subprocess_compat import windows_detach_popen_kwargs - # Invoke the updater as a module under this interpreter rather - # than through hermes_cmd (venv\Scripts\hermes.exe): the shim - # launcher holds its own file open for the whole run, and the - # update has to replace it. Going through python.exe maps no - # shim, so the entry points can be rewritten freely. + # Invoke the updater as a module under this interpreter rather than through + # hermes_cmd (venv\Scripts\hermes.exe): the shim launcher holds its own file open + # for the whole run, and the update has to replace it. helper = textwrap.dedent( """ import os, subprocess, sys diff --git a/gateway/status_phrases.py b/gateway/status_phrases.py index b7c490273e..c7d64b898d 100644 --- a/gateway/status_phrases.py +++ b/gateway/status_phrases.py @@ -1,17 +1,12 @@ """Human-friendly generic gateway status phrases. -These helpers deliberately avoid relaying raw model scratch text. They turn -Hermes' long-running gateway status surface into short status lines suitable -for chat surfaces. +Turns Hermes' long-running gateway status surface into short chat-safe lines +without relaying raw model scratch text: only configured phrase strings are +used — tool args, commands, previews, and reasoning are never interpolated. Built-in defaults live in ``gateway/assets/status_phrases.yaml``. Users can add -portable, profile-relative phrase catalogs under ``HERMES_HOME`` either by using -conventional paths:: - - ~/.hermes/status_phrases.yaml - ~/.hermes/status_phrases/*.yaml - -or by pointing config at a relative file/directory:: +profile-relative catalogs under ``HERMES_HOME`` via the conventional paths +``status_phrases.yaml`` / ``status_phrases/*.yaml`` or via config:: display: status_phrases: @@ -19,10 +14,7 @@ or by pointing config at a relative file/directory:: mode: append # append (default) or replace Absolute paths and ``..`` escapes are ignored on purpose so config stays -profile-portable and does not accidentally read arbitrary files. - -Only configured phrase strings are used; raw tool args, commands, previews, and -reasoning text are never interpolated into the returned phrase. +profile-portable and cannot read arbitrary files. """ from __future__ import annotations @@ -36,13 +28,14 @@ import yaml from hermes_constants import get_hermes_home -# These are Hermes UI surfaces, not app/vendor/domain buckets. Keep this -# long-running-only: regular tool/thinking/interim chatter is intentionally not -# rewritten into generic placeholders because that gets noisy fast in chat. +# Hermes UI surfaces, not app/vendor buckets. Long-running-only: regular +# tool/thinking/interim chatter is deliberately not rewritten (too noisy in chat). _STATUS_SURFACES = ("status", "generic") _MAX_CUSTOM_PHRASES_PER_SURFACE = 80 _MAX_PHRASE_CHARS = 160 _CONVENTIONAL_RELATIVE_PATHS = ("status_phrases.yaml", "status_phrases") +_YAML_SUFFIXES = {".yaml", ".yml"} +_CONFIG_KEYS = ("generic_status_phrases", "status_phrases") # legacy alias first _FALLBACK_PHRASES: dict[str, list[str]] = { "status": ["still on it", "still working through it", "waiting for the result"], @@ -54,25 +47,20 @@ def _clean_phrase_list(value: Any) -> list[str]: if not isinstance(value, list): return [] cleaned: list[str] = [] - seen: set[str] = set() for item in value[:_MAX_CUSTOM_PHRASES_PER_SURFACE]: phrase = str(item or "").strip() - if not phrase or len(phrase) > _MAX_PHRASE_CHARS or phrase in seen: - continue - cleaned.append(phrase) - seen.add(phrase) + if phrase and len(phrase) <= _MAX_PHRASE_CHARS and phrase not in cleaned: + cleaned.append(phrase) return cleaned def _merge_phrase_mapping(catalog: dict[str, list[str]], section: Mapping[str, Any], *, inherited_mode: str | None = None) -> None: - mode = str(section.get("mode") or inherited_mode or "append").strip().lower() - replace = mode == "replace" + replace = str(section.get("mode") or inherited_mode or "append").strip().lower() == "replace" phrase_map = section.get("phrases") if isinstance(section.get("phrases"), Mapping) else section for surface in _STATUS_SURFACES: phrases = _clean_phrase_list(phrase_map.get(surface) if isinstance(phrase_map, Mapping) else None) - if not phrases: - continue - catalog[surface] = phrases if replace else [*catalog.get(surface, []), *phrases] + if phrases: + catalog[surface] = phrases if replace else [*catalog.get(surface, []), *phrases] def _merge_phrase_file(catalog: dict[str, list[str]], path: Path, *, inherited_mode: str | None = None) -> None: @@ -101,56 +89,43 @@ def _relative_path_under(base_dir: Path, raw_path: Any) -> Path | None: def _iter_phrase_files(path: Path) -> list[Path]: - if path.is_file() and path.suffix.lower() in {".yaml", ".yml"}: + if path.is_file() and path.suffix.lower() in _YAML_SUFFIXES: return [path] if path.is_dir(): - return sorted( - child for child in path.iterdir() - if child.is_file() and child.suffix.lower() in {".yaml", ".yml"} - ) + return sorted(c for c in path.iterdir() if c.is_file() and c.suffix.lower() in _YAML_SUFFIXES) return [] -def _merge_phrase_paths( - catalog: dict[str, list[str]], - paths: Any, - *, - base_dir: Path, - inherited_mode: str | None = None, -) -> None: +def _merge_phrase_paths(catalog: dict[str, list[str]], paths: Any, *, base_dir: Path, inherited_mode: str | None = None) -> None: if paths is None: return - raw_paths = paths if isinstance(paths, list) else [paths] - for raw_path in raw_paths: + for raw_path in paths if isinstance(paths, list) else [paths]: resolved = _relative_path_under(base_dir, raw_path) - if resolved is None: - continue - for phrase_file in _iter_phrase_files(resolved): - _merge_phrase_file(catalog, phrase_file, inherited_mode=inherited_mode) + if resolved is not None: + for phrase_file in _iter_phrase_files(resolved): + _merge_phrase_file(catalog, phrase_file, inherited_mode=inherited_mode) + + +def _copy_catalog(catalog: Mapping[str, list[str]]) -> dict[str, list[str]]: + return {surface: list(phrases) for surface, phrases in catalog.items()} def _load_builtin_catalog() -> dict[str, list[str]]: - catalog = {surface: list(phrases) for surface, phrases in _FALLBACK_PHRASES.items()} - catalog_path = Path(__file__).resolve().parent / "assets" / "status_phrases.yaml" - _merge_phrase_file(catalog, catalog_path, inherited_mode="replace") + catalog = _copy_catalog(_FALLBACK_PHRASES) + _merge_phrase_file(catalog, Path(__file__).resolve().parent / "assets" / "status_phrases.yaml", inherited_mode="replace") return catalog _DEFAULT_PHRASES: dict[str, list[str]] = _load_builtin_catalog() -def _copy_default_catalog() -> dict[str, list[str]]: - return {surface: list(phrases) for surface, phrases in _DEFAULT_PHRASES.items()} - - -def _merge_phrase_config(catalog: dict[str, list[str]], section: Any, *, base_dir: Path | None = None) -> None: - """Merge one display.status_phrases-style section into ``catalog``.""" +def _merge_phrase_config(catalog: dict[str, list[str]], section: Any, *, base_dir: Path) -> None: + """Merge one display.status_phrases-style section (files first, then inline phrases).""" if not isinstance(section, Mapping): return mode = str(section.get("mode") or "append").strip().lower() - if base_dir is not None: - _merge_phrase_paths(catalog, section.get("path"), base_dir=base_dir, inherited_mode=mode) - _merge_phrase_paths(catalog, section.get("paths"), base_dir=base_dir, inherited_mode=mode) + for key in ("path", "paths"): + _merge_phrase_paths(catalog, section.get(key), base_dir=base_dir, inherited_mode=mode) _merge_phrase_mapping(catalog, section) @@ -162,7 +137,7 @@ def resolve_status_phrase_catalog(user_config: Mapping[str, Any] | None, platfor alias ``generic_status_phrases``), then ``display.platforms..status_phrases``. """ - catalog = _copy_default_catalog() + catalog = _copy_catalog(_DEFAULT_PHRASES) hermes_home = get_hermes_home() _merge_phrase_paths(catalog, list(_CONVENTIONAL_RELATIVE_PATHS), base_dir=hermes_home) @@ -170,15 +145,13 @@ def resolve_status_phrase_catalog(user_config: Mapping[str, Any] | None, platfor if not isinstance(display, Mapping): return catalog - _merge_phrase_config(catalog, display.get("generic_status_phrases"), base_dir=hermes_home) - _merge_phrase_config(catalog, display.get("status_phrases"), base_dir=hermes_home) - + sections = [display] platforms = display.get("platforms") - if platform_key and isinstance(platforms, Mapping): - platform_display = platforms.get(platform_key) - if isinstance(platform_display, Mapping): - _merge_phrase_config(catalog, platform_display.get("generic_status_phrases"), base_dir=hermes_home) - _merge_phrase_config(catalog, platform_display.get("status_phrases"), base_dir=hermes_home) + if platform_key and isinstance(platforms, Mapping) and isinstance(platforms.get(platform_key), Mapping): + sections.append(platforms[platform_key]) + for section in sections: + for key in _CONFIG_KEYS: + _merge_phrase_config(catalog, section.get(key), base_dir=hermes_home) return catalog @@ -190,8 +163,7 @@ def classify_status_context( args: Any = None, ) -> str: """Classify an internal gateway event into a Hermes UI-surface bucket.""" - normalized = str(kind or "").strip().lower() - if normalized in {"heartbeat", "waiting", "long_running", "status"}: + if str(kind or "").strip().lower() in {"heartbeat", "waiting", "long_running", "status"}: return "status" return "generic" @@ -216,11 +188,8 @@ def choose_status_phrase( candidates = list(phrase_catalog.get(category) or phrase_catalog.get("generic") or _DEFAULT_PHRASES["generic"]) if recent: recent_set = set(recent) - fresh = [phrase for phrase in candidates if phrase not in recent_set] - if fresh: - candidates = fresh - picker = rng or _random - phrase = picker.choice(candidates) + candidates = [p for p in candidates if p not in recent_set] or candidates + phrase = (rng or _random).choice(candidates) if recent is not None: recent.append(phrase) del recent[:-6] diff --git a/gateway/sticker_cache.py b/gateway/sticker_cache.py index c536817306..58f1ee1c87 100644 --- a/gateway/sticker_cache.py +++ b/gateway/sticker_cache.py @@ -1,11 +1,7 @@ -""" -Sticker description cache for Telegram. +"""Sticker description cache for Telegram. -When users send stickers, we describe them via the vision tool and cache -the descriptions keyed by file_unique_id so we don't re-analyze the same -sticker image on every send. Descriptions are concise (1-2 sentences). - -Cache location: ~/.hermes/sticker_cache.json +Stickers are described via the vision tool once and cached by file_unique_id +(``~/.hermes/sticker_cache.json``) so the same image is never re-analyzed. """ import json @@ -19,7 +15,7 @@ from hermes_cli.config import get_hermes_home CACHE_PATH = get_hermes_home() / "sticker_cache.json" -# Vision prompt for describing stickers -- kept concise to save tokens +# Kept concise to save tokens. STICKER_VISION_PROMPT = ( "Describe this sticker in 1-2 sentences. Focus on what it depicts -- " "character, action, emotion. Be concise and objective." @@ -27,7 +23,6 @@ STICKER_VISION_PROMPT = ( def _load_cache() -> dict: - """Load the sticker cache from disk.""" if CACHE_PATH.exists(): try: return json.loads(CACHE_PATH.read_text(encoding="utf-8")) @@ -37,11 +32,9 @@ def _load_cache() -> dict: def _save_cache(cache: dict) -> None: - """Save the sticker cache to disk atomically.""" + """Write the cache atomically (temp file + fsync + replace).""" CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) - fd, tmp_path = tempfile.mkstemp( - dir=str(CACHE_PATH.parent), suffix=".tmp" - ) + fd, tmp_path = tempfile.mkstemp(dir=str(CACHE_PATH.parent), suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(cache, f, indent=2, ensure_ascii=False) @@ -57,65 +50,32 @@ def _save_cache(cache: dict) -> None: def get_cached_description(file_unique_id: str) -> Optional[dict]: - """ - Look up a cached sticker description. + """Return ``{description, emoji, set_name, cached_at}`` or None.""" + return _load_cache().get(file_unique_id) - Returns: - dict with keys {description, emoji, set_name, cached_at} or None. - """ + +def cache_sticker_description(file_unique_id: str, description: str, emoji: str = "", set_name: str = "") -> None: + """Store a vision-generated description under Telegram's stable sticker id.""" cache = _load_cache() - return cache.get(file_unique_id) - - -def cache_sticker_description( - file_unique_id: str, - description: str, - emoji: str = "", - set_name: str = "", -) -> None: - """ - Store a sticker description in the cache. - - Args: - file_unique_id: Telegram's stable sticker identifier. - description: Vision-generated description text. - emoji: Associated emoji (e.g. "😀"). - set_name: Sticker set name if available. - """ - cache = _load_cache() - cache[file_unique_id] = { - "description": description, - "emoji": emoji, - "set_name": set_name, - "cached_at": time.time(), - } + cache[file_unique_id] = {"description": description, "emoji": emoji, "set_name": set_name, "cached_at": time.time()} _save_cache(cache) -def build_sticker_injection( - description: str, - emoji: str = "", - set_name: str = "", -) -> str: - """ - Build the warm-style injection text for a sticker description. - - Returns a string like: - [The user sent a sticker 😀 from "MyPack"~ It shows: "A cat waving" (=^.w.^=)] +def build_sticker_injection(description: str, emoji: str = "", set_name: str = "") -> str: + """Warm-style injection text, e.g. + ``[The user sent a sticker 😀 from "MyPack"~ It shows: "A cat waving" (=^.w.^=)]``. + ``set_name`` is only shown together with an emoji. """ context = "" if set_name and emoji: context = f" {emoji} from \"{set_name}\"" elif emoji: context = f" {emoji}" - return f"[The user sent a sticker{context}~ It shows: \"{description}\" (=^.w.^=)]" def build_animated_sticker_injection(emoji: str = "") -> str: - """ - Build injection text for animated/video stickers we can't analyze. - """ + """Injection text for animated/video stickers we can't analyze.""" if emoji: return ( f"[The user sent an animated sticker {emoji}~ " diff --git a/gateway/stream_dispatch.py b/gateway/stream_dispatch.py index 94587149b7..0a6b63a53f 100644 --- a/gateway/stream_dispatch.py +++ b/gateway/stream_dispatch.py @@ -1,21 +1,14 @@ """Adapter-driven dispatch of structured stream events to a delivery sink. -``GatewayEventDispatcher`` is the seam Tobi asked for: the agent emits typed -events (gateway/stream_events.py), and the *adapter* decides how each one is -delivered. The dispatcher holds an adapter + the stream consumer (sink) + the -resolved per-channel presentation settings (tool-progress mode, preview length) -and routes each event through the adapter's render hooks. +``GatewayEventDispatcher`` holds an adapter, the stream consumer (sink) and the +resolved per-channel presentation settings, and routes each typed event +(gateway/stream_events.py) through the adapter's render hooks. Message events +flow into the consumer; tool events are formatted by the adapter — which may +return None to *eat* them on platforms without tool chrome — and enqueued onto +the same tool-progress queue the gateway drains, so the two paths never race. -Message/commentary/segment events flow into the consumer (native draft on -Telegram DMs, edit-in-place elsewhere). Tool events are formatted by the -adapter — which may return None to *eat* the event on platforms that can't -render tool chrome — and the rendered line is enqueued onto the same tool -progress queue the gateway already drains, so the two no longer race through -independent code paths. - -This module deliberately has no platform knowledge and no asyncio: it is a thin -synchronous router callable from the agent's worker thread, exactly like the -callbacks it replaces. +No platform knowledge and no asyncio: a thin synchronous router callable from +the agent's worker thread, exactly like the callbacks it replaced. """ from __future__ import annotations @@ -24,14 +17,7 @@ import logging from typing import Any, Callable, Optional from gateway.stream_events import ( - Commentary, - GatewayNotice, - LongToolHint, - MessageChunk, - MessageStop, - StreamEvent, - ToolCallChunk, - ToolCallFinished, + Commentary, GatewayNotice, LongToolHint, MessageChunk, MessageStop, StreamEvent, ToolCallChunk, ) logger = logging.getLogger("gateway.stream_events") @@ -40,28 +26,16 @@ logger = logging.getLogger("gateway.stream_events") class GatewayEventDispatcher: """Route typed stream events through an adapter onto a delivery sink. - Parameters - ---------- - adapter: - The platform adapter. Provides ``render_message_event`` and - ``format_tool_event`` (BasePlatformAdapter defaults reproduce today's - behavior; adapters may override for native rendering). - sink: - The GatewayStreamConsumer for assistant-text delivery. May be None - when streaming is disabled, in which case message events are dropped - (the final response still goes out via the normal send path). - enqueue_tool_line: - Callback that places a rendered tool-progress line onto the gateway's - progress queue (the same queue ``send_progress_messages`` drains). May - be None when tool progress is disabled for this channel. - tool_mode: - Resolved tool-progress mode for this channel ("all" / "new" / "verbose" - / "off"). - preview_max_len: - Resolved ``tool_preview_length`` (0 = no cap in verbose mode). - on_long_tool / on_notice: - Optional hooks for LongToolHint / GatewayNotice events, letting the - gateway own the "should I surface this here?" decision. + adapter: provides ``render_message_event`` / ``format_tool_event`` + (BasePlatformAdapter defaults reproduce legacy behavior). + sink: the GatewayStreamConsumer; None when streaming is disabled (message + events are dropped — the final response still goes out normally). + enqueue_tool_line: puts a rendered tool-progress line on the gateway's + progress queue; None when tool progress is disabled for the channel. + tool_mode: "all" / "new" / "verbose" / "off". + preview_max_len: resolved ``tool_preview_length`` (0 = no cap in verbose). + on_long_tool / on_notice: optional hooks so the gateway owns the + "should I surface this here?" decision. """ def __init__( @@ -82,8 +56,7 @@ class GatewayEventDispatcher: self.preview_max_len = preview_max_len self._on_long_tool = on_long_tool self._on_notice = on_notice - # "new" mode dedup — only report when the tool changes. - self._last_tool: Optional[str] = None + self._last_tool: Optional[str] = None # "new"-mode dedup def dispatch(self, event: StreamEvent) -> None: """Route a single event. Never raises into the agent's worker thread.""" @@ -96,37 +69,26 @@ class GatewayEventDispatcher: if isinstance(event, (MessageChunk, MessageStop, Commentary)): if self.sink is not None: self.adapter.render_message_event(event, self.sink) - return + elif isinstance(event, ToolCallChunk): + self._dispatch_tool_call(event) + elif isinstance(event, LongToolHint) and self._on_long_tool is not None: + self._on_long_tool(event) + elif isinstance(event, GatewayNotice) and self._on_notice is not None: + self._on_notice(event) + # ToolCallFinished: no chrome on completion (only "started" is rendered); + # completion only drives onboarding hints (LongToolHint). - if isinstance(event, ToolCallChunk): - if self.tool_mode == "off" or self._enqueue_tool_line is None: - return - # "new" mode: only emit when the tool changes. - if self.tool_mode == "new" and event.tool_name == self._last_tool: - return - self._last_tool = event.tool_name - line = self.adapter.format_tool_event( - event, mode=self.tool_mode, preview_max_len=self.preview_max_len, - ) - # None == adapter chose to eat this event (can't render tool chrome). - if line: - self._enqueue_tool_line(line) + def _dispatch_tool_call(self, event: ToolCallChunk) -> None: + if self.tool_mode == "off" or self._enqueue_tool_line is None: return - - if isinstance(event, ToolCallFinished): - # Default: no chrome on completion (matches today — the gateway only - # rendered "started" events). Completion drives onboarding hints. - return - - if isinstance(event, LongToolHint): - if self._on_long_tool is not None: - self._on_long_tool(event) - return - - if isinstance(event, GatewayNotice): - if self._on_notice is not None: - self._on_notice(event) + if self.tool_mode == "new" and event.tool_name == self._last_tool: return + self._last_tool = event.tool_name + line = self.adapter.format_tool_event( + event, mode=self.tool_mode, preview_max_len=self.preview_max_len, + ) + if line: # None/"" == adapter chose to eat this event + self._enqueue_tool_line(line) __all__ = ["GatewayEventDispatcher"] diff --git a/gateway/stream_events.py b/gateway/stream_events.py index 206d2d7875..ac0df25f28 100644 --- a/gateway/stream_events.py +++ b/gateway/stream_events.py @@ -1,35 +1,23 @@ """Structured streaming events — the agent→gateway delivery contract. -Historically the agent drove gateway delivery through a fan of loosely-typed -callbacks (``stream_delta_callback(text)``, ``tool_progress_callback(event_type, -tool_name, preview, args)``, ``interim_assistant_callback(text)`` …) and each -gateway callback decided *both* what to render and how to send it. That -coupling is why tool-progress bubbles and the streaming draft raced each other -on Telegram, and why tool-call formatting lived agent-side even though only the -gateway knows what a given platform can render. +A small typed vocabulary naming *what happened* without prescribing *how it is +delivered*. The agent emits these from its worker thread; the gateway's stream +consumer (``GatewayStreamConsumer``) is the single sink and the platform adapter +decides rendering (Telegram may stream a native draft; iMessage may drop tool +chrome). This replaced a fan of loosely-typed callbacks whose gateway side +decided both rendering and sending — the cause of tool-progress bubbles racing +the streaming draft. -This module defines a small, typed event vocabulary that names *what happened* -without prescribing *how it is delivered*. The gateway's stream consumer -(``GatewayStreamConsumer``) is the single sink; the platform adapter decides how -to render each event (Telegram can stream a MarkdownV2 ```bash``` block as a -native draft; iMessage has no rich formatting and may collapse or drop tool -chrome). Separation of concerns: smart agent emits structured data, smart -gateway decides delivery. +Plain frozen dataclasses: no behavior, no platform knowledge, no I/O, safe to +hand across the thread/async boundary. -These are intentionally plain frozen dataclasses — no behavior, no platform -knowledge, no I/O. They are cheap to construct on the agent's worker thread and -safe to hand across the thread/async boundary into the consumer queue. - -Design constraints (see hermes-agent-dev skill — message-flow + cache -invariants): - * Events describe *transport*, never *context*. Nothing here is persisted to - conversation history; what the gateway chooses to "eat" (e.g. tool chrome on - a platform that can't render it) must never diverge from the bytes stored in - the agent's message history. History is owned by the agent; these events are - a presentation-layer stream only. - * Backward compatible by construction. The gateway adapts its existing - callbacks into these events at the boundary; adapters that don't opt into - event-native rendering get identical behavior via the base-class default. +Invariants: + * Events describe *transport*, never *context*. Nothing here is persisted; + whatever the gateway chooses to "eat" must never diverge from the bytes in + the agent's message history, which the agent alone owns. + * Backward compatible by construction: the gateway adapts existing callbacks + into events at the boundary; adapters that don't opt in get identical + behavior via the base-class default. """ from __future__ import annotations @@ -38,94 +26,62 @@ from dataclasses import dataclass, field from typing import Any, Dict, Optional, Union -# ── Message (assistant text) events ────────────────────────────────────────── - @dataclass(frozen=True) class MessageChunk: - """A delta of streamed assistant text. - - ``text`` is the incremental content as it arrives from the model. The - consumer accumulates chunks and progressively renders them (native draft on - Telegram DMs, edit-in-place elsewhere). Reasoning/think-block content is - filtered upstream and never arrives as a MessageChunk. - """ + """A delta of streamed assistant text (reasoning/think-block content is + filtered upstream and never arrives here); the consumer accumulates chunks + and renders progressively (native draft on Telegram DMs, edit-in-place + elsewhere).""" text: str @dataclass(frozen=True) class MessageStop: - """The current assistant message segment is complete. + """The current assistant text segment is complete. - Emitted when a contiguous run of assistant text ends — either the whole - response finished, or a tool boundary interrupts the text so the next - segment should render as a fresh message *below* any tool chrome. - - ``final`` is True only for the terminal stop of the whole turn; an - intermediate stop (text → tool call → more text) carries ``final=False`` so - the consumer finalizes the current bubble and prepares a new segment without - treating the turn as done. + ``final`` is True only for the terminal stop of the turn; an intermediate + stop (text → tool call → more text) carries ``final=False`` so the consumer + finalizes the current bubble and starts a fresh segment below tool chrome. """ final: bool = False @dataclass(frozen=True) class Commentary: - """A complete interim assistant message emitted between tool iterations. - - Example: the model says "I'll inspect the repo first." before issuing a tool - call. Unlike a MessageChunk this is already-complete text (not a delta); the - consumer renders it as its own message so it reads as a distinct beat. - """ + """A complete interim assistant message between tool iterations (not a delta).""" text: str -# ── Tool-call events ───────────────────────────────────────────────────────── - @dataclass(frozen=True) class ToolCallChunk: - """A tool invocation has started (or its in-progress state changed). - - Carries the raw facts about the call — name, a short argument ``preview``, - and the full ``args`` dict — and lets the *gateway* decide presentation - (emoji, truncation, verbose vs compact, or eat it entirely on platforms that - don't show tool chrome). Previously the agent's gateway callback baked the - emoji + preview formatting in; that decision now belongs to the adapter. - """ + """A tool invocation started. Raw facts only; the adapter decides presentation.""" tool_name: str preview: Optional[str] = None args: Optional[Dict[str, Any]] = None - # Monotonic per-turn index, so the consumer can correlate a finish with its - # start and so "new"-mode dedup (only report when the tool changes) works - # without the consumer tracking call order itself. + # Monotonic per-turn index: correlates a finish with its start and lets + # "new"-mode dedup work without the consumer tracking call order. index: int = 0 @dataclass(frozen=True) class ToolCallFinished: - """A tool invocation completed. + """A tool invocation completed. Tool *output* never travels here (it is history). - ``duration`` is wall-clock seconds. ``ok`` reflects whether the tool - returned without raising. The gateway uses this to clear/settle a progress - bubble and to drive one-time onboarding hints (e.g. suggest /verbose after a - long tool run). No tool *output* travels here — output is the agent's - concern and is persisted to history, not streamed as presentation. + The gateway uses it to clear/settle a progress bubble and to drive one-time + onboarding hints (e.g. suggest /verbose after a long tool run). """ tool_name: str - duration: float = 0.0 - ok: bool = True + duration: float = 0.0 # wall-clock seconds + ok: bool = True # returned without raising index: int = 0 -# ── Gateway control / lifecycle events ─────────────────────────────────────── - @dataclass(frozen=True) class LongToolHint: """One-shot onboarding nudge when a tool runs longer than the threshold. - The gateway gates this on platform capability (the /verbose command must be - usable) and on the user not having seen the hint before. Modeled as an - event so the *gateway* owns the "should I surface this here?" decision rather - than the agent. + The gateway (not the agent) gates it on platform capability (the /verbose + command must be usable) and on the user not having seen it before. """ tool_name: str = "" duration: float = 0.0 @@ -133,39 +89,24 @@ class LongToolHint: @dataclass(frozen=True) class GatewayNotice: - """A gateway-originated control message (restart, online, long-run notice). + """A gateway-originated control message. - ``kind`` is a stable string the adapter can switch on - (``"restart"`` / ``"online"`` / ``"long_run"`` / …). ``text`` is the - human-readable default the base class renders when an adapter has no - platform-specific treatment. + ``kind`` is a stable string adapters switch on (``"restart"`` / ``"online"`` + / ``"long_run"`` / …); ``text`` is the default rendering. """ kind: str text: str = "" extra: Dict[str, Any] = field(default_factory=dict) -# Union of every event the consumer's dispatcher accepts. Kept explicit (rather -# than a marker base class) so a missing ``case`` in an exhaustive match is a -# visible type error rather than a silent fall-through. +# Explicit union (not a marker base class) so a missing ``case`` in an +# exhaustive match is a visible type error rather than a silent fall-through. StreamEvent = Union[ - MessageChunk, - MessageStop, - Commentary, - ToolCallChunk, - ToolCallFinished, - LongToolHint, - GatewayNotice, + MessageChunk, MessageStop, Commentary, + ToolCallChunk, ToolCallFinished, LongToolHint, GatewayNotice, ] - __all__ = [ - "MessageChunk", - "MessageStop", - "Commentary", - "ToolCallChunk", - "ToolCallFinished", - "LongToolHint", - "GatewayNotice", - "StreamEvent", + "MessageChunk", "MessageStop", "Commentary", "ToolCallChunk", + "ToolCallFinished", "LongToolHint", "GatewayNotice", "StreamEvent", ] diff --git a/gateway/streaming_tts_consumer.py b/gateway/streaming_tts_consumer.py index 86cafd4545..b796385c59 100644 --- a/gateway/streaming_tts_consumer.py +++ b/gateway/streaming_tts_consumer.py @@ -1,39 +1,27 @@ """Gateway streaming-TTS consumer — LLM deltas to adapter PCM audio sink. -Bridges the synchronous agent ``stream_delta_callback`` (fired from the -worker thread) to a voice-capable platform adapter's streaming-audio -contract, so playback begins while the LLM is still generating. +Bridges the synchronous agent ``stream_delta_callback`` (worker thread) to a +voice-capable adapter's streaming-audio contract so playback begins while the +LLM is still generating. Lifecycle:: consumer = StreamingTTSConsumer(adapter, chat_id, tts_config, loop, metadata) agent.stream_delta_callback = consumer.on_delta # sync, non-blocking ... agent runs in executor ... - consumer.finish() # signal end-of-text + consumer.finish() # signal end-of-text success = await consumer.wait_complete(timeout=10) - if consumer.suppress_whole_file: - # suppress whole-file auto-TTS for this turn - consumer.abort("cancelled") # idempotent cancellation + if consumer.suppress_whole_file: ... # skip whole-file auto-TTS + consumer.abort("cancelled") # idempotent cancellation -Design: -- ``on_delta`` is synchronous and never blocks the agent thread. It feeds - deltas into a ``SentenceChunker`` and queues completed clauses onto a - thread-safe ``queue.Queue``. -- An asyncio task (``run``) runs on the gateway event loop, draining the - queue, synthesising each clause via a ``StreamingTTSProvider``, and - writing PCM chunks to the adapter. -- Per-turn state is isolated: each consumer instance owns its own chunker, - queue, handle, and flags. Concurrent chats cannot cross-contaminate. -- On successful completion (all clauses synthesised and written), the - consumer reports ``completed=True`` so the gateway can suppress the - duplicate whole-file auto-TTS. -- On failure before any audible output, the consumer reports - ``completed=False`` and clears ``suppress_whole_file`` so the gateway can - fall back to whole-file TTS. -- On failure after partial audible output, the consumer reports - ``completed=False`` but keeps ``suppress_whole_file=True`` so the gateway - does NOT replay the whole response from the beginning. -- Cancellation/abort is idempotent: late chunks are silently dropped. +``on_delta`` never blocks: it feeds a ``SentenceChunker`` and queues clauses on a +thread-safe ``queue.Queue``; the ``_run`` task on the gateway loop drains it, +synthesises via a ``StreamingTTSProvider`` and writes PCM to the adapter. State +is per instance (concurrent chats cannot cross-contaminate); abort is idempotent +and late chunks are dropped. Outcome contract: full success -> ``completed``; +failure before any audible output -> ``suppress_whole_file=False`` (gateway falls +back to whole-file TTS); failure after partial audio -> ``partial`` and +``suppress_whole_file=True`` (never replay the response from the beginning). """ from __future__ import annotations @@ -69,12 +57,10 @@ class StreamingTTSConsumer: self._adapter = adapter self._chat_id = chat_id - self._tts_config = tts_config self._loop = loop self._metadata = metadata - # Resolve the streaming provider once. If unavailable, the consumer is - # inactive and the gateway falls back to whole-file TTS. + # Resolved once; None => inactive, gateway falls back to whole-file TTS. self._streamer = resolve_streaming_provider(tts_config) self._chunker = SentenceChunker() @@ -87,12 +73,10 @@ class StreamingTTSConsumer: else: self._audio_format = audio_format or AudioFormat() - # Thread-safe queue: completed clauses and the occasional abort sentinel. + # Thread-safe queue of completed clauses plus the _DONE/_ABORT sentinels. self._queue: "queue.Queue[Any]" = queue.Queue(maxsize=256) - # Per-turn state. self._handle: Optional[StreamingTTSHandle] = None - self._started = False self._completed = False self._partial = False self._aborted = False @@ -101,58 +85,36 @@ class StreamingTTSConsumer: self._suppress_whole_file = False self._task: Optional[asyncio.Task] = None self._lock = threading.Lock() - - # Pre-allocate the strip-markdown helper lazily to avoid import cycles. - self._strip_markdown = None - - # ------------------------------------------------------------------ - # Public properties - # ------------------------------------------------------------------ + self._strip_markdown = None # lazily imported to avoid import cycles @property - def active(self) -> bool: - """True when this consumer has a usable streaming provider.""" + def active(self) -> bool: # usable streaming provider resolved return self._streamer is not None @property - def completed(self) -> bool: - """True when streaming audio was fully delivered.""" + def completed(self) -> bool: # streaming audio fully delivered return self._completed @property - def partial(self) -> bool: - """True when some audio was audible before a failure or drop.""" + def partial(self) -> bool: # some audio was audible before a failure/drop return self._partial @property - def started(self) -> bool: - """True when the adapter accepted the streaming session.""" - return self._started - - @property - def audible(self) -> bool: - """True once the first PCM chunk has been written.""" + def audible(self) -> bool: # first PCM chunk has been written return bool(self._handle and self._handle.audible) @property - def dropped(self) -> bool: - """True when queue saturation dropped at least one clause.""" + def dropped(self) -> bool: # queue saturation dropped at least one clause return self._dropped @property - def suppress_whole_file(self) -> bool: - """True when the gateway should skip the legacy whole-file TTS fallback.""" + def suppress_whole_file(self) -> bool: # gateway should skip whole-file TTS fallback return self._suppress_whole_file @property - def done(self) -> bool: - """True once the async drain task has terminated.""" + def done(self) -> bool: # async drain task has terminated return self._task is not None and self._task.done() - # ------------------------------------------------------------------ - # Sync callback (agent worker thread) - # ------------------------------------------------------------------ - def on_delta(self, text: str) -> None: """Receive a text delta from the agent. Non-blocking.""" if self._aborted or not self.active or self._finished: @@ -167,11 +129,10 @@ class StreamingTTSConsumer: logger.debug("streaming TTS on_delta error", exc_info=True) def finish(self) -> None: - """Signal end-of-text and flush the chunker tail. + """Signal end-of-text, flush the chunker tail, then enqueue ``_DONE``. - Enqueues a ``_DONE`` sentinel after all flushed clauses so the - drain loop has a deterministic termination signal that cannot - race with a late ``on_delta`` or be lost when the queue is full. + The sentinel follows all flushed clauses so the drain loop has a + deterministic termination that cannot race a late ``on_delta``. """ if self._finished: return @@ -186,13 +147,7 @@ class StreamingTTSConsumer: logger.debug("streaming TTS queue full while flushing tail") except Exception: pass - # Guarantee the _DONE sentinel reaches the queue. If the bounded - # queue is full, drain one item to make room — the sentinel is - # load-bearing and must not be lost (#60671 hardening). - self._enqueue_done() - - def _enqueue_done(self) -> None: - """Enqueue the _DONE sentinel, evicting a queued clause if necessary.""" + # The load-bearing _DONE sentinel must never be lost: evict a clause if full. while True: try: self._queue.put_nowait(_DONE) @@ -204,106 +159,72 @@ class StreamingTTSConsumer: except queue.Empty: continue - # ------------------------------------------------------------------ - # Async lifecycle (gateway event loop) - # ------------------------------------------------------------------ - def start(self) -> asyncio.Task: - """Create and return the async drain task on the gateway loop.""" - if self._task is not None: - return self._task - self._task = self._loop.create_task(self._run()) + """Create (once) and return the async drain task on the gateway loop.""" + if self._task is None: + self._task = self._loop.create_task(self._run()) return self._task + def _settle(self, *, failed: bool) -> None: + """Set the outcome flags from what was audible. + + Never report completion after a failure or a dropped clause; keep + suppression whenever audio was audible so the gateway does not replay + the response from the beginning. + """ + audible = self._handle.audible + degraded = failed or self._dropped + self._completed = audible and not degraded + if audible and degraded: + self._partial = True + self._suppress_whole_file = audible + async def _run(self) -> None: """Drain clauses from the queue, synthesise, and write to the adapter.""" if not self.active: return - if not self._adapter.supports_streaming_tts(self._chat_id, self._audio_format): logger.debug("adapter %s does not support streaming TTS", getattr(self._adapter, "name", "?")) return - try: self._handle = await self._adapter.begin_streaming_tts( - self._chat_id, - self._audio_format, - metadata=self._metadata, + self._chat_id, self._audio_format, metadata=self._metadata, ) except Exception as exc: logger.debug("begin_streaming_tts failed: %s", exc) self._handle = None return - if self._handle is None: return - self._started = True self._suppress_whole_file = False - try: - while True: - if self._aborted: - break + while not self._aborted: try: item = await asyncio.to_thread(self._queue.get, True, 0.1) except queue.Empty: continue - - if item is _ABORT: - break - if item is _DONE: + if item is _ABORT or item is _DONE or self._aborted: break if not isinstance(item, str): continue - if self._aborted: - break - try: await self._synthesise_and_write(item) except Exception as exc: logger.warning("streaming TTS clause failed: %s", exc) - if self._handle and self._handle.audible: - self._partial = True - self._suppress_whole_file = True - else: - self._suppress_whole_file = False - self._completed = False + self._settle(failed=True) await self._safe_abort(str(exc)) return if not self._aborted and self._handle is not None: - _finish_failed = False try: await self._adapter.finish_streaming_tts(self._handle, interrupted=self._aborted) except Exception as exc: logger.debug("finish_streaming_tts error: %s", exc) - _finish_failed = True - - if _finish_failed: - # finish_streaming_tts() raised — never report full - # completion. If audio was already audible, report - # partial and preserve suppression so the gateway - # does not replay from the beginning. If no audio - # was audible, permit whole-file fallback. - if self._handle.audible: - self._partial = True - self._completed = False - self._suppress_whole_file = True - else: - self._completed = False - self._suppress_whole_file = False + self._settle(failed=True) await self._safe_abort("finish_streaming_tts failed") - elif self._handle.audible and not self._dropped: - self._completed = True - self._suppress_whole_file = True - elif self._handle.audible and self._dropped: - self._partial = True - self._completed = False - self._suppress_whole_file = True else: - self._completed = False - self._suppress_whole_file = False + self._settle(failed=False) except Exception as exc: logger.warning("streaming TTS consumer error: %s", exc) await self._safe_abort(str(exc)) @@ -316,17 +237,17 @@ class StreamingTTSConsumer: async def _synthesise_and_write(self, clause: str) -> None: """Synthesise one clause via the streamer and write PCM chunks.""" - if self._handle is None or self._handle.aborted: + if self._handle is None or self._handle.aborted or self._streamer is None: return - cleaned = self._strip_markdown_for_tts(clause) - if not cleaned or not cleaned.strip(): + if not cleaned.strip(): return - - if self._streamer is None: - return - - async for chunk in self._iter_stream_chunks(cleaned): + iterator = iter(self._streamer.stream(cleaned)) + while True: + # next() runs in a thread so a blocking provider never stalls the loop. + chunk = await asyncio.to_thread(next, iterator, _DONE) + if chunk is _DONE: + return if self._aborted or self._handle.aborted: return if not chunk: @@ -337,24 +258,6 @@ class StreamingTTSConsumer: self._handle.audible = True self._suppress_whole_file = True - async def _iter_stream_chunks(self, text: str): - """Yield provider PCM chunks one at a time without blocking the loop.""" - if self._streamer is None: - return - iterator = iter(self._streamer.stream(text)) - while True: - has_chunk, chunk = await asyncio.to_thread(self._next_stream_chunk, iterator) - if not has_chunk: - break - yield chunk - - @staticmethod - def _next_stream_chunk(iterator: Any) -> tuple[bool, Optional[bytes]]: - try: - return True, next(iterator) - except StopIteration: - return False, None - def _strip_markdown_for_tts(self, text: str) -> str: """Lazy-import and apply the TTS markdown stripper.""" if self._strip_markdown is None: @@ -377,19 +280,14 @@ class StreamingTTSConsumer: if self._handle: self._handle.aborted = True - # ------------------------------------------------------------------ - # Cancellation and completion - # ------------------------------------------------------------------ - def abort(self, reason: str = "cancelled") -> None: """Idempotent cancellation from any thread.""" with self._lock: if self._aborted: return self._aborted = True - # Guarantee the _ABORT sentinel reaches the queue. If the bounded - # queue is full, drain one item to make room — the sentinel must - # not be lost (#60671 hardening). + # The _ABORT sentinel is load-bearing and must reach the queue even when + # the bounded queue is full: evict an item to make room. for _attempt in range(3): try: self._queue.put_nowait(_ABORT) @@ -403,21 +301,15 @@ class StreamingTTSConsumer: logger.debug("streaming TTS _ABORT sentinel could not be enqueued") if self._handle is not None and not self._handle.aborted: try: - self._loop.call_soon_threadsafe( - asyncio.create_task, - self._safe_abort(reason), - ) + self._loop.call_soon_threadsafe(asyncio.create_task, self._safe_abort(reason)) except Exception: pass async def wait_complete(self, timeout: float = 10.0) -> bool: """Wait for the drain task to finish. Returns True only on full success.""" - if self._task is None: - return self._completed - try: - await asyncio.wait_for(asyncio.shield(self._task), timeout=timeout) - except (asyncio.TimeoutError, asyncio.CancelledError): - pass - except Exception: - pass + if self._task is not None: + try: + await asyncio.wait_for(asyncio.shield(self._task), timeout=timeout) + except (asyncio.CancelledError, Exception): + pass return self._completed diff --git a/gateway/turn_lease.py b/gateway/turn_lease.py index 0f640f9fd3..60e3441e39 100644 --- a/gateway/turn_lease.py +++ b/gateway/turn_lease.py @@ -1,51 +1,36 @@ """Per-session turn lease — serializes the [load history → run → flush] region. -Why this exists (#64934): the gateway's busy guards are keyed by ROUTING KEY -(``_active_sessions`` in the adapter, ``_running_agents`` in the runner), but -the durable transcript is owned by SESSION_ID — and ``switch_session()`` makes -the key→id mapping many-to-one (``/resume`` of a named session from a second -chat/topic, CLI-continuity rebinding, async-delegation completion pinning, -Telegram topic-binding tip-walks). Two routing keys mapped to one session_id -run concurrent turns on two different agent objects, so no per-key guard ever -sees the collision. The two turns then interleave their flushes on one -transcript: rows persist in completion order instead of arrival order, the -identity-marker dedup over shared history dicts can swallow a row outright, -and the second turn runs on a history base that never saw the first turn's -exchange — leaving a permanent ``user;user`` alternation wedge that -``repair_message_sequence`` re-repairs on every request forever. +Why: the gateway's busy guards are keyed by ROUTING KEY (adapter +``_active_sessions``, runner ``_running_agents``), but the durable transcript is +owned by SESSION_ID, and ``switch_session()`` makes key→id many-to-one (/resume +of a named session from a second chat, CLI-continuity rebinding, async-delegation +completion pinning, Telegram topic-binding tip-walks). Two routing keys on one +session_id ran concurrent turns on two agent objects that no per-key guard could +see; their flushes interleaved on one transcript (rows in completion order, +identity-marker dedup swallowing rows, a permanent ``user;user`` alternation +wedge that ``repair_message_sequence`` re-repaired forever). -The lease closes that route by serializing per RESOLVED session_id: it is -acquired after session resolution is final (post ``switch_session``/tip-walk), -immediately before the transcript load, and released in the dispatch layer's -``finally`` on every exit path. Same-key messages never reach the acquisition -point while a turn runs (both routing-key guards hold them), so the lock is -uncontended everywhere except the alias-key route — where the second turn now -waits for the first turn's flush and logs one WARNING naming the session and -both routing keys (pairing with the cross-agent tripwire in -``agent/agent_runtime_helpers.note_turn_start``). +The lease serializes per RESOLVED session_id: acquired after resolution is final +(post switch_session/tip-walk), immediately before the transcript load, and +released in the dispatch layer's ``finally`` on every exit path. Same-key +messages never reach acquisition while a turn runs (routing-key guards hold +them), so the lock is uncontended except on the alias-key route, where the +second turn waits for the first's flush and logs one WARNING naming the session +and both keys (pairing with ``agent_runtime_helpers.note_turn_start``). Safety properties: +- Generation-scoped, identity-checked, idempotent release: a token records its + owner (routing key, run generation) and only the exact current holder frees + the lease — a stale unwind can never release a newer turn's lease. +- Fail-closed on timeout: a timed-out waiter raises :class:`TurnLeaseTimeoutError` + and must be rejected with a visible resend notice; it never runs against the + still-held lease. +- Bounded registry: eviction only removes idle (unheld, uncontended) entries. -- **Generation-scoped, identity-checked release.** A token records its owner - (routing key, run generation) and release only frees the lease when that - exact token is the current holder — a stale unwind can never release a - newer turn's lease (the #28686 ownership lesson applied). Release is - idempotent. -- **Fail-closed on timeout.** A timed-out waiter raises - :class:`TurnLeaseTimeoutError` and must be rejected by the dispatch layer - with a visible resend notice. It never runs concurrently against the - still-held lease and therefore cannot defeat the serialization invariant. -- **Bounded registry.** The per-session lease map is size-capped; eviction - only ever removes idle (unheld, uncontended) entries, never a live lease. - -Known limits (deliberate, flagged on #64934): - -- A CLI process sharing the session via CLI-continuity is outside any - in-process lock — that pair needs a DB-level lease (separate design). -- Mid-turn compression rotation leaves a small alias window: the tip-walk can - resolve a fresh child id while the parent-holding turn is still in flight. - The mid-turn binding-sync sites are the right place to alias the lease in a - follow-up. +Known limits: a CLI process sharing the session via CLI-continuity is outside +any in-process lock (needs a DB-level lease); mid-turn compression rotation +leaves a small alias window, closed by :meth:`SessionTurnLeaseRegistry.rebind` +at the mid-turn binding-sync sites. """ import asyncio @@ -55,29 +40,26 @@ from typing import Dict, Optional logger = logging.getLogger(__name__) -# Upper bound on tracked per-session leases. Idle entries (no holder or -# pending acquire) are evicted oldest-first once the cap is reached; live -# leases are never evicted, so a burst of distinct sessions can transiently -# exceed the cap rather than break serialization. +# Cap on tracked per-session leases. Idle entries are evicted oldest-first; live +# leases never are, so a burst of distinct sessions may transiently exceed the cap +# rather than break serialization. DEFAULT_MAX_LEASES = 512 -# Fallback wait (seconds) when the caller passes no positive timeout. The -# gateway carries this independently through its internal -# HERMES_TURN_LEASE_TIMEOUT bridge because lease contention is not agent -# inactivity. A caller that reaches this bound must reject the turn rather than -# run it concurrently with the holder. -# Keep contention fail-closed, but never pin a sequential platform updater for -# minutes. A waiter that cannot acquire promptly is rejected with a resend -# notice; it is never authorized to run without serialization. +# Fallback wait (seconds) when the caller passes no positive timeout. The gateway +# carries this through its internal HERMES_TURN_LEASE_TIMEOUT bridge because lease +# contention is not agent inactivity. Fail-closed but short: never pin a +# sequential platform updater for minutes — a waiter that cannot acquire promptly +# is rejected with a resend notice, never authorized to run unserialized. DEFAULT_LEASE_WAIT = 5.0 -class TurnLeaseTimeoutError(TimeoutError): - """The session lease stayed held for the caller's full wait budget. +def _holder_desc(holder: Optional["TurnLeaseToken"]) -> tuple: + return (holder.owner_key, holder.generation) if holder else ("?", "?") - This is a fail-closed signal: the caller did not acquire the lease and - must not enter the transcript load/run/flush region for this turn. - """ + +class TurnLeaseTimeoutError(TimeoutError): + """The lease stayed held for the caller's full wait budget (fail-closed: the + caller must not enter the transcript load/run/flush region).""" def __init__( self, @@ -100,19 +82,13 @@ class TurnLeaseTimeoutError(TimeoutError): class TurnLeaseToken: """Handle returned by :meth:`SessionTurnLeaseRegistry.acquire`. - A timeout raises :class:`TurnLeaseTimeoutError` instead of returning a - token, so every token handed out is a held lease. ``released`` makes - release idempotent. + Every token handed out is a held lease (timeouts raise instead); ``released`` + makes release idempotent. """ __slots__ = ("session_id", "owner_key", "generation", "released") - def __init__( - self, - session_id: str, - owner_key: str, - generation: int, - ) -> None: + def __init__(self, session_id: str, owner_key: str, generation: int) -> None: self.session_id = session_id self.owner_key = owner_key self.generation = generation @@ -127,13 +103,7 @@ class TurnLeaseToken: class _SessionLease: - __slots__ = ( - "lock", - "holder", - "acquired_at", - "last_used", - "pending_acquires", - ) + __slots__ = ("lock", "holder", "acquired_at", "last_used", "pending_acquires") def __init__(self) -> None: self.lock = asyncio.Lock() @@ -144,20 +114,15 @@ class _SessionLease: @property def idle(self) -> bool: - """True when this lease can be evicted: nobody holds or awaits it.""" - return ( - self.holder is None - and not self.lock.locked() - and self.pending_acquires == 0 - ) + """True when evictable: nobody holds or awaits it.""" + return self.holder is None and not self.lock.locked() and self.pending_acquires == 0 class SessionTurnLeaseRegistry: """Asyncio lease per resolved session_id serializing transcript turns. - Process-local and single-event-loop by design — the same visibility scope - as the routing-key guards it extends. All methods must be called from the - gateway's event loop. + Process-local and single-event-loop by design — the same visibility scope as + the routing-key guards it extends. Call only from the gateway's event loop. """ def __init__(self, max_entries: int = DEFAULT_MAX_LEASES) -> None: @@ -171,16 +136,13 @@ class SessionTurnLeaseRegistry: lease = self._leases.get(session_id) if lease is None: self._evict_idle() - lease = _SessionLease() - self._leases[session_id] = lease + lease = self._leases[session_id] = _SessionLease() lease.last_used = time.time() return lease def _evict_idle(self) -> None: - """Drop oldest idle entries so a new lease fits under the cap. - - Never evicts a held or contended lease — correctness beats the cap. - """ + """Drop oldest idle entries so a new lease fits under the cap. Never + evicts a held or contended lease — correctness beats the cap.""" overflow = len(self._leases) - self._max_entries + 1 if overflow <= 0: return @@ -201,10 +163,8 @@ class SessionTurnLeaseRegistry: ) -> Optional[TurnLeaseToken]: """Acquire the turn lease for ``session_id``, waiting if held. - Returns a held :class:`TurnLeaseToken`. Raises - :class:`TurnLeaseTimeoutError` when the wait budget expires; the caller - must reject rather than enter the serialized region. Returns ``None`` - for a falsy ``session_id``. + Raises :class:`TurnLeaseTimeoutError` when the wait budget expires (the + caller must reject the turn). Returns None for a falsy ``session_id``. """ if not session_id: return None @@ -213,7 +173,6 @@ class SessionTurnLeaseRegistry: lease = self._get_or_create(session_id) if lease.lock.locked(): - holder = lease.holder logger.warning( "turn lease contention on session %s: routing key %s (gen %s) " "waiting behind in-flight turn held by routing key %s (gen %s, " @@ -223,21 +182,19 @@ class SessionTurnLeaseRegistry: session_id, owner_key, generation, - holder.owner_key if holder else "?", - holder.generation if holder else "?", + *_holder_desc(lease.holder), time.time() - lease.acquired_at if lease.acquired_at else -1.0, ) # Lock.release() wakes a waiter while leaving the lock momentarily - # unlocked. Track every in-progress acquire across that handoff so + # unlocked. Count every in-progress acquire across that handoff so # eviction cannot orphan the old lock and create a second lock for the - # same session. Count even apparently-uncontended acquires: wait_for() - # may schedule them before the underlying lock coroutine runs. + # same session — even apparently-uncontended ones, since wait_for() may + # schedule them before the underlying lock coroutine runs. lease.pending_acquires += 1 try: await asyncio.wait_for(lease.lock.acquire(), timeout=wait) except asyncio.TimeoutError: - holder = lease.holder logger.error( "turn lease wait timed out after %.0fs on session %s " "(waiter: routing key %s gen %s; holder: routing key %s " @@ -247,47 +204,37 @@ class SessionTurnLeaseRegistry: session_id, owner_key, generation, - holder.owner_key if holder else "?", - holder.generation if holder else "?", + *_holder_desc(lease.holder), ) raise TurnLeaseTimeoutError( - session_id, - owner_key=owner_key, - generation=generation, - wait_seconds=wait, + session_id, owner_key=owner_key, generation=generation, wait_seconds=wait ) from None finally: lease.pending_acquires -= 1 - # The lock is held and there is no await before holder publication, so - # the lease cannot become evictable after the pending count is cleared. + # Lock held and no await before holder publication, so the lease cannot + # become evictable after the pending count is cleared. lease.holder = token - lease.acquired_at = time.time() - lease.last_used = lease.acquired_at + lease.acquired_at = lease.last_used = time.time() return token def rebind(self, token: Optional[TurnLeaseToken], new_session_id: str) -> bool: """Alias a HELD lease onto ``new_session_id`` after mid-turn rotation. - Compression can rotate the durable session_id while a turn is in - flight (session-hygiene pre-compression, in-agent compression). The - turn's flush then targets the NEW id — so the serialization boundary - must follow it, or an alias routing key resolving the new id (e.g. a - topic tip-walk landing on the fresh child) could start a concurrent - turn the lease never sees. This closes the rotation-alias window - flagged on #64934. + Compression can rotate the durable session_id mid-turn (session-hygiene + pre-compression, in-agent compression); the flush then targets the NEW + id, so the serialization boundary must follow it or an alias key + resolving the new id could start a turn the lease never sees. - Mechanism: the SAME ``_SessionLease`` object is registered under the - new id (the old mapping stays until it goes idle and is evicted), so - acquirers on either id serialize against one lock — no lock state is - moved, no asyncio internals are touched. Only the current holder can - rebind (identity-checked like release), and the token follows to the - new id so release frees the shared object. + Mechanism: the SAME ``_SessionLease`` is registered under the new id (the + old mapping stays until idle and evicted), so acquirers on either id + serialize on one lock — no lock state moves. Only the current holder can + rebind (identity-checked like release), and the token follows so release + frees the shared object. - Edge: if the new id already has a live lease of its own (another - turn is running on the target session), the two serialization - domains cannot be merged mid-wait — log loudly and keep the token on - the old id. Fail-open, never deadlock: a holder cannot wait mid-turn. + Edge: if the new id already has a live lease of its own, the two domains + cannot be merged mid-wait — log loudly and keep the token on the old id. + Fail-open, never deadlock: a holder cannot wait mid-turn. """ if ( token is None @@ -302,7 +249,6 @@ class SessionTurnLeaseRegistry: existing = self._leases.get(new_session_id) if existing is not None and existing is not lease and not existing.idle: - holder = existing.holder logger.warning( "turn lease rebind blocked: session %s rotated to %s mid-turn " "(holder: routing key %s gen %s) but the target session's " @@ -313,8 +259,7 @@ class SessionTurnLeaseRegistry: new_session_id, token.owner_key, token.generation, - holder.owner_key if holder else "?", - holder.generation if holder else "?", + *_holder_desc(existing.holder), new_session_id, ) return False @@ -325,12 +270,10 @@ class SessionTurnLeaseRegistry: return True def release(self, token: Optional[TurnLeaseToken]) -> bool: - """Release ``token``'s lease. Idempotent; ownership-checked. + """Release ``token``'s lease. Idempotent; ownership-checked. - Returns True only when this exact token was the current holder and - the lock was freed. A re-release or a stale token whose slot has - since been granted to a newer turn are both safe no-ops — a stale - unwind can never release a newer turn's lease. + True only when this exact token was the current holder. A re-release or + a stale token whose slot went to a newer turn are safe no-ops. """ if token is None or token.released: return False diff --git a/tests/gateway/relay/test_relay_multiplatform.py b/tests/gateway/relay/test_relay_multiplatform.py index 59516aadb7..2302f2b09a 100644 --- a/tests/gateway/relay/test_relay_multiplatform.py +++ b/tests/gateway/relay/test_relay_multiplatform.py @@ -3,7 +3,6 @@ Covers the agent half of Shape A (gateway-gateway D-Q1.5b.1 / D-Q1.5c): - relay_platform_identities() parsing the GATEWAY_RELAY_PLATFORMS list + GATEWAY_RELAY_BOT_IDS keyed map (the cut-over shape — no scalar fallback), - - relay_bot_username() reading the per-platform username, - self_provision_relay() looping one /relay/provision POST per platform under one gatewayId + one secret, partial-failure-tolerant, - the RelayAdapter stamping the per-frame egress platform on outbound from the @@ -55,9 +54,6 @@ def test_identities_multi_platform_keyed_map(monkeypatch): assert relay.relay_platform_identities() == [("discord", "app-1"), ("telegram", "bot-9")] # The PRIMARY is the first listed platform. assert relay.relay_platform_identity() == ("discord", "app-1") - # Username folded into the per-platform entry; the leading @ is stripped. - assert relay.relay_bot_username("telegram") == "my_bot" - assert relay.relay_bot_username("discord") is None def test_bot_ids_malformed_json_degrades_to_empty(monkeypatch): diff --git a/tests/gateway/test_browser_control_broker_hardening.py b/tests/gateway/test_browser_control_broker_hardening.py index 912ebb1d3c..f76c88c0d3 100644 --- a/tests/gateway/test_browser_control_broker_hardening.py +++ b/tests/gateway/test_browser_control_broker_hardening.py @@ -402,7 +402,7 @@ def test_session_lane_replacement_and_owner_detach_are_scoped(): transport_family="cloud-ticket-ws", ) == other - assert broker.detach_owner("owner-shared") == 1 + broker.detach(second, owner="owner-shared") assert broker.scope_for_session( session_id="session-fixture", principal_id="principal-fixture", @@ -413,7 +413,6 @@ def test_session_lane_replacement_and_owner_detach_are_scoped(): principal_id="principal-fixture", transport_family="cloud-ticket-ws", ) == other - assert broker.detach_owner("missing-owner") == 0 broker.reset() assert broker.scope_for_session( @@ -476,7 +475,7 @@ def test_transport_teardown_can_cancel_waiters_without_writing_to_closing_peer() scope = _scope() thread, outcome, frames = _start_pending(broker, scope) - assert broker.detach_owner("owner-fixture", notify_controller=False) == 1 + broker.detach(scope, owner="owner-fixture", notify_controller=False) thread.join(timeout=1.0) assert isinstance(outcome.get("error"), ControllerCancelled) diff --git a/tests/gateway/test_delivery_ledger.py b/tests/gateway/test_delivery_ledger.py index e371ea80e2..28fb20e1cc 100644 --- a/tests/gateway/test_delivery_ledger.py +++ b/tests/gateway/test_delivery_ledger.py @@ -610,8 +610,6 @@ class TestAttemptsOnlySpentOnRealSends: _orphan("ob-1") assert dl.sweep_recoverable(deliverable_platforms={"discord"}) == [] - row = dl.debug_rows() - assert "abandoned" not in row with dl._connect() as conn: state, attempts = conn.execute( "SELECT state, attempts FROM delivery_obligations " diff --git a/tests/gateway/test_delivery_ledger_fd_leak.py b/tests/gateway/test_delivery_ledger_fd_leak.py index 8d1d6ae64e..4a5bbf4da1 100644 --- a/tests/gateway/test_delivery_ledger_fd_leak.py +++ b/tests/gateway/test_delivery_ledger_fd_leak.py @@ -76,7 +76,6 @@ def test_ledger_operations_close_every_connection(monkeypatch, tmp_path): dl.mark_attempting(oid) dl.mark_delivered(oid) dl.sweep_recoverable() - dl.debug_rows() assert opened, "expected at least one connection to be opened" assert len(opened) == len(closed) diff --git a/tests/gateway/test_hosted_room_peer.py b/tests/gateway/test_hosted_room_peer.py index 49698c4cc4..8b4f6c160d 100644 --- a/tests/gateway/test_hosted_room_peer.py +++ b/tests/gateway/test_hosted_room_peer.py @@ -17,13 +17,11 @@ from gateway.hosted_room_peer import ( HostedRoomGrantError, HostedRoomPeerError, PROTOCOL_VERSION, - RoomLinkProbe, catalog_mapping, derive_room_grant_secret, gateway_room_grant_secret, issue_room_grant, local_room_link_endpoint, - select_room_link, verify_room_grant, ) from hermes_constants import reset_hermes_home_override, set_hermes_home_override @@ -292,33 +290,6 @@ def test_room_grant_fails_closed_for_tamper_expiry_and_permission(): verify_room_grant(SECRET, token[:-1] + "A", dispatch, now=105) -def test_link_selection_prefers_safe_direct_then_overlay_then_relay_then_pull(): - selected = select_room_link( - [ - RoomLinkProbe("relay", True, True, 10), - RoomLinkProbe("direct", True, True, 50), - RoomLinkProbe("overlay", True, True, 5), - RoomLinkProbe("pull", True, True, 1), - ], - desktop_available=False, - ) - assert selected is not None - assert selected.mode == "direct" - - -def test_link_selection_never_falls_back_to_unencrypted_route(): - assert ( - select_room_link( - [RoomLinkProbe("direct", True, False, 1)], - desktop_available=False, - ) - is None - ) - fallback = select_room_link([], desktop_available=True) - assert fallback is not None - assert fallback.mode == "desktop" - - def test_local_catalog_is_honest_for_app_managed_process(monkeypatch): from gateway.hosted_room_peer import local_catalog_mapping diff --git a/tests/gateway/test_session_db_recovery.py b/tests/gateway/test_session_db_recovery.py index d29ca76431..10b3f05ae8 100644 --- a/tests/gateway/test_session_db_recovery.py +++ b/tests/gateway/test_session_db_recovery.py @@ -33,7 +33,6 @@ def test_failed_open_obeys_backoff_then_recovers() -> None: return handle assert cache.get(path, opener) is None - assert cache.status_for(path) == "unavailable" clock.now = 1.99 assert cache.get(path, opener) is None assert calls == 1 @@ -42,7 +41,6 @@ def test_failed_open_obeys_backoff_then_recovers() -> None: assert cache.get(path, opener) is handle assert cache.get(path, opener) is handle assert calls == 2 - assert cache.status_for(path) == "ok" def test_retry_is_single_flight_for_concurrent_callers() -> None: @@ -74,7 +72,6 @@ def test_retry_is_single_flight_for_concurrent_callers() -> None: # and keep using the fallback rather than opening or blocking behind it. assert cache.get(path, opener) is None assert calls == 2 - assert cache.status_for(path) == "retrying" release.set() thread.join(timeout=5) @@ -196,7 +193,6 @@ def test_non_cacheable_guard_is_retried_immediately() -> None: except RuntimeError: pass assert calls == 2 - assert cache.status_for(path) == "unknown" def test_close_all_rejects_and_closes_inflight_success() -> None: diff --git a/tests/gateway/test_streaming_tts_consumer.py b/tests/gateway/test_streaming_tts_consumer.py index a614c949f8..69218d78ee 100644 --- a/tests/gateway/test_streaming_tts_consumer.py +++ b/tests/gateway/test_streaming_tts_consumer.py @@ -171,7 +171,6 @@ def _make_consumer(adapter, chat_id, loop, streamer): consumer = StreamingTTSConsumer.__new__(StreamingTTSConsumer) consumer._adapter = adapter consumer._chat_id = chat_id - consumer._tts_config = {} consumer._loop = loop consumer._metadata = None consumer._audio_format = AudioFormat( @@ -183,7 +182,6 @@ def _make_consumer(adapter, chat_id, loop, streamer): consumer._chunker = SentenceChunker() consumer._queue = queue.Queue(maxsize=256) consumer._handle = None - consumer._started = False consumer._completed = False consumer._partial = False consumer._aborted = False diff --git a/tests/hermes_cli/test_gateway_restart_loop.py b/tests/hermes_cli/test_gateway_restart_loop.py index a005b9c2e8..d310d7b4ca 100644 --- a/tests/hermes_cli/test_gateway_restart_loop.py +++ b/tests/hermes_cli/test_gateway_restart_loop.py @@ -1750,17 +1750,6 @@ class TestRestartLoopGuard: import gateway.restart_loop_guard as rlg rlg.clear() - - - - def test_is_tripped_reads_without_recording(self): - import gateway.restart_loop_guard as rlg - rlg.record_restart_interrupted_boot(60, now=1000.0) - rlg.record_restart_interrupted_boot(60, now=1001.0) - assert rlg.is_restart_loop_tripped(3, 60, now=1002.0) is False - rlg.record_restart_interrupted_boot(60, now=1002.0) - assert rlg.is_restart_loop_tripped(3, 60, now=1003.0) is True - def test_clear_resets(self): import gateway.restart_loop_guard as rlg rlg.check_and_record(3, 60, now=1000.0) @@ -1794,7 +1783,6 @@ class TestRestartLoopGuard: rlg.check_and_record(3, 60, now=1150.0) # 1h later: unrelated restart, chain reset to a single boot. assert rlg.check_and_record(3, 60, now=4800.0) is False - assert rlg.is_restart_loop_tripped(3, 60, now=4801.0) is False def test_fast_respawn_loop_still_trips(self): """#30719 regression: the original ~10s loop must keep tripping.""" @@ -1823,7 +1811,6 @@ class TestRestartLoopGuard: import gateway.restart_loop_guard as rlg for ts in (1000.0, 1150.0, 1300.0, 1450.0): assert rlg.check_and_record(0, 60, now=ts) is False - assert rlg.is_restart_loop_tripped(0, 60, now=1451.0) is False class TestTerminalToolGatewayLifecycleGuardRemote: """Remote-backend and two-session cwd regression coverage."""